diff options
| author |  RaNaN <Mast3rRaNaN@hotmail.de>
| 2013-06-09 18:10:22 +0200 |
|---|---|---|
| committer | RaNaN <Mast3rRaNaN@hotmail.de>
| 2013-06-09 18:10:23 +0200 |
| commit | 16af85004c84d0d6c626b4f8424ce9647669a0c1 (patch) | |
| tree | 025d479862d376dbc17e934f4ed20031c8cd97d1 /module | |
| parent | adapted to jshint config (diff) | |
| download | pyload-16af85004c84d0d6c626b4f8424ce9647669a0c1.tar.xz | |
moved everything from module to pyload
Diffstat (limited to 'module')
575 files changed, 0 insertions, 68328 deletions
diff --git a/module/AccountManager.py b/module/AccountManager.py deleted file mode 100644 index 5cdcfb57a..000000000 --- a/module/AccountManager.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -from threading import Lock -from random import choice - -from module.utils import lock, json - -class AccountManager: - """manages all accounts""" - - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.loadAccounts() - - def loadAccounts(self): - """loads all accounts available""" - - self.accounts = {} - - for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): - # put into options as used in other context - options = json.loads(options) if options else {} - options["activated"] = activated - - self.createAccount(plugin, loginname, password, options) - - - def iterAccounts(self): - """ yields login, account for all accounts""" - for name, data in self.accounts.iteritems(): - for login, account in data.iteritems(): - yield login, account - - def saveAccounts(self): - """save all account information""" - # TODO: multi user - # TODO: activated - - data = [] - for name, plugin in self.accounts.iteritems(): - data.extend( - [(name, acc.loginname, 1 if acc.activated else 0, acc.password, json.dumps(acc.options)) for acc in - plugin.itervalues()]) - self.core.db.saveAccounts(data) - - def createAccount(self, plugin, loginname, password, options): - klass = self.core.pluginManager.loadClass("accounts", plugin) - if not klass: - self.core.log.warning(_("Unknown account plugin %s") % plugin) - return - - if plugin not in self.accounts: - self.accounts[plugin] = {} - - self.core.log.debug("Create account %s:%s" % (plugin, loginname)) - - self.accounts[plugin][loginname] = klass(self, loginname, password, options) - - - def getAccount(self, plugin, user): - return self.accounts[plugin].get(user, None) - - @lock - def updateAccount(self, plugin, user, password=None, options={}): - """add or update account""" - if plugin in self.accounts and user in self.accounts[plugin]: - acc = self.accounts[plugin][user] - updated = acc.update(password, options) - - self.saveAccounts() - if updated: acc.scheduleRefresh(force=True) - else: - self.createAccount(plugin, user, password, options) - self.saveAccounts() - - self.sendChange(plugin, user) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - if plugin in self.accounts and user in self.accounts[plugin]: - del self.accounts[plugin][user] - self.core.db.removeAccount(plugin, user) - self.core.eventManager.dispatchEvent("account:deleted", plugin, user) - else: - self.core.log.debug("Remove non existent account %s %s" % (plugin, user)) - - - @lock - def getAccountForPlugin(self, plugin): - if plugin in self.accounts: - accs = [x for x in self.accounts[plugin].values() if x.isUsable()] - if accs: return choice(accs) - - return None - - @lock - def getAllAccounts(self, refresh=False): - """ Return account info, refresh afterwards if needed - - :param refresh: - :return: - """ - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) - - # load unavailable account info - for p_dict in self.accounts.itervalues(): - for acc in p_dict.itervalues(): - acc.getAccountInfo() - - return self.accounts - - def refreshAllAccounts(self): - """ Force a refresh of every account """ - for p in self.accounts.itervalues(): - for acc in p.itervalues(): - acc.getAccountInfo(True) - - def sendChange(self, plugin, name): - self.core.eventManager.dispatchEvent("account:updated", plugin, name)
\ No newline at end of file diff --git a/module/AddonManager.py b/module/AddonManager.py deleted file mode 100644 index 9a8ad44ac..000000000 --- a/module/AddonManager.py +++ /dev/null @@ -1,248 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import __builtin__ - -from thread import start_new_thread -from threading import RLock - -from types import MethodType - -from module.threads.AddonThread import AddonThread -from module.PluginManager import literal_eval -from utils import lock, to_string - -class AddonManager: - """ Manages addons, loading, unloading. """ - - def __init__(self, core): - self.core = core - self.config = self.core.config - - __builtin__.addonManager = self #needed to let addons register themselves - - self.log = self.core.log - # TODO: multiuser, addons can store the user itself, probably not needed here - self.plugins = {} - self.methods = {} # dict of names and list of methods usable by rpc - self.events = {} # Contains event that will be registered - - self.lock = RLock() - self.createIndex() - - # manage addons on config change - self.addEvent("config:changed", self.manageAddons) - - @lock - def callInHooks(self, event, eventName, *args): - """ Calls a method in all addons and catch / log errors""" - for plugin in self.plugins.itervalues(): - self.call(plugin, event, *args) - self.dispatchEvent(eventName, *args) - - def call(self, addon, f, *args): - try: - func = getattr(addon, f) - return func(*args) - except Exception, e: - addon.logError(_("Error when executing %s" % f), e) - self.core.print_exc() - - def addRPC(self, plugin, func, doc): - doc = doc.strip() if doc else "" - - if plugin in self.methods: - self.methods[plugin][func] = doc - else: - self.methods[plugin] = {func: doc} - - def callRPC(self, plugin, func, args): - if not args: args = [] - else: - args = literal_eval(args) - - plugin = self.plugins[plugin] - f = getattr(plugin, func) - return f(*args) - - @lock - def createIndex(self): - active = [] - deactive = [] - - for pluginname in self.core.pluginManager.getPlugins("addons"): - try: - # check first for builtin plugin - attrs = self.core.pluginManager.loadAttributes("addons", pluginname) - internal = attrs.get("internal", False) - - if internal or self.core.config.get(pluginname, "activated"): - pluginClass = self.core.pluginManager.loadClass("addons", pluginname) - - if not pluginClass: continue - - plugin = pluginClass(self.core, self) - self.plugins[pluginClass.__name__] = plugin - - # hide internals from printing - if not internal and plugin.isActivated(): - active.append(pluginClass.__name__) - else: - self.log.debug("Loaded internal plugin: %s" % pluginClass.__name__) - else: - deactive.append(pluginname) - - - except: - self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) - self.core.print_exc() - - self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivate addons: %s") % ", ".join(sorted(deactive))) - - def manageAddons(self, plugin, name, value): - # TODO: user - - # check if section was a plugin - if plugin not in self.core.pluginManager.getPlugins("addons"): - return - - if name == "activated" and value: - self.activateAddon(plugin) - elif name == "activated" and not value: - self.deactivateAddon(plugin) - - @lock - def activateAddon(self, plugin): - #check if already loaded - if plugin in self.plugins: - return - - pluginClass = self.core.pluginManager.loadClass("addons", plugin) - - if not pluginClass: return - - self.log.debug("Plugin loaded: %s" % plugin) - - plugin = pluginClass(self.core, self) - self.plugins[pluginClass.__name__] = plugin - - # active the addon in new thread - start_new_thread(plugin.activate, tuple()) - self.registerEvents() # TODO: BUG: events will be destroyed and not re-registered - - @lock - def deactivateAddon(self, plugin): - if plugin not in self.plugins: - return - else: - addon = self.plugins[plugin] - - if addon.__internal__: return - - self.call(addon, "deactivate") - self.log.debug("Plugin deactivated: %s" % plugin) - - #remove periodic call - self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(addon.cb)) - del self.plugins[addon.__name__] - - #remove event listener - for f in dir(addon): - if f.startswith("__") or type(getattr(addon, f)) != MethodType: - continue - self.core.eventManager.removeFromEvents(getattr(addon, f)) - - def activateAddons(self): - self.log.info(_("Activating Plugins...")) - for plugin in self.plugins.itervalues(): - if plugin.isActivated(): - self.call(plugin, "activate") - - self.registerEvents() - - def deactivateAddons(self): - """ Called when core is shutting down """ - self.log.info(_("Deactivating Plugins...")) - for plugin in self.plugins.itervalues(): - self.call(plugin, "deactivate") - - def downloadPreparing(self, pyfile): - self.callInHooks("downloadPreparing", "download:preparing", pyfile) - - def downloadFinished(self, pyfile): - self.callInHooks("downloadFinished", "download:finished", pyfile) - - def downloadFailed(self, pyfile): - self.callInHooks("downloadFailed", "download:failed", pyfile) - - def packageFinished(self, package): - self.callInHooks("packageFinished", "package:finished", package) - - def beforeReconnecting(self, ip): - self.callInHooks("beforeReconnecting", "reconnecting:before", ip) - - def afterReconnecting(self, ip): - self.callInHooks("afterReconnecting", "reconnecting:after", ip) - - @lock - def startThread(self, function, *args, **kwargs): - AddonThread(self.core.threadManager, function, args, kwargs) - - def activePlugins(self): - """ returns all active plugins """ - return [x for x in self.plugins.itervalues() if x.isActivated()] - - def getAllInfo(self): - """returns info stored by addon plugins""" - info = {} - for name, plugin in self.plugins.iteritems(): - if plugin.info: - #copy and convert so str - info[name] = dict( - [(x, to_string(y)) for x, y in plugin.info.iteritems()]) - return info - - def getInfo(self, plugin): - info = {} - if plugin in self.plugins and self.plugins[plugin].info: - info = dict([(x, to_string(y)) - for x, y in self.plugins[plugin].info.iteritems()]) - - return info - - def addEventListener(self, plugin, func, event): - """ add the event to the list """ - if plugin not in self.events: - self.events[plugin] = [] - self.events[plugin].append((func, event)) - - def registerEvents(self): - """ actually register all saved events """ - for name, plugin in self.plugins.iteritems(): - if name in self.events: - for func, event in self.events[name]: - self.addEvent(event, getattr(plugin, func)) - # clean up - del self.events[name] - - def addEvent(self, *args): - self.core.eventManager.addEvent(*args) - - def dispatchEvent(self, *args): - self.core.eventManager.dispatchEvent(*args) - diff --git a/module/Api.py b/module/Api.py deleted file mode 100644 index bfeeff10c..000000000 --- a/module/Api.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import re -from types import MethodType - -from remote.apitypes import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -perm_map = {} - -# decorator only called on init, never initialized, so has no effect on runtime -def RequirePerm(bits): - class _Dec(object): - def __new__(cls, func, *args, **kwargs): - perm_map[func.__name__] = bits - return func - - return _Dec - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) - -stateMap = { - DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), - DownloadState.Finished: frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), - DownloadState.Unfinished: None, # set below - DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted)), - DownloadState.Unmanaged: None, #TODO -} - -stateMap[DownloadState.Unfinished] = frozenset(stateMap[DownloadState.All].difference(stateMap[DownloadState.Finished])) - -def state_string(state): - return ",".join(str(x) for x in stateMap[state]) - -from datatypes.User import User - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api, websocket backend or json api. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via web interface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - - EXTERNAL = Iface # let the json api know which methods are external - EXTEND = False # only extendable when set too true - - def __init__(self, core): - self.core = core - self.user_apis = {} - - @property - def user(self): - return None #TODO return default user? - - @property - def primaryUID(self): - return self.user.primary if self.user else None - - @classmethod - def initComponents(cls): - # Allow extending the api - # This prevents unintentionally registering of the components, - # but will only work once when they are imported - cls.EXTEND = True - # Import all Api modules, they register themselves. - import module.api - # they will vanish from the namespace afterwards - - - @classmethod - def extend(cls, api): - """Takes all params from api and extends cls with it. - api class can be removed afterwards - - :param api: Class with methods to extend - """ - if cls.EXTEND: - for name, func in api.__dict__.iteritems(): - if name.startswith("_"): continue - setattr(cls, name, MethodType(func, None, cls)) - - return cls.EXTEND - - def withUserContext(self, uid): - """ Returns a proxy version of the api, to call method in user context - - :param uid: user or userData instance or uid - :return: :class:`UserApi` - """ - if isinstance(uid, User): - uid = uid.uid - - if uid not in self.user_apis: - user = self.core.db.getUserData(uid=uid) - if not user: #TODO: anonymous user? - return None - - self.user_apis[uid] = UserApi(self.core, User.fromUserData(self, user)) - - return self.user_apis[uid] - - - ############################# - # Auth+User Information - ############################# - - # TODO - - @RequirePerm(Permission.All) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return True if self.checkAuth(username, password, remoteip) else False - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - self.core.log.info(_("User '%s' tries to log in") % username) - - return self.core.db.checkAuth(username, password) - - def isAuthorized(self, func, user): - """checks if the user is authorized for specific method - - :param func: function name - :param user: `User` - :return: boolean - """ - if user.isAdmin(): - return True - elif func in perm_map and user.hasPermission(perm_map[func]): - return True - else: - return False - - # TODO - @RequirePerm(Permission.All) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if not user: - raise UserDoesNotExists(username) - - return user.toUserData() - - def getAllUserData(self): - """returns all known user and info""" - return self.core.db.getAllUserData() - - def changePassword(self, username, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(username, oldpw, newpw) - - def setUserPermission(self, user, permission, role): - self.core.db.setPermission(user, permission) - self.core.db.setRole(user, role) - - -class UserApi(Api): - """ Proxy object for api that provides all methods in user context """ - - def __init__(self, core, user): - # No need to init super class - self.core = core - self._user = user - - def withUserContext(self, uid): - raise Exception("Not allowed") - - @property - def user(self): - return self._user
\ No newline at end of file diff --git a/module/FileManager.py b/module/FileManager.py deleted file mode 100644 index 7b14613f7..000000000 --- a/module/FileManager.py +++ /dev/null @@ -1,582 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import time -from ReadWriteLock import ReadWriteLock - -from module.utils import lock, read_lock - -from Api import PackageStatus, DownloadStatus as DS, TreeCollection, PackageDoesNotExists -from datatypes.PyFile import PyFile -from datatypes.PyPackage import PyPackage, RootPackage - -# invalidates the cache -def invalidate(func): - def new(*args): - args[0].downloadstats = {} - args[0].queuestats = {} - args[0].jobCache = {} - return func(*args) - - return new - -# TODO: needs to be replaced later -OWNER = 0 - -class FileManager: - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - ROOT_PACKAGE = -1 - - def __init__(self, core): - """Constructor""" - self.core = core - self.evm = core.eventManager - - # translations - self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), - _("finished"), _("skipped"), _("failed"), _("starting"), - _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), - _("decrypting"), _("processing"), _("custom"), _("unknown")] - - self.files = {} # holds instances for files - self.packages = {} # same for packages - - self.jobCache = {} - - # locking the caches, db is already locked implicit - self.lock = ReadWriteLock() - #self.lock._Verbose__verbose = True - - self.downloadstats = {} # cached dl stats - self.queuestats = {} # cached queue stats - - self.db = self.core.db - - def save(self): - """saves all data to backend""" - self.db.commit() - - @read_lock - def syncSave(self): - """saves all data to backend and waits until all data are written""" - for pyfile in self.files.values(): - pyfile.sync() - - for pypack in self.packages.values(): - pypack.sync() - - self.db.syncSave() - - def cachedFiles(self): - return self.files.values() - - def cachedPackages(self): - return self.packages.values() - - def getCollector(self): - pass - - @invalidate - def addLinks(self, data, package): - """Add links, data = (plugin, url) tuple. Internal method should use API.""" - self.db.addLinks(data, package, OWNER) - self.evm.dispatchEvent("package:updated", package) - - - @invalidate - def addPackage(self, name, folder, root, password, site, comment, paused): - """Adds a package to database""" - pid = self.db.addPackage(name, folder, root, password, site, comment, - PackageStatus.Paused if paused else PackageStatus.Ok, OWNER) - p = self.db.getPackageInfo(pid) - - self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder) - return pid - - - @lock - def getPackage(self, pid): - """return package instance""" - if pid == self.ROOT_PACKAGE: - return RootPackage(self, OWNER) - elif pid in self.packages: - pack = self.packages[pid] - pack.timestamp = time() - return pack - else: - info = self.db.getPackageInfo(pid, False) - if not info: return None - - pack = PyPackage.fromInfoData(self, info) - self.packages[pid] = pack - - return pack - - @read_lock - def getPackageInfo(self, pid): - """returns dict with package information""" - if pid == self.ROOT_PACKAGE: - pack = RootPackage(self, OWNER).toInfoData() - elif pid in self.packages: - pack = self.packages[pid].toInfoData() - pack.stats = self.db.getStatsForPackage(pid) - else: - pack = self.db.getPackageInfo(pid) - - if not pack: return None - - # todo: what does this todo mean?! - #todo: fill child packs and files - packs = self.db.getAllPackages(root=pid) - if pid in packs: del packs[pid] - pack.pids = packs.keys() - - files = self.db.getAllFiles(package=pid) - pack.fids = files.keys() - - return pack - - @lock - def getFile(self, fid): - """returns pyfile instance""" - if fid in self.files: - return self.files[fid] - else: - info = self.db.getFileInfo(fid) - if not info: return None - - f = PyFile.fromInfoData(self, info) - self.files[fid] = f - return f - - @read_lock - def getFileInfo(self, fid): - """returns dict with file information""" - if fid in self.files: - return self.files[fid].toInfoData() - - return self.db.getFileInfo(fid) - - @read_lock - def getTree(self, pid, full, state, search=None): - """ return a TreeCollection and fill the info data of containing packages. - optional filter only unfnished files - """ - view = TreeCollection(pid) - - # for depth=1, we don't need to retrieve all files/packages - root = pid if not full else None - - packs = self.db.getAllPackages(root) - files = self.db.getAllFiles(package=root, state=state, search=search) - - # updating from cache - for fid, f in self.files.iteritems(): - if fid in files: - files[fid] = f.toInfoData() - - # foreign pid, don't overwrite local pid ! - for fpid, p in self.packages.iteritems(): - if fpid in packs: - # copy the stats data - stats = packs[fpid].stats - packs[fpid] = p.toInfoData() - packs[fpid].stats = stats - - # root package is not in database, create an instance - if pid == self.ROOT_PACKAGE: - view.root = RootPackage(self, OWNER).toInfoData() - packs[self.ROOT_PACKAGE] = view.root - elif pid in packs: - view.root = packs[pid] - else: # package does not exists - return view - - # linear traversal over all data - for fpid, p in packs.iteritems(): - if p.fids is None: p.fids = [] - if p.pids is None: p.pids = [] - - root = packs.get(p.root, None) - if root: - if root.pids is None: root.pids = [] - root.pids.append(fpid) - - for fid, f in files.iteritems(): - p = packs.get(f.package, None) - if p: p.fids.append(fid) - - - # cutting of tree is not good in runtime, only saves bandwidth - # need to remove some entries - if full and pid > -1: - keep = [] - queue = [pid] - while queue: - fpid = queue.pop() - keep.append(fpid) - queue.extend(packs[fpid].pids) - - # now remove unneeded data - for fpid in packs.keys(): - if fpid not in keep: - del packs[fpid] - - for fid, f in files.items(): - if f.package not in keep: - del files[fid] - - #remove root - del packs[pid] - view.files = files - view.packages = packs - - return view - - - @lock - def getJob(self, occ): - """get suitable job""" - - #TODO only accessed by one thread, should not need a lock - #TODO needs to be approved for new database - #TODO clean mess - #TODO improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - - return pyfile - - def getDownloadStats(self, user=None): - """ return number of downloads """ - if user not in self.downloadstats: - self.downloadstats[user] = self.db.downloadstats(user) - - return self.downloadstats[user] - - def getQueueStats(self, user=None, force=False): - """number of files that have to be processed, failed files will not be included""" - if user not in self.queuestats or force: - self.queuestats[user] = self.db.queuestats(user) - - return self.queuestats[user] - - def scanDownloadFolder(self): - pass - - @lock - @invalidate - def deletePackage(self, pid): - """delete package and all contained links""" - - p = self.getPackage(pid) - if not p: return - - oldorder = p.packageorder - root = p.root - - for pyfile in self.cachedFiles(): - if pyfile.packageid == pid: - pyfile.abortDownload() - - # TODO: delete child packages - # TODO: delete folder - - self.db.deletePackage(pid) - self.releasePackage(pid) - - for pack in self.cachedPackages(): - if pack.root == root and pack.packageorder > oldorder: - pack.packageorder -= 1 - - self.evm.dispatchEvent("package:deleted", pid) - - @lock - @invalidate - def deleteFile(self, fid): - """deletes links""" - - f = self.getFile(fid) - if not f: return - - pid = f.packageid - order = f.fileorder - - if fid in self.core.threadManager.processingIds(): - f.abortDownload() - - # TODO: delete real file - - self.db.deleteFile(fid, f.fileorder, f.packageid) - self.releaseFile(fid) - - for pyfile in self.files.itervalues(): - if pyfile.packageid == pid and pyfile.fileorder > order: - pyfile.fileorder -= 1 - - self.evm.dispatchEvent("file:deleted", fid, pid) - - @lock - def releaseFile(self, fid): - """removes pyfile from cache""" - if fid in self.files: - del self.files[fid] - - @lock - def releasePackage(self, pid): - """removes package from cache""" - if pid in self.packages: - del self.packages[pid] - - def updateFile(self, pyfile): - """updates file""" - self.db.updateFile(pyfile) - - # This event is thrown with pyfile or only fid - self.evm.dispatchEvent("file:updated", pyfile) - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - self.evm.dispatchEvent("package:updated", pypack.pid) - - @invalidate - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status,[ hash,] url)""" - self.db.updateLinkInfo(data) - self.evm.dispatchEvent("package:updated", pid) - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - # TODO: user context? - if not self.db.queuestats()[0]: - self.core.addonManager.dispatchEvent("download:allFinished") - self.core.log.debug("All downloads finished") - return True - - return False - - def checkAllLinksProcessed(self, fid=-1): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - # TODO: user context? - if not self.db.processcount(fid): - self.core.addonManager.dispatchEvent("download:allProcessed") - self.core.log.debug("All downloads processed") - return True - - return False - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls addonmanager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.addonManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - def resetCount(self): - self.queuecount = -1 - - @read_lock - @invalidate - def restartPackage(self, pid): - """restart package""" - for pyfile in self.cachedFiles(): - if pyfile.packageid == pid: - self.restartFile(pyfile.id) - - self.db.restartPackage(pid) - - if pid in self.packages: - self.packages[pid].setFinished = False - - self.evm.dispatchEvent("package:updated", pid) - - @read_lock - @invalidate - def restartFile(self, fid): - """ restart file""" - if fid in self.files: - f = self.files[fid] - f.status = DS.Queued - f.name = f.url - f.error = "" - f.abortDownload() - - self.db.restartFile(fid) - self.evm.dispatchEvent("file:updated", fid) - - - @lock - @invalidate - def orderPackage(self, pid, position): - - p = self.getPackageInfo(pid) - self.db.orderPackage(pid, p.root, p.packageorder, position) - - for pack in self.packages.itervalues(): - if pack.root != p.root or pack.packageorder < 0: continue - if pack.pid == pid: - pack.packageorder = position - if p.packageorder > position: - if position <= pack.packageorder < p.packageorder: - pack.packageorder += 1 - elif p.order < position: - if position >= pack.packageorder > p.packageorder: - pack.packageorder -= 1 - - self.db.commit() - - self.evm.dispatchEvent("package:reordered", pid, position, p.root) - - @lock - @invalidate - def orderFiles(self, fids, pid, position): - - files = [self.getFileInfo(fid) for fid in fids] - orders = [f.fileorder for f in files] - if min(orders) + len(files) != max(orders) + 1: - raise Exception("Tried to reorder non continous block of files") - - # minimum fileorder - f = reduce(lambda x,y: x if x.fileorder < y.fileorder else y, files) - order = f.fileorder - - self.db.orderFiles(pid, fids, order, position) - diff = len(fids) - - if f.fileorder > position: - for pyfile in self.files.itervalues(): - if pyfile.packageid != f.package or pyfile.fileorder < 0: continue - if position <= pyfile.fileorder < f.fileorder: - pyfile.fileorder += diff - - for i, fid in enumerate(fids): - if fid in self.files: - self.files[fid].fileorder = position + i - - elif f.fileorder < position: - for pyfile in self.files.itervalues(): - if pyfile.packageid != f.package or pyfile.fileorder < 0: continue - if position >= pyfile.fileorder >= f.fileorder+diff: - pyfile.fileorder -= diff - - for i, fid in enumerate(fids): - if fid in self.files: - self.files[fid].fileorder = position -diff + i + 1 - - self.db.commit() - - self.evm.dispatchEvent("file:reordered", pid) - - @read_lock - @invalidate - def movePackage(self, pid, root): - """ move pid - root """ - - p = self.getPackageInfo(pid) - dest = self.getPackageInfo(root) - if not p: raise PackageDoesNotExists(pid) - if not dest: raise PackageDoesNotExists(root) - - # cantor won't be happy if we put the package in itself - if pid == root or p.root == root: return False - - # TODO move real folders - - # we assume pack is not in use anyway, so we can release it - self.releasePackage(pid) - self.db.movePackage(p.root, p.packageorder, pid, root) - - return True - - @read_lock - @invalidate - def moveFiles(self, fids, pid): - """ move all fids to pid """ - - f = self.getFileInfo(fids[0]) - if not f or f.package == pid: - return False - if not self.getPackageInfo(pid): - raise PackageDoesNotExists(pid) - - # TODO move real files - - self.db.moveFiles(f.package, fids, pid) - - return True - - - @invalidate - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile.status not in (DS.NA, DS.Finished, DS.Skipped): - urls.append((pyfile.url, pyfile.pluginname)) - - self.core.threadManager.createInfoThread(urls, pid) - - - @invalidate - def restartFailed(self): - """ restart all failed links """ - # failed should not be in cache anymore, so working on db is sufficient - self.db.restartFailed() diff --git a/module/InitHomeDir.py b/module/InitHomeDir.py deleted file mode 100644 index ccf525576..000000000 --- a/module/InitHomeDir.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - - This modules inits working directories and global variables, pydir and homedir -""" - -from os import makedirs, path, chdir -from os.path import join -import sys -from sys import argv, platform - -import __builtin__ - -__builtin__.owd = path.abspath("") #original working directory -__builtin__.pypath = path.abspath(path.join(__file__, "..", "..")) - -sys.path.append(join(pypath, "module", "lib")) - -homedir = "" - -if platform == 'nt': - homedir = path.expanduser("~") - if homedir == "~": - import ctypes - - CSIDL_APPDATA = 26 - _SHGetFolderPath = ctypes.windll.shell32.SHGetFolderPathW - _SHGetFolderPath.argtypes = [ctypes.wintypes.HWND, - ctypes.c_int, - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, ctypes.wintypes.LPCWSTR] - - path_buf = ctypes.wintypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) - result = _SHGetFolderPath(0, CSIDL_APPDATA, 0, 0, path_buf) - homedir = path_buf.value -else: - homedir = path.expanduser("~") - -__builtin__.homedir = homedir - -configdir = None -args = " ".join(argv) -# dirty method to set configdir from commandline arguments -if "--configdir=" in args: - for arg in argv: - if arg.startswith("--configdir="): - configdir = arg.replace('--configdir=', '').strip() - -elif "nosetests" in args: - print "Running in test mode" - configdir = join(pypath, "tests", "config") - -elif path.exists(path.join(pypath, "module", "config", "configdir")): - f = open(path.join(pypath, "module", "config", "configdir"), "rb") - c = f.read().strip() - f.close() - configdir = path.join(pypath, c) - -# default config dir -if not configdir: - if platform in ("posix", "linux2", "darwin"): - configdir = path.join(homedir, ".pyload") - else: - configdir = path.join(homedir, "pyload") - -if not path.exists(configdir): - makedirs(configdir, 0700) - -__builtin__.configdir = configdir -chdir(configdir) - -#print "Using %s as working directory." % configdir diff --git a/module/PluginManager.py b/module/PluginManager.py deleted file mode 100644 index bb1c76bf3..000000000 --- a/module/PluginManager.py +++ /dev/null @@ -1,407 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2013 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -import re -import sys - -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath, basename -from sys import version_info -from time import time - -from module.lib.SafeEval import const_eval as literal_eval -from module.plugins.Base import Base - -from new_collections import namedtuple - -#TODO: ignores not updatable - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -PluginTuple = namedtuple("PluginTuple", "version re deps category user path") - -class PluginManager: - ROOT = "module.plugins." - LOCALROOT = "localplugins." - TYPES = ("crypter", "hoster", "accounts", "addons", "internal") - - BUILTIN = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) - SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', - re.I) - # note the nongreedy character: that means we can not embed list and dicts - MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) - - NO_MATCH = re.compile(r'^no_match$') - - def __init__(self, core): - self.core = core - - #self.config = self.core.config - self.log = core.log - - self.plugins = {} - self.modules = {} # cached modules - self.history = [] # match history to speedup parsing (type, name) - self.user_context = {} # plugins working with user context - self.createIndex() - - #register for import addon - sys.meta_path.append(self) - - def logDebug(self, type, plugin, msg): - self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) - - def createIndex(self): - """create information for all plugins available""" - # add to path, so we can import from userplugins - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - a = time() - for type in self.TYPES: - self.plugins[type] = self.parse(type) - - self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - - def parse(self, folder, home=None): - """ Analyze and parses all plugins in folder """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "module", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - # replace suffix and version tag - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - plugin = self.parsePlugin(join(pfolder, f), folder, name, home) - if plugin: - plugins[name] = plugin - - if not home: - temp = self.parse(folder, plugins) - plugins.update(temp) - - return plugins - - def parseAttributes(self, filename, name, folder=""): - """ Parse attribute dict from plugin""" - data = open(filename, "rb") - content = data.read() - data.close() - - attrs = {} - for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): - #replace gettext function and eval result - try: - attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) - except: - self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) - self.core.print_exc() - - if not hasattr(Base, "__%s__" % m[0]): - if m[0] != "type": #TODO remove type from all plugins, its not needed - self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - - return attrs - - def parsePlugin(self, filename, folder, name, home=None): - """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - - :arg home: dict with plugins, of which the found one will be matched against (according version) - :returns PluginTuple""" - - attrs = self.parseAttributes(filename, name, folder) - if not attrs: return - - version = 0 - - if "version" in attrs: - try: - version = float(attrs["version"]) - except ValueError: - self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) - version = 9 #TODO remove when plugins are fixed, causing update loops - else: - self.logDebug(folder, name, "No version attribute") - - # home contains plugins from pyload root - if home and name in home: - if home[name].version >= version: - return - - if name in IGNORE or (folder, name) in IGNORE: - return - - if "pattern" in attrs and attrs["pattern"]: - try: - plugin_re = re.compile(attrs["pattern"], re.I) - except: - self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) - plugin_re = self.NO_MATCH - else: plugin_re = self.NO_MATCH - - deps = attrs.get("dependencies", None) - category = attrs.get("category", None) if folder == "addons" else None - - # create plugin tuple - plugin = PluginTuple(version, plugin_re, deps, category, bool(home), filename) - - # internals have no config - if folder == "internal": - return plugin - - if folder == "addons" and "config" not in attrs and not attrs.get("internal", False): - attrs["config"] = (["activated", "bool", "Activated", False],) - - if "config" in attrs and attrs["config"]: - config = attrs["config"] - desc = attrs.get("description", "") - long_desc = attrs.get("long_description", "") - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "addons" and not attrs.get("internal", False): - for item in config: - if item[0] == "activated": break - else: # activated flag missing - config.insert(0, ("activated", "bool", "Activated", False)) - - # Everything that is no addon and user_context=True, is added to dict - if folder != "addons" or attrs.get("user_context", False): - self.user_context[name] = True - - try: - self.core.config.addConfigSection(name, name, desc, long_desc, config) - except: - self.logDebug(folder, name, "Invalid config %s" % config) - - return plugin - - - def parseUrls(self, urls): - """parse plugins for given list of urls, separate to crypter and hoster""" - - res = {"hoster": [], "crypter": []} # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): - self.log.debug("Parsing invalid type %s" % type(url)) - continue - - found = False - - for ptype, name in self.history: - if self.plugins[ptype][name].re.match(url): - res[ptype].append((url, name)) - found = (ptype, name) - break # need to exit this loop first - - if found: # found match - if self.history[0] != found: #update history - self.history.remove(found) - self.history.insert(0, found) - continue - - for ptype in ("crypter", "hoster"): - for name, plugin in self.plugins[ptype].iteritems(): - if plugin.re.match(url): - res[ptype].append((url, name)) - self.history.insert(0, (ptype, name)) - del self.history[10:] # cut down to size of 10 - found = True - break - - if not found: - res["hoster"].append((url, "BasePlugin")) - - return res["hoster"], res["crypter"] - - def getPlugins(self, type): - return self.plugins.get(type, None) - - def findPlugin(self, name, pluginlist=("hoster", "crypter")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return ptype, self.plugins[ptype][name] - return None, None - - def getPluginModule(self, name): - """ Decprecated: return plugin module from hoster|crypter""" - self.log.debug("Deprecated method: .getPluginModule()") - type, plugin = self.findPlugin(name) - return self.loadModule(type, name) - - def getPluginClass(self, name): - """ return plugin class from hoster|crypter, always the not overwritten one """ - type, plugin = self.findPlugin(name) - return self.loadClass(type, name) - - # MultiHoster will overwrite this - getPlugin = getPluginClass - - def loadAttributes(self, type, name): - plugin = self.plugins[type][name] - return self.parseAttributes(plugin.path, name, type) - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of module.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if (type, name) in self.modules: return self.modules[(type, name)] - try: - # convert path to python recognizable import - path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") - module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) - self.modules[(type, name)] = module # cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - self.core.print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.LOCALROOT): #separate pyload plugins - if fullname.startswith(self.LOCALROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name].user: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name].user: - return self - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.LOCALROOT) - else: - newname = name.replace(self.LOCALROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - def reloadPlugins(self, type_plugins): - """ reloads and reindexes plugins """ - if not type_plugins: return False - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - as_dict = {} - for t, n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - # we do not reload addons or internals, would cause to much side effects - if "addons" in as_dict or "internal" in as_dict: - return False - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type]: - if (type, plugin) in self.modules: - self.log.debug("Reloading %s" % plugin) - reload(self.modules[(type, plugin)]) - - # index re-creation - for type in ("crypter", "container", "hoster", "captcha", "accounts"): - self.plugins[type] = self.parse(type) - - if "accounts" in as_dict: #accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return True - - def isUserPlugin(self, plugin): - """ A plugin suitable for multiple user """ - return plugin in self.user_context - - def isPluginType(self, plugin, type): - return plugin in self.plugins[type] - - def getCategory(self, plugin): - if plugin in self.plugins["addons"]: - return self.plugins["addons"][plugin].category or "addon" - - def loadIcon(self, name): - """ load icon for single plugin, base64 encoded""" - pass - - def checkDependencies(self, type, name): - """ Check deps for given plugin - - :return: List of unfullfilled dependencies - """ - pass - diff --git a/module/Scheduler.py b/module/Scheduler.py deleted file mode 100644 index 0bc396b69..000000000 --- a/module/Scheduler.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from time import time -from heapq import heappop, heappush -from thread import start_new_thread -from threading import Lock - -class AlreadyCalled(Exception): - pass - - -class Deferred(): - def __init__(self): - self.call = [] - self.result = () - - def addCallback(self, f, *cargs, **ckwargs): - self.call.append((f, cargs, ckwargs)) - - def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled - self.result = (args, kwargs) - for f, cargs, ckwargs in self.call: - args += tuple(cargs) - kwargs.update(ckwargs) - f(*args ** kwargs) - - -class Scheduler(): - def __init__(self, core): - self.core = core - - self.queue = PriorityQueue() - - def addJob(self, t, call, args=[], kwargs={}, threaded=True): - d = Deferred() - t += time() - j = Job(t, call, args, kwargs, d, threaded) - self.queue.put((t, j)) - return d - - - def removeJob(self, d): - """ - :param d: defered object - :return: if job was deleted - """ - index = -1 - - for i, j in enumerate(self.queue): - if j[1].deferred == d: - index = i - - if index >= 0: - del self.queue[index] - return True - - return False - - def work(self): - while True: - t, j = self.queue.get() - if not j: - break - else: - if t <= time(): - j.start() - else: - self.queue.put((t, j)) - break - - -class Job(): - def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): - self.time = float(time) - self.call = call - self.args = args - self.kwargs = kwargs - self.deferred = deferred - self.threaded = threaded - - def run(self): - ret = self.call(*self.args, **self.kwargs) - if self.deferred is None: - return - else: - self.deferred.callback(ret) - - def start(self): - if self.threaded: - start_new_thread(self.run, ()) - else: - self.run() - - -class PriorityQueue(): - """ a non blocking priority queue """ - - def __init__(self): - self.queue = [] - self.lock = Lock() - - def __iter__(self): - return iter(self.queue) - - def __delitem__(self, key): - del self.queue[key] - - def put(self, element): - self.lock.acquire() - heappush(self.queue, element) - self.lock.release() - - def get(self): - """ return element or None """ - self.lock.acquire() - try: - el = heappop(self.queue) - return el - except IndexError: - return None, None - finally: - self.lock.release()
\ No newline at end of file diff --git a/module/Setup.py b/module/Setup.py deleted file mode 100644 index eef7a88ca..000000000 --- a/module/Setup.py +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import module.utils.pylgettext as gettext -import os -import sys -import socket -import webbrowser - -from getpass import getpass -from time import time -from sys import exit - -from module.utils.fs import abspath, dirname, exists, join, makedirs -from module.utils import get_console_encoding -from module.web.ServerThread import WebServer - - -class Setup(): - """ - pyLoads initial setup configuration assistant - """ - - def __init__(self, path, config): - self.path = path - self.config = config - self.stdin_encoding = get_console_encoding(sys.stdin.encoding) - self.lang = None - # We will create a timestamp so that the setup will be completed in a specific interval - self.timestamp = time() - - # TODO: probably unneeded - self.yes = "yes" - self.no = "no" - - - def start(self): - web = WebServer(pysetup=self) - web.start() - - error = web.check_error() - if error: #todo errno 44 port already in use - print error - - url = "http://%s:%d/" % (socket.gethostbyname(socket.gethostname()), web.port) - - print "Setup is started" - - opened = webbrowser.open_new_tab(url) - if not opened: - print "Please point your browser to %s" % url - - - self.ask_lang() - - print "" - print _("Would you like to configure pyLoad via Webinterface?") - print _("You need a Browser and a connection to this PC for it.") - print _("Url would be: http://hostname:8000/") - viaweb = self.ask(_("Start initial webinterface for configuration?"), self.yes, bool=True) - if viaweb: - self.start_web() - else: - self.start_cli() - - - - def start_cli(self): - - - print _("Welcome to the pyLoad Configuration Assistent.") - print _("It will check your system and make a basic setup in order to run pyLoad.") - print "" - print _("The value in brackets [] always is the default value,") - print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") - print _( - "Don't forget: You can always rerun this assistent with --setup or -s parameter, when you start pyLoadCore.") - print _("If you have any problems with this assistent hit STRG-C,") - print _("to abort and don't let him start with pyLoadCore automatically anymore.") - print "" - print _("When you are ready for system check, hit enter.") - raw_input() - - #self.get_page_next() - - - if len(avail) < 5: - print _("Features missing: ") - print - - if not self.check_module("Crypto"): - print _("no py-crypto available") - print _("You need this if you want to decrypt container files.") - print "" - - if not ssl: - print _("no SSL available") - print _("This is needed if you want to establish a secure connection to core or webinterface.") - print _("If you only want to access locally to pyLoad ssl is not useful.") - print "" - - if not captcha: - print _("no Captcha Recognition available") - print _("Only needed for some hosters and as freeuser.") - print "" - - if not js: - print _("no JavaScript engine found") - print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") - - print _("You can abort the setup now and fix some dependencies if you want.") - - con = self.ask(_("Continue with setup?"), self.yes, bool=True) - - if not con: - return False - - print "" - print _("Do you want to change the config path? Current is %s") % abspath("") - print _( - "If you use pyLoad on a server or the home partition lives on an internal flash it may be a good idea to change it.") - path = self.ask(_("Change config path?"), self.no, bool=True) - if path: - self.conf_path() - #calls exit when changed - - print "" - print _("Do you want to configure login data and basic settings?") - print _("This is recommend for first run.") - con = self.ask(_("Make basic setup?"), self.yes, bool=True) - - if con: - self.conf_basic() - - if ssl: - print "" - print _("Do you want to configure ssl?") - ssl = self.ask(_("Configure ssl?"), self.no, bool=True) - if ssl: - self.conf_ssl() - - if web: - print "" - print _("Do you want to configure webinterface?") - web = self.ask(_("Configure webinterface?"), self.yes, bool=True) - if web: - self.conf_web() - - print "" - print _("Setup finished successfully.") - print _("Hit enter to exit and restart pyLoad") - raw_input() - return True - - - def start_web(self): - print "" - print _("Webinterface running for setup.") - # TODO start browser? - try: - from module.web import ServerThread - ServerThread.setup = self - from module.web import webinterface - webinterface.run_simple() - self.web = True - return True - except Exception, e: - print "Webinterface failed with this error: ", e - print "Falling back to commandline setup." - self.start_cli() - - - def conf_basic(self): - print "" - print _("## Basic Setup ##") - - print "" - print _("The following logindata is valid for CLI, GUI and webinterface.") - - from module.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - db.shutdown() - - print "" - print _("External clients (GUI, CLI or other) need remote access to work over the network.") - print _("However, if you only want to use the webinterface you may disable it to save ram.") - self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.yes, bool=True) - - print "" - langs = self.config.getMetaData("general", "language") - self.config["general"]["language"] = self.ask(_("Language"), "en", langs.type.split(";")) - - self.config["general"]["download_folder"] = self.ask(_("Downloadfolder"), "Downloads") - self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") - #print _("You should disable checksum proofing, if you have low hardware requirements.") - #self.config["general"]["checksum"] = self.ask(_("Proof checksum?"), "y", bool=True) - - reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) - self.config["reconnect"]["activated"] = reconnect - if reconnect: - self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") - - - def conf_web(self): - print "" - print _("## Webinterface Setup ##") - - print "" - self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) - print "" - print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") - self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") - self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") - print "" - print _("pyLoad offers several server backends, now following a short explanation.") - print "threaded:", _("Default server, this server offers SSL and is a good alternative to builtin.") - print "fastcgi:", _( - "Can be used by apache, lighttpd, requires you to configure them, which is not too easy job.") - print "lightweight:", _("Very fast alternative written in C, requires libev and linux knowledge.") - print "\t", _("Get it from here: https://github.com/jonashaag/bjoern, compile it") - print "\t", _("and copy bjoern.so to module/lib") - - print - print _( - "Attention: In some rare cases the builtin server is not working, if you notice problems with the webinterface") - print _("come back here and change the builtin server to the threaded one here.") - - self.config["webinterface"]["server"] = self.ask(_("Server"), "threaded", - ["builtin", "threaded", "fastcgi", "lightweight"]) - - def conf_ssl(self): - print "" - print _("## SSL Setup ##") - print "" - print _("Execute these commands from pyLoad config folder to make ssl certificates:") - print "" - print "openssl genrsa -out ssl.key 1024" - print "openssl req -new -key ssl.key -out ssl.csr" - print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " - print "" - print _("If you're done and everything went fine, you can activate ssl now.") - self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) - - def set_user(self): - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - from module.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - - noaction = True - try: - while True: - print _("Select action") - print _("1 - Create/Edit user") - print _("2 - List users") - print _("3 - Remove user") - print _("4 - Quit") - action = raw_input("[1]/2/3/4: ") - if not action in ("1", "2", "3", "4"): - continue - elif action == "1": - print "" - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - noaction = False - elif action == "2": - print "" - print _("Users") - print "-----" - users = db.getAllUserData() - noaction = False - for user in users.itervalues(): - print user.name - print "-----" - print "" - elif action == "3": - print "" - username = self.ask(_("Username"), "") - if username: - db.removeUser(username) - noaction = False - elif action == "4": - db.syncSave() - break - finally: - if not noaction: - db.shutdown() - - def conf_path(self, trans=False): - if trans: - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - print _("Setting new configpath, current configuration will not be transferred!") - path = self.ask(_("Configpath"), abspath("")) - try: - path = join(pypath, path) - if not exists(path): - makedirs(path) - f = open(join(pypath, "module", "config", "configdir"), "wb") - f.write(path) - f.close() - print _("Configpath changed, setup will now close, please restart to go on.") - print _("Press Enter to exit.") - raw_input() - exit() - except Exception, e: - print _("Setting config path failed: %s") % str(e) - - - def ask_lang(self): - langs = self.config.getMetaData("general", "language").type.split(";") - self.lang = self.ask(u"Choose your Language / WÀhle deine Sprache", "en", langs) - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True) - translation.install(True) - - #l10n Input shorthand for yes - self.yes = _("y") - #l10n Input shorthand for no - self.no = _("n") - - def ask(self, qst, default, answers=[], bool=False, password=False): - """ Generate dialog on command line """ - - if answers: - info = "(" - for i, answer in enumerate(answers): - info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) - - info += ")" - elif bool: - if default == self.yes: - info = "([%s]/%s)" % (self.yes, self.no) - else: - info = "(%s/[%s])" % (self.yes, self.no) - else: - info = "[%s]" % default - - if password: - p1 = True - p2 = False - while p1 != p2: - # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) - sys.stdout.write(_("Password: ")) - p1 = getpass("") - - if len(p1) < 4: - print _("Password too short. Use at least 4 symbols.") - continue - - sys.stdout.write(_("Password (again): ")) - p2 = getpass("") - - if p1 == p2: - return p1 - else: - print _("Passwords did not match.") - - while True: - input = raw_input(qst + " %s: " % info) - input = input.decode(self.stdin_encoding) - - if input.strip() == "": - input = default - - if bool: - #l10n yes, true,t are inputs for booleans with value true - if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: - return True - #l10n no, false,f are inputs for booleans with value false - elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: - return False - else: - print _("Invalid Input") - continue - - if not answers: - return input - - else: - if input in answers: - return input - else: - print _("Invalid Input") - - -if __name__ == "__main__": - test = Setup(join(abspath(dirname(__file__)), ".."), None) - test.start() diff --git a/module/__init__.py b/module/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/__init__.py +++ /dev/null diff --git a/module/api/AccountApi.py b/module/api/AccountApi.py deleted file mode 100644 index 981842b5c..000000000 --- a/module/api/AccountApi.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - - -class AccountApi(ApiComponent): - """ All methods to control accounts """ - - @RequirePerm(Permission.Accounts) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAllAccounts(refresh) - accounts = [] - for plugin in accs.itervalues(): - accounts.extend([acc.toInfoData() for acc in plugin.values()]) - - return accounts - - @RequirePerm(Permission.All) - def getAccountTypes(self): - """All available account types. - - :return: string list - """ - return self.core.pluginManager.getPlugins("accounts").keys() - - @RequirePerm(Permission.Accounts) - def updateAccount(self, plugin, login, password): - """Changes pw/options for specific account.""" - # TODO: options - self.core.accountManager.updateAccount(plugin, login, password, {}) - - def updateAccountInfo(self, account): - """ Update account from :class:`AccountInfo` """ - #TODO - - @RequirePerm(Permission.Accounts) - def removeAccount(self, account): - """Remove account from pyload. - - :param account: :class:`ÃccountInfo` instance - """ - self.core.accountManager.removeAccount(account.plugin, account.loginname) - - -if Api.extend(AccountApi): - del AccountApi
\ No newline at end of file diff --git a/module/api/AddonApi.py b/module/api/AddonApi.py deleted file mode 100644 index 917c7dc4c..000000000 --- a/module/api/AddonApi.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - -class AddonApi(ApiComponent): - """ Methods to interact with addons """ - - def getAllInfo(self): - """Returns all information stored by addon plugins. Values are always strings - - :return: {"plugin": {"name": value } } - """ - return self.core.addonManager.getAllInfo() - - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.addonManager.getInfo(plugin) - -if Api.extend(AddonApi): - del AddonApi
\ No newline at end of file diff --git a/module/api/ApiComponent.py b/module/api/ApiComponent.py deleted file mode 100644 index 3948086c2..000000000 --- a/module/api/ApiComponent.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.remote.apitypes import Iface - -# Workaround to let code-completion think, this is subclass of Iface -Iface = object -class ApiComponent(Iface): - - __slots__ = [] - - def __init__(self, core, user): - # Only for auto completion, this class can not be instantiated - from pyload import Core - from module.datatypes.User import User - assert isinstance(core, Core) - assert issubclass(ApiComponent, Iface) - self.core = core - assert isinstance(user, User) - self.user = user - self.primaryUID = 0 - # No instantiating! - raise Exception()
\ No newline at end of file diff --git a/module/api/CollectorApi.py b/module/api/CollectorApi.py deleted file mode 100644 index eb36f7a21..000000000 --- a/module/api/CollectorApi.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission - -from ApiComponent import ApiComponent - -class CollectorApi(ApiComponent): - """ Link collector """ - - @RequirePerm(Permission.All) - def getCollector(self): - pass - - @RequirePerm(Permission.Add) - def addToCollector(self, links): - pass - - @RequirePerm(Permission.Add) - def addFromCollector(self, name, new_name): - pass - - @RequirePerm(Permission.Delete) - def deleteCollPack(self, name): - pass - - @RequirePerm(Permission.Add) - def renameCollPack(self, name, new_name): - pass - - @RequirePerm(Permission.Delete) - def deleteCollLink(self, url): - pass - - -if Api.extend(CollectorApi): - del CollectorApi
\ No newline at end of file diff --git a/module/api/ConfigApi.py b/module/api/ConfigApi.py deleted file mode 100644 index 527f494ce..000000000 --- a/module/api/ConfigApi.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, ConfigHolder, ConfigItem, ConfigInfo -from module.utils import to_string - -from ApiComponent import ApiComponent - -# helper function to create a ConfigHolder -def toConfigHolder(section, config, values): - holder = ConfigHolder(section, config.name, config.description, config.long_desc) - holder.items = [ConfigItem(option, x.name, x.description, x.type, to_string(x.default), - to_string(values.get(option, x.default))) for option, x in - config.config.iteritems()] - return holder - - -class ConfigApi(ApiComponent): - """ Everything related to configuration """ - - def getConfigValue(self, section, option): - """Retrieve config value. - - :param section: name of category, or plugin - :param option: config option - :rtype: str - :return: config value as string - """ - value = self.core.config.get(section, option, self.primaryUID) - return to_string(value) - - def setConfigValue(self, section, option, value): - """Set new config value. - - :param section: - :param option: - :param value: new config value - """ - if option in ("limit_speed", "max_speed"): #not so nice to update the limit - self.core.requestFactory.updateBucket() - - self.core.config.set(section, option, value, self.primaryUID) - - def getConfig(self): - """Retrieves complete config of core. - - :rtype: dict of section -> ConfigHolder - """ - data = {} - for section, config, values in self.core.config.iterCoreSections(): - data[section] = toConfigHolder(section, config, values) - return data - - def getCoreConfig(self): - """ Retrieves core config sections - - :rtype: list of PluginInfo - """ - return [ConfigInfo(section, config.name, config.description, False, False) - for section, config, values in self.core.config.iterCoreSections()] - - @RequirePerm(Permission.Plugins) - def getPluginConfig(self): - """All plugins and addons the current user has configured - - :rtype: list of PluginInfo - """ - # TODO: include addons that are activated by default - # TODO: multi user - # TODO: better plugin / addon activated config - data = [] - active = [x.getName() for x in self.core.addonManager.activePlugins()] - for name, config, values in self.core.config.iterSections(self.primaryUID): - # skip unmodified and inactive addons - if not values and name not in active: continue - - item = ConfigInfo(name, config.name, config.description, - self.core.pluginManager.getCategory(name), - self.core.pluginManager.isUserPlugin(name), - values.get("activated", None if "activated" not in config.config else config.config[ - "activated"].default)) - data.append(item) - - return data - - @RequirePerm(Permission.Plugins) - def getAvailablePlugins(self): - """List of all available plugins, that are configurable - - :rtype: list of PluginInfo - """ - # TODO: filter user_context / addons when not allowed - plugins = [ConfigInfo(name, config.name, config.description, - self.core.pluginManager.getCategory(name), - self.core.pluginManager.isUserPlugin(name)) - for name, config, values in self.core.config.iterSections(self.primaryUID)] - - return plugins - - @RequirePerm(Permission.Plugins) - def loadConfig(self, name): - """Get complete config options for desired section - - :param name: Name of plugin or config section - :rtype: ConfigHolder - """ - # requires at least plugin permissions, but only admin can load core config - config, values = self.core.config.getSection(name, self.primaryUID) - return toConfigHolder(name, config, values) - - - @RequirePerm(Permission.Plugins) - def saveConfig(self, config): - """Used to save a configuration, core config can only be saved by admins - - :param config: :class:`ConfigHolder` - """ - for item in config.items: - self.core.config.set(config.name, item.name, item.value, sync=False, user=self.primaryUID) - # save the changes - self.core.config.saveValues(self.primaryUID, config.name) - - @RequirePerm(Permission.Plugins) - def deleteConfig(self, plugin): - """Deletes modified config - - :param plugin: plugin name - """ - #TODO: delete should deactivate addons? - self.core.config.delete(plugin, self.primaryUID) - - -if Api.extend(ConfigApi): - del ConfigApi
\ No newline at end of file diff --git a/module/api/CoreApi.py b/module/api/CoreApi.py deleted file mode 100644 index e5c5e8b41..000000000 --- a/module/api/CoreApi.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, ServerStatus, Interaction -from module.utils.fs import join, free_space -from module.utils import compare_time - -from ApiComponent import ApiComponent - -class CoreApi(ApiComponent): - """ This module provides methods for general interaction with the core, like status or progress retrieval """ - - @RequirePerm(Permission.All) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - @RequirePerm(Permission.All) - def getWSAddress(self): - """Gets and address for the websocket based on configuration""" - # TODO SSL (wss) - return "ws://%%s:%d" % self.core.config['remote']['port'] - - @RequirePerm(Permission.All) - def getServerStatus(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - queue = self.core.files.getQueueStats(self.primaryUID) - total = self.core.files.getDownloadStats(self.primaryUID) - - serverStatus = ServerStatus(0, - total[0], queue[0], - total[1], queue[1], - self.isInteractionWaiting(Interaction.All), - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.threadManager.pause, - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) - - - for pyfile in self.core.threadManager.getActiveDownloads(self.primaryUID): - serverStatus.speed += pyfile.getSpeed() #bytes/s - - return serverStatus - - @RequirePerm(Permission.All) - def getProgressInfo(self): - """ Status of all currently running tasks - - :rtype: list of :class:`ProgressInfo` - """ - return self.core.threadManager.getProgressList(self.primaryUID) - - def pauseServer(self): - """Pause server: It won't start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config["reconnect"]["activated"] ^= True - return self.core.config["reconnect"]["activated"] - - def freeSpace(self): - """Available free space at download directory in bytes""" - return free_space(self.core.config["general"]["download_folder"]) - - - def quit(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = join(self.core.config['log']['log_folder'], 'log.txt') - try: - fh = open(filename, "r") - lines = fh.readlines() - fh.close() - if offset >= len(lines): - return [] - return lines[offset:] - except: - return ['No log available'] - - @RequirePerm(Permission.All) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config['downloadTime']['start'].split(":") - end = self.core.config['downloadTime']['end'].split(":") - return compare_time(start, end) - - @RequirePerm(Permission.All) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config['reconnect']['startTime'].split(":") - end = self.core.config['reconnect']['endTime'].split(":") - return compare_time(start, end) and self.core.config["reconnect"]["activated"] - - -if Api.extend(CoreApi): - del CoreApi
\ No newline at end of file diff --git a/module/api/DownloadApi.py b/module/api/DownloadApi.py deleted file mode 100644 index ba49435b3..000000000 --- a/module/api/DownloadApi.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from os.path import isabs - -from module.Api import Api, RequirePerm, Permission -from module.utils.fs import join - -from ApiComponent import ApiComponent - -class DownloadApi(ApiComponent): - """ Component to create, add, delete or modify downloads.""" - - @RequirePerm(Permission.Add) - def generateAndAddPackages(self, links, paused=False): - """Generates and add packages - - :param links: list of urls - :param paused: paused package - :return: list of package ids - """ - return [self.addPackageP(name, urls, "", paused) for name, urls - in self.generatePackages(links).iteritems()] - - @RequirePerm(Permission.Add) - def createPackage(self, name, folder, root, password="", site="", comment="", paused=False): - """Create a new package. - - :param name: display name of the package - :param folder: folder name or relative path, abs path are not allowed - :param root: package id of root package, -1 for top level package - :param password: single pw or list of passwords separated with new line - :param site: arbitrary url to site for more information - :param comment: arbitrary comment - :param paused: No downloads will be started when True - :return: pid of newly created package - """ - - if isabs(folder): - folder = folder.replace("/", "_") - - folder = folder.replace("http://", "").replace(":", "").replace("\\", "_").replace("..", "") - - self.core.log.info(_("Added package %(name)s as folder %(folder)s") % {"name": name, "folder": folder}) - pid = self.core.files.addPackage(name, folder, root, password, site, comment, paused) - - return pid - - - @RequirePerm(Permission.Add) - def addPackage(self, name, links, password=""): - """Convenient method to add a package to the top-level and for adding links. - - :return: package id - """ - return self.addPackageChild(name, links, password, -1, False) - - @RequirePerm(Permission.Add) - def addPackageP(self, name, links, password, paused): - """ Same as above with additional paused attribute. """ - return self.addPackageChild(name, links, password, -1, paused) - - @RequirePerm(Permission.Add) - def addPackageChild(self, name, links, password, root, paused): - """Adds a package, with links to desired package. - - :param root: parents package id - :return: package id of the new package - """ - if self.core.config['general']['folder_per_package']: - folder = name - else: - folder = "" - - pid = self.createPackage(name, folder, root, password) - self.addLinks(pid, links) - - return pid - - @RequirePerm(Permission.Add) - def addLinks(self, pid, links): - """Adds links to specific package. Initiates online status fetching. - - :param pid: package id - :param links: list of urls - """ - hoster, crypter = self.core.pluginManager.parseUrls(links) - - if hoster: - self.core.files.addLinks(hoster, pid) - self.core.threadManager.createInfoThread(hoster, pid) - - self.core.threadManager.createDecryptThread(crypter, pid) - - self.core.log.info((_("Added %d links to package") + " #%d" % pid) % len(hoster)) - self.core.files.save() - - @RequirePerm(Permission.Add) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") - th.write(str(data)) - th.close() - - return self.addPackage(th.name, [th.name]) - - @RequirePerm(Permission.Delete) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for fid in fids: - self.core.files.deleteFile(fid) - - self.core.files.save() - - @RequirePerm(Permission.Delete) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for pid in pids: - self.core.files.deletePackage(pid) - - self.core.files.save() - - - @RequirePerm(Permission.Modify) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(pid) - - @RequirePerm(Permission.Modify) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(fid) - - @RequirePerm(Permission.Modify) - def recheckPackage(self, pid): - """Check online status of all files in a package, also a default action when package is added. """ - self.core.files.reCheckPackage(pid) - - @RequirePerm(Permission.Modify) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - @RequirePerm(Permission.Modify) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cachedFiles() - for pyfile in pyfiles: - pyfile.abortDownload() - - @RequirePerm(Permission.Modify) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cachedFiles() - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - -if Api.extend(DownloadApi): - del DownloadApi
\ No newline at end of file diff --git a/module/api/DownloadPreparingApi.py b/module/api/DownloadPreparingApi.py deleted file mode 100644 index edd5d362c..000000000 --- a/module/api/DownloadPreparingApi.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from itertools import chain - -from module.Api import Api, RequirePerm, Permission, OnlineCheck, LinkStatus, urlmatcher -from module.utils.fs import join -from module.utils.packagetools import parseNames -from module.network.RequestFactory import getURL - -from ApiComponent import ApiComponent - -class DownloadPreparingApi(ApiComponent): - """ All kind of methods to parse links or retrieve online status """ - - @RequirePerm(Permission.Add) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitrary text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - - # remove duplicates - return self.checkURLs(set(urls)) - - - @RequirePerm(Permission.Add) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matching urls. - - :param urls: - :return: {plugin: urls} - """ - data, crypter = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugin in chain(data, crypter): - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - return plugins - - @RequirePerm(Permission.Add) - def checkOnlineStatus(self, urls): - """ initiates online status check, will also decrypt files. - - :param urls: - :return: initial set of data as :class:`OnlineCheck` instance containing the result id - """ - data, crypter = self.core.pluginManager.parseUrls(urls) - - # initial result does not contain the crypter links - tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] - data = parseNames(tmp) - result = {} - - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - data.update(crypter) # hoster and crypter will be processed - rid = self.core.threadManager.createResultThread(data, False) - - return OnlineCheck(rid, result) - - @RequirePerm(Permission.Add) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submitted container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: :class:`OnlineCheck` - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") - th.write(str(data)) - th.close() - urls.append(th.name) - return self.checkOnlineStatus(urls) - - @RequirePerm(Permission.Add) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then there is no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @RequirePerm(Permission.Add) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - result = parseNames((x, x) for x in links) - return result - - -if Api.extend(DownloadPreparingApi): - del DownloadPreparingApi
\ No newline at end of file diff --git a/module/api/FileApi.py b/module/api/FileApi.py deleted file mode 100644 index 8a55d9dfd..000000000 --- a/module/api/FileApi.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, DownloadState, PackageDoesNotExists, FileDoesNotExists -from module.utils import uniqify - -from ApiComponent import ApiComponent - -# TODO: user context -class FileApi(ApiComponent): - """Everything related to available packages or files. Deleting, Modifying and so on.""" - - @RequirePerm(Permission.All) - def getAllFiles(self): - """ same as `getFileTree` for toplevel root and full tree""" - return self.getFileTree(-1, True) - - @RequirePerm(Permission.All) - def getFilteredFiles(self, state): - """ same as `getFilteredFileTree` for toplevel root and full tree""" - return self.getFilteredFileTree(-1, state, True) - - @RequirePerm(Permission.All) - def getFileTree(self, pid, full): - """ Retrieve data for specific package. full=True will retrieve all data available - and can result in greater delays. - - :param pid: package id - :param full: go down the complete tree or only the first layer - :return: :class:`TreeCollection` - """ - return self.core.files.getTree(pid, full, DownloadState.All) - - @RequirePerm(Permission.All) - def getFilteredFileTree(self, pid, full, state): - """ Same as `getFileTree` but only contains files with specific download state. - - :param pid: package id - :param full: go down the complete tree or only the first layer - :param state: :class:`DownloadState`, the attributes used for filtering - :return: :class:`TreeCollection` - """ - return self.core.files.getTree(pid, full, state) - - @RequirePerm(Permission.All) - def getPackageContent(self, pid): - """ Only retrieve content of a specific package. see `getFileTree`""" - return self.getFileTree(pid, False) - - @RequirePerm(Permission.All) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :raises PackageDoesNotExists: - :return: :class:`PackageInfo` - """ - info = self.core.files.getPackageInfo(pid) - if not info: - raise PackageDoesNotExists(pid) - return info - - @RequirePerm(Permission.All) - def getFileInfo(self, fid): - """ Info for specific file - - :param fid: file id - :raises FileDoesNotExists: - :return: :class:`FileInfo` - - """ - info = self.core.files.getFileInfo(fid) - if not info: - raise FileDoesNotExists(fid) - return info - - @RequirePerm(Permission.Download) - def getFilePath(self, fid): - """ Internal method to get the filepath""" - info = self.getFileInfo(fid) - pack = self.core.files.getPackage(info.package) - return pack.getPath(), info.name - - - @RequirePerm(Permission.All) - def findFiles(self, pattern): - return self.core.files.getTree(-1, True, DownloadState.All, pattern) - - @RequirePerm(Permission.All) - def searchSuggestions(self, pattern): - names = self.core.db.getMatchingFilenames(pattern, self.primaryUID) - # TODO: stemming and reducing the names to provide better suggestions - return uniqify(names) - - @RequirePerm(Permission.All) - def findPackages(self, tags): - pass - - @RequirePerm(Permission.Modify) - def updatePackage(self, pack): - """Allows to modify several package attributes. - - :param pid: package id - :param data: :class:`PackageInfo` - """ - pid = pack.pid - p = self.core.files.getPackage(pid) - if not p: raise PackageDoesNotExists(pid) - - #TODO: fix - for key, value in data.iteritems(): - if key == "id": continue - setattr(p, key, value) - - p.sync() - self.core.files.save() - - @RequirePerm(Permission.Modify) - def setPackageFolder(self, pid, path): - pass - - @RequirePerm(Permission.Modify) - def movePackage(self, pid, root): - """ Set a new root for specific package. This will also moves the files on disk\ - and will only work when no file is currently downloading. - - :param pid: package id - :param root: package id of new root - :raises PackageDoesNotExists: When pid or root is missing - :return: False if package can't be moved - """ - return self.core.files.movePackage(pid, root) - - @RequirePerm(Permission.Modify) - def moveFiles(self, fids, pid): - """Move multiple files to another package. This will move the files on disk and\ - only work when files are not downloading. All files needs to be continuous ordered - in the current package. - - :param fids: list of file ids - :param pid: destination package - :return: False if files can't be moved - """ - return self.core.files.moveFiles(fids, pid) - - @RequirePerm(Permission.Modify) - def orderPackage(self, pid, position): - """Set new position for a package. - - :param pid: package id - :param position: new position, 0 for very beginning - """ - self.core.files.orderPackage(pid, position) - - @RequirePerm(Permission.Modify) - def orderFiles(self, fids, pid, position): - """ Set a new position for a bunch of files within a package. - All files have to be in the same package and must be **continuous**\ - in the package. That means no gaps between them. - - :param fids: list of file ids - :param pid: package id of parent package - :param position: new position: 0 for very beginning - """ - self.core.files.orderFiles(fids, pid, position) - - -if Api.extend(FileApi): - del FileApi
\ No newline at end of file diff --git a/module/api/UserInteractionApi.py b/module/api/UserInteractionApi.py deleted file mode 100644 index b95b7c468..000000000 --- a/module/api/UserInteractionApi.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.Api import Api, RequirePerm, Permission, Interaction - -from ApiComponent import ApiComponent - -class UserInteractionApi(ApiComponent): - """ Everything needed for user interaction """ - - @RequirePerm(Permission.Interaction) - def isInteractionWaiting(self, mode): - """ Check if task is waiting. - - :param mode: binary or'ed output type - :return: boolean - """ - return self.core.interactionManager.isTaskWaiting(self.primaryUID, mode) - - @RequirePerm(Permission.Interaction) - def getInteractionTasks(self, mode): - """Retrieve task for specific mode. - - :param mode: binary or'ed interaction types which should be retrieved - :rtype list of :class:`InteractionTask` - """ - tasks = self.core.interactionManager.getTasks(self.primaryUID, mode) - # retrieved tasks count as seen - for t in tasks: - t.seen = True - if t.type == Interaction.Notification: - t.setWaiting(self.core.interactionManager.CLIENT_THRESHOLD) - - return tasks - - @RequirePerm(Permission.Interaction) - def setInteractionResult(self, iid, result): - """Set Result for a interaction task. It will be immediately removed from task queue afterwards - - :param iid: interaction id - :param result: result as json string - """ - task = self.core.interactionManager.getTaskByID(iid) - if task and self.primaryUID == task.owner: - task.setResult(result) - - @RequirePerm(Permission.Interaction) - def getAddonHandler(self): - pass - - @RequirePerm(Permission.Interaction) - def callAddonHandler(self, plugin, func, pid_or_fid): - pass - - @RequirePerm(Permission.Download) - def generateDownloadLink(self, fid, timeout): - pass - - -if Api.extend(UserInteractionApi): - del UserInteractionApi
\ No newline at end of file diff --git a/module/api/__init__.py b/module/api/__init__.py deleted file mode 100644 index 1348fd26f..000000000 --- a/module/api/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -__all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi", - "CollectorApi", "UserInteractionApi", "AccountApi", "AddonApi"] - -# Import all components -# from .import * -# Above does not work in py 2.5 -for name in __all__: - __import__(__name__ + "." + name)
\ No newline at end of file diff --git a/module/cli/AddPackage.py b/module/cli/AddPackage.py deleted file mode 100644 index a73401586..000000000 --- a/module/cli/AddPackage.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### - -from Handler import Handler -from printer import * - -class AddPackage(Handler): - """ let the user add packages """ - - def init(self): - self.name = "" - self.urls = [] - - def onEnter(self, inp): - if inp == "0": - self.cli.reset() - - if not self.name: - self.name = inp - self.setInput() - elif inp == "END": - #add package - self.client.addPackage(self.name, self.urls, 1) - self.cli.reset() - else: - if inp.strip(): - self.urls.append(inp) - self.setInput() - - def renderBody(self, line): - println(line, white(_("Add Package:"))) - println(line + 1, "") - line += 2 - - if not self.name: - println(line, _("Enter a name for the new package")) - println(line + 1, "") - line += 2 - else: - println(line, _("Package: %s") % self.name) - println(line + 1, _("Parse the links you want to add.")) - println(line + 2, _("Type %s when done.") % mag("END")) - println(line + 3, _("Links added: ") + mag(len(self.urls))) - line += 4 - - println(line, "") - println(line + 1, mag("0.") + _(" back to main menu")) - - return line + 2
\ No newline at end of file diff --git a/module/cli/Handler.py b/module/cli/Handler.py deleted file mode 100644 index 476d09386..000000000 --- a/module/cli/Handler.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### -class Handler: - def __init__(self, cli): - self.cli = cli - self.init() - - client = property(lambda self: self.cli.client) - input = property(lambda self: self.cli.input) - - def init(self): - pass - - def onChar(self, char): - pass - - def onBackSpace(self): - pass - - def onEnter(self, inp): - pass - - def setInput(self, inp=""): - self.cli.setInput(inp) - - def backspace(self): - self.cli.setInput(self.input[:-1]) - - def renderBody(self, line): - """ gets the line where to render output and should return the line number below its content """ - return line + 1
\ No newline at end of file diff --git a/module/cli/ManageFiles.py b/module/cli/ManageFiles.py deleted file mode 100644 index c133d9959..000000000 --- a/module/cli/ManageFiles.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2011 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### - -from itertools import islice -from time import time - -from Handler import Handler -from printer import * - -from module.Api import Destination, PackageData - -class ManageFiles(Handler): - """ possibility to manage queue/collector """ - - def init(self): - self.target = Destination.Queue - self.pos = 0 #position in queue - self.package = -1 #chosen package - self.mode = "" # move/delete/restart - - self.cache = None - self.links = None - self.time = 0 - - def onChar(self, char): - if char in ("m", "d", "r"): - self.mode = char - self.setInput() - elif char == "p": - self.pos = max(0, self.pos - 5) - self.backspace() - elif char == "n": - self.pos += 5 - self.backspace() - - def onBackSpace(self): - if not self.input and self.mode: - self.mode = "" - if not self.input and self.package > -1: - self.package = -1 - - def onEnter(self, input): - if input == "0": - self.cli.reset() - elif self.package < 0 and self.mode: - #mode select - packs = self.parseInput(input) - if self.mode == "m": - [self.client.movePackage((self.target + 1) % 2, x) for x in packs] - elif self.mode == "d": - self.client.deletePackages(packs) - elif self.mode == "r": - [self.client.restartPackage(x) for x in packs] - - elif self.mode: - #edit links - links = self.parseInput(input, False) - - if self.mode == "d": - self.client.deleteFiles(links) - elif self.mode == "r": - map(self.client.restartFile, links) - - else: - #look into package - try: - self.package = int(input) - except: - pass - - self.cache = None - self.links = None - self.pos = 0 - self.mode = "" - self.setInput() - - - def renderBody(self, line): - if self.package < 0: - println(line, white(_("Manage Packages:"))) - else: - println(line, white((_("Manage Links:")))) - line += 1 - - if self.mode: - if self.mode == "m": - println(line, _("What do you want to move?")) - elif self.mode == "d": - println(line, _("What do you want to delete?")) - elif self.mode == "r": - println(line, _("What do you want to restart?")) - - println(line + 1, "Enter a single number, comma separated numbers or ranges. e.g.: 1,2,3 or 1-3.") - line += 2 - else: - println(line, _("Choose what you want to do, or enter package number.")) - println(line + 1, ("%s - %%s, %s - %%s, %s - %%s" % (mag("d"), mag("m"), mag("r"))) % ( - _("delete"), _("move"), _("restart"))) - line += 2 - - if self.package < 0: - #print package info - pack = self.getPackages() - i = 0 - for value in islice(pack, self.pos, self.pos + 5): - try: - println(line, mag(str(value.pid)) + ": " + value.name) - line += 1 - i += 1 - except Exception, e: - pass - for x in range(5 - i): - println(line, "") - line += 1 - else: - #print links info - pack = self.getLinks() - i = 0 - for value in islice(pack.links, self.pos, self.pos + 5): - try: - println(line, mag(value.fid) + ": %s | %s | %s" % ( - value.name, value.statusmsg, value.plugin)) - line += 1 - i += 1 - except Exception, e: - pass - for x in range(5 - i): - println(line, "") - line += 1 - - println(line, mag("p") + _(" - previous") + " | " + mag("n") + _(" - next")) - println(line + 1, mag("0.") + _(" back to main menu")) - - return line + 2 - - - def getPackages(self): - if self.cache and self.time + 2 < time(): - return self.cache - - if self.target == Destination.Queue: - data = self.client.getQueue() - else: - data = self.client.getCollector() - - - self.cache = data - self.time = time() - - return data - - def getLinks(self): - if self.links and self.time + 1 < time(): - return self.links - - try: - data = self.client.getPackageData(self.package) - except: - data = PackageData(links=[]) - - self.links = data - self.time = time() - - return data - - def parseInput(self, inp, package=True): - inp = inp.strip() - if "-" in inp: - l, n, h = inp.partition("-") - l = int(l) - h = int(h) - r = range(l, h + 1) - - ret = [] - if package: - for p in self.cache: - if p.pid in r: - ret.append(p.pid) - else: - for l in self.links.links: - if l.lid in r: - ret.append(l.lid) - - return ret - - else: - return [int(x) for x in inp.split(",")] diff --git a/module/cli/__init__.py b/module/cli/__init__.py deleted file mode 100644 index fa8a09291..000000000 --- a/module/cli/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from AddPackage import AddPackage -from ManageFiles import ManageFiles
\ No newline at end of file diff --git a/module/cli/printer.py b/module/cli/printer.py deleted file mode 100644 index c62c1800e..000000000 --- a/module/cli/printer.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -def blue(string): - return "\033[1;34m" + unicode(string) + "\033[0m" - -def green(string): - return "\033[1;32m" + unicode(string) + "\033[0m" - -def yellow(string): - return "\033[1;33m" + unicode(string) + "\033[0m" - -def red(string): - return "\033[1;31m" + unicode(string) + "\033[0m" - -def cyan(string): - return "\033[1;36m" + unicode(string) + "\033[0m" - -def mag(string): - return "\033[1;35m" + unicode(string) + "\033[0m" - -def white(string): - return "\033[1;37m" + unicode(string) + "\033[0m" - -def println(line, content): - print "\033[" + str(line) + ";0H\033[2K" + content
\ No newline at end of file diff --git a/module/config/ConfigManager.py b/module/config/ConfigManager.py deleted file mode 100644 index 3290ed4ec..000000000 --- a/module/config/ConfigManager.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from new_collections import OrderedDict - -from module.Api import InvalidConfigSection -from module.utils import from_string, json - -from ConfigParser import ConfigParser - - -def convertKeyError(func): - """ converts KeyError into InvalidConfigSection """ - - def conv(*args, **kwargs): - try: - return func(*args, **kwargs) - except KeyError: - raise InvalidConfigSection(args[1]) - - return conv - - -class ConfigManager(ConfigParser): - """ Manages the core config and configs for addons and single user. - Has similar interface to ConfigParser. """ - - def __init__(self, core, parser): - # No __init__ call to super class is needed! - - self.core = core - self.db = core.db - # The config parser, holding the core config - self.parser = parser - - # similar to parser, separated meta data and values - self.config = OrderedDict() - - # Value cache for multiple user configs - # Values are populated from db on first access - # Entries are saved as (user, section) keys - self.values = {} - # TODO: similar to a cache, could be deleted periodically - - def save(self): - self.parser.save() - - @convertKeyError - def get(self, section, option, user=None): - """get config value, core config only available for admins. - if user is not valid default value will be returned""" - - # Core config loaded from parser, when no user is given or he is admin - if section in self.parser and user is None: - return self.parser.get(section, option) - else: - # We need the id and not the instance - # Will be None for admin user and so the same as internal access - try: - # Check if this config exists - # Configs without meta data can not be loaded! - data = self.config[section].config[option] - return self.loadValues(user, section)[option] - except KeyError: - pass # Returns default value later - - return self.config[section].config[option].default - - def loadValues(self, user, section): - if (user, section) not in self.values: - conf = self.db.loadConfig(section, user) - try: - self.values[user, section] = json.loads(conf) if conf else {} - except ValueError: # Something did go wrong when parsing - self.values[user, section] = {} - self.core.print_exc() - - return self.values[user, section] - - @convertKeyError - def set(self, section, option, value, sync=True, user=None): - """ set config value """ - - changed = False - if section in self.parser and user is None: - changed = self.parser.set(section, option, value, sync) - else: - data = self.config[section].config[option] - value = from_string(value, data.type) - old_value = self.get(section, option) - - # Values will always be saved to db, sync is ignored - if value != old_value: - changed = True - self.values[user, section][option] = value - if sync: self.saveValues(user, section) - - if changed: self.core.evm.dispatchEvent("config:changed", section, option, value) - return changed - - def saveValues(self, user, section): - if section in self.parser and user is None: - self.save() - elif (user, section) in self.values: - self.db.saveConfig(section, json.dumps(self.values[user, section]), user) - - def delete(self, section, user=None): - """ Deletes values saved in db and cached values for given user, NOT meta data - Does not trigger an error when nothing was deleted. """ - if (user, section) in self.values: - del self.values[user, section] - - self.db.deleteConfig(section, user) - self.core.evm.dispatchEvent("config:deleted", section, user) - - def iterCoreSections(self): - return self.parser.iterSections() - - def iterSections(self, user=None): - """ Yields: section, metadata, values """ - values = self.db.loadConfigsForUser(user) - - # Every section needs to be json decoded - for section, data in values.items(): - try: - values[section] = json.loads(data) if data else {} - except ValueError: - values[section] = {} - self.core.print_exc() - - for name, config in self.config.iteritems(): - yield name, config, values[name] if name in values else {} - - def getSection(self, section, user=None): - if section in self.parser and user is None: - return self.parser.getSection(section) - - values = self.loadValues(user, section) - return self.config.get(section), values diff --git a/module/config/ConfigParser.py b/module/config/ConfigParser.py deleted file mode 100644 index bf9192270..000000000 --- a/module/config/ConfigParser.py +++ /dev/null @@ -1,207 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from time import sleep -from os.path import exists -from gettext import gettext -from new_collections import namedtuple, OrderedDict - -from module.utils import from_string -from module.utils.fs import chmod - -from default import make_config - -CONF_VERSION = 2 -SectionTuple = namedtuple("SectionTuple", "name description long_desc config") -ConfigData = namedtuple("ConfigData", "name type description default") - -class ConfigParser: - """ - Holds and manages the configuration + meta data for config read from file. - """ - - CONFIG = "pyload.conf" - - def __init__(self, config=None): - - if config: self.CONFIG = config - - # Meta data information - self.config = OrderedDict() - # The actual config values - self.values = {} - - self.checkVersion() - - self.loadDefault() - self.parseValues(self.CONFIG) - - def loadDefault(self): - make_config(self) - - def checkVersion(self): - """Determines if config needs to be deleted""" - e = None - # workaround conflict, with GUI (which also accesses the config) so try read in 3 times - for i in range(0, 3): - try: - if exists(self.CONFIG): - f = open(self.CONFIG, "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - f = open(self.CONFIG, "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - print "Old version of %s deleted" % self.CONFIG - else: - f = open(self.CONFIG, "wb") - f.write("version:" + str(CONF_VERSION)) - f.close() - - except Exception, ex: - e = ex - sleep(0.3) - if e: raise e - - - def parseValues(self, filename): - """read config values from file""" - f = open(filename, "rb") - config = f.readlines()[1:] - - # save the current section - section = "" - - for line in config: - line = line.strip() - - # comment line, different variants - if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): continue - - if line.startswith("["): - section = line.replace("[", "").replace("]", "") - - if section not in self.config: - print "Unrecognized section", section - section = "" - - else: - name, non, value = line.rpartition("=") - name = name.strip() - value = value.strip() - - if not section: - print "Value without section", name - continue - - if name in self.config[section].config: - self.set(section, name, value, sync=False) - else: - print "Unrecognized option", section, name - - - def save(self): - """saves config to filename""" - - configs = [] - f = open(self.CONFIG, "wb") - configs.append(f) - chmod(self.CONFIG, 0600) - f.write("version: %i\n\n" % CONF_VERSION) - - for section, data in self.config.iteritems(): - f.write("[%s]\n" % section) - - for option, data in data.config.iteritems(): - value = self.get(section, option) - if type(value) == unicode: value = value.encode("utf8") - else: value = str(value) - - f.write('%s = %s\n' % (option, value)) - - f.write("\n") - - f.close() - - def __getitem__(self, section): - """provides dictionary like access: c['section']['option']""" - return Section(self, section) - - def __contains__(self, section): - """ checks if parser contains section """ - return section in self.config - - def get(self, section, option): - """get value or default""" - try: - return self.values[section][option] - except KeyError: - return self.config[section].config[option].default - - def set(self, section, option, value, sync=True): - """set value""" - - data = self.config[section].config[option] - value = from_string(value, data.type) - old_value = self.get(section, option) - - # only save when different values - if value != old_value: - if section not in self.values: self.values[section] = {} - self.values[section][option] = value - if sync: - self.save() - return True - - return False - - def getMetaData(self, section, option): - """ get all config data for an option """ - return self.config[section].config[option] - - def iterSections(self): - """ Yields section, config info, values, for all sections """ - - for name, config in self.config.iteritems(): - yield name, config, self.values[name] if name in self.values else {} - - def getSection(self, section): - """ Retrieves single config as tuple (section, values) """ - return self.config[section], self.values[section] if section in self.values else {} - - def addConfigSection(self, section, name, desc, long_desc, config): - """Adds a section to the config. `config` is a list of config tuples as used in plugin api defined as: - The order of the config elements is preserved with OrderedDict - """ - d = OrderedDict() - - for entry in config: - if len(entry) == 5: - conf_name, type, conf_desc, conf_verbose, default = entry - else: # config options without description - conf_name, type, conf_desc, default = entry - conf_verbose = "" - - d[conf_name] = ConfigData(gettext(conf_desc), type, gettext(conf_verbose), from_string(default, type)) - - data = SectionTuple(gettext(name), gettext(desc), gettext(long_desc), d) - self.config[section] = data - -class Section: - """provides dictionary like access for configparser""" - - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) - - def __setitem__(self, item, value): - """setitem""" - self.parser.set(self.section, item, value) diff --git a/module/config/__init__.py b/module/config/__init__.py deleted file mode 100644 index 4b31e848b..000000000 --- a/module/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'christian' diff --git a/module/config/default.py b/module/config/default.py deleted file mode 100644 index 8a2044281..000000000 --- a/module/config/default.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Configuration layout for default base config -""" - -#TODO: write tooltips and descriptions -#TODO: use apis config related classes - -def make_config(config): - # Check if gettext is installed - _ = lambda x: x - - config.addConfigSection("remote", _("Remote"), _("Description"), _("Long description"), - [ - ("activated", "bool", _("Activated"), _("Tooltip"), True), - ("port", "int", _("Port"), _("Tooltip"), 7227), - ("listenaddr", "ip", _("Adress"), _("Tooltip"), "0.0.0.0"), - ]) - - config.addConfigSection("log", _("Log"), _("Description"), _("Long description"), - [ - ("log_size", "int", _("Size in kb"), _("Tooltip"), 100), - ("log_folder", "folder", _("Folder"), _("Tooltip"), "Logs"), - ("file_log", "bool", _("File Log"), _("Tooltip"), True), - ("log_count", "int", _("Count"), _("Tooltip"), 5), - ("log_rotate", "bool", _("Log Rotate"), _("Tooltip"), True), - ]) - - config.addConfigSection("permission", _("Permissions"), _("Description"), _("Long description"), - [ - ("group", "str", _("Groupname"), _("Tooltip"), "users"), - ("change_dl", "bool", _("Change Group and User of Downloads"), _("Tooltip"), False), - ("change_file", "bool", _("Change file mode of downloads"), _("Tooltip"), False), - ("user", "str", _("Username"), _("Tooltip"), "user"), - ("file", "str", _("Filemode for Downloads"), _("Tooltip"), "0644"), - ("change_group", "bool", _("Change group of running process"), _("Tooltip"), False), - ("folder", "str", _("Folder Permission mode"), _("Tooltip"), "0755"), - ("change_user", "bool", _("Change user of running process"), _("Tooltip"), False), - ]) - - config.addConfigSection("general", _("General"), _("Description"), _("Long description"), - [ - ("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), _("Tooltip"), "en"), - ("download_folder", "folder", _("Download Folder"), _("Tooltip"), "Downloads"), - ("checksum", "bool", _("Use Checksum"), _("Tooltip"), False), - ("folder_per_package", "bool", _("Create folder for each package"), _("Tooltip"), True), - ("debug_mode", "bool", _("Debug Mode"), _("Tooltip"), False), - ("min_free_space", "int", _("Min Free Space (MB)"), _("Tooltip"), 200), - ("renice", "int", _("CPU Priority"), _("Tooltip"), 0), - ]) - - config.addConfigSection("ssl", _("SSL"), _("Description"), _("Long description"), - [ - ("cert", "file", _("SSL Certificate"), _("Tooltip"), "ssl.crt"), - ("activated", "bool", _("Activated"), _("Tooltip"), False), - ("key", "file", _("SSL Key"), _("Tooltip"), "ssl.key"), - ]) - - config.addConfigSection("webinterface", _("Webinterface"), _("Description"), _("Long description"), - [ - ("template", "str", _("Template"), _("Tooltip"), "default"), - ("activated", "bool", _("Activated"), _("Tooltip"), True), - ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""), - ("server", "auto;threaded;fallback;fastcgi", _("Server"), _("Tooltip"), "auto"), - ("force_server", "str", _("Favor specific server"), _("Tooltip"), ""), - ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"), - ("https", "bool", _("Use HTTPS"), _("Tooltip"), False), - ("port", "int", _("Port"), _("Tooltip"), 8001), - ("develop", "str", _("Development mode"), _(""), False), - ]) - - config.addConfigSection("proxy", _("Proxy"), _("Description"), _("Long description"), - [ - ("username", "str", _("Username"), _("Tooltip"), ""), - ("proxy", "bool", _("Use Proxy"), _("Tooltip"), False), - ("address", "str", _("Address"), _("Tooltip"), "localhost"), - ("password", "password", _("Password"), _("Tooltip"), ""), - ("type", "http;socks4;socks5", _("Protocol"), _("Tooltip"), "http"), - ("port", "int", _("Port"), _("Tooltip"), 7070), - ]) - - config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"), - [ - ("endTime", "time", _("End"), _("Tooltip"), "0:00"), - ("activated", "bool", _("Use Reconnect"), _("Tooltip"), False), - ("method", "str", _("Method"), _("Tooltip"), "./reconnect.sh"), - ("startTime", "time", _("Start"), _("Tooltip"), "0:00"), - ]) - - config.addConfigSection("download", _("Download"), _("Description"), _("Long description"), - [ - ("max_downloads", "int", _("Max Parallel Downloads"), _("Tooltip"), 3), - ("limit_speed", "bool", _("Limit Download Speed"), _("Tooltip"), False), - ("interface", "str", _("Download interface to bind (ip or Name)"), _("Tooltip"), ""), - ("skip_existing", "bool", _("Skip already existing files"), _("Tooltip"), False), - ("max_speed", "int", _("Max Download Speed in kb/s"), _("Tooltip"), -1), - ("ipv6", "bool", _("Allow IPv6"), _("Tooltip"), False), - ("chunks", "int", _("Max connections for one download"), _("Tooltip"), 3), - ("restart_failed", "bool", _("Restart failed downloads on startup"), _("Tooltip"), False), - ]) - - config.addConfigSection("downloadTime", _("Download Time"), _("Description"), _("Long description"), - [ - ("start", "time", _("Start"), _("Tooltip"), "0:00"), - ("end", "time", _("End"), _("Tooltip"), "0:00"), - ]) diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py deleted file mode 100644 index 518674951..000000000 --- a/module/database/AccountDatabase.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.database import queue, async -from module.database import DatabaseBackend - - -class AccountMethods: - @queue - def loadAccounts(db): - db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') - return db.c.fetchall() - - @async - def saveAccounts(db, data): - # TODO: owner, shared - - db.c.executemany( - 'INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) - - @async - def removeAccount(db, plugin, loginname): - db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) - - -DatabaseBackend.registerSub(AccountMethods)
\ No newline at end of file diff --git a/module/database/ConfigDatabase.py b/module/database/ConfigDatabase.py deleted file mode 100644 index 554e07132..000000000 --- a/module/database/ConfigDatabase.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.database import DatabaseMethods, queue, async - -class ConfigMethods(DatabaseMethods): - - @async - def saveConfig(self, plugin, config, user=None): - if user is None: user = -1 - self.c.execute('INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', (plugin, config, user)) - - - @queue - def loadConfig(self, plugin, user=None): - if user is None: user = -1 - self.c.execute('SELECT config FROM settings WHERE plugin=? AND user=?', (plugin, user)) - - r = self.c.fetchone() - return r[0] if r else "" - - @async - def deleteConfig(self, plugin, user=None): - if user is None: - self.c.execute('DELETE FROM settings WHERE plugin=?', (plugin, )) - else: - self.c.execute('DELETE FROM settings WHERE plugin=? AND user=?', (plugin, user)) - - @queue - def loadAllConfigs(self): - self.c.execute('SELECT user, plugin, config FROM settings') - configs = {} - for r in self.c: - if r[0] in configs: - configs[r[0]][r[1]] = r[2] - else: - configs[r[0]] = {r[1]: r[2]} - - return configs - - @queue - def loadConfigsForUser(self, user=None): - if user is None: user = -1 - self.c.execute('SELECT plugin, config FROM settings WHERE user=?', (user,)) - configs = {} - for r in self.c: - configs[r[0]] = r[1] - - return configs - - @async - def clearAllConfigs(self): - self.c.execute('DELETE FROM settings') - - -ConfigMethods.register()
\ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py deleted file mode 100644 index 54d1c54bb..000000000 --- a/module/database/DatabaseBackend.py +++ /dev/null @@ -1,492 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN, mkaay -############################################################################### - -from threading import Thread, Event -from shutil import move - -from Queue import Queue -from traceback import print_exc - -from module.utils.fs import chmod, exists, remove - -try: - from pysqlite2 import dbapi2 as sqlite3 -except: - import sqlite3 - -DB = None -DB_VERSION = 6 - -def set_DB(db): - global DB - DB = db - - -def queue(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return DB.queue(f, *args, **kwargs) - - return x - - -def async(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return DB.async(f, *args, **kwargs) - - return x - - -def inner(f): - @staticmethod - def x(*args, **kwargs): - if DB: - return f(DB, *args, **kwargs) - - return x - - -class DatabaseMethods: - # stubs for autocompletion - core = None - manager = None - conn = None - c = None - - @classmethod - def register(cls): - DatabaseBackend.registerSub(cls) - - -class DatabaseJob(): - def __init__(self, f, *args, **kwargs): - self.done = Event() - - self.f = f - self.args = args - self.kwargs = kwargs - - self.result = None - self.exception = False - - # import inspect - # self.frame = inspect.currentframe() - - def __repr__(self): - from os.path import basename - - frame = self.frame.f_back - output = "" - for i in range(5): - output += "\t%s:%s, %s\n" % (basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) - frame = frame.f_back - del frame - del self.frame - - return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) - - def processJob(self): - try: - self.result = self.f(*self.args, **self.kwargs) - except Exception, e: - print_exc() - try: - print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e - except: - pass - - self.exception = e - finally: - self.done.set() - - def wait(self): - self.done.wait() - - -class DatabaseBackend(Thread): - subs = [] - - DB_FILE = "pyload.db" - VERSION_FILE = "db.version" - - def __init__(self, core): - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.manager = None # set later - self.running = Event() - - self.jobs = Queue() - - set_DB(self) - - def setup(self): - """ *MUST* be called before db can be used !""" - self.start() - self.running.wait() - - def init(self): - """main loop, which executes commands""" - - version = self._checkVersion() - - self.conn = sqlite3.connect(self.DB_FILE) - chmod(self.DB_FILE, 0600) - - self.c = self.conn.cursor() - - if version is not None and version < DB_VERSION: - success = self._convertDB(version) - - # delete database - if not success: - self.c.close() - self.conn.close() - - try: - self.manager.core.log.warning(_("Database was deleted due to incompatible version.")) - except: - print "Database was deleted due to incompatible version." - - remove(self.VERSION_FILE) - move(self.DB_FILE, self.DB_FILE + ".backup") - f = open(self.VERSION_FILE, "wb") - f.write(str(DB_VERSION)) - f.close() - - self.conn = sqlite3.connect(self.DB_FILE) - chmod(self.DB_FILE, 0600) - self.c = self.conn.cursor() - - self._createTables() - self.conn.commit() - - - def run(self): - try: - self.init() - finally: - self.running.set() - - while True: - j = self.jobs.get() - if j == "quit": - self.c.close() - self.conn.commit() - self.conn.close() - self.closing.set() - break - j.processJob() - - - def shutdown(self): - self.running.clear() - self.closing = Event() - self.jobs.put("quit") - self.closing.wait(1) - - def _checkVersion(self): - """ get db version""" - if not exists(self.VERSION_FILE): - f = open(self.VERSION_FILE, "wb") - f.write(str(DB_VERSION)) - f.close() - return - - f = open(self.VERSION_FILE, "rb") - v = int(f.read().strip()) - f.close() - - return v - - def _convertDB(self, v): - try: - return getattr(self, "_convertV%i" % v)() - except: - return False - - #--convert scripts start - - def _convertV6(self): - return False - - #--convert scripts end - - def _createTables(self): - """create tables for database""" - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "packages" (' - '"pid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL, ' - '"folder" TEXT DEFAULT "" NOT NULL, ' - '"site" TEXT DEFAULT "" NOT NULL, ' - '"comment" TEXT DEFAULT "" NOT NULL, ' - '"password" TEXT DEFAULT "" NOT NULL, ' - '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger - '"status" INTEGER DEFAULT 0 NOT NULL,' - '"tags" TEXT DEFAULT "" NOT NULL,' - '"shared" INTEGER DEFAULT 0 NOT NULL,' - '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger - '"root" INTEGER DEFAULT -1 NOT NULL, ' - '"owner" INTEGER NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'CHECK (root != pid)' - ')' - ) - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_package" AFTER INSERT ON "packages"' - 'BEGIN ' - 'UPDATE packages SET added = strftime("%s", "now"), ' - 'packageorder = (SELECT max(p.packageorder) + 1 FROM packages p WHERE p.root=new.root) ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "delete_package" AFTER DELETE ON "packages"' - 'BEGIN ' - 'DELETE FROM files WHERE package = old.pid;' - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' - 'END' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') - self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "files" (' - '"fid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL, ' - '"size" INTEGER DEFAULT 0 NOT NULL, ' - '"status" INTEGER DEFAULT 0 NOT NULL, ' - '"media" INTEGER DEFAULT 1 NOT NULL,' - '"added" INTEGER DEFAULT 0 NOT NULL,' - '"fileorder" INTEGER DEFAULT -1 NOT NULL, ' - '"url" TEXT DEFAULT "" NOT NULL, ' - '"plugin" TEXT DEFAULT "" NOT NULL, ' - '"hash" TEXT DEFAULT "" NOT NULL, ' - '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' - '"error" TEXT DEFAULT "" NOT NULL, ' - '"package" INTEGER NOT NULL, ' - '"owner" INTEGER NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'FOREIGN KEY(package) REFERENCES packages(id)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') - self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' - 'BEGIN ' - 'UPDATE files SET added = strftime("%s", "now"), ' - 'fileorder = (SELECT max(f.fileorder) + 1 FROM files f WHERE f.package=new.package) ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "collector" (' - '"owner" INTEGER NOT NULL, ' - '"data" TEXT NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY(owner) ON CONFLICT REPLACE' - ') ' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" (' - '"identifier" TEXT NOT NULL, ' - '"key" TEXT NOT NULL, ' - '"value" TEXT DEFAULT "", ' - 'PRIMARY KEY (identifier, key) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" (' - '"uid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL UNIQUE, ' - '"email" TEXT DEFAULT "" NOT NULL, ' - '"password" TEXT NOT NULL, ' - '"role" INTEGER DEFAULT 0 NOT NULL, ' - '"permission" INTEGER DEFAULT 0 NOT NULL, ' - '"folder" TEXT DEFAULT "" NOT NULL, ' - '"traffic" INTEGER DEFAULT -1 NOT NULL, ' - '"dllimit" INTEGER DEFAULT -1 NOT NULL, ' - '"dlquota" TEXT DEFAULT "" NOT NULL, ' - '"hddquota" INTEGER DEFAULT -1 NOT NULL, ' - '"template" TEXT DEFAULT "default" NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self - 'FOREIGN KEY(user) REFERENCES users(uid)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "username_index" ON users(name)') - - self.c.execute( - 'CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER INSERT ON "users"' - 'BEGIN ' - 'UPDATE users SET user = new.uid, folder=new.name ' - 'WHERE rowid = new.rowid;' - 'END' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "settings" (' - '"plugin" TEXT NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' - '"config" TEXT NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "accounts" (' - '"plugin" TEXT NOT NULL, ' - '"loginname" TEXT NOT NULL, ' - '"owner" INTEGER NOT NULL DEFAULT -1, ' - '"activated" INTEGER DEFAULT 1, ' - '"password" TEXT DEFAULT "", ' - '"shared" INTEGER DEFAULT 0, ' - '"options" TEXT DEFAULT "", ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' - ')' - ) - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "stats" (' - '"user" INTEGER NOT NULL, ' - '"plugin" TEXT NOT NULL, ' - '"time" INTEGER NOT NULL, ' - '"premium" INTEGER DEFAULT 0 NOT NULL, ' - '"amount" INTEGER DEFAULT 0 NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(uid), ' - 'PRIMARY KEY(user, plugin, time)' - ')' - ) - self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)') - - #try to lower ids - self.c.execute('SELECT max(fid) FROM files') - fid = self.c.fetchone()[0] - fid = int(fid) if fid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "files")) - - self.c.execute('SELECT max(pid) FROM packages') - pid = self.c.fetchone()[0] - pid = int(pid) if pid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) - - self.c.execute('VACUUM') - - - def createCursor(self): - return self.conn.cursor() - - @async - def commit(self): - self.conn.commit() - - @queue - def syncSave(self): - self.conn.commit() - - @async - def rollback(self): - self.conn.rollback() - - def async(self, f, *args, **kwargs): - args = (self, ) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - - def queue(self, f, *args, **kwargs): - args = (self, ) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - # only wait when db is running - if self.running.isSet(): job.wait() - return job.result - - @classmethod - def registerSub(cls, klass): - cls.subs.append(klass) - - @classmethod - def unregisterSub(cls, klass): - cls.subs.remove(klass) - - def __getattr__(self, attr): - for sub in DatabaseBackend.subs: - if hasattr(sub, attr): - return getattr(sub, attr) - raise AttributeError(attr) - -if __name__ == "__main__": - db = DatabaseBackend() - db.setup() - - class Test(): - @queue - def insert(db): - c = db.createCursor() - for i in range(1000): - c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @async - def insert2(db): - c = db.createCursor() - for i in range(1000 * 1000): - c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @queue - def select(db): - c = db.createCursor() - for i in range(10): - res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) - print res.fetchone() - - @queue - def error(db): - c = db.createCursor() - print "a" - c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) - print "e" - - db.registerSub(Test) - from time import time - - start = time() - for i in range(100): - db.insert() - end = time() - print end - start - - start = time() - db.insert2() - end = time() - print end - start - - db.error() - diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py deleted file mode 100644 index 023dd1dc5..000000000 --- a/module/database/FileDatabase.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from new_collections import OrderedDict - -from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats, DownloadState as DS, state_string -from module.database import DatabaseMethods, queue, async, inner - -zero_stats = PackageStats(0, 0, 0, 0) - - -class FileMethods(DatabaseMethods): - - @queue - def filecount(self): - """returns number of files, currently only used for debugging""" - self.c.execute("SELECT COUNT(*) FROM files") - return self.c.fetchone()[0] - - @queue - def downloadstats(self, user=None): - """ number of downloads and size """ - if user is None: - self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus != 0") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus != 0", - user) - - r = self.c.fetchone() - # sum is None when no elements are added - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - @queue - def queuestats(self, user=None): - """ number and size of files in queue not finished yet""" - # status not in NA, finished, skipped - if user is None: - self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus NOT IN (0,5,6)") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, package p WHERE f.package = p.pid AND p.owner=? AND dlstatus NOT IN (0,5,6)", - user) - - r = self.c.fetchone() - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - - # TODO: multi user? - @queue - def processcount(self, fid=-1, user=None): - """ number of files which have to be processed """ - # status in online, queued, starting, waiting, downloading - self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) - return self.c.fetchone()[0] - - @queue - def processstats(self, user=None): - if user is None: - self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10)") - else: - self.c.execute( - "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus IN (2,3,8,9,10)", - user) - r = self.c.fetchone() - return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) - - @queue - def addLink(self, url, name, plugin, package, owner): - # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', - (url, name, plugin, package, owner)) - return self.c.lastrowid - - @async - def addLinks(self, links, package, owner): - """ links is a list of tuples (url, plugin)""" - links = [(x[0], x[0], x[1], package, owner) for x in links] - self.c.executemany( - 'INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', - links) - - @queue - def addFile(self, name, size, media, package, owner): - # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', - (name, size, media, package, owner)) - return self.c.lastrowid - - @queue - def addPackage(self, name, folder, root, password, site, comment, status, owner): - self.c.execute( - 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' - , (name, folder, root, password, site, comment, status, owner)) - return self.c.lastrowid - - @async - def deletePackage(self, pid, owner=None): - # order updated by trigger, as well as links deleted - if owner is None: - self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) - else: - self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) - - @async - def deleteFile(self, fid, order, package, owner=None): - """ To delete a file order and package of it is needed """ - if owner is None: - self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) - else: - self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', - (order, package, owner)) - - @async - def saveCollector(self, owner, data): - """ simply save the json string to database """ - self.c.execute("INSERT INTO collector(owner, data) VALUES (?,?)", (owner, data)) - - @queue - def retrieveCollector(self, owner): - """ retrieve the saved string """ - self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) - r = self.c.fetchone() - if not r: return None - return r[0] - - @async - def deleteCollector(self, owner): - """ drop saved user collector """ - self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) - - @queue - def getAllFiles(self, package=None, search=None, state=None, owner=None): - """ Return dict with file information - - :param package: optional package to filter out - :param search: or search string for file name - :param unfinished: filter by dlstatus not finished - :param owner: only specific owner - """ - qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') - - arg = [] - - if state is not None and state != DS.All: - qry += 'dlstatus IN (%s) AND ' % state_string(state) - if owner is not None: - qry += 'owner=? AND ' - arg.append(owner) - - if package is not None: - arg.append(package) - qry += 'package=? AND ' - if search is not None: - search = "%%%s%%" % search.strip("%") - arg.append(search) - qry += "name LIKE ? " - - # make qry valid - if qry.endswith("WHERE "): qry = qry[:-6] - if qry.endswith("AND "): qry = qry[:-4] - - self.c.execute(qry + "ORDER BY package, fileorder", arg) - - data = OrderedDict() - for r in self.c: - f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) - if r[11] > 0: # dl status != NA - f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) - - data[r[0]] = f - - return data - - @queue - def getMatchingFilenames(self, pattern, owner=None): - """ Return matching file names for pattern, useful for search suggestions """ - qry = 'SELECT name FROM files WHERE name LIKE ?' - args = ["%%%s%%" % pattern.strip("%")] - if owner: - qry += " AND owner=?" - args.append(owner) - - self.c.execute(qry, args) - return [r[0] for r in self.c] - - @queue - def getAllPackages(self, root=None, owner=None, tags=None): - """ Return dict with package information - - :param root: optional root to filter - :param owner: optional user id - :param tags: optional tag list - """ - qry = ( - 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' - 'FROM packages%s ORDER BY root, packageorder') - - if root is None: - stats = self.getPackageStats(owner=owner) - if owner is None: - self.c.execute(qry % "") - else: - self.c.execute(qry % " WHERE owner=?", (owner,)) - else: - stats = self.getPackageStats(root=root, owner=owner) - if owner is None: - self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) - else: - self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) - - data = OrderedDict() - for r in self.c: - data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], - stats.get(r[0], zero_stats) - ) - - return data - - @inner - def getPackageStats(self, pid=None, root=None, owner=None): - qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " - "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " - "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " - "FROM packages p JOIN files f ON p.pid = f.package %(sub)s AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " - "GROUP BY p.pid") - - # status in (finished, skipped, processing) - - if root is not None: - self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) - elif pid is not None: - self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) - elif owner is not None: - self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) - else: - self.c.execute(qry % {"sub": ""}) - - data = {} - for r in self.c: - data[r[0]] = PackageStats( - r[2] if r[2] else 0, - r[4] if r[4] else 0, - int(r[1]) if r[1] else 0, - int(r[3]) if r[3] else 0, - ) - - return data - - @queue - def getStatsForPackage(self, pid): - return self.getPackageStats(pid=pid)[pid] - - @queue - def getFileInfo(self, fid, force=False): - """get data for specific file, when force is true download info will be appended""" - self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files ' - 'WHERE fid=?', (fid,)) - r = self.c.fetchone() - if not r: - return None - else: - f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) - if r[11] > 0 or force: - f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) - - return f - - @queue - def getPackageInfo(self, pid, stats=True): - """get data for a specific package, optionally with package stats""" - if stats: - stats = self.getPackageStats(pid=pid) - - self.c.execute( - 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' - 'FROM packages WHERE pid=?', (pid,)) - - r = self.c.fetchone() - if not r: - return None - else: - return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], - stats.get(r[0], zero_stats) if stats else None - ) - - # TODO: does this need owner? - @async - def updateLinkInfo(self, data): - """ data is list of tuples (name, size, status,[ hash,] url)""" - # status in (NA, Offline, Online, Queued, TempOffline) - if data and len(data[0]) == 4: - self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', - data) - else: - self.c.executemany( - 'UPDATE files SET name=?, size=?, dlstatus=?, hash=? WHERE url=? AND dlstatus IN (0,1,2,3,11)', data) - - @async - def updateFile(self, f): - self.c.execute('UPDATE files SET name=?, size=?, status=?,' - 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', - (f.name, f.size, f.filestatus, f.media, f.url, - f.hash, f.status, f.error, f.fid)) - - @async - def updatePackage(self, p): - self.c.execute( - 'UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=?, shared=? WHERE pid=?', - (p.name, p.folder, p.site, p.comment, p.password, ",".join(p.tags), p.status, p.shared, p.pid)) - - # TODO: most modifying methods needs owner argument to avoid checking beforehand - @async - def orderPackage(self, pid, root, oldorder, order): - if oldorder > order: # package moved upwards - self.c.execute( - 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0' - , (order, oldorder, root)) - elif oldorder < order: # moved downwards - self.c.execute( - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0' - , (order, oldorder, root)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) - - @async - def orderFiles(self, pid, fids, oldorder, order): - diff = len(fids) - data = [] - - if oldorder > order: # moved upwards - self.c.execute('UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?' - , (diff, order, oldorder, pid)) - data = [(order + i, fid) for i, fid in enumerate(fids)] - elif oldorder < order: - self.c.execute( - 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?' - , (diff, order, oldorder + diff, pid)) - data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] - - self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) - - @async - def moveFiles(self, pid, fids, package): - self.c.execute('SELECT max(fileorder) FROM files WHERE package=?', (package,)) - r = self.c.fetchone() - order = (r[0] if r[0] else 0) + 1 - - self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', - (len(fids), order, pid)) - - data = [(package, order + i, fid) for i, fid in enumerate(fids)] - self.c.executemany('UPDATE files SET package=?, fileorder=? WHERE fid=?', data) - - @async - def movePackage(self, root, order, pid, dpid): - self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) - r = self.c.fetchone() - max = (r[0] if r[0] else 0) + 1 - - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', - (order, root)) - - self.c.execute('UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) - - @async - def restartFile(self, fid): - # status -> queued - self.c.execute('UPDATE files SET dlstatus=3, error="" WHERE fid=?', (fid,)) - - @async - def restartPackage(self, pid): - # status -> queued - self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) - - - # TODO: multi user approach - @queue - def getJob(self, occ): - """return pyfile ids, which are suitable for download and don't use a occupied plugin""" - cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) - #TODO - - # dlstatus in online, queued | package status = ok - cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " - "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " - "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd - - self.c.execute(cmd) - - return [x[0] for x in self.c] - - @queue - def getUnfinished(self, pid): - """return list of max length 3 ids with pyfiles in package not finished or processed""" - - # status in finished, skipped, processing - self.c.execute("SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) - return [r[0] for r in self.c] - - @queue - def restartFailed(self, owner): - # status=queued, where status in failed, aborted, temp offline - self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") - - @queue - def findDuplicates(self, id, folder, filename): - """ checks if filename exists with different id and same package, dlstatus = finished """ - # TODO: also check root of package - self.c.execute( - "SELECT f.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND f.dlstatus=5 AND f.name=?" - , (folder, id, filename)) - return self.c.fetchone() - - @queue - def purgeLinks(self): - # fstatus = missing - self.c.execute("DELETE FROM files WHERE status == 1") - - @queue - def purgeAll(self): # only used for debugging - self.c.execute("DELETE FROM packages") - self.c.execute("DELETE FROM files") - self.c.execute("DELETE FROM collector") - - -FileMethods.register()
\ No newline at end of file diff --git a/module/database/StatisticDatabase.py b/module/database/StatisticDatabase.py deleted file mode 100644 index 10619eb5b..000000000 --- a/module/database/StatisticDatabase.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.database import DatabaseMethods, queue, async, inner - -# TODO - -class StatisticMethods(DatabaseMethods): - pass - - - -StatisticMethods.register()
\ No newline at end of file diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py deleted file mode 100644 index ffaf51763..000000000 --- a/module/database/StorageDatabase.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.database import DatabaseBackend, queue - -class StorageMethods(): - @queue - def setStorage(db, identifier, key, value): - db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) - if db.c.fetchone() is not None: - db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) - else: - db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - - @queue - def getStorage(db, identifier, key=None): - if key is not None: - db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) - row = db.c.fetchone() - if row is not None: - return row[0] - else: - db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier, )) - d = {} - for row in db.c: - d[row[0]] = row[1] - return d - - @queue - def delStorage(db, identifier, key): - db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) - -DatabaseBackend.registerSub(StorageMethods) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py deleted file mode 100644 index 0df94e0eb..000000000 --- a/module/database/UserDatabase.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from hashlib import sha1 -from string import letters, digits -from random import choice - -alphnum = letters+digits - -from module.Api import UserData - -from DatabaseBackend import DatabaseMethods, queue, async - -def random_salt(): - return "".join(choice(alphnum) for x in range(0,5)) - -class UserMethods(DatabaseMethods): - - @queue - def addUser(self, user, password): - salt = random_salt() - h = sha1(salt + password) - password = salt + h.hexdigest() - - self.c.execute('SELECT name FROM users WHERE name=?', (user, )) - if self.c.fetchone() is not None: - self.c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) - else: - self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - - @queue - def getUserData(self, name=None, uid=None): - qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users" WHERE ') - - if name is not None: - self.c.execute(qry + "name=?", (name,)) - r = self.c.fetchone() - if r: - return UserData(*r) - - elif uid is not None: - self.c.execute(qry + "uid=?", (uid,)) - r = self.c.fetchone() - if r: - return UserData(*r) - - return None - - @queue - def getAllUserData(self): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users"') - user = {} - for r in self.c: - user[r[0]] = UserData(*r) - - return user - - - @queue - def checkAuth(self, user, password): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template, password FROM "users" WHERE name=?', (user, )) - r = self.c.fetchone() - if not r: - return None - salt = r[-1][:5] - pw = r[-1][5:] - h = sha1(salt + password) - if h.hexdigest() == pw: - return UserData(*r[:-1]) - else: - return None - - @queue #TODO - def changePassword(self, user, oldpw, newpw): - self.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) - r = self.c.fetchone() - if not r: - return False - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + oldpw) - if h.hexdigest() == pw: - salt = random_salt() - h = sha1(salt + newpw) - password = salt + h.hexdigest() - - self.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) - return True - - return False - - @async - def setPermission(self, user, perms): - self.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - - @async - def setRole(self, user, role): - self.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - - # TODO update methods - - @async - def removeUser(self, uid=None): - # deletes user and all associated accounts - self.c.execute('DELETE FROM users WHERE user=?', (uid, )) - -UserMethods.register() diff --git a/module/database/__init__.py b/module/database/__init__.py deleted file mode 100644 index d3f97fb53..000000000 --- a/module/database/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from DatabaseBackend import DatabaseMethods, DatabaseBackend, queue, async, inner - -from FileDatabase import FileMethods -from UserDatabase import UserMethods -from StorageDatabase import StorageMethods -from AccountDatabase import AccountMethods -from ConfigDatabase import ConfigMethods -from StatisticDatabase import StatisticMethods
\ No newline at end of file diff --git a/module/datatypes/PyFile.py b/module/datatypes/PyFile.py deleted file mode 100644 index 5f2be8769..000000000 --- a/module/datatypes/PyFile.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import sleep, time -from ReadWriteLock import ReadWriteLock - -from module.Api import ProgressInfo, DownloadProgress, FileInfo, DownloadInfo, DownloadStatus -from module.utils import lock, read_lock - -statusMap = { - "none": 0, - "offline": 1, - "online": 2, - "queued": 3, - "paused": 4, - "finished": 5, - "skipped": 6, - "failed": 7, - "starting": 8, - "waiting": 9, - "downloading": 10, - "temp. offline": 11, - "aborted": 12, - "decrypting": 13, - "processing": 14, - "custom": 15, - "unknown": 16, -} - - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", - "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", - "lock", "plugin", "waitUntil", "abort", "statusname", - "reconnected", "pluginclass") - - @staticmethod - def fromInfoData(m, info): - f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, - "", "", "", DownloadStatus.NA, "", info.package, info.owner) - if info.download: - f.url = info.download.url - f.pluginname = info.download.plugin - f.hash = info.download.hash - f.status = info.download.status - f.error = info.download.error - - return f - - def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, - url, pluginname, hash, status, error, package, owner): - - self.m = manager - - self.fid = int(fid) - self._name = name - self._size = size - self.filestatus = filestatus - self.media = media - self.added = added - self.fileorder = fileorder - self.url = url - self.pluginname = pluginname - self.hash = hash - self.status = status - self.error = error - self.ownerid = owner - self.packageid = package - # database information ends here - - self.lock = ReadWriteLock() - - self.plugin = None - - self.waitUntil = 0 # time() + time to wait - - # status attributes - self.abort = False - self.reconnected = False - self.statusname = None - - - @property - def id(self): - self.m.core.log.debug("Deprecated attr .id, use .fid instead") - return self.fid - - def setSize(self, value): - self._size = int(value) - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - def getName(self): - try: - if self.plugin.req.name: - return self.plugin.req.name - else: - return self._name - except: - return self._name - - def setName(self, name): - """ Only set unicode or utf8 strings as name """ - if type(name) == str: - name = name.decode("utf8") - - self._name = name - - name = property(getName, setName) - - def __repr__(self): - return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname) - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) - self.plugin = self.pluginclass(self) - - @read_lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute""" - return hasattr(self, "plugin") and self.plugin - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - def setStatus(self, status): - self.status = statusMap[status] - # needs to sync so status is written to database - self.sync() - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - def hasStatus(self, status): - return statusMap[status] == self.status - - def sync(self): - """sync PyFile instance with database""" - self.m.updateFile(self) - - @lock - def release(self): - """sync and remove from cache""" - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseFile(self.fid) - - - def toInfoData(self): - return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, - self.media, self.added, self.fileorder, DownloadInfo( - self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error - ) - ) - - def getPath(self): - pass - - def move(self, pid): - pass - - @read_lock - def abortDownload(self): - """abort pyfile if possible""" - # TODO: abort timeout, currently dead locks - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - sleep(0.1) - - self.abort = False - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except: - return 0 - - def getETA(self): - """ gets established time of arrival / or waiting time""" - try: - if self.status == DownloadStatus.Waiting: - return self.waitUntil - time() - - return self.getBytesLeft() / self.getSpeed() - except: - return 0 - - def getBytesArrived(self): - """ gets bytes arrived """ - try: - return self.plugin.req.arrived - except: - return 0 - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.plugin.req.size - self.plugin.req.arrived - except: - return 0 - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except: - return self.size - - def getProgressInfo(self): - return ProgressInfo(self.pluginname, self.name, self.getStatusName(), self.getETA(), - self.getBytesArrived(), self.getSize(), - DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.status)) diff --git a/module/datatypes/PyPackage.py b/module/datatypes/PyPackage.py deleted file mode 100644 index 6ba37ee12..000000000 --- a/module/datatypes/PyPackage.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from time import time - -from module.Api import PackageInfo, PackageStatus -from module.utils.fs import join - -class PyPackage: - """ - Represents a package object at runtime - """ - - @staticmethod - def fromInfoData(m, info): - return PyPackage(m, info.pid, info.name, info.folder, info.root, info.owner, - info.site, info.comment, info.password, info.added, info.tags, info.status, info.shared, info.packageorder) - - def __init__(self, manager, pid, name, folder, root, owner, site, comment, password, added, tags, status, - shared, packageorder): - self.m = manager - - self.pid = pid - self.name = name - self.folder = folder - self.root = root - self.ownerid = owner - self.site = site - self.comment = comment - self.password = password - self.added = added - self.tags = tags - self.status = status - self.shared = shared - self.packageorder = packageorder - self.timestamp = time() - - #: Finish event already fired - self.setFinished = False - - @property - def id(self): - self.m.core.log.debug("Deprecated package attr .id, use .pid instead") - return self.pid - - def isStale(self): - return self.timestamp + 30 * 60 > time() - - def toInfoData(self): - return PackageInfo(self.pid, self.name, self.folder, self.root, self.ownerid, self.site, - self.comment, self.password, self.added, self.tags, self.status, self.shared, self.packageorder - ) - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.pid)["links"] - - def getPath(self, name=""): - self.timestamp = time() - return join(self.m.getPackage(self.root).getPath(), self.folder, name) - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - def delete(self): - self.m.deletePackage(self.id) - - def deleteIfEmpty(self): - """ True if deleted """ - if not len(self.getChildren()): - self.delete() - return True - return False - - def notifyChange(self): - self.m.core.eventManager.dispatchEvent("packageUpdated", self.id) - - -class RootPackage(PyPackage): - def __init__(self, m, owner): - PyPackage.__init__(self, m, -1, "root", "", owner, -2, "", "", "", 0, [], PackageStatus.Ok, False, 0) - - def getPath(self, name=""): - return join(self.m.core.config["general"]["download_folder"], name) - - # no database operations - def sync(self): - pass - - def delete(self): - pass - - def release(self): - pass
\ No newline at end of file diff --git a/module/datatypes/User.py b/module/datatypes/User.py deleted file mode 100644 index 141191df4..000000000 --- a/module/datatypes/User.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - - -from module.Api import UserData, Permission, Role -from module.utils import bits_set - -#TODO: activate user -#noinspection PyUnresolvedReferences -class User(UserData): - - @staticmethod - def fromUserData(api, user): - return User(api, user.uid, user.name, user.email, user.role, user.permission, user.folder, - user.traffic, user.dllimit, user.dlquota, user.hddquota, user.user, user.templateName) - - def __init__(self, api, *args, **kwargs): - UserData.__init__(self, *args, **kwargs) - self.api = api - - - def toUserData(self): - # TODO - return UserData() - - def hasPermission(self, perms): - """ Accepts permission bit or name """ - if isinstance(perms, basestring) and hasattr(Permission, perms): - perms = getattr(Permission, perms) - - return bits_set(perms, self.permission) - - def hasRole(self, role): - if isinstance(role, basestring) and hasattr(Role, role): - role = getattr(Role, role) - - return self.role == role - - def isAdmin(self): - return self.hasRole(Role.Admin) - - @property - def primary(self): - """ Primary user id, Internal user handle used for most operations - Secondary user account share id with primary user. Only Admins have no primary id. """ - if self.hasRole(Role.Admin): - return None - return self.user if self.user else self.uid
\ No newline at end of file diff --git a/module/datatypes/__init__.py b/module/datatypes/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/datatypes/__init__.py +++ /dev/null diff --git a/module/interaction/EventManager.py b/module/interaction/EventManager.py deleted file mode 100644 index 7d37ca6b9..000000000 --- a/module/interaction/EventManager.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- - -from threading import Lock -from traceback import print_exc - -class EventManager: - """ - Handles all event-related tasks, also stores an event queue for clients, so they can retrieve them later. - - **Known Events:** - Most addon methods exist as events. These are some additional known events. - - ===================== ================ =========================================================== - Name Arguments Description - ===================== ================ =========================================================== - event eventName, *args Called for every event, with eventName and original args - download:preparing fid A download was just queued and will be prepared now. - download:start fid A plugin will immediately start the download afterwards. - download:allProcessed All links were handled, pyLoad would idle afterwards. - download:allFinished All downloads in the queue are finished. - config:changed sec, opt, value The config was changed. - ===================== ================ =========================================================== - - | Notes: - | download:allProcessed is *always* called before download:allFinished. - """ - - def __init__(self, core): - self.core = core - self.log = core.log - - # uuid : list of events - self.clients = {} - self.events = {"event": []} - - self.lock = Lock() - - def getEvents(self, uuid): - """ Get accumulated events for uuid since last call, this also registers a new client """ - if uuid not in self.clients: - self.clients[uuid] = Client() - return self.clients[uuid].get() - - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - if func in self.events[event]: - self.log.debug("Function already registered %s" % func) - else: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def removeFromEvents(self, func): - """ Removes func from all known events """ - for name, events in self.events.iteritems(): - if func in events: - events.remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - for f in self.events["event"]: - try: - f(event, *args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % ("event", f, args, str(e))) - if self.core.debug: - print_exc() - - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - print_exc()
\ No newline at end of file diff --git a/module/interaction/InteractionManager.py b/module/interaction/InteractionManager.py deleted file mode 100644 index e4ae05501..000000000 --- a/module/interaction/InteractionManager.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -from threading import Lock -from time import time -from base64 import standard_b64encode - -from new_collections import OrderedDict - -from module.utils import lock, bits_set -from module.Api import Interaction as IA -from module.Api import InputType, Input - -from InteractionTask import InteractionTask - -class InteractionManager: - """ - Class that gives ability to interact with the user. - Arbitrary tasks with predefined output and input types can be set off. - """ - - # number of seconds a client is classified as active - CLIENT_THRESHOLD = 60 - NOTIFICATION_TIMEOUT = 60 * 60 * 30 - MAX_NOTIFICATIONS = 50 - - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = OrderedDict() #task store, for all outgoing tasks - - self.last_clients = {} - self.ids = 0 #uniue interaction ids - - def isClientConnected(self, user): - return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time() - - @lock - def work(self): - # old notifications will be removed - for n in [k for k, v in self.tasks.iteritems() if v.timedOut()]: - del self.tasks[n] - - # keep notifications count limited - n = [k for k,v in self.tasks.iteritems() if v.type == IA.Notification] - n.reverse() - for v in n[:self.MAX_NOTIFICATIONS]: - del self.tasks[v] - - @lock - def createNotification(self, title, content, desc="", plugin="", owner=None): - """ Creates and queues a new Notification - - :param title: short title - :param content: text content - :param desc: short form of the notification - :param plugin: plugin name - :return: :class:`InteractionTask` - """ - task = InteractionTask(self.ids, IA.Notification, Input(InputType.Text, content), "", title, desc, plugin, - owner=owner) - self.ids += 1 - self.queueTask(task) - return task - - @lock - def createQueryTask(self, input, desc, default="", plugin="", owner=None): - # input type was given, create a input widget - if type(input) == int: - input = Input(input) - if not isinstance(input, Input): - raise TypeError("'Input' class expected not '%s'" % type(input)) - - task = InteractionTask(self.ids, IA.Query, input, default, _("Query"), desc, plugin, owner=owner) - self.ids += 1 - self.queueTask(task) - return task - - @lock - def createCaptchaTask(self, img, format, filename, plugin="", type=InputType.Text, owner=None): - """ Createss a new captcha task. - - :param img: image content (not base encoded) - :param format: img format - :param type: :class:`InputType` - :return: - """ - if type == 'textual': - type = InputType.Text - elif type == 'positional': - type = InputType.Click - - input = Input(type, [standard_b64encode(img), format, filename]) - - #todo: title desc plugin - task = InteractionTask(self.ids, IA.Captcha, input, - None, _("Captcha request"), _("Please solve the captcha."), plugin, owner=owner) - - self.ids += 1 - self.queueTask(task) - return task - - @lock - def removeTask(self, task): - if task.iid in self.tasks: - del self.tasks[task.iid] - self.core.evm.dispatchEvent("interaction:deleted", task.iid) - - @lock - def getTaskByID(self, iid): - return self.tasks.get(iid, None) - - @lock - def getTasks(self, user, mode=IA.All): - # update last active clients - self.last_clients[user] = time() - - # filter current mode - tasks = [t for t in self.tasks.itervalues() if mode == IA.All or bits_set(t.type, mode)] - # filter correct user / or shared - tasks = [t for t in tasks if user is None or user == t.owner or t.shared] - - return tasks - - def isTaskWaiting(self, user, mode=IA.All): - tasks = [t for t in self.getTasks(user, mode) if not t.type == IA.Notification or not t.seen] - return len(tasks) > 0 - - def queueTask(self, task): - cli = self.isClientConnected(task.owner) - - # set waiting times based on threshold - if cli: - task.setWaiting(self.CLIENT_THRESHOLD) - else: # TODO: higher threshold after client connects? - task.setWaiting(self.CLIENT_THRESHOLD / 3) - - if task.type == IA.Notification: - task.setWaiting(self.NOTIFICATION_TIMEOUT) # notifications are valid for 30h - - for plugin in self.core.addonManager.activePlugins(): - try: - plugin.newInteractionTask(task) - except: - self.core.print_exc() - - self.tasks[task.iid] = task - self.core.evm.dispatchEvent("interaction:added", task) - - -if __name__ == "__main__": - it = InteractionTask()
\ No newline at end of file diff --git a/module/interaction/InteractionTask.py b/module/interaction/InteractionTask.py deleted file mode 100644 index d2877b2b0..000000000 --- a/module/interaction/InteractionTask.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from time import time - -from module.Api import InteractionTask as BaseInteractionTask -from module.Api import Interaction, InputType, Input - -#noinspection PyUnresolvedReferences -class InteractionTask(BaseInteractionTask): - """ - General Interaction Task extends ITask defined by api with additional fields and methods. - """ - #: Plugins can put needed data here - storage = None - #: Timestamp when task expires - wait_until = 0 - #: The received result - result = None - #: List of registered handles - handler = None - #: Error Message - error = None - #: Timeout locked - locked = False - #: A task that was retrieved counts as seen - seen = False - #: A task that is relevant to every user - shared = False - #: primary uid of the owner - owner = None - - def __init__(self, *args, **kwargs): - if 'owner' in kwargs: - self.owner = kwargs['owner'] - del kwargs['owner'] - if 'shared' in kwargs: - self.shared = kwargs['shared'] - del kwargs['shared'] - - BaseInteractionTask.__init__(self, *args, **kwargs) - - # additional internal attributes - self.storage = {} - self.handler = [] - self.wait_until = 0 - - def convertResult(self, value): - #TODO: convert based on input/output - return value - - def getResult(self): - return self.result - - def setShared(self): - """ enable shared mode, should not be reversed""" - self.shared = True - - def setResult(self, value): - self.result = self.convertResult(value) - - def setWaiting(self, sec, lock=False): - """ sets waiting in seconds from now, < 0 can be used as infinitive """ - if not self.locked: - if sec < 0: - self.wait_until = -1 - else: - self.wait_until = max(time() + sec, self.wait_until) - - if lock: self.locked = True - - def isWaiting(self): - if self.result or self.error or self.timedOut(): - return False - - return True - - def timedOut(self): - return time() > self.wait_until > -1 - - def correct(self): - [x.taskCorrect(self) for x in self.handler] - - def invalid(self): - [x.taskInvalid(self) for x in self.handler]
\ No newline at end of file diff --git a/module/interaction/__init__.py b/module/interaction/__init__.py deleted file mode 100644 index de6d13128..000000000 --- a/module/interaction/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'christian' -
\ No newline at end of file diff --git a/module/lib/Getch.py b/module/lib/Getch.py deleted file mode 100644 index 22b7ea7f8..000000000 --- a/module/lib/Getch.py +++ /dev/null @@ -1,76 +0,0 @@ -class Getch: - """ - Gets a single character from standard input. Does not echo to - the screen. - """ - - def __init__(self): - try: - self.impl = _GetchWindows() - except ImportError: - try: - self.impl = _GetchMacCarbon() - except(AttributeError, ImportError): - self.impl = _GetchUnix() - - def __call__(self): return self.impl() - - -class _GetchUnix: - def __init__(self): - import tty - import sys - - def __call__(self): - import sys - import tty - import termios - - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - -class _GetchWindows: - def __init__(self): - import msvcrt - - def __call__(self): - import msvcrt - - return msvcrt.getch() - -class _GetchMacCarbon: - """ - A function which returns the current ASCII key that is down; - if no ASCII key is down, the null string is returned. The - page http://www.mactech.com/macintosh-c/chap02-1.html was - very helpful in figuring out how to do this. - """ - - def __init__(self): - import Carbon - Carbon.Evt #see if it has this (in Unix, it doesn't) - - def __call__(self): - import Carbon - - if Carbon.Evt.EventAvail(0x0008)[0] == 0: # 0x0008 is the keyDownMask - return '' - else: - # - # The event contains the following info: - # (what,msg,when,where,mod)=Carbon.Evt.GetNextEvent(0x0008)[1] - # - # The message (msg) contains the ASCII char which is - # extracted with the 0x000000FF charCodeMask; this - # number is converted to an ASCII character with chr() and - # returned - # - (what, msg, when, where, mod) = Carbon.Evt.GetNextEvent(0x0008)[1] - return chr(msg)
\ No newline at end of file diff --git a/module/lib/ReadWriteLock.py b/module/lib/ReadWriteLock.py deleted file mode 100644 index cc82f3d48..000000000 --- a/module/lib/ReadWriteLock.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding: iso-8859-15 -*- -"""locks.py - Read-Write lock thread lock implementation - -See the class documentation for more info. - -Copyright (C) 2007, Heiko Wundram. -Released under the BSD-license. - -http://code.activestate.com/recipes/502283-read-write-lock-class-rlock-like/ -""" - -# Imports -# ------- - -from threading import Condition, Lock, currentThread -from time import time - - -# Read write lock -# --------------- - -class ReadWriteLock(object): - """Read-Write lock class. A read-write lock differs from a standard - threading.RLock() by allowing multiple threads to simultaneously hold a - read lock, while allowing only a single thread to hold a write lock at the - same point of time. - - When a read lock is requested while a write lock is held, the reader - is blocked; when a write lock is requested while another write lock is - held or there are read locks, the writer is blocked. - - Writers are always preferred by this implementation: if there are blocked - threads waiting for a write lock, current readers may request more read - locks (which they eventually should free, as they starve the waiting - writers otherwise), but a new thread requesting a read lock will not - be granted one, and block. This might mean starvation for readers if - two writer threads interweave their calls to acquireWrite() without - leaving a window only for readers. - - In case a current reader requests a write lock, this can and will be - satisfied without giving up the read locks first, but, only one thread - may perform this kind of lock upgrade, as a deadlock would otherwise - occur. After the write lock has been granted, the thread will hold a - full write lock, and not be downgraded after the upgrading call to - acquireWrite() has been match by a corresponding release(). - """ - - def __init__(self): - """Initialize this read-write lock.""" - - # Condition variable, used to signal waiters of a change in object - # state. - self.__condition = Condition(Lock()) - - # Initialize with no writers. - self.__writer = None - self.__upgradewritercount = 0 - self.__pendingwriters = [] - - # Initialize with no readers. - self.__readers = {} - - def acquire(self, blocking=True, timeout=None, shared=False): - if shared: - self.acquireRead(timeout) - else: - self.acquireWrite(timeout) - - def acquireRead(self, timeout=None): - """Acquire a read lock for the current thread, waiting at most - timeout seconds or doing a non-blocking check in case timeout is <= 0. - - In case timeout is None, the call to acquireRead blocks until the - lock request can be serviced. - - In case the timeout expires before the lock could be serviced, a - RuntimeError is thrown.""" - - if timeout is not None: - endtime = time() + timeout - me = currentThread() - self.__condition.acquire() - try: - if self.__writer is me: - # If we are the writer, grant a new read lock, always. - self.__writercount += 1 - return - while True: - if self.__writer is None: - # Only test anything if there is no current writer. - if self.__upgradewritercount or self.__pendingwriters: - if me in self.__readers: - # Only grant a read lock if we already have one - # in case writers are waiting for their turn. - # This means that writers can't easily get starved - # (but see below, readers can). - self.__readers[me] += 1 - return - # No, we aren't a reader (yet), wait for our turn. - else: - # Grant a new read lock, always, in case there are - # no pending writers (and no writer). - self.__readers[me] = self.__readers.get(me, 0) + 1 - return - if timeout is not None: - remaining = endtime - time() - if remaining <= 0: - # Timeout has expired, signal caller of this. - raise RuntimeError("Acquiring read lock timed out") - self.__condition.wait(remaining) - else: - self.__condition.wait() - finally: - self.__condition.release() - - def acquireWrite(self, timeout=None): - """Acquire a write lock for the current thread, waiting at most - timeout seconds or doing a non-blocking check in case timeout is <= 0. - - In case the write lock cannot be serviced due to the deadlock - condition mentioned above, a ValueError is raised. - - In case timeout is None, the call to acquireWrite blocks until the - lock request can be serviced. - - In case the timeout expires before the lock could be serviced, a - RuntimeError is thrown.""" - - if timeout is not None: - endtime = time() + timeout - me, upgradewriter = currentThread(), False - self.__condition.acquire() - try: - if self.__writer is me: - # If we are the writer, grant a new write lock, always. - self.__writercount += 1 - return - elif me in self.__readers: - # If we are a reader, no need to add us to pendingwriters, - # we get the upgradewriter slot. - if self.__upgradewritercount: - # If we are a reader and want to upgrade, and someone - # else also wants to upgrade, there is no way we can do - # this except if one of us releases all his read locks. - # Signal this to user. - raise ValueError( - "Inevitable dead lock, denying write lock" - ) - upgradewriter = True - self.__upgradewritercount = self.__readers.pop(me) - else: - # We aren't a reader, so add us to the pending writers queue - # for synchronization with the readers. - self.__pendingwriters.append(me) - while True: - if not self.__readers and self.__writer is None: - # Only test anything if there are no readers and writers. - if self.__upgradewritercount: - if upgradewriter: - # There is a writer to upgrade, and it's us. Take - # the write lock. - self.__writer = me - self.__writercount = self.__upgradewritercount + 1 - self.__upgradewritercount = 0 - return - # There is a writer to upgrade, but it's not us. - # Always leave the upgrade writer the advance slot, - # because he presumes he'll get a write lock directly - # from a previously held read lock. - elif self.__pendingwriters[0] is me: - # If there are no readers and writers, it's always - # fine for us to take the writer slot, removing us - # from the pending writers queue. - # This might mean starvation for readers, though. - self.__writer = me - self.__writercount = 1 - self.__pendingwriters = self.__pendingwriters[1:] - return - if timeout is not None: - remaining = endtime - time() - if remaining <= 0: - # Timeout has expired, signal caller of this. - if upgradewriter: - # Put us back on the reader queue. No need to - # signal anyone of this change, because no other - # writer could've taken our spot before we got - # here (because of remaining readers), as the test - # for proper conditions is at the start of the - # loop, not at the end. - self.__readers[me] = self.__upgradewritercount - self.__upgradewritercount = 0 - else: - # We were a simple pending writer, just remove us - # from the FIFO list. - self.__pendingwriters.remove(me) - raise RuntimeError("Acquiring write lock timed out") - self.__condition.wait(remaining) - else: - self.__condition.wait() - finally: - self.__condition.release() - - def release(self): - """Release the currently held lock. - - In case the current thread holds no lock, a ValueError is thrown.""" - - me = currentThread() - self.__condition.acquire() - try: - if self.__writer is me: - # We are the writer, take one nesting depth away. - self.__writercount -= 1 - if not self.__writercount: - # No more write locks; take our writer position away and - # notify waiters of the new circumstances. - self.__writer = None - self.__condition.notifyAll() - elif me in self.__readers: - # We are a reader currently, take one nesting depth away. - self.__readers[me] -= 1 - if not self.__readers[me]: - # No more read locks, take our reader position away. - del self.__readers[me] - if not self.__readers: - # No more readers, notify waiters of the new - # circumstances. - self.__condition.notifyAll() - else: - raise ValueError("Trying to release unheld lock") - finally: - self.__condition.release() diff --git a/module/lib/SafeEval.py b/module/lib/SafeEval.py deleted file mode 100644 index 8fc57f261..000000000 --- a/module/lib/SafeEval.py +++ /dev/null @@ -1,47 +0,0 @@ -## {{{ http://code.activestate.com/recipes/286134/ (r3) (modified) -import dis - -_const_codes = map(dis.opmap.__getitem__, [ - 'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP', - 'BUILD_LIST','BUILD_MAP','BUILD_TUPLE', - 'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR' - ]) - - -_load_names = ['False', 'True', 'null', 'true', 'false'] - -_locals = {'null': None, 'true': True, 'false': False} - -def _get_opcodes(codeobj): - i = 0 - opcodes = [] - s = codeobj.co_code - names = codeobj.co_names - while i < len(s): - code = ord(s[i]) - opcodes.append(code) - if code >= dis.HAVE_ARGUMENT: - i += 3 - else: - i += 1 - return opcodes, names - -def test_expr(expr, allowed_codes): - try: - c = compile(expr, "", "eval") - except: - raise ValueError, "%s is not a valid expression" % expr - codes, names = _get_opcodes(c) - for code in codes: - if code not in allowed_codes: - for n in names: - if n not in _load_names: - raise ValueError, "opcode %s not allowed" % dis.opname[code] - return c - - -def const_eval(expr): - c = test_expr(expr, _const_codes) - return eval(c, None, _locals) - -## end of http://code.activestate.com/recipes/286134/ }}} diff --git a/module/lib/__init__.py b/module/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/lib/__init__.py +++ /dev/null diff --git a/module/lib/beaker/__init__.py b/module/lib/beaker/__init__.py deleted file mode 100644 index 792d60054..000000000 --- a/module/lib/beaker/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/module/lib/beaker/cache.py b/module/lib/beaker/cache.py deleted file mode 100644 index 4a96537ff..000000000 --- a/module/lib/beaker/cache.py +++ /dev/null @@ -1,459 +0,0 @@ -"""Cache object - -The Cache object is used to manage a set of cache files and their -associated backend. The backends can be rotated on the fly by -specifying an alternate type when used. - -Advanced users can add new backends in beaker.backends - -""" - -import warnings - -import beaker.container as container -import beaker.util as util -from beaker.exceptions import BeakerException, InvalidCacheBackendError - -import beaker.ext.memcached as memcached -import beaker.ext.database as database -import beaker.ext.sqla as sqla -import beaker.ext.google as google - -# Initialize the basic available backends -clsmap = { - 'memory':container.MemoryNamespaceManager, - 'dbm':container.DBMNamespaceManager, - 'file':container.FileNamespaceManager, - 'ext:memcached':memcached.MemcachedNamespaceManager, - 'ext:database':database.DatabaseNamespaceManager, - 'ext:sqla': sqla.SqlaNamespaceManager, - 'ext:google': google.GoogleNamespaceManager, - } - -# Initialize the cache region dict -cache_regions = {} -cache_managers = {} - -try: - import pkg_resources - - # Load up the additional entry point defined backends - for entry_point in pkg_resources.iter_entry_points('beaker.backends'): - try: - NamespaceManager = entry_point.load() - name = entry_point.name - if name in clsmap: - raise BeakerException("NamespaceManager name conflict,'%s' " - "already loaded" % name) - clsmap[name] = NamespaceManager - except (InvalidCacheBackendError, SyntaxError): - # Ignore invalid backends - pass - except: - import sys - from pkg_resources import DistributionNotFound - # Warn when there's a problem loading a NamespaceManager - if not isinstance(sys.exc_info()[1], DistributionNotFound): - import traceback - from StringIO import StringIO - tb = StringIO() - traceback.print_exc(file=tb) - warnings.warn("Unable to load NamespaceManager entry point: '%s': " - "%s" % (entry_point, tb.getvalue()), RuntimeWarning, - 2) -except ImportError: - pass - - - - -def cache_region(region, *deco_args): - """Decorate a function to cache itself using a cache region - - The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is - because the namespace used for the functions cache is based on - the functions name and the module. - - - Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - @cache_region('short_term', 'some_data') - def populate_things(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - cache = [None] - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - reg = cache_regions[region] - if not reg.get('enabled', True): - return func(*args) - - if not cache[0]: - if region not in cache_regions: - raise BeakerException('Cache region not configured: %s' % region) - cache[0] = Cache._get_cache(namespace, reg) - - cache_key = " ".join(map(str, deco_args + args)) - def go(): - return func(*args) - - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - cached._arg_region = region - return cached - return decorate - - -def region_invalidate(namespace, region, *args): - """Invalidate a cache region namespace or decorated function - - This function only invalidates cache spaces created with the - cache_region decorator. - - :param namespace: Either the namespace of the result to invalidate, or the - cached function reference - - :param region: The region the function was cached to. If the function was - cached to a single region then this argument can be None - - :param args: Arguments that were used to differentiate the cached - function as well as the arguments passed to the decorated - function - - Example:: - - # Add cache region settings to beaker: - beaker.cache.cache_regions.update(dict_of_config_region_options)) - - def populate_things(invalidate=False): - - @cache_region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - region_invalidate(load, None, 'some_data', - 'rabbits', 20, 0) - return load('rabbits', 20, 0) - - """ - if callable(namespace): - if not region: - region = namespace._arg_region - namespace = namespace._arg_namespace - - if not region: - raise BeakerException("Region or callable function " - "namespace is required") - else: - region = cache_regions[region] - - cache = Cache._get_cache(namespace, region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) - - -class Cache(object): - """Front-end to the containment API implementing a data cache. - - :param namespace: the namespace of this Cache - - :param type: type of cache to use - - :param expire: seconds to keep cached data - - :param expiretime: seconds to keep cached data (legacy support) - - :param starttime: time when cache was cache was - - """ - def __init__(self, namespace, type='memory', expiretime=None, - starttime=None, expire=None, **nsargs): - try: - cls = clsmap[type] - if isinstance(cls, InvalidCacheBackendError): - raise cls - except KeyError: - raise TypeError("Unknown cache implementation %r" % type) - - self.namespace = cls(namespace, **nsargs) - self.expiretime = expiretime or expire - self.starttime = starttime - self.nsargs = nsargs - - @classmethod - def _get_cache(cls, namespace, kw): - key = namespace + str(kw) - try: - return cache_managers[key] - except KeyError: - cache_managers[key] = cache = cls(namespace, **kw) - return cache - - def put(self, key, value, **kw): - self._get_value(key, **kw).set_value(value) - set_value = put - - def get(self, key, **kw): - """Retrieve a cached value from the container""" - return self._get_value(key, **kw).get_value() - get_value = get - - def remove_value(self, key, **kw): - mycontainer = self._get_value(key, **kw) - if mycontainer.has_current_value(): - mycontainer.clear_value() - remove = remove_value - - def _get_value(self, key, **kw): - if isinstance(key, unicode): - key = key.encode('ascii', 'backslashreplace') - - if 'type' in kw: - return self._legacy_get_value(key, **kw) - - kw.setdefault('expiretime', self.expiretime) - kw.setdefault('starttime', self.starttime) - - return container.Value(key, self.namespace, **kw) - - @util.deprecated("Specifying a " - "'type' and other namespace configuration with cache.get()/put()/etc. " - "is deprecated. Specify 'type' and other namespace configuration to " - "cache_manager.get_cache() and/or the Cache constructor instead.") - def _legacy_get_value(self, key, type, **kw): - expiretime = kw.pop('expiretime', self.expiretime) - starttime = kw.pop('starttime', None) - createfunc = kw.pop('createfunc', None) - kwargs = self.nsargs.copy() - kwargs.update(kw) - c = Cache(self.namespace.namespace, type=type, **kwargs) - return c._get_value(key, expiretime=expiretime, createfunc=createfunc, - starttime=starttime) - - def clear(self): - """Clear all the values from the namespace""" - self.namespace.remove() - - # dict interface - def __getitem__(self, key): - return self.get(key) - - def __contains__(self, key): - return self._get_value(key).has_current_value() - - def has_key(self, key): - return key in self - - def __delitem__(self, key): - self.remove_value(key) - - def __setitem__(self, key, value): - self.put(key, value) - - -class CacheManager(object): - def __init__(self, **kwargs): - """Initialize a CacheManager object with a set of options - - Options should be parsed with the - :func:`~beaker.util.parse_cache_config_options` function to - ensure only valid options are used. - - """ - self.kwargs = kwargs - self.regions = kwargs.pop('cache_regions', {}) - - # Add these regions to the module global - cache_regions.update(self.regions) - - def get_cache(self, name, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - return Cache._get_cache(name, kw) - - def get_cache_region(self, name, region): - if region not in self.regions: - raise BeakerException('Cache region not configured: %s' % region) - kw = self.regions[region] - return Cache._get_cache(name, kw) - - def region(self, region, *args): - """Decorate a function to cache itself using a cache region - - The region decorator requires arguments if there are more than - 2 of the same named function, in the same module. This is - because the namespace used for the functions cache is based on - the functions name and the module. - - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(): - - @cache.region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - return cache_region(region, *args) - - def region_invalidate(self, namespace, region, *args): - """Invalidate a cache region namespace or decorated function - - This function only invalidates cache spaces created with the - cache_region decorator. - - :param namespace: Either the namespace of the result to invalidate, or the - name of the cached function - - :param region: The region the function was cached to. If the function was - cached to a single region then this argument can be None - - :param args: Arguments that were used to differentiate the cached - function as well as the arguments passed to the decorated - function - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - def populate_things(invalidate=False): - - @cache.region('short_term', 'some_data') - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - cache.region_invalidate(load, None, 'some_data', - 'rabbits', 20, 0) - return load('rabbits', 20, 0) - - - """ - return region_invalidate(namespace, region, *args) - if callable(namespace): - if not region: - region = namespace._arg_region - namespace = namespace._arg_namespace - - if not region: - raise BeakerException("Region or callable function " - "namespace is required") - else: - region = self.regions[region] - - cache = self.get_cache(namespace, **region) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) - - def cache(self, *args, **kwargs): - """Decorate a function to cache itself with supplied parameters - - :param args: Used to make the key unique for this function, as in region() - above. - - :param kwargs: Parameters to be passed to get_cache(), will override defaults - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(): - - @cache.cache('mycache', expire=15) - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - return load('rabbits', 20, 0) - - .. note:: - - The function being decorated must only be called with - positional arguments. - - """ - cache = [None] - key = " ".join(str(x) for x in args) - - def decorate(func): - namespace = util.func_namespace(func) - def cached(*args): - if not cache[0]: - cache[0] = self.get_cache(namespace, **kwargs) - cache_key = key + " " + " ".join(str(x) for x in args) - def go(): - return func(*args) - return cache[0].get_value(cache_key, createfunc=go) - cached._arg_namespace = namespace - return cached - return decorate - - def invalidate(self, func, *args, **kwargs): - """Invalidate a cache decorated function - - This function only invalidates cache spaces created with the - cache decorator. - - :param func: Decorated function to invalidate - - :param args: Used to make the key unique for this function, as in region() - above. - - :param kwargs: Parameters that were passed for use by get_cache(), note that - this is only required if a ``type`` was specified for the - function - - Example:: - - # Assuming a cache object is available like: - cache = CacheManager(dict_of_config_options) - - - def populate_things(invalidate=False): - - @cache.cache('mycache', type="file", expire=15) - def load(search_term, limit, offset): - return load_the_data(search_term, limit, offset) - - # If the results should be invalidated first - if invalidate: - cache.invalidate(load, 'mycache', 'rabbits', 20, 0, type="file") - return load('rabbits', 20, 0) - - """ - namespace = func._arg_namespace - - cache = self.get_cache(namespace, **kwargs) - cache_key = " ".join(str(x) for x in args) - cache.remove_value(cache_key) diff --git a/module/lib/beaker/container.py b/module/lib/beaker/container.py deleted file mode 100644 index 515e97af6..000000000 --- a/module/lib/beaker/container.py +++ /dev/null @@ -1,633 +0,0 @@ -"""Container and Namespace classes""" -import anydbm -import cPickle -import logging -import os -import time - -import beaker.util as util -from beaker.exceptions import CreationAbortedError, MissingCacheParameter -from beaker.synchronization import _threading, file_synchronizer, \ - mutex_synchronizer, NameLock, null_synchronizer - -__all__ = ['Value', 'Container', 'ContainerContext', - 'MemoryContainer', 'DBMContainer', 'NamespaceManager', - 'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer', - 'OpenResourceNamespaceManager', - 'FileNamespaceManager', 'CreationAbortedError'] - - -logger = logging.getLogger('beaker.container') -if logger.isEnabledFor(logging.DEBUG): - debug = logger.debug -else: - def debug(message, *args): - pass - - -class NamespaceManager(object): - """Handles dictionary operations and locking for a namespace of - values. - - The implementation for setting and retrieving the namespace data is - handled by subclasses. - - NamespaceManager may be used alone, or may be privately accessed by - one or more Container objects. Container objects provide per-key - services like expiration times and automatic recreation of values. - - Multiple NamespaceManagers created with a particular name will all - share access to the same underlying datasource and will attempt to - synchronize against a common mutex object. The scope of this - sharing may be within a single process or across multiple - processes, depending on the type of NamespaceManager used. - - The NamespaceManager itself is generally threadsafe, except in the - case of the DBMNamespaceManager in conjunction with the gdbm dbm - implementation. - - """ - - @classmethod - def _init_dependencies(cls): - pass - - def __init__(self, namespace): - self._init_dependencies() - self.namespace = namespace - - def get_creation_lock(self, key): - raise NotImplementedError() - - def do_remove(self): - raise NotImplementedError() - - def acquire_read_lock(self): - pass - - def release_read_lock(self): - pass - - def acquire_write_lock(self, wait=True): - return True - - def release_write_lock(self): - pass - - def has_key(self, key): - return self.__contains__(key) - - def __getitem__(self, key): - raise NotImplementedError() - - def __setitem__(self, key, value): - raise NotImplementedError() - - def set_value(self, key, value, expiretime=None): - """Optional set_value() method called by Value. - - Allows an expiretime to be passed, for namespace - implementations which can prune their collections - using expiretime. - - """ - self[key] = value - - def __contains__(self, key): - raise NotImplementedError() - - def __delitem__(self, key): - raise NotImplementedError() - - def keys(self): - raise NotImplementedError() - - def remove(self): - self.do_remove() - - -class OpenResourceNamespaceManager(NamespaceManager): - """A NamespaceManager where read/write operations require opening/ - closing of a resource which is possibly mutexed. - - """ - def __init__(self, namespace): - NamespaceManager.__init__(self, namespace) - self.access_lock = self.get_access_lock() - self.openers = 0 - self.mutex = _threading.Lock() - - def get_access_lock(self): - raise NotImplementedError() - - def do_open(self, flags): - raise NotImplementedError() - - def do_close(self): - raise NotImplementedError() - - def acquire_read_lock(self): - self.access_lock.acquire_read_lock() - try: - self.open('r', checkcount = True) - except: - self.access_lock.release_read_lock() - raise - - def release_read_lock(self): - try: - self.close(checkcount = True) - finally: - self.access_lock.release_read_lock() - - def acquire_write_lock(self, wait=True): - r = self.access_lock.acquire_write_lock(wait) - try: - if (wait or r): - self.open('c', checkcount = True) - return r - except: - self.access_lock.release_write_lock() - raise - - def release_write_lock(self): - try: - self.close(checkcount=True) - finally: - self.access_lock.release_write_lock() - - def open(self, flags, checkcount=False): - self.mutex.acquire() - try: - if checkcount: - if self.openers == 0: - self.do_open(flags) - self.openers += 1 - else: - self.do_open(flags) - self.openers = 1 - finally: - self.mutex.release() - - def close(self, checkcount=False): - self.mutex.acquire() - try: - if checkcount: - self.openers -= 1 - if self.openers == 0: - self.do_close() - else: - if self.openers > 0: - self.do_close() - self.openers = 0 - finally: - self.mutex.release() - - def remove(self): - self.access_lock.acquire_write_lock() - try: - self.close(checkcount=False) - self.do_remove() - finally: - self.access_lock.release_write_lock() - -class Value(object): - __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\ - 'namespace' - - def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None): - self.key = key - self.createfunc = createfunc - self.expire_argument = expiretime - self.starttime = starttime - self.storedtime = -1 - self.namespace = namespace - - def has_value(self): - """return true if the container has a value stored. - - This is regardless of it being expired or not. - - """ - self.namespace.acquire_read_lock() - try: - return self.namespace.has_key(self.key) - finally: - self.namespace.release_read_lock() - - def can_have_value(self): - return self.has_current_value() or self.createfunc is not None - - def has_current_value(self): - self.namespace.acquire_read_lock() - try: - has_value = self.namespace.has_key(self.key) - if has_value: - try: - stored, expired, value = self._get_value() - return not self._is_expired(stored, expired) - except KeyError: - pass - return False - finally: - self.namespace.release_read_lock() - - def _is_expired(self, storedtime, expiretime): - """Return true if this container's value is expired.""" - return ( - ( - self.starttime is not None and - storedtime < self.starttime - ) - or - ( - expiretime is not None and - time.time() >= expiretime + storedtime - ) - ) - - def get_value(self): - self.namespace.acquire_read_lock() - try: - has_value = self.has_value() - if has_value: - try: - stored, expired, value = self._get_value() - if not self._is_expired(stored, expired): - return value - except KeyError: - # guard against un-mutexed backends raising KeyError - has_value = False - - if not self.createfunc: - raise KeyError(self.key) - finally: - self.namespace.release_read_lock() - - has_createlock = False - creation_lock = self.namespace.get_creation_lock(self.key) - if has_value: - if not creation_lock.acquire(wait=False): - debug("get_value returning old value while new one is created") - return value - else: - debug("lock_creatfunc (didnt wait)") - has_createlock = True - - if not has_createlock: - debug("lock_createfunc (waiting)") - creation_lock.acquire() - debug("lock_createfunc (waited)") - - try: - # see if someone created the value already - self.namespace.acquire_read_lock() - try: - if self.has_value(): - try: - stored, expired, value = self._get_value() - if not self._is_expired(stored, expired): - return value - except KeyError: - # guard against un-mutexed backends raising KeyError - pass - finally: - self.namespace.release_read_lock() - - debug("get_value creating new value") - v = self.createfunc() - self.set_value(v) - return v - finally: - creation_lock.release() - debug("released create lock") - - def _get_value(self): - value = self.namespace[self.key] - try: - stored, expired, value = value - except ValueError: - if not len(value) == 2: - raise - # Old format: upgrade - stored, value = value - expired = self.expire_argument - debug("get_value upgrading time %r expire time %r", stored, self.expire_argument) - self.namespace.release_read_lock() - self.set_value(value, stored) - self.namespace.acquire_read_lock() - except TypeError: - # occurs when the value is None. memcached - # may yank the rug from under us in which case - # that's the result - raise KeyError(self.key) - return stored, expired, value - - def set_value(self, value, storedtime=None): - self.namespace.acquire_write_lock() - try: - if storedtime is None: - storedtime = time.time() - debug("set_value stored time %r expire time %r", storedtime, self.expire_argument) - self.namespace.set_value(self.key, (storedtime, self.expire_argument, value)) - finally: - self.namespace.release_write_lock() - - def clear_value(self): - self.namespace.acquire_write_lock() - try: - debug("clear_value") - if self.namespace.has_key(self.key): - try: - del self.namespace[self.key] - except KeyError: - # guard against un-mutexed backends raising KeyError - pass - self.storedtime = -1 - finally: - self.namespace.release_write_lock() - -class AbstractDictionaryNSManager(NamespaceManager): - """A subclassable NamespaceManager that places data in a dictionary. - - Subclasses should provide a "dictionary" attribute or descriptor - which returns a dict-like object. The dictionary will store keys - that are local to the "namespace" attribute of this manager, so - ensure that the dictionary will not be used by any other namespace. - - e.g.:: - - import collections - cached_data = collections.defaultdict(dict) - - class MyDictionaryManager(AbstractDictionaryNSManager): - def __init__(self, namespace): - AbstractDictionaryNSManager.__init__(self, namespace) - self.dictionary = cached_data[self.namespace] - - The above stores data in a global dictionary called "cached_data", - which is structured as a dictionary of dictionaries, keyed - first on namespace name to a sub-dictionary, then on actual - cache key to value. - - """ - - def get_creation_lock(self, key): - return NameLock( - identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key), - reentrant=True - ) - - def __getitem__(self, key): - return self.dictionary[key] - - def __contains__(self, key): - return self.dictionary.__contains__(key) - - def has_key(self, key): - return self.dictionary.__contains__(key) - - def __setitem__(self, key, value): - self.dictionary[key] = value - - def __delitem__(self, key): - del self.dictionary[key] - - def do_remove(self): - self.dictionary.clear() - - def keys(self): - return self.dictionary.keys() - -class MemoryNamespaceManager(AbstractDictionaryNSManager): - namespaces = util.SyncDict() - - def __init__(self, namespace, **kwargs): - AbstractDictionaryNSManager.__init__(self, namespace) - self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace, - dict) - -class DBMNamespaceManager(OpenResourceNamespaceManager): - def __init__(self, namespace, dbmmodule=None, data_dir=None, - dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs): - self.digest_filenames = digest_filenames - - if not dbm_dir and not data_dir: - raise MissingCacheParameter("data_dir or dbm_dir is required") - elif dbm_dir: - self.dbm_dir = dbm_dir - else: - self.dbm_dir = data_dir + "/container_dbm" - util.verify_directory(self.dbm_dir) - - if not lock_dir and not data_dir: - raise MissingCacheParameter("data_dir or lock_dir is required") - elif lock_dir: - self.lock_dir = lock_dir - else: - self.lock_dir = data_dir + "/container_dbm_lock" - util.verify_directory(self.lock_dir) - - self.dbmmodule = dbmmodule or anydbm - - self.dbm = None - OpenResourceNamespaceManager.__init__(self, namespace) - - self.file = util.encoded_path(root= self.dbm_dir, - identifiers=[self.namespace], - extension='.dbm', - digest_filenames=self.digest_filenames) - - debug("data file %s", self.file) - self._checkfile() - - def get_access_lock(self): - return file_synchronizer(identifier=self.namespace, - lock_dir=self.lock_dir) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier = "dbmcontainer/funclock/%s" % self.namespace, - lock_dir=self.lock_dir - ) - - def file_exists(self, file): - if os.access(file, os.F_OK): - return True - else: - for ext in ('db', 'dat', 'pag', 'dir'): - if os.access(file + os.extsep + ext, os.F_OK): - return True - - return False - - def _checkfile(self): - if not self.file_exists(self.file): - g = self.dbmmodule.open(self.file, 'c') - g.close() - - def get_filenames(self): - list = [] - if os.access(self.file, os.F_OK): - list.append(self.file) - - for ext in ('pag', 'dir', 'db', 'dat'): - if os.access(self.file + os.extsep + ext, os.F_OK): - list.append(self.file + os.extsep + ext) - return list - - def do_open(self, flags): - debug("opening dbm file %s", self.file) - try: - self.dbm = self.dbmmodule.open(self.file, flags) - except: - self._checkfile() - self.dbm = self.dbmmodule.open(self.file, flags) - - def do_close(self): - if self.dbm is not None: - debug("closing dbm file %s", self.file) - self.dbm.close() - - def do_remove(self): - for f in self.get_filenames(): - os.remove(f) - - def __getitem__(self, key): - return cPickle.loads(self.dbm[key]) - - def __contains__(self, key): - return self.dbm.has_key(key) - - def __setitem__(self, key, value): - self.dbm[key] = cPickle.dumps(value) - - def __delitem__(self, key): - del self.dbm[key] - - def keys(self): - return self.dbm.keys() - - -class FileNamespaceManager(OpenResourceNamespaceManager): - def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, - digest_filenames=True, **kwargs): - self.digest_filenames = digest_filenames - - if not file_dir and not data_dir: - raise MissingCacheParameter("data_dir or file_dir is required") - elif file_dir: - self.file_dir = file_dir - else: - self.file_dir = data_dir + "/container_file" - util.verify_directory(self.file_dir) - - if not lock_dir and not data_dir: - raise MissingCacheParameter("data_dir or lock_dir is required") - elif lock_dir: - self.lock_dir = lock_dir - else: - self.lock_dir = data_dir + "/container_file_lock" - util.verify_directory(self.lock_dir) - OpenResourceNamespaceManager.__init__(self, namespace) - - self.file = util.encoded_path(root=self.file_dir, - identifiers=[self.namespace], - extension='.cache', - digest_filenames=self.digest_filenames) - self.hash = {} - - debug("data file %s", self.file) - - def get_access_lock(self): - return file_synchronizer(identifier=self.namespace, - lock_dir=self.lock_dir) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier = "filecontainer/funclock/%s" % self.namespace, - lock_dir = self.lock_dir - ) - - def file_exists(self, file): - return os.access(file, os.F_OK) - - def do_open(self, flags): - if self.file_exists(self.file): - fh = open(self.file, 'rb') - try: - self.hash = cPickle.load(fh) - except (IOError, OSError, EOFError, cPickle.PickleError, ValueError): - pass - fh.close() - - self.flags = flags - - def do_close(self): - if self.flags == 'c' or self.flags == 'w': - fh = open(self.file, 'wb') - cPickle.dump(self.hash, fh) - fh.close() - - self.hash = {} - self.flags = None - - def do_remove(self): - try: - os.remove(self.file) - except OSError, err: - # for instance, because we haven't yet used this cache, - # but client code has asked for a clear() operation... - pass - self.hash = {} - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -#### legacy stuff to support the old "Container" class interface - -namespace_classes = {} - -ContainerContext = dict - -class ContainerMeta(type): - def __init__(cls, classname, bases, dict_): - namespace_classes[cls] = cls.namespace_class - return type.__init__(cls, classname, bases, dict_) - def __call__(self, key, context, namespace, createfunc=None, - expiretime=None, starttime=None, **kwargs): - if namespace in context: - ns = context[namespace] - else: - nscls = namespace_classes[self] - context[namespace] = ns = nscls(namespace, **kwargs) - return Value(key, ns, createfunc=createfunc, - expiretime=expiretime, starttime=starttime) - -class Container(object): - __metaclass__ = ContainerMeta - namespace_class = NamespaceManager - -class FileContainer(Container): - namespace_class = FileNamespaceManager - -class MemoryContainer(Container): - namespace_class = MemoryNamespaceManager - -class DBMContainer(Container): - namespace_class = DBMNamespaceManager - -DbmContainer = DBMContainer diff --git a/module/lib/beaker/converters.py b/module/lib/beaker/converters.py deleted file mode 100644 index f0ad34963..000000000 --- a/module/lib/beaker/converters.py +++ /dev/null @@ -1,26 +0,0 @@ -# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -def asbool(obj): - if isinstance(obj, (str, unicode)): - obj = obj.strip().lower() - if obj in ['true', 'yes', 'on', 'y', 't', '1']: - return True - elif obj in ['false', 'no', 'off', 'n', 'f', '0']: - return False - else: - raise ValueError( - "String is not true/false: %r" % obj) - return bool(obj) - -def aslist(obj, sep=None, strip=True): - if isinstance(obj, (str, unicode)): - lst = obj.split(sep) - if strip: - lst = [v.strip() for v in lst] - return lst - elif isinstance(obj, (list, tuple)): - return obj - elif obj is None: - return [] - else: - return [obj] diff --git a/module/lib/beaker/crypto/__init__.py b/module/lib/beaker/crypto/__init__.py deleted file mode 100644 index 3e26b0c13..000000000 --- a/module/lib/beaker/crypto/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from warnings import warn - -from beaker.crypto.pbkdf2 import PBKDF2, strxor -from beaker.crypto.util import hmac, sha1, hmac_sha1, md5 -from beaker import util - -keyLength = None - -if util.jython: - try: - from beaker.crypto.jcecrypto import getKeyLength, aesEncrypt - keyLength = getKeyLength() - except ImportError: - pass -else: - try: - from beaker.crypto.pycrypto import getKeyLength, aesEncrypt, aesDecrypt - keyLength = getKeyLength() - except ImportError: - pass - -if not keyLength: - has_aes = False -else: - has_aes = True - -if has_aes and keyLength < 32: - warn('Crypto implementation only supports key lengths up to %d bits. ' - 'Generated session cookies may be incompatible with other ' - 'environments' % (keyLength * 8)) - - -def generateCryptoKeys(master_key, salt, iterations): - # NB: We XOR parts of the keystream into the randomly-generated parts, just - # in case os.urandom() isn't as random as it should be. Note that if - # os.urandom() returns truly random data, this will have no effect on the - # overall security. - keystream = PBKDF2(master_key, salt, iterations=iterations) - cipher_key = keystream.read(keyLength) - return cipher_key diff --git a/module/lib/beaker/crypto/jcecrypto.py b/module/lib/beaker/crypto/jcecrypto.py deleted file mode 100644 index 4062d513e..000000000 --- a/module/lib/beaker/crypto/jcecrypto.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Encryption module that uses the Java Cryptography Extensions (JCE). - -Note that in default installations of the Java Runtime Environment, the -maximum key length is limited to 128 bits due to US export -restrictions. This makes the generated keys incompatible with the ones -generated by pycryptopp, which has no such restrictions. To fix this, -download the "Unlimited Strength Jurisdiction Policy Files" from Sun, -which will allow encryption using 256 bit AES keys. -""" -from javax.crypto import Cipher -from javax.crypto.spec import SecretKeySpec, IvParameterSpec - -import jarray - -# Initialization vector filled with zeros -_iv = IvParameterSpec(jarray.zeros(16, 'b')) - -def aesEncrypt(data, key): - cipher = Cipher.getInstance('AES/CTR/NoPadding') - skeySpec = SecretKeySpec(key, 'AES') - cipher.init(Cipher.ENCRYPT_MODE, skeySpec, _iv) - return cipher.doFinal(data).tostring() - -# magic. -aesDecrypt = aesEncrypt - -def getKeyLength(): - maxlen = Cipher.getMaxAllowedKeyLength('AES/CTR/NoPadding') - return min(maxlen, 256) / 8 diff --git a/module/lib/beaker/crypto/pbkdf2.py b/module/lib/beaker/crypto/pbkdf2.py deleted file mode 100644 index 96dc5fbb2..000000000 --- a/module/lib/beaker/crypto/pbkdf2.py +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/python -# -*- coding: ascii -*- -########################################################################### -# PBKDF2.py - PKCS#5 v2.0 Password-Based Key Derivation -# -# Copyright (C) 2007 Dwayne C. Litzenberger <dlitz@dlitz.net> -# All rights reserved. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose and without fee is hereby granted, -# provided that the above copyright notice appear in all copies and that -# both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR PROVIDES THIS SOFTWARE ``AS IS'' AND ANY EXPRESSED OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Country of origin: Canada -# -########################################################################### -# Sample PBKDF2 usage: -# from Crypto.Cipher import AES -# from PBKDF2 import PBKDF2 -# import os -# -# salt = os.urandom(8) # 64-bit salt -# key = PBKDF2("This passphrase is a secret.", salt).read(32) # 256-bit key -# iv = os.urandom(16) # 128-bit IV -# cipher = AES.new(key, AES.MODE_CBC, iv) -# ... -# -# Sample crypt() usage: -# from PBKDF2 import crypt -# pwhash = crypt("secret") -# alleged_pw = raw_input("Enter password: ") -# if pwhash == crypt(alleged_pw, pwhash): -# print "Password good" -# else: -# print "Invalid password" -# -########################################################################### -# History: -# -# 2007-07-27 Dwayne C. Litzenberger <dlitz@dlitz.net> -# - Initial Release (v1.0) -# -# 2007-07-31 Dwayne C. Litzenberger <dlitz@dlitz.net> -# - Bugfix release (v1.1) -# - SECURITY: The PyCrypto XOR cipher (used, if available, in the _strxor -# function in the previous release) silently truncates all keys to 64 -# bytes. The way it was used in the previous release, this would only be -# problem if the pseudorandom function that returned values larger than -# 64 bytes (so SHA1, SHA256 and SHA512 are fine), but I don't like -# anything that silently reduces the security margin from what is -# expected. -# -########################################################################### - -__version__ = "1.1" - -from struct import pack -from binascii import b2a_hex -from random import randint - -from base64 import b64encode - -from beaker.crypto.util import hmac as HMAC, hmac_sha1 as SHA1 - -def strxor(a, b): - return "".join([chr(ord(x) ^ ord(y)) for (x, y) in zip(a, b)]) - -class PBKDF2(object): - """PBKDF2.py : PKCS#5 v2.0 Password-Based Key Derivation - - This implementation takes a passphrase and a salt (and optionally an - iteration count, a digest module, and a MAC module) and provides a - file-like object from which an arbitrarily-sized key can be read. - - If the passphrase and/or salt are unicode objects, they are encoded as - UTF-8 before they are processed. - - The idea behind PBKDF2 is to derive a cryptographic key from a - passphrase and a salt. - - PBKDF2 may also be used as a strong salted password hash. The - 'crypt' function is provided for that purpose. - - Remember: Keys generated using PBKDF2 are only as strong as the - passphrases they are derived from. - """ - - def __init__(self, passphrase, salt, iterations=1000, - digestmodule=SHA1, macmodule=HMAC): - if not callable(macmodule): - macmodule = macmodule.new - self.__macmodule = macmodule - self.__digestmodule = digestmodule - self._setup(passphrase, salt, iterations, self._pseudorandom) - - def _pseudorandom(self, key, msg): - """Pseudorandom function. e.g. HMAC-SHA1""" - return self.__macmodule(key=key, msg=msg, - digestmod=self.__digestmodule).digest() - - def read(self, bytes): - """Read the specified number of key bytes.""" - if self.closed: - raise ValueError("file-like object is closed") - - size = len(self.__buf) - blocks = [self.__buf] - i = self.__blockNum - while size < bytes: - i += 1 - if i > 0xffffffff: - # We could return "" here, but - raise OverflowError("derived key too long") - block = self.__f(i) - blocks.append(block) - size += len(block) - buf = "".join(blocks) - retval = buf[:bytes] - self.__buf = buf[bytes:] - self.__blockNum = i - return retval - - def __f(self, i): - # i must fit within 32 bits - assert (1 <= i <= 0xffffffff) - U = self.__prf(self.__passphrase, self.__salt + pack("!L", i)) - result = U - for j in xrange(2, 1+self.__iterations): - U = self.__prf(self.__passphrase, U) - result = strxor(result, U) - return result - - def hexread(self, octets): - """Read the specified number of octets. Return them as hexadecimal. - - Note that len(obj.hexread(n)) == 2*n. - """ - return b2a_hex(self.read(octets)) - - def _setup(self, passphrase, salt, iterations, prf): - # Sanity checks: - - # passphrase and salt must be str or unicode (in the latter - # case, we convert to UTF-8) - if isinstance(passphrase, unicode): - passphrase = passphrase.encode("UTF-8") - if not isinstance(passphrase, str): - raise TypeError("passphrase must be str or unicode") - if isinstance(salt, unicode): - salt = salt.encode("UTF-8") - if not isinstance(salt, str): - raise TypeError("salt must be str or unicode") - - # iterations must be an integer >= 1 - if not isinstance(iterations, (int, long)): - raise TypeError("iterations must be an integer") - if iterations < 1: - raise ValueError("iterations must be at least 1") - - # prf must be callable - if not callable(prf): - raise TypeError("prf must be callable") - - self.__passphrase = passphrase - self.__salt = salt - self.__iterations = iterations - self.__prf = prf - self.__blockNum = 0 - self.__buf = "" - self.closed = False - - def close(self): - """Close the stream.""" - if not self.closed: - del self.__passphrase - del self.__salt - del self.__iterations - del self.__prf - del self.__blockNum - del self.__buf - self.closed = True - -def crypt(word, salt=None, iterations=None): - """PBKDF2-based unix crypt(3) replacement. - - The number of iterations specified in the salt overrides the 'iterations' - parameter. - - The effective hash length is 192 bits. - """ - - # Generate a (pseudo-)random salt if the user hasn't provided one. - if salt is None: - salt = _makesalt() - - # salt must be a string or the us-ascii subset of unicode - if isinstance(salt, unicode): - salt = salt.encode("us-ascii") - if not isinstance(salt, str): - raise TypeError("salt must be a string") - - # word must be a string or unicode (in the latter case, we convert to UTF-8) - if isinstance(word, unicode): - word = word.encode("UTF-8") - if not isinstance(word, str): - raise TypeError("word must be a string or unicode") - - # Try to extract the real salt and iteration count from the salt - if salt.startswith("$p5k2$"): - (iterations, salt, dummy) = salt.split("$")[2:5] - if iterations == "": - iterations = 400 - else: - converted = int(iterations, 16) - if iterations != "%x" % converted: # lowercase hex, minimum digits - raise ValueError("Invalid salt") - iterations = converted - if not (iterations >= 1): - raise ValueError("Invalid salt") - - # Make sure the salt matches the allowed character set - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789./" - for ch in salt: - if ch not in allowed: - raise ValueError("Illegal character %r in salt" % (ch,)) - - if iterations is None or iterations == 400: - iterations = 400 - salt = "$p5k2$$" + salt - else: - salt = "$p5k2$%x$%s" % (iterations, salt) - rawhash = PBKDF2(word, salt, iterations).read(24) - return salt + "$" + b64encode(rawhash, "./") - -# Add crypt as a static method of the PBKDF2 class -# This makes it easier to do "from PBKDF2 import PBKDF2" and still use -# crypt. -PBKDF2.crypt = staticmethod(crypt) - -def _makesalt(): - """Return a 48-bit pseudorandom salt for crypt(). - - This function is not suitable for generating cryptographic secrets. - """ - binarysalt = "".join([pack("@H", randint(0, 0xffff)) for i in range(3)]) - return b64encode(binarysalt, "./") - -def test_pbkdf2(): - """Module self-test""" - from binascii import a2b_hex - - # - # Test vectors from RFC 3962 - # - - # Test 1 - result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1).read(16) - expected = a2b_hex("cdedb5281bb2f801565a1122b2563515") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 2 - result = PBKDF2("password", "ATHENA.MIT.EDUraeburn", 1200).hexread(32) - expected = ("5c08eb61fdf71e4e4ec3cf6ba1f5512b" - "a7e52ddbc5e5142f708a31e2e62b1e13") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 3 - result = PBKDF2("X"*64, "pass phrase equals block size", 1200).hexread(32) - expected = ("139c30c0966bc32ba55fdbf212530ac9" - "c5ec59f1a452f5cc9ad940fea0598ed1") - if result != expected: - raise RuntimeError("self-test failed") - - # Test 4 - result = PBKDF2("X"*65, "pass phrase exceeds block size", 1200).hexread(32) - expected = ("9ccad6d468770cd51b10e6a68721be61" - "1a8b4d282601db3b36be9246915ec82a") - if result != expected: - raise RuntimeError("self-test failed") - - # - # Other test vectors - # - - # Chunked read - f = PBKDF2("kickstart", "workbench", 256) - result = f.read(17) - result += f.read(17) - result += f.read(1) - result += f.read(2) - result += f.read(3) - expected = PBKDF2("kickstart", "workbench", 256).read(40) - if result != expected: - raise RuntimeError("self-test failed") - - # - # crypt() test vectors - # - - # crypt 1 - result = crypt("cloadm", "exec") - expected = '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql' - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 2 - result = crypt("gnu", '$p5k2$c$u9HvcT4d$.....') - expected = '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g' - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 3 - result = crypt("dcl", "tUsch7fU", iterations=13) - expected = "$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL" - if result != expected: - raise RuntimeError("self-test failed") - - # crypt 4 (unicode) - result = crypt(u'\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2', - '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ') - expected = '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ' - if result != expected: - raise RuntimeError("self-test failed") - -if __name__ == '__main__': - test_pbkdf2() - -# vim:set ts=4 sw=4 sts=4 expandtab: diff --git a/module/lib/beaker/crypto/pycrypto.py b/module/lib/beaker/crypto/pycrypto.py deleted file mode 100644 index a3eb4d9db..000000000 --- a/module/lib/beaker/crypto/pycrypto.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Encryption module that uses pycryptopp or pycrypto""" -try: - # Pycryptopp is preferred over Crypto because Crypto has had - # various periods of not being maintained, and pycryptopp uses - # the Crypto++ library which is generally considered the 'gold standard' - # of crypto implementations - from pycryptopp.cipher import aes - - def aesEncrypt(data, key): - cipher = aes.AES(key) - return cipher.process(data) - - # magic. - aesDecrypt = aesEncrypt - -except ImportError: - from Crypto.Cipher import AES - - def aesEncrypt(data, key): - cipher = AES.new(key) - - data = data + (" " * (16 - (len(data) % 16))) - return cipher.encrypt(data) - - def aesDecrypt(data, key): - cipher = AES.new(key) - - return cipher.decrypt(data).rstrip() - -def getKeyLength(): - return 32 diff --git a/module/lib/beaker/crypto/util.py b/module/lib/beaker/crypto/util.py deleted file mode 100644 index d97e8ce6f..000000000 --- a/module/lib/beaker/crypto/util.py +++ /dev/null @@ -1,30 +0,0 @@ -from warnings import warn -from beaker import util - - -try: - # Use PyCrypto (if available) - from Crypto.Hash import HMAC as hmac, SHA as hmac_sha1 - sha1 = hmac_sha1.new - -except ImportError: - - # PyCrypto not available. Use the Python standard library. - import hmac - - # When using the stdlib, we have to make sure the hmac version and sha - # version are compatible - if util.py24: - from sha import sha as sha1 - import sha as hmac_sha1 - else: - # NOTE: We have to use the callable with hashlib (hashlib.sha1), - # otherwise hmac only accepts the sha module object itself - from hashlib import sha1 - hmac_sha1 = sha1 - - -if util.py24: - from md5 import md5 -else: - from hashlib import md5 diff --git a/module/lib/beaker/exceptions.py b/module/lib/beaker/exceptions.py deleted file mode 100644 index cc0eed286..000000000 --- a/module/lib/beaker/exceptions.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Beaker exception classes""" - -class BeakerException(Exception): - pass - - -class CreationAbortedError(Exception): - """Deprecated.""" - - -class InvalidCacheBackendError(BeakerException, ImportError): - pass - - -class MissingCacheParameter(BeakerException): - pass - - -class LockError(BeakerException): - pass - - -class InvalidCryptoBackendError(BeakerException): - pass diff --git a/module/lib/beaker/ext/__init__.py b/module/lib/beaker/ext/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/lib/beaker/ext/__init__.py +++ /dev/null diff --git a/module/lib/beaker/ext/database.py b/module/lib/beaker/ext/database.py deleted file mode 100644 index 701e6f7d2..000000000 --- a/module/lib/beaker/ext/database.py +++ /dev/null @@ -1,165 +0,0 @@ -import cPickle -import logging -import pickle -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict - -log = logging.getLogger(__name__) - -sa = None -pool = None -types = None - -class DatabaseNamespaceManager(OpenResourceNamespaceManager): - metadatas = SyncDict() - tables = SyncDict() - - @classmethod - def _init_dependencies(cls): - global sa, pool, types - if sa is not None: - return - try: - import sqlalchemy as sa - import sqlalchemy.pool as pool - from sqlalchemy import types - except ImportError: - raise InvalidCacheBackendError("Database cache backend requires " - "the 'sqlalchemy' library") - - def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, - table_name='beaker_cache', data_dir=None, lock_dir=None, - **params): - """Creates a database namespace manager - - ``url`` - SQLAlchemy compliant db url - ``sa_opts`` - A dictionary of SQLAlchemy keyword options to initialize the engine - with. - ``optimistic`` - Use optimistic session locking, note that this will result in an - additional select when updating a cache value to compare version - numbers. - ``table_name`` - The table name to use in the database for the cache. - """ - OpenResourceNamespaceManager.__init__(self, namespace) - - if sa_opts is None: - sa_opts = params - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_db_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - # Check to see if the table's been created before - url = url or sa_opts['sa.url'] - table_key = url + table_name - def make_cache(): - # Check to see if we have a connection pool open already - meta_key = url + table_name - def make_meta(): - # SQLAlchemy pops the url, this ensures it sticks around - # later - sa_opts['sa.url'] = url - engine = sa.engine_from_config(sa_opts, 'sa.') - meta = sa.MetaData() - meta.bind = engine - return meta - meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta) - # Create the table object and cache it now - cache = sa.Table(table_name, meta, - sa.Column('id', types.Integer, primary_key=True), - sa.Column('namespace', types.String(255), nullable=False), - sa.Column('accessed', types.DateTime, nullable=False), - sa.Column('created', types.DateTime, nullable=False), - sa.Column('data', types.PickleType, nullable=False), - sa.UniqueConstraint('namespace') - ) - cache.create(checkfirst=True) - return cache - self.hash = {} - self._is_new = False - self.loaded = False - self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - return file_synchronizer( - identifier ="databasecontainer/funclock/%s" % self.namespace, - lock_dir = self.lock_dir) - - def do_open(self, flags): - # If we already loaded the data, don't bother loading it again - if self.loaded: - self.flags = flags - return - - cache = self.cache - result = sa.select([cache.c.data], - cache.c.namespace==self.namespace - ).execute().fetchone() - if not result: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = result['data'] - except (IOError, OSError, EOFError, cPickle.PickleError, - pickle.PickleError): - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - cache = self.cache - if self._is_new: - cache.insert().execute(namespace=self.namespace, data=self.hash, - accessed=datetime.now(), - created=datetime.now()) - self._is_new = False - else: - cache.update(cache.c.namespace==self.namespace).execute( - data=self.hash, accessed=datetime.now()) - self.flags = None - - def do_remove(self): - cache = self.cache - cache.delete(cache.c.namespace==self.namespace).execute() - self.hash = {} - - # We can retain the fact that we did a load attempt, but since the - # file is gone this will be a new namespace should it be saved. - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - -class DatabaseContainer(Container): - namespace_manager = DatabaseNamespaceManager diff --git a/module/lib/beaker/ext/google.py b/module/lib/beaker/ext/google.py deleted file mode 100644 index dd8380d7f..000000000 --- a/module/lib/beaker/ext/google.py +++ /dev/null @@ -1,120 +0,0 @@ -import cPickle -import logging -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError -from beaker.synchronization import null_synchronizer - -log = logging.getLogger(__name__) - -db = None - -class GoogleNamespaceManager(OpenResourceNamespaceManager): - tables = {} - - @classmethod - def _init_dependencies(cls): - global db - if db is not None: - return - try: - db = __import__('google.appengine.ext.db').appengine.ext.db - except ImportError: - raise InvalidCacheBackendError("Datastore cache backend requires the " - "'google.appengine.ext' library") - - def __init__(self, namespace, table_name='beaker_cache', **params): - """Creates a datastore namespace manager""" - OpenResourceNamespaceManager.__init__(self, namespace) - - def make_cache(): - table_dict = dict(created=db.DateTimeProperty(), - accessed=db.DateTimeProperty(), - data=db.BlobProperty()) - table = type(table_name, (db.Model,), table_dict) - return table - self.table_name = table_name - self.cache = GoogleNamespaceManager.tables.setdefault(table_name, make_cache()) - self.hash = {} - self._is_new = False - self.loaded = False - self.log_debug = logging.DEBUG >= log.getEffectiveLevel() - - # Google wants namespaces to start with letters, change the namespace - # to start with a letter - self.namespace = 'p%s' % self.namespace - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - # this is weird, should probably be present - return null_synchronizer() - - def do_open(self, flags): - # If we already loaded the data, don't bother loading it again - if self.loaded: - self.flags = flags - return - - item = self.cache.get_by_key_name(self.namespace) - - if not item: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = cPickle.loads(str(item.data)) - except (IOError, OSError, EOFError, cPickle.PickleError): - if self.log_debug: - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - if self._is_new: - item = self.cache(key_name=self.namespace) - item.data = cPickle.dumps(self.hash) - item.created = datetime.now() - item.accessed = datetime.now() - item.put() - self._is_new = False - else: - item = self.cache.get_by_key_name(self.namespace) - item.data = cPickle.dumps(self.hash) - item.accessed = datetime.now() - item.put() - self.flags = None - - def do_remove(self): - item = self.cache.get_by_key_name(self.namespace) - item.delete() - self.hash = {} - - # We can retain the fact that we did a load attempt, but since the - # file is gone this will be a new namespace should it be saved. - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -class GoogleContainer(Container): - namespace_class = GoogleNamespaceManager diff --git a/module/lib/beaker/ext/memcached.py b/module/lib/beaker/ext/memcached.py deleted file mode 100644 index 96516953f..000000000 --- a/module/lib/beaker/ext/memcached.py +++ /dev/null @@ -1,82 +0,0 @@ -from beaker.container import NamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict -import warnings - -memcache = None - -class MemcachedNamespaceManager(NamespaceManager): - clients = SyncDict() - - @classmethod - def _init_dependencies(cls): - global memcache - if memcache is not None: - return - try: - import pylibmc as memcache - except ImportError: - try: - import cmemcache as memcache - warnings.warn("cmemcache is known to have serious " - "concurrency issues; consider using 'memcache' or 'pylibmc'") - except ImportError: - try: - import memcache - except ImportError: - raise InvalidCacheBackendError("Memcached cache backend requires either " - "the 'memcache' or 'cmemcache' library") - - def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): - NamespaceManager.__init__(self, namespace) - - if not url: - raise MissingCacheParameter("url is required") - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_mcd_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';')) - - def get_creation_lock(self, key): - return file_synchronizer( - identifier="memcachedcontainer/funclock/%s" % self.namespace,lock_dir = self.lock_dir) - - def _format_key(self, key): - return self.namespace + '_' + key.replace(' ', '\302\267') - - def __getitem__(self, key): - return self.mc.get(self._format_key(key)) - - def __contains__(self, key): - value = self.mc.get(self._format_key(key)) - return value is not None - - def has_key(self, key): - return key in self - - def set_value(self, key, value, expiretime=None): - if expiretime: - self.mc.set(self._format_key(key), value, time=expiretime) - else: - self.mc.set(self._format_key(key), value) - - def __setitem__(self, key, value): - self.set_value(key, value) - - def __delitem__(self, key): - self.mc.delete(self._format_key(key)) - - def do_remove(self): - self.mc.flush_all() - - def keys(self): - raise NotImplementedError("Memcache caching does not support iteration of all cache keys") - -class MemcachedContainer(Container): - namespace_class = MemcachedNamespaceManager diff --git a/module/lib/beaker/ext/sqla.py b/module/lib/beaker/ext/sqla.py deleted file mode 100644 index 8c79633c1..000000000 --- a/module/lib/beaker/ext/sqla.py +++ /dev/null @@ -1,133 +0,0 @@ -import cPickle -import logging -import pickle -from datetime import datetime - -from beaker.container import OpenResourceNamespaceManager, Container -from beaker.exceptions import InvalidCacheBackendError, MissingCacheParameter -from beaker.synchronization import file_synchronizer, null_synchronizer -from beaker.util import verify_directory, SyncDict - - -log = logging.getLogger(__name__) - -sa = None - -class SqlaNamespaceManager(OpenResourceNamespaceManager): - binds = SyncDict() - tables = SyncDict() - - @classmethod - def _init_dependencies(cls): - global sa - if sa is not None: - return - try: - import sqlalchemy as sa - except ImportError: - raise InvalidCacheBackendError("SQLAlchemy, which is required by " - "this backend, is not installed") - - def __init__(self, namespace, bind, table, data_dir=None, lock_dir=None, - **kwargs): - """Create a namespace manager for use with a database table via - SQLAlchemy. - - ``bind`` - SQLAlchemy ``Engine`` or ``Connection`` object - - ``table`` - SQLAlchemy ``Table`` object in which to store namespace data. - This should usually be something created by ``make_cache_table``. - """ - OpenResourceNamespaceManager.__init__(self, namespace) - - if lock_dir: - self.lock_dir = lock_dir - elif data_dir: - self.lock_dir = data_dir + "/container_db_lock" - if self.lock_dir: - verify_directory(self.lock_dir) - - self.bind = self.__class__.binds.get(str(bind.url), lambda: bind) - self.table = self.__class__.tables.get('%s:%s' % (bind.url, table.name), - lambda: table) - self.hash = {} - self._is_new = False - self.loaded = False - - def get_access_lock(self): - return null_synchronizer() - - def get_creation_lock(self, key): - return file_synchronizer( - identifier ="databasecontainer/funclock/%s" % self.namespace, - lock_dir=self.lock_dir) - - def do_open(self, flags): - if self.loaded: - self.flags = flags - return - select = sa.select([self.table.c.data], - (self.table.c.namespace == self.namespace)) - result = self.bind.execute(select).fetchone() - if not result: - self._is_new = True - self.hash = {} - else: - self._is_new = False - try: - self.hash = result['data'] - except (IOError, OSError, EOFError, cPickle.PickleError, - pickle.PickleError): - log.debug("Couln't load pickle data, creating new storage") - self.hash = {} - self._is_new = True - self.flags = flags - self.loaded = True - - def do_close(self): - if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): - if self._is_new: - insert = self.table.insert() - self.bind.execute(insert, namespace=self.namespace, data=self.hash, - accessed=datetime.now(), created=datetime.now()) - self._is_new = False - else: - update = self.table.update(self.table.c.namespace == self.namespace) - self.bind.execute(update, data=self.hash, accessed=datetime.now()) - self.flags = None - - def do_remove(self): - delete = self.table.delete(self.table.c.namespace == self.namespace) - self.bind.execute(delete) - self.hash = {} - self._is_new = True - - def __getitem__(self, key): - return self.hash[key] - - def __contains__(self, key): - return self.hash.has_key(key) - - def __setitem__(self, key, value): - self.hash[key] = value - - def __delitem__(self, key): - del self.hash[key] - - def keys(self): - return self.hash.keys() - - -class SqlaContainer(Container): - namespace_manager = SqlaNamespaceManager - -def make_cache_table(metadata, table_name='beaker_cache'): - """Return a ``Table`` object suitable for storing cached values for the - namespace manager. Do not create the table.""" - return sa.Table(table_name, metadata, - sa.Column('namespace', sa.String(255), primary_key=True), - sa.Column('accessed', sa.DateTime, nullable=False), - sa.Column('created', sa.DateTime, nullable=False), - sa.Column('data', sa.PickleType, nullable=False)) diff --git a/module/lib/beaker/middleware.py b/module/lib/beaker/middleware.py deleted file mode 100644 index 7ba88b37d..000000000 --- a/module/lib/beaker/middleware.py +++ /dev/null @@ -1,165 +0,0 @@ -import warnings - -try: - from paste.registry import StackedObjectProxy - beaker_session = StackedObjectProxy(name="Beaker Session") - beaker_cache = StackedObjectProxy(name="Cache Manager") -except: - beaker_cache = None - beaker_session = None - -from beaker.cache import CacheManager -from beaker.session import Session, SessionObject -from beaker.util import coerce_cache_params, coerce_session_params, \ - parse_cache_config_options - - -class CacheMiddleware(object): - cache = beaker_cache - - def __init__(self, app, config=None, environ_key='beaker.cache', **kwargs): - """Initialize the Cache Middleware - - The Cache middleware will make a Cache instance available - every request under the ``environ['beaker.cache']`` key by - default. The location in environ can be changed by setting - ``environ_key``. - - ``config`` - dict All settings should be prefixed by 'cache.'. This - method of passing variables is intended for Paste and other - setups that accumulate multiple component settings in a - single dictionary. If config contains *no cache. prefixed - args*, then *all* of the config options will be used to - intialize the Cache objects. - - ``environ_key`` - Location where the Cache instance will keyed in the WSGI - environ - - ``**kwargs`` - All keyword arguments are assumed to be cache settings and - will override any settings found in ``config`` - - """ - self.app = app - config = config or {} - - self.options = {} - - # Update the options with the parsed config - self.options.update(parse_cache_config_options(config)) - - # Add any options from kwargs, but leave out the defaults this - # time - self.options.update( - parse_cache_config_options(kwargs, include_defaults=False)) - - # Assume all keys are intended for cache if none are prefixed with - # 'cache.' - if not self.options and config: - self.options = config - - self.options.update(kwargs) - self.cache_manager = CacheManager(**self.options) - self.environ_key = environ_key - - def __call__(self, environ, start_response): - if environ.get('paste.registry'): - if environ['paste.registry'].reglist: - environ['paste.registry'].register(self.cache, - self.cache_manager) - environ[self.environ_key] = self.cache_manager - return self.app(environ, start_response) - - -class SessionMiddleware(object): - session = beaker_session - - def __init__(self, wrap_app, config=None, environ_key='beaker.session', - **kwargs): - """Initialize the Session Middleware - - The Session middleware will make a lazy session instance - available every request under the ``environ['beaker.session']`` - key by default. The location in environ can be changed by - setting ``environ_key``. - - ``config`` - dict All settings should be prefixed by 'session.'. This - method of passing variables is intended for Paste and other - setups that accumulate multiple component settings in a - single dictionary. If config contains *no cache. prefixed - args*, then *all* of the config options will be used to - intialize the Cache objects. - - ``environ_key`` - Location where the Session instance will keyed in the WSGI - environ - - ``**kwargs`` - All keyword arguments are assumed to be session settings and - will override any settings found in ``config`` - - """ - config = config or {} - - # Load up the default params - self.options = dict(invalidate_corrupt=True, type=None, - data_dir=None, key='beaker.session.id', - timeout=None, secret=None, log_file=None) - - # Pull out any config args meant for beaker session. if there are any - for dct in [config, kwargs]: - for key, val in dct.iteritems(): - if key.startswith('beaker.session.'): - self.options[key[15:]] = val - if key.startswith('session.'): - self.options[key[8:]] = val - if key.startswith('session_'): - warnings.warn('Session options should start with session. ' - 'instead of session_.', DeprecationWarning, 2) - self.options[key[8:]] = val - - # Coerce and validate session params - coerce_session_params(self.options) - - # Assume all keys are intended for cache if none are prefixed with - # 'cache.' - if not self.options and config: - self.options = config - - self.options.update(kwargs) - self.wrap_app = wrap_app - self.environ_key = environ_key - - def __call__(self, environ, start_response): - session = SessionObject(environ, **self.options) - if environ.get('paste.registry'): - if environ['paste.registry'].reglist: - environ['paste.registry'].register(self.session, session) - environ[self.environ_key] = session - environ['beaker.get_session'] = self._get_session - - def session_start_response(status, headers, exc_info = None): - if session.accessed(): - session.persist() - if session.__dict__['_headers']['set_cookie']: - cookie = session.__dict__['_headers']['cookie_out'] - if cookie: - headers.append(('Set-cookie', cookie)) - return start_response(status, headers, exc_info) - return self.wrap_app(environ, session_start_response) - - def _get_session(self): - return Session({}, use_cookies=False, **self.options) - - -def session_filter_factory(global_conf, **kwargs): - def filter(app): - return SessionMiddleware(app, global_conf, **kwargs) - return filter - - -def session_filter_app_factory(app, global_conf, **kwargs): - return SessionMiddleware(app, global_conf, **kwargs) diff --git a/module/lib/beaker/session.py b/module/lib/beaker/session.py deleted file mode 100644 index 7d465530b..000000000 --- a/module/lib/beaker/session.py +++ /dev/null @@ -1,618 +0,0 @@ -import Cookie -import os -import random -import time -from datetime import datetime, timedelta - -from beaker.crypto import hmac as HMAC, hmac_sha1 as SHA1, md5 -from beaker.util import pickle - -from beaker import crypto -from beaker.cache import clsmap -from beaker.exceptions import BeakerException, InvalidCryptoBackendError -from base64 import b64encode, b64decode - - -__all__ = ['SignedCookie', 'Session'] - -getpid = hasattr(os, 'getpid') and os.getpid or (lambda : '') - -class SignedCookie(Cookie.BaseCookie): - """Extends python cookie to give digital signature support""" - def __init__(self, secret, input=None): - self.secret = secret - Cookie.BaseCookie.__init__(self, input) - - def value_decode(self, val): - val = val.strip('"') - sig = HMAC.new(self.secret, val[40:], SHA1).hexdigest() - - # Avoid timing attacks - invalid_bits = 0 - input_sig = val[:40] - if len(sig) != len(input_sig): - return None, val - - for a, b in zip(sig, input_sig): - invalid_bits += a != b - - if invalid_bits: - return None, val - else: - return val[40:], val - - def value_encode(self, val): - sig = HMAC.new(self.secret, val, SHA1).hexdigest() - return str(val), ("%s%s" % (sig, val)) - - -class Session(dict): - """Session object that uses container package for storage. - - ``key`` - The name the cookie should be set to. - ``timeout`` - How long session data is considered valid. This is used - regardless of the cookie being present or not to determine - whether session data is still valid. - ``cookie_domain`` - Domain to use for the cookie. - ``secure`` - Whether or not the cookie should only be sent over SSL. - """ - def __init__(self, request, id=None, invalidate_corrupt=False, - use_cookies=True, type=None, data_dir=None, - key='beaker.session.id', timeout=None, cookie_expires=True, - cookie_domain=None, secret=None, secure=False, - namespace_class=None, **namespace_args): - if not type: - if data_dir: - self.type = 'file' - else: - self.type = 'memory' - else: - self.type = type - - self.namespace_class = namespace_class or clsmap[self.type] - - self.namespace_args = namespace_args - - self.request = request - self.data_dir = data_dir - self.key = key - - self.timeout = timeout - self.use_cookies = use_cookies - self.cookie_expires = cookie_expires - - # Default cookie domain/path - self._domain = cookie_domain - self._path = '/' - self.was_invalidated = False - self.secret = secret - self.secure = secure - self.id = id - self.accessed_dict = {} - - if self.use_cookies: - cookieheader = request.get('cookie', '') - if secret: - try: - self.cookie = SignedCookie(secret, input=cookieheader) - except Cookie.CookieError: - self.cookie = SignedCookie(secret, input=None) - else: - self.cookie = Cookie.SimpleCookie(input=cookieheader) - - if not self.id and self.key in self.cookie: - self.id = self.cookie[self.key].value - - self.is_new = self.id is None - if self.is_new: - self._create_id() - self['_accessed_time'] = self['_creation_time'] = time.time() - else: - try: - self.load() - except: - if invalidate_corrupt: - self.invalidate() - else: - raise - - def _create_id(self): - self.id = md5( - md5("%f%s%f%s" % (time.time(), id({}), random.random(), - getpid())).hexdigest(), - ).hexdigest() - self.is_new = True - self.last_accessed = None - if self.use_cookies: - self.cookie[self.key] = self.id - if self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - self.cookie[self.key]['path'] = self._path - if self.cookie_expires is not True: - if self.cookie_expires is False: - expires = datetime.fromtimestamp( 0x7FFFFFFF ) - elif isinstance(self.cookie_expires, timedelta): - expires = datetime.today() + self.cookie_expires - elif isinstance(self.cookie_expires, datetime): - expires = self.cookie_expires - else: - raise ValueError("Invalid argument for cookie_expires: %s" - % repr(self.cookie_expires)) - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = False - - def created(self): - return self['_creation_time'] - created = property(created) - - def _set_domain(self, domain): - self['_domain'] = domain - self.cookie[self.key]['domain'] = domain - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def _get_domain(self): - return self._domain - - domain = property(_get_domain, _set_domain) - - def _set_path(self, path): - self['_path'] = path - self.cookie[self.key]['path'] = path - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def _get_path(self): - return self._path - - path = property(_get_path, _set_path) - - def _delete_cookie(self): - self.request['set_cookie'] = True - self.cookie[self.key] = self.id - if self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - self.cookie[self.key]['path'] = '/' - expires = datetime.today().replace(year=2003) - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def delete(self): - """Deletes the session from the persistent storage, and sends - an expired cookie out""" - if self.use_cookies: - self._delete_cookie() - self.clear() - - def invalidate(self): - """Invalidates this session, creates a new session id, returns - to the is_new state""" - self.clear() - self.was_invalidated = True - self._create_id() - self.load() - - def load(self): - "Loads the data from this session from persistent storage" - self.namespace = self.namespace_class(self.id, - data_dir=self.data_dir, digest_filenames=False, - **self.namespace_args) - now = time.time() - self.request['set_cookie'] = True - - self.namespace.acquire_read_lock() - timed_out = False - try: - self.clear() - try: - session_data = self.namespace['session'] - - # Memcached always returns a key, its None when its not - # present - if session_data is None: - session_data = { - '_creation_time':now, - '_accessed_time':now - } - self.is_new = True - except (KeyError, TypeError): - session_data = { - '_creation_time':now, - '_accessed_time':now - } - self.is_new = True - - if self.timeout is not None and \ - now - session_data['_accessed_time'] > self.timeout: - timed_out= True - else: - # Properly set the last_accessed time, which is different - # than the *currently* _accessed_time - if self.is_new or '_accessed_time' not in session_data: - self.last_accessed = None - else: - self.last_accessed = session_data['_accessed_time'] - - # Update the current _accessed_time - session_data['_accessed_time'] = now - self.update(session_data) - self.accessed_dict = session_data.copy() - finally: - self.namespace.release_read_lock() - if timed_out: - self.invalidate() - - def save(self, accessed_only=False): - """Saves the data for this session to persistent storage - - If accessed_only is True, then only the original data loaded - at the beginning of the request will be saved, with the updated - last accessed time. - - """ - # Look to see if its a new session that was only accessed - # Don't save it under that case - if accessed_only and self.is_new: - return None - - if not hasattr(self, 'namespace'): - self.namespace = self.namespace_class( - self.id, - data_dir=self.data_dir, - digest_filenames=False, - **self.namespace_args) - - self.namespace.acquire_write_lock() - try: - if accessed_only: - data = dict(self.accessed_dict.items()) - else: - data = dict(self.items()) - - # Save the data - if not data and 'session' in self.namespace: - del self.namespace['session'] - else: - self.namespace['session'] = data - finally: - self.namespace.release_write_lock() - if self.is_new: - self.request['set_cookie'] = True - - def revert(self): - """Revert the session to its original state from its first - access in the request""" - self.clear() - self.update(self.accessed_dict) - - # TODO: I think both these methods should be removed. They're from - # the original mod_python code i was ripping off but they really - # have no use here. - def lock(self): - """Locks this session against other processes/threads. This is - automatic when load/save is called. - - ***use with caution*** and always with a corresponding 'unlock' - inside a "finally:" block, as a stray lock typically cannot be - unlocked without shutting down the whole application. - - """ - self.namespace.acquire_write_lock() - - def unlock(self): - """Unlocks this session against other processes/threads. This - is automatic when load/save is called. - - ***use with caution*** and always within a "finally:" block, as - a stray lock typically cannot be unlocked without shutting down - the whole application. - - """ - self.namespace.release_write_lock() - -class CookieSession(Session): - """Pure cookie-based session - - Options recognized when using cookie-based sessions are slightly - more restricted than general sessions. - - ``key`` - The name the cookie should be set to. - ``timeout`` - How long session data is considered valid. This is used - regardless of the cookie being present or not to determine - whether session data is still valid. - ``encrypt_key`` - The key to use for the session encryption, if not provided the - session will not be encrypted. - ``validate_key`` - The key used to sign the encrypted session - ``cookie_domain`` - Domain to use for the cookie. - ``secure`` - Whether or not the cookie should only be sent over SSL. - - """ - def __init__(self, request, key='beaker.session.id', timeout=None, - cookie_expires=True, cookie_domain=None, encrypt_key=None, - validate_key=None, secure=False, **kwargs): - - if not crypto.has_aes and encrypt_key: - raise InvalidCryptoBackendError("No AES library is installed, can't generate " - "encrypted cookie-only Session.") - - self.request = request - self.key = key - self.timeout = timeout - self.cookie_expires = cookie_expires - self.encrypt_key = encrypt_key - self.validate_key = validate_key - self.request['set_cookie'] = False - self.secure = secure - self._domain = cookie_domain - self._path = '/' - - try: - cookieheader = request['cookie'] - except KeyError: - cookieheader = '' - - if validate_key is None: - raise BeakerException("No validate_key specified for Cookie only " - "Session.") - - try: - self.cookie = SignedCookie(validate_key, input=cookieheader) - except Cookie.CookieError: - self.cookie = SignedCookie(validate_key, input=None) - - self['_id'] = self._make_id() - self.is_new = True - - # If we have a cookie, load it - if self.key in self.cookie and self.cookie[self.key].value is not None: - self.is_new = False - try: - self.update(self._decrypt_data()) - except: - pass - if self.timeout is not None and time.time() - \ - self['_accessed_time'] > self.timeout: - self.clear() - self.accessed_dict = self.copy() - self._create_cookie() - - def created(self): - return self['_creation_time'] - created = property(created) - - def id(self): - return self['_id'] - id = property(id) - - def _set_domain(self, domain): - self['_domain'] = domain - self._domain = domain - - def _get_domain(self): - return self._domain - - domain = property(_get_domain, _set_domain) - - def _set_path(self, path): - self['_path'] = path - self._path = path - - def _get_path(self): - return self._path - - path = property(_get_path, _set_path) - - def _encrypt_data(self): - """Serialize, encipher, and base64 the session dict""" - if self.encrypt_key: - nonce = b64encode(os.urandom(40))[:8] - encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, - self.validate_key + nonce, 1) - data = pickle.dumps(self.copy(), 2) - return nonce + b64encode(crypto.aesEncrypt(data, encrypt_key)) - else: - data = pickle.dumps(self.copy(), 2) - return b64encode(data) - - def _decrypt_data(self): - """Bas64, decipher, then un-serialize the data for the session - dict""" - if self.encrypt_key: - nonce = self.cookie[self.key].value[:8] - encrypt_key = crypto.generateCryptoKeys(self.encrypt_key, - self.validate_key + nonce, 1) - payload = b64decode(self.cookie[self.key].value[8:]) - data = crypto.aesDecrypt(payload, encrypt_key) - return pickle.loads(data) - else: - data = b64decode(self.cookie[self.key].value) - return pickle.loads(data) - - def _make_id(self): - return md5(md5( - "%f%s%f%s" % (time.time(), id({}), random.random(), getpid()) - ).hexdigest() - ).hexdigest() - - def save(self, accessed_only=False): - """Saves the data for this session to persistent storage""" - if accessed_only and self.is_new: - return - if accessed_only: - self.clear() - self.update(self.accessed_dict) - self._create_cookie() - - def expire(self): - """Delete the 'expires' attribute on this Session, if any.""" - - self.pop('_expires', None) - - def _create_cookie(self): - if '_creation_time' not in self: - self['_creation_time'] = time.time() - if '_id' not in self: - self['_id'] = self._make_id() - self['_accessed_time'] = time.time() - - if self.cookie_expires is not True: - if self.cookie_expires is False: - expires = datetime.fromtimestamp( 0x7FFFFFFF ) - elif isinstance(self.cookie_expires, timedelta): - expires = datetime.today() + self.cookie_expires - elif isinstance(self.cookie_expires, datetime): - expires = self.cookie_expires - else: - raise ValueError("Invalid argument for cookie_expires: %s" - % repr(self.cookie_expires)) - self['_expires'] = expires - elif '_expires' in self: - expires = self['_expires'] - else: - expires = None - - val = self._encrypt_data() - if len(val) > 4064: - raise BeakerException("Cookie value is too long to store") - - self.cookie[self.key] = val - if '_domain' in self: - self.cookie[self.key]['domain'] = self['_domain'] - elif self._domain: - self.cookie[self.key]['domain'] = self._domain - if self.secure: - self.cookie[self.key]['secure'] = True - - self.cookie[self.key]['path'] = self.get('_path', '/') - - if expires: - self.cookie[self.key]['expires'] = \ - expires.strftime("%a, %d-%b-%Y %H:%M:%S GMT" ) - self.request['cookie_out'] = self.cookie[self.key].output(header='') - self.request['set_cookie'] = True - - def delete(self): - """Delete the cookie, and clear the session""" - # Send a delete cookie request - self._delete_cookie() - self.clear() - - def invalidate(self): - """Clear the contents and start a new session""" - self.delete() - self['_id'] = self._make_id() - - -class SessionObject(object): - """Session proxy/lazy creator - - This object proxies access to the actual session object, so that in - the case that the session hasn't been used before, it will be - setup. This avoid creating and loading the session from persistent - storage unless its actually used during the request. - - """ - def __init__(self, environ, **params): - self.__dict__['_params'] = params - self.__dict__['_environ'] = environ - self.__dict__['_sess'] = None - self.__dict__['_headers'] = [] - - def _session(self): - """Lazy initial creation of session object""" - if self.__dict__['_sess'] is None: - params = self.__dict__['_params'] - environ = self.__dict__['_environ'] - self.__dict__['_headers'] = req = {'cookie_out':None} - req['cookie'] = environ.get('HTTP_COOKIE') - if params.get('type') == 'cookie': - self.__dict__['_sess'] = CookieSession(req, **params) - else: - self.__dict__['_sess'] = Session(req, use_cookies=True, - **params) - return self.__dict__['_sess'] - - def __getattr__(self, attr): - return getattr(self._session(), attr) - - def __setattr__(self, attr, value): - setattr(self._session(), attr, value) - - def __delattr__(self, name): - self._session().__delattr__(name) - - def __getitem__(self, key): - return self._session()[key] - - def __setitem__(self, key, value): - self._session()[key] = value - - def __delitem__(self, key): - self._session().__delitem__(key) - - def __repr__(self): - return self._session().__repr__() - - def __iter__(self): - """Only works for proxying to a dict""" - return iter(self._session().keys()) - - def __contains__(self, key): - return self._session().has_key(key) - - def get_by_id(self, id): - """Loads a session given a session ID""" - params = self.__dict__['_params'] - session = Session({}, use_cookies=False, id=id, **params) - if session.is_new: - return None - return session - - def save(self): - self.__dict__['_dirty'] = True - - def delete(self): - self.__dict__['_dirty'] = True - self._session().delete() - - def persist(self): - """Persist the session to the storage - - If its set to autosave, then the entire session will be saved - regardless of if save() has been called. Otherwise, just the - accessed time will be updated if save() was not called, or - the session will be saved if save() was called. - - """ - if self.__dict__['_params'].get('auto'): - self._session().save() - else: - if self.__dict__.get('_dirty'): - self._session().save() - else: - self._session().save(accessed_only=True) - - def dirty(self): - return self.__dict__.get('_dirty', False) - - def accessed(self): - """Returns whether or not the session has been accessed""" - return self.__dict__['_sess'] is not None diff --git a/module/lib/beaker/synchronization.py b/module/lib/beaker/synchronization.py deleted file mode 100644 index 761303707..000000000 --- a/module/lib/beaker/synchronization.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Synchronization functions. - -File- and mutex-based mutual exclusion synchronizers are provided, -as well as a name-based mutex which locks within an application -based on a string name. - -""" - -import os -import sys -import tempfile - -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading - -# check for fcntl module -try: - sys.getwindowsversion() - has_flock = False -except: - try: - import fcntl - has_flock = True - except ImportError: - has_flock = False - -from beaker import util -from beaker.exceptions import LockError - -__all__ = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer", - "NameLock", "_threading"] - - -class NameLock(object): - """a proxy for an RLock object that is stored in a name based - registry. - - Multiple threads can get a reference to the same RLock based on the - name alone, and synchronize operations related to that name. - - """ - locks = util.WeakValuedRegistry() - - class NLContainer(object): - def __init__(self, reentrant): - if reentrant: - self.lock = _threading.RLock() - else: - self.lock = _threading.Lock() - def __call__(self): - return self.lock - - def __init__(self, identifier = None, reentrant = False): - if identifier is None: - self._lock = NameLock.NLContainer(reentrant) - else: - self._lock = NameLock.locks.get(identifier, NameLock.NLContainer, - reentrant) - - def acquire(self, wait = True): - return self._lock().acquire(wait) - - def release(self): - self._lock().release() - - -_synchronizers = util.WeakValuedRegistry() -def _synchronizer(identifier, cls, **kwargs): - return _synchronizers.sync_get((identifier, cls), cls, identifier, **kwargs) - - -def file_synchronizer(identifier, **kwargs): - if not has_flock or 'lock_dir' not in kwargs: - return mutex_synchronizer(identifier) - else: - return _synchronizer(identifier, FileSynchronizer, **kwargs) - - -def mutex_synchronizer(identifier, **kwargs): - return _synchronizer(identifier, ConditionSynchronizer, **kwargs) - - -class null_synchronizer(object): - def acquire_write_lock(self, wait=True): - return True - def acquire_read_lock(self): - pass - def release_write_lock(self): - pass - def release_read_lock(self): - pass - acquire = acquire_write_lock - release = release_write_lock - - -class SynchronizerImpl(object): - def __init__(self): - self._state = util.ThreadLocal() - - class SyncState(object): - __slots__ = 'reentrantcount', 'writing', 'reading' - - def __init__(self): - self.reentrantcount = 0 - self.writing = False - self.reading = False - - def state(self): - if not self._state.has(): - state = SynchronizerImpl.SyncState() - self._state.put(state) - return state - else: - return self._state.get() - state = property(state) - - def release_read_lock(self): - state = self.state - - if state.writing: - raise LockError("lock is in writing state") - if not state.reading: - raise LockError("lock is not in reading state") - - if state.reentrantcount == 1: - self.do_release_read_lock() - state.reading = False - - state.reentrantcount -= 1 - - def acquire_read_lock(self, wait = True): - state = self.state - - if state.writing: - raise LockError("lock is in writing state") - - if state.reentrantcount == 0: - x = self.do_acquire_read_lock(wait) - if (wait or x): - state.reentrantcount += 1 - state.reading = True - return x - elif state.reading: - state.reentrantcount += 1 - return True - - def release_write_lock(self): - state = self.state - - if state.reading: - raise LockError("lock is in reading state") - if not state.writing: - raise LockError("lock is not in writing state") - - if state.reentrantcount == 1: - self.do_release_write_lock() - state.writing = False - - state.reentrantcount -= 1 - - release = release_write_lock - - def acquire_write_lock(self, wait = True): - state = self.state - - if state.reading: - raise LockError("lock is in reading state") - - if state.reentrantcount == 0: - x = self.do_acquire_write_lock(wait) - if (wait or x): - state.reentrantcount += 1 - state.writing = True - return x - elif state.writing: - state.reentrantcount += 1 - return True - - acquire = acquire_write_lock - - def do_release_read_lock(self): - raise NotImplementedError() - - def do_acquire_read_lock(self): - raise NotImplementedError() - - def do_release_write_lock(self): - raise NotImplementedError() - - def do_acquire_write_lock(self): - raise NotImplementedError() - - -class FileSynchronizer(SynchronizerImpl): - """a synchronizer which locks using flock(). - - Adapted for Python/multithreads from Apache::Session::Lock::File, - http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm - - This module does not unlink temporary files, - because it interferes with proper locking. This can cause - problems on certain systems (Linux) whose file systems (ext2) do not - perform well with lots of files in one directory. To prevent this - you should use a script to clean out old files from your lock directory. - - """ - def __init__(self, identifier, lock_dir): - super(FileSynchronizer, self).__init__() - self._filedescriptor = util.ThreadLocal() - - if lock_dir is None: - lock_dir = tempfile.gettempdir() - else: - lock_dir = lock_dir - - self.filename = util.encoded_path( - lock_dir, - [identifier], - extension='.lock' - ) - - def _filedesc(self): - return self._filedescriptor.get() - _filedesc = property(_filedesc) - - def _open(self, mode): - filedescriptor = self._filedesc - if filedescriptor is None: - filedescriptor = os.open(self.filename, mode) - self._filedescriptor.put(filedescriptor) - return filedescriptor - - def do_acquire_read_lock(self, wait): - filedescriptor = self._open(os.O_CREAT | os.O_RDONLY) - if not wait: - try: - fcntl.flock(filedescriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) - return True - except IOError: - os.close(filedescriptor) - self._filedescriptor.remove() - return False - else: - fcntl.flock(filedescriptor, fcntl.LOCK_SH) - return True - - def do_acquire_write_lock(self, wait): - filedescriptor = self._open(os.O_CREAT | os.O_WRONLY) - if not wait: - try: - fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - return True - except IOError: - os.close(filedescriptor) - self._filedescriptor.remove() - return False - else: - fcntl.flock(filedescriptor, fcntl.LOCK_EX) - return True - - def do_release_read_lock(self): - self._release_all_locks() - - def do_release_write_lock(self): - self._release_all_locks() - - def _release_all_locks(self): - filedescriptor = self._filedesc - if filedescriptor is not None: - fcntl.flock(filedescriptor, fcntl.LOCK_UN) - os.close(filedescriptor) - self._filedescriptor.remove() - - -class ConditionSynchronizer(SynchronizerImpl): - """a synchronizer using a Condition.""" - - def __init__(self, identifier): - super(ConditionSynchronizer, self).__init__() - - # counts how many asynchronous methods are executing - self.async = 0 - - # pointer to thread that is the current sync operation - self.current_sync_operation = None - - # condition object to lock on - self.condition = _threading.Condition(_threading.Lock()) - - def do_acquire_read_lock(self, wait = True): - self.condition.acquire() - try: - # see if a synchronous operation is waiting to start - # or is already running, in which case we wait (or just - # give up and return) - if wait: - while self.current_sync_operation is not None: - self.condition.wait() - else: - if self.current_sync_operation is not None: - return False - - self.async += 1 - finally: - self.condition.release() - - if not wait: - return True - - def do_release_read_lock(self): - self.condition.acquire() - try: - self.async -= 1 - - # check if we are the last asynchronous reader thread - # out the door. - if self.async == 0: - # yes. so if a sync operation is waiting, notifyAll to wake - # it up - if self.current_sync_operation is not None: - self.condition.notifyAll() - elif self.async < 0: - raise LockError("Synchronizer error - too many " - "release_read_locks called") - finally: - self.condition.release() - - def do_acquire_write_lock(self, wait = True): - self.condition.acquire() - try: - # here, we are not a synchronous reader, and after returning, - # assuming waiting or immediate availability, we will be. - - if wait: - # if another sync is working, wait - while self.current_sync_operation is not None: - self.condition.wait() - else: - # if another sync is working, - # we dont want to wait, so forget it - if self.current_sync_operation is not None: - return False - - # establish ourselves as the current sync - # this indicates to other read/write operations - # that they should wait until this is None again - self.current_sync_operation = _threading.currentThread() - - # now wait again for asyncs to finish - if self.async > 0: - if wait: - # wait - self.condition.wait() - else: - # we dont want to wait, so forget it - self.current_sync_operation = None - return False - finally: - self.condition.release() - - if not wait: - return True - - def do_release_write_lock(self): - self.condition.acquire() - try: - if self.current_sync_operation is not _threading.currentThread(): - raise LockError("Synchronizer error - current thread doesnt " - "have the write lock") - - # reset the current sync operation so - # another can get it - self.current_sync_operation = None - - # tell everyone to get ready - self.condition.notifyAll() - finally: - # everyone go !! - self.condition.release() diff --git a/module/lib/beaker/util.py b/module/lib/beaker/util.py deleted file mode 100644 index 04c9617c5..000000000 --- a/module/lib/beaker/util.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Beaker utilities""" - -try: - import thread as _thread - import threading as _threading -except ImportError: - import dummy_thread as _thread - import dummy_threading as _threading - -from datetime import datetime, timedelta -import os -import string -import types -import weakref -import warnings -import sys - -py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) -py24 = sys.version_info < (2,5) -jython = sys.platform.startswith('java') - -if py3k or jython: - import pickle -else: - import cPickle as pickle - -from beaker.converters import asbool -from threading import local as _tlocal - - -__all__ = ["ThreadLocal", "Registry", "WeakValuedRegistry", "SyncDict", - "encoded_path", "verify_directory"] - - -def verify_directory(dir): - """verifies and creates a directory. tries to - ignore collisions with other threads and processes.""" - - tries = 0 - while not os.access(dir, os.F_OK): - try: - tries += 1 - os.makedirs(dir) - except: - if tries > 5: - raise - - -def deprecated(message): - def wrapper(fn): - def deprecated_method(*args, **kargs): - warnings.warn(message, DeprecationWarning, 2) - return fn(*args, **kargs) - # TODO: use decorator ? functools.wrapper ? - deprecated_method.__name__ = fn.__name__ - deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) - return deprecated_method - return wrapper - -class ThreadLocal(object): - """stores a value on a per-thread basis""" - - __slots__ = '_tlocal' - - def __init__(self): - self._tlocal = _tlocal() - - def put(self, value): - self._tlocal.value = value - - def has(self): - return hasattr(self._tlocal, 'value') - - def get(self, default=None): - return getattr(self._tlocal, 'value', default) - - def remove(self): - del self._tlocal.value - -class SyncDict(object): - """ - An efficient/threadsafe singleton map algorithm, a.k.a. - "get a value based on this key, and create if not found or not - valid" paradigm: - - exists && isvalid ? get : create - - Designed to work with weakref dictionaries to expect items - to asynchronously disappear from the dictionary. - - Use python 2.3.3 or greater ! a major bug was just fixed in Nov. - 2003 that was driving me nuts with garbage collection/weakrefs in - this section. - - """ - def __init__(self): - self.mutex = _thread.allocate_lock() - self.dict = {} - - def get(self, key, createfunc, *args, **kwargs): - try: - if self.has_key(key): - return self.dict[key] - else: - return self.sync_get(key, createfunc, *args, **kwargs) - except KeyError: - return self.sync_get(key, createfunc, *args, **kwargs) - - def sync_get(self, key, createfunc, *args, **kwargs): - self.mutex.acquire() - try: - try: - if self.has_key(key): - return self.dict[key] - else: - return self._create(key, createfunc, *args, **kwargs) - except KeyError: - return self._create(key, createfunc, *args, **kwargs) - finally: - self.mutex.release() - - def _create(self, key, createfunc, *args, **kwargs): - self[key] = obj = createfunc(*args, **kwargs) - return obj - - def has_key(self, key): - return self.dict.has_key(key) - - def __contains__(self, key): - return self.dict.__contains__(key) - def __getitem__(self, key): - return self.dict.__getitem__(key) - def __setitem__(self, key, value): - self.dict.__setitem__(key, value) - def __delitem__(self, key): - return self.dict.__delitem__(key) - def clear(self): - self.dict.clear() - - -class WeakValuedRegistry(SyncDict): - def __init__(self): - self.mutex = _threading.RLock() - self.dict = weakref.WeakValueDictionary() - -sha1 = None -def encoded_path(root, identifiers, extension = ".enc", depth = 3, - digest_filenames=True): - - """Generate a unique file-accessible path from the given list of - identifiers starting at the given root directory.""" - ident = "_".join(identifiers) - - global sha1 - if sha1 is None: - from beaker.crypto import sha1 - - if digest_filenames: - if py3k: - ident = sha1(ident.encode('utf-8')).hexdigest() - else: - ident = sha1(ident).hexdigest() - - ident = os.path.basename(ident) - - tokens = [] - for d in range(1, depth): - tokens.append(ident[0:d]) - - dir = os.path.join(root, *tokens) - verify_directory(dir) - - return os.path.join(dir, ident + extension) - - -def verify_options(opt, types, error): - if not isinstance(opt, types): - if not isinstance(types, tuple): - types = (types,) - coerced = False - for typ in types: - try: - if typ in (list, tuple): - opt = [x.strip() for x in opt.split(',')] - else: - if typ == bool: - typ = asbool - opt = typ(opt) - coerced = True - except: - pass - if coerced: - break - if not coerced: - raise Exception(error) - elif isinstance(opt, str) and not opt.strip(): - raise Exception("Empty strings are invalid for: %s" % error) - return opt - - -def verify_rules(params, ruleset): - for key, types, message in ruleset: - if key in params: - params[key] = verify_options(params[key], types, message) - return params - - -def coerce_session_params(params): - rules = [ - ('data_dir', (str, types.NoneType), "data_dir must be a string " - "referring to a directory."), - ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " - "directory."), - ('type', (str, types.NoneType), "Session type must be a string."), - ('cookie_expires', (bool, datetime, timedelta), "Cookie expires was " - "not a boolean, datetime, or timedelta instance."), - ('cookie_domain', (str, types.NoneType), "Cookie domain must be a " - "string."), - ('id', (str,), "Session id must be a string."), - ('key', (str,), "Session key must be a string."), - ('secret', (str, types.NoneType), "Session secret must be a string."), - ('validate_key', (str, types.NoneType), "Session encrypt_key must be " - "a string."), - ('encrypt_key', (str, types.NoneType), "Session validate_key must be " - "a string."), - ('secure', (bool, types.NoneType), "Session secure must be a boolean."), - ('timeout', (int, types.NoneType), "Session timeout must be an " - "integer."), - ('auto', (bool, types.NoneType), "Session is created if accessed."), - ] - return verify_rules(params, rules) - - -def coerce_cache_params(params): - rules = [ - ('data_dir', (str, types.NoneType), "data_dir must be a string " - "referring to a directory."), - ('lock_dir', (str, types.NoneType), "lock_dir must be a string referring to a " - "directory."), - ('type', (str,), "Cache type must be a string."), - ('enabled', (bool, types.NoneType), "enabled must be true/false " - "if present."), - ('expire', (int, types.NoneType), "expire must be an integer representing " - "how many seconds the cache is valid for"), - ('regions', (list, tuple, types.NoneType), "Regions must be a " - "comma seperated list of valid regions") - ] - return verify_rules(params, rules) - - -def parse_cache_config_options(config, include_defaults=True): - """Parse configuration options and validate for use with the - CacheManager""" - - # Load default cache options - if include_defaults: - options= dict(type='memory', data_dir=None, expire=None, - log_file=None) - else: - options = {} - for key, val in config.iteritems(): - if key.startswith('beaker.cache.'): - options[key[13:]] = val - if key.startswith('cache.'): - options[key[6:]] = val - coerce_cache_params(options) - - # Set cache to enabled if not turned off - if 'enabled' not in options: - options['enabled'] = True - - # Configure region dict if regions are available - regions = options.pop('regions', None) - if regions: - region_configs = {} - for region in regions: - # Setup the default cache options - region_options = dict(data_dir=options.get('data_dir'), - lock_dir=options.get('lock_dir'), - type=options.get('type'), - enabled=options['enabled'], - expire=options.get('expire')) - region_len = len(region) + 1 - for key in options.keys(): - if key.startswith('%s.' % region): - region_options[key[region_len:]] = options.pop(key) - coerce_cache_params(region_options) - region_configs[region] = region_options - options['cache_regions'] = region_configs - return options - -def func_namespace(func): - """Generates a unique namespace for a function""" - kls = None - if hasattr(func, 'im_func'): - kls = func.im_class - func = func.im_func - - if kls: - return '%s.%s' % (kls.__module__, kls.__name__) - else: - return '%s.%s' % (func.__module__, func.__name__) diff --git a/module/lib/bottle.py b/module/lib/bottle.py deleted file mode 100644 index b00bda1c9..000000000 --- a/module/lib/bottle.py +++ /dev/null @@ -1,3251 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Bottle is a fast and simple micro-framework for small web applications. It -offers request dispatching (Routes) with url parameter support, templates, -a built-in HTTP Server and adapters for many third party WSGI/HTTP-server and -template engines - all in a single file and with no dependencies other than the -Python Standard Library. - -Homepage and documentation: http://bottlepy.org/ - -Copyright (c) 2012, Marcel Hellkamp. -License: MIT (see LICENSE for details) -""" - -from __future__ import with_statement - -__author__ = 'Marcel Hellkamp' -__version__ = '0.11.4' -__license__ = 'MIT' - -# The gevent server adapter needs to patch some modules before they are imported -# This is why we parse the commandline parameters here but handle them later -if __name__ == '__main__': - from optparse import OptionParser - _cmd_parser = OptionParser(usage="usage: %prog [options] package.module:app") - _opt = _cmd_parser.add_option - _opt("--version", action="store_true", help="show version number.") - _opt("-b", "--bind", metavar="ADDRESS", help="bind socket to ADDRESS.") - _opt("-s", "--server", default='wsgiref', help="use SERVER as backend.") - _opt("-p", "--plugin", action="append", help="install additional plugin/s.") - _opt("--debug", action="store_true", help="start server in debug mode.") - _opt("--reload", action="store_true", help="auto-reload on file changes.") - _cmd_options, _cmd_args = _cmd_parser.parse_args() - if _cmd_options.server and _cmd_options.server.startswith('gevent'): - import gevent.monkey; gevent.monkey.patch_all() - -import base64, cgi, email.utils, functools, hmac, imp, itertools, mimetypes,\ - os, re, subprocess, sys, tempfile, threading, time, urllib, warnings - -from datetime import date as datedate, datetime, timedelta -from tempfile import TemporaryFile -from traceback import format_exc, print_exc - -try: from json import dumps as json_dumps, loads as json_lds -except ImportError: # pragma: no cover - try: from simplejson import dumps as json_dumps, loads as json_lds - except ImportError: - try: from django.utils.simplejson import dumps as json_dumps, loads as json_lds - except ImportError: - def json_dumps(data): - raise ImportError("JSON support requires Python 2.6 or simplejson.") - json_lds = json_dumps - - - -# We now try to fix 2.5/2.6/3.1/3.2 incompatibilities. -# It ain't pretty but it works... Sorry for the mess. - -py = sys.version_info -py3k = py >= (3,0,0) -py25 = py < (2,6,0) -py31 = (3,1,0) <= py < (3,2,0) - -# Workaround for the missing "as" keyword in py3k. -def _e(): return sys.exc_info()[1] - -# Workaround for the "print is a keyword/function" Python 2/3 dilemma -# and a fallback for mod_wsgi (resticts stdout/err attribute access) -try: - _stdout, _stderr = sys.stdout.write, sys.stderr.write -except IOError: - _stdout = lambda x: sys.stdout.write(x) - _stderr = lambda x: sys.stderr.write(x) - -# Lots of stdlib and builtin differences. -if py3k: - import http.client as httplib - import _thread as thread - from urllib.parse import urljoin, SplitResult as UrlSplitResult - from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote - urlunquote = functools.partial(urlunquote, encoding='latin1') - from http.cookies import SimpleCookie - from collections import MutableMapping as DictMixin - import pickle - from io import BytesIO - basestring = str - unicode = str - json_loads = lambda s: json_lds(touni(s)) - callable = lambda x: hasattr(x, '__call__') - imap = map -else: # 2.x - import httplib - import thread - from urlparse import urljoin, SplitResult as UrlSplitResult - from urllib import urlencode, quote as urlquote, unquote as urlunquote - from Cookie import SimpleCookie - from itertools import imap - import cPickle as pickle - from StringIO import StringIO as BytesIO - if py25: - from UserDict import DictMixin - def next(it): return it.next() - bytes = str - else: # 2.6, 2.7 - from collections import MutableMapping as DictMixin - json_loads = json_lds - -# Some helpers for string/byte handling -def tob(s, enc='utf8'): - return s.encode(enc) if isinstance(s, unicode) else bytes(s) -def touni(s, enc='utf8', err='strict'): - return s.decode(enc, err) if isinstance(s, bytes) else unicode(s) -tonat = touni if py3k else tob - -# 3.2 fixes cgi.FieldStorage to accept bytes (which makes a lot of sense). -# 3.1 needs a workaround. -if py31: - from io import TextIOWrapper - class NCTextIOWrapper(TextIOWrapper): - def close(self): pass # Keep wrapped buffer open. - -# File uploads (which are implemented as empty FiledStorage instances...) -# have a negative truth value. That makes no sense, here is a fix. -class FieldStorage(cgi.FieldStorage): - def __nonzero__(self): return bool(self.list or self.file) - if py3k: __bool__ = __nonzero__ - -# A bug in functools causes it to break if the wrapper is an instance method -def update_wrapper(wrapper, wrapped, *a, **ka): - try: functools.update_wrapper(wrapper, wrapped, *a, **ka) - except AttributeError: pass - - - -# These helpers are used at module level and need to be defined first. -# And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. - -def depr(message): - warnings.warn(message, DeprecationWarning, stacklevel=3) - -def makelist(data): # This is just to handy - if isinstance(data, (tuple, list, set, dict)): return list(data) - elif data: return [data] - else: return [] - - -class DictProperty(object): - ''' Property that maps to a key in a local dict-like attribute. ''' - def __init__(self, attr, key=None, read_only=False): - self.attr, self.key, self.read_only = attr, key, read_only - - def __call__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter, self.key = func, self.key or func.__name__ - return self - - def __get__(self, obj, cls): - if obj is None: return self - key, storage = self.key, getattr(obj, self.attr) - if key not in storage: storage[key] = self.getter(obj) - return storage[key] - - def __set__(self, obj, value): - if self.read_only: raise AttributeError("Read-Only property.") - getattr(obj, self.attr)[self.key] = value - - def __delete__(self, obj): - if self.read_only: raise AttributeError("Read-Only property.") - del getattr(obj, self.attr)[self.key] - - -class cached_property(object): - ''' A property that is only computed once per instance and then replaces - itself with an ordinary attribute. Deleting the attribute resets the - property. ''' - - def __init__(self, func): - self.func = func - - def __get__(self, obj, cls): - if obj is None: return self - value = obj.__dict__[self.func.__name__] = self.func(obj) - return value - - -class lazy_attribute(object): - ''' A property that caches itself to the class object. ''' - def __init__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter = func - - def __get__(self, obj, cls): - value = self.getter(cls) - setattr(cls, self.__name__, value) - return value - - - - - - -############################################################################### -# Exceptions and Events ######################################################## -############################################################################### - - -class BottleException(Exception): - """ A base class for exceptions used by bottle. """ - pass - - - - - - -############################################################################### -# Routing ###################################################################### -############################################################################### - - -class RouteError(BottleException): - """ This is a base class for all routing related exceptions """ - - -class RouteReset(BottleException): - """ If raised by a plugin or request handler, the route is reset and all - plugins are re-applied. """ - -class RouterUnknownModeError(RouteError): pass - - -class RouteSyntaxError(RouteError): - """ The route parser found something not supported by this router """ - - -class RouteBuildError(RouteError): - """ The route could not been built """ - - -class Router(object): - ''' A Router is an ordered collection of route->target pairs. It is used to - efficiently match WSGI requests against a number of routes and return - the first target that satisfies the request. The target may be anything, - usually a string, ID or callable object. A route consists of a path-rule - and a HTTP method. - - The path-rule is either a static path (e.g. `/contact`) or a dynamic - path that contains wildcards (e.g. `/wiki/<page>`). The wildcard syntax - and details on the matching order are described in docs:`routing`. - ''' - - default_pattern = '[^/]+' - default_filter = 're' - #: Sorry for the mess. It works. Trust me. - rule_syntax = re.compile('(\\\\*)'\ - '(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)'\ - '|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)'\ - '(?::((?:\\\\.|[^\\\\>]+)+)?)?)?>))') - - def __init__(self, strict=False): - self.rules = {} # A {rule: Rule} mapping - self.builder = {} # A rule/name->build_info mapping - self.static = {} # Cache for static routes: {path: {method: target}} - self.dynamic = [] # Cache for dynamic routes. See _compile() - #: If true, static routes are no longer checked first. - self.strict_order = strict - self.filters = {'re': self.re_filter, 'int': self.int_filter, - 'float': self.float_filter, 'path': self.path_filter} - - def re_filter(self, conf): - return conf or self.default_pattern, None, None - - def int_filter(self, conf): - return r'-?\d+', int, lambda x: str(int(x)) - - def float_filter(self, conf): - return r'-?[\d.]+', float, lambda x: str(float(x)) - - def path_filter(self, conf): - return r'.+?', None, None - - def add_filter(self, name, func): - ''' Add a filter. The provided function is called with the configuration - string as parameter and must return a (regexp, to_python, to_url) tuple. - The first element is a string, the last two are callables or None. ''' - self.filters[name] = func - - def parse_rule(self, rule): - ''' Parses a rule into a (name, filter, conf) token stream. If mode is - None, name contains a static rule part. ''' - offset, prefix = 0, '' - for match in self.rule_syntax.finditer(rule): - prefix += rule[offset:match.start()] - g = match.groups() - if len(g[0])%2: # Escaped wildcard - prefix += match.group(0)[len(g[0]):] - offset = match.end() - continue - if prefix: yield prefix, None, None - name, filtr, conf = g[1:4] if not g[2] is None else g[4:7] - if not filtr: filtr = self.default_filter - yield name, filtr, conf or None - offset, prefix = match.end(), '' - if offset <= len(rule) or prefix: - yield prefix+rule[offset:], None, None - - def add(self, rule, method, target, name=None): - ''' Add a new route or replace the target for an existing route. ''' - if rule in self.rules: - self.rules[rule][method] = target - if name: self.builder[name] = self.builder[rule] - return - - target = self.rules[rule] = {method: target} - - # Build pattern and other structures for dynamic routes - anons = 0 # Number of anonymous wildcards - pattern = '' # Regular expression pattern - filters = [] # Lists of wildcard input filters - builder = [] # Data structure for the URL builder - is_static = True - for key, mode, conf in self.parse_rule(rule): - if mode: - is_static = False - mask, in_filter, out_filter = self.filters[mode](conf) - if key: - pattern += '(?P<%s>%s)' % (key, mask) - else: - pattern += '(?:%s)' % mask - key = 'anon%d' % anons; anons += 1 - if in_filter: filters.append((key, in_filter)) - builder.append((key, out_filter or str)) - elif key: - pattern += re.escape(key) - builder.append((None, key)) - self.builder[rule] = builder - if name: self.builder[name] = builder - - if is_static and not self.strict_order: - self.static[self.build(rule)] = target - return - - def fpat_sub(m): - return m.group(0) if len(m.group(1)) % 2 else m.group(1) + '(?:' - flat_pattern = re.sub(r'(\\*)(\(\?P<[^>]*>|\((?!\?))', fpat_sub, pattern) - - try: - re_match = re.compile('^(%s)$' % pattern).match - except re.error: - raise RouteSyntaxError("Could not add Route: %s (%s)" % (rule, _e())) - - def match(path): - """ Return an url-argument dictionary. """ - url_args = re_match(path).groupdict() - for name, wildcard_filter in filters: - try: - url_args[name] = wildcard_filter(url_args[name]) - except ValueError: - raise HTTPError(400, 'Path has wrong format.') - return url_args - - try: - combined = '%s|(^%s$)' % (self.dynamic[-1][0].pattern, flat_pattern) - self.dynamic[-1] = (re.compile(combined), self.dynamic[-1][1]) - self.dynamic[-1][1].append((match, target)) - except (AssertionError, IndexError): # AssertionError: Too many groups - self.dynamic.append((re.compile('(^%s$)' % flat_pattern), - [(match, target)])) - return match - - def build(self, _name, *anons, **query): - ''' Build an URL by filling the wildcards in a rule. ''' - builder = self.builder.get(_name) - if not builder: raise RouteBuildError("No route with that name.", _name) - try: - for i, value in enumerate(anons): query['anon%d'%i] = value - url = ''.join([f(query.pop(n)) if n else f for (n,f) in builder]) - return url if not query else url+'?'+urlencode(query) - except KeyError: - raise RouteBuildError('Missing URL argument: %r' % _e().args[0]) - - def match(self, environ): - ''' Return a (target, url_agrs) tuple or raise HTTPError(400/404/405). ''' - path, targets, urlargs = environ['PATH_INFO'] or '/', None, {} - if path in self.static: - targets = self.static[path] - else: - for combined, rules in self.dynamic: - match = combined.match(path) - if not match: continue - getargs, targets = rules[match.lastindex - 1] - urlargs = getargs(path) if getargs else {} - break - - if not targets: - raise HTTPError(404, "Not found: " + repr(environ['PATH_INFO'])) - method = environ['REQUEST_METHOD'].upper() - if method in targets: - return targets[method], urlargs - if method == 'HEAD' and 'GET' in targets: - return targets['GET'], urlargs - if 'ANY' in targets: - return targets['ANY'], urlargs - allowed = [verb for verb in targets if verb != 'ANY'] - if 'GET' in allowed and 'HEAD' not in allowed: - allowed.append('HEAD') - raise HTTPError(405, "Method not allowed.", Allow=",".join(allowed)) - - -class Route(object): - ''' This class wraps a route callback along with route specific metadata and - configuration and applies Plugins on demand. It is also responsible for - turing an URL path rule into a regular expression usable by the Router. - ''' - - def __init__(self, app, rule, method, callback, name=None, - plugins=None, skiplist=None, **config): - #: The application this route is installed to. - self.app = app - #: The path-rule string (e.g. ``/wiki/:page``). - self.rule = rule - #: The HTTP method as a string (e.g. ``GET``). - self.method = method - #: The original callback with no plugins applied. Useful for introspection. - self.callback = callback - #: The name of the route (if specified) or ``None``. - self.name = name or None - #: A list of route-specific plugins (see :meth:`Bottle.route`). - self.plugins = plugins or [] - #: A list of plugins to not apply to this route (see :meth:`Bottle.route`). - self.skiplist = skiplist or [] - #: Additional keyword arguments passed to the :meth:`Bottle.route` - #: decorator are stored in this dictionary. Used for route-specific - #: plugin configuration and meta-data. - self.config = ConfigDict(config) - - def __call__(self, *a, **ka): - depr("Some APIs changed to return Route() instances instead of"\ - " callables. Make sure to use the Route.call method and not to"\ - " call Route instances directly.") - return self.call(*a, **ka) - - @cached_property - def call(self): - ''' The route callback with all plugins applied. This property is - created on demand and then cached to speed up subsequent requests.''' - return self._make_callback() - - def reset(self): - ''' Forget any cached values. The next time :attr:`call` is accessed, - all plugins are re-applied. ''' - self.__dict__.pop('call', None) - - def prepare(self): - ''' Do all on-demand work immediately (useful for debugging).''' - self.call - - @property - def _context(self): - depr('Switch to Plugin API v2 and access the Route object directly.') - return dict(rule=self.rule, method=self.method, callback=self.callback, - name=self.name, app=self.app, config=self.config, - apply=self.plugins, skip=self.skiplist) - - def all_plugins(self): - ''' Yield all Plugins affecting this route. ''' - unique = set() - for p in reversed(self.app.plugins + self.plugins): - if True in self.skiplist: break - name = getattr(p, 'name', False) - if name and (name in self.skiplist or name in unique): continue - if p in self.skiplist or type(p) in self.skiplist: continue - if name: unique.add(name) - yield p - - def _make_callback(self): - callback = self.callback - for plugin in self.all_plugins(): - try: - if hasattr(plugin, 'apply'): - api = getattr(plugin, 'api', 1) - context = self if api > 1 else self._context - callback = plugin.apply(callback, context) - else: - callback = plugin(callback) - except RouteReset: # Try again with changed configuration. - return self._make_callback() - if not callback is self.callback: - update_wrapper(callback, self.callback) - return callback - - def __repr__(self): - return '<%s %r %r>' % (self.method, self.rule, self.callback) - - - - - - -############################################################################### -# Application Object ########################################################### -############################################################################### - - -class Bottle(object): - """ Each Bottle object represents a single, distinct web application and - consists of routes, callbacks, plugins, resources and configuration. - Instances are callable WSGI applications. - - :param catchall: If true (default), handle all exceptions. Turn off to - let debugging middleware handle exceptions. - """ - - def __init__(self, catchall=True, autojson=True): - #: If true, most exceptions are caught and returned as :exc:`HTTPError` - self.catchall = catchall - - #: A :class:`ResourceManager` for application files - self.resources = ResourceManager() - - #: A :class:`ConfigDict` for app specific configuration. - self.config = ConfigDict() - self.config.autojson = autojson - - self.routes = [] # List of installed :class:`Route` instances. - self.router = Router() # Maps requests to :class:`Route` instances. - self.error_handler = {} - - # Core plugins - self.plugins = [] # List of installed plugins. - self.hooks = HooksPlugin() - self.install(self.hooks) - if self.config.autojson: - self.install(JSONPlugin()) - self.install(TemplatePlugin()) - - - def mount(self, prefix, app, **options): - ''' Mount an application (:class:`Bottle` or plain WSGI) to a specific - URL prefix. Example:: - - root_app.mount('/admin/', admin_app) - - :param prefix: path prefix or `mount-point`. If it ends in a slash, - that slash is mandatory. - :param app: an instance of :class:`Bottle` or a WSGI application. - - All other parameters are passed to the underlying :meth:`route` call. - ''' - if isinstance(app, basestring): - prefix, app = app, prefix - depr('Parameter order of Bottle.mount() changed.') # 0.10 - - segments = [p for p in prefix.split('/') if p] - if not segments: raise ValueError('Empty path prefix.') - path_depth = len(segments) - - def mountpoint_wrapper(): - try: - request.path_shift(path_depth) - rs = BaseResponse([], 200) - def start_response(status, header): - rs.status = status - for name, value in header: rs.add_header(name, value) - return rs.body.append - body = app(request.environ, start_response) - body = itertools.chain(rs.body, body) - return HTTPResponse(body, rs.status_code, **rs.headers) - finally: - request.path_shift(-path_depth) - - options.setdefault('skip', True) - options.setdefault('method', 'ANY') - options.setdefault('mountpoint', {'prefix': prefix, 'target': app}) - options['callback'] = mountpoint_wrapper - - self.route('/%s/<:re:.*>' % '/'.join(segments), **options) - if not prefix.endswith('/'): - self.route('/' + '/'.join(segments), **options) - - def merge(self, routes): - ''' Merge the routes of another :class:`Bottle` application or a list of - :class:`Route` objects into this application. The routes keep their - 'owner', meaning that the :data:`Route.app` attribute is not - changed. ''' - if isinstance(routes, Bottle): - routes = routes.routes - for route in routes: - self.add_route(route) - - def install(self, plugin): - ''' Add a plugin to the list of plugins and prepare it for being - applied to all routes of this application. A plugin may be a simple - decorator or an object that implements the :class:`Plugin` API. - ''' - if hasattr(plugin, 'setup'): plugin.setup(self) - if not callable(plugin) and not hasattr(plugin, 'apply'): - raise TypeError("Plugins must be callable or implement .apply()") - self.plugins.append(plugin) - self.reset() - return plugin - - def uninstall(self, plugin): - ''' Uninstall plugins. Pass an instance to remove a specific plugin, a type - object to remove all plugins that match that type, a string to remove - all plugins with a matching ``name`` attribute or ``True`` to remove all - plugins. Return the list of removed plugins. ''' - removed, remove = [], plugin - for i, plugin in list(enumerate(self.plugins))[::-1]: - if remove is True or remove is plugin or remove is type(plugin) \ - or getattr(plugin, 'name', True) == remove: - removed.append(plugin) - del self.plugins[i] - if hasattr(plugin, 'close'): plugin.close() - if removed: self.reset() - return removed - - def run(self, **kwargs): - ''' Calls :func:`run` with the same parameters. ''' - run(self, **kwargs) - - def reset(self, route=None): - ''' Reset all routes (force plugins to be re-applied) and clear all - caches. If an ID or route object is given, only that specific route - is affected. ''' - if route is None: routes = self.routes - elif isinstance(route, Route): routes = [route] - else: routes = [self.routes[route]] - for route in routes: route.reset() - if DEBUG: - for route in routes: route.prepare() - self.hooks.trigger('app_reset') - - def close(self): - ''' Close the application and all installed plugins. ''' - for plugin in self.plugins: - if hasattr(plugin, 'close'): plugin.close() - self.stopped = True - - def match(self, environ): - """ Search for a matching route and return a (:class:`Route` , urlargs) - tuple. The second value is a dictionary with parameters extracted - from the URL. Raise :exc:`HTTPError` (404/405) on a non-match.""" - return self.router.match(environ) - - def get_url(self, routename, **kargs): - """ Return a string that matches a named route """ - scriptname = request.environ.get('SCRIPT_NAME', '').strip('/') + '/' - location = self.router.build(routename, **kargs).lstrip('/') - return urljoin(urljoin('/', scriptname), location) - - def add_route(self, route): - ''' Add a route object, but do not change the :data:`Route.app` - attribute.''' - self.routes.append(route) - self.router.add(route.rule, route.method, route, name=route.name) - if DEBUG: route.prepare() - - def route(self, path=None, method='GET', callback=None, name=None, - apply=None, skip=None, **config): - """ A decorator to bind a function to a request URL. Example:: - - @app.route('/hello/:name') - def hello(name): - return 'Hello %s' % name - - The ``:name`` part is a wildcard. See :class:`Router` for syntax - details. - - :param path: Request path or a list of paths to listen to. If no - path is specified, it is automatically generated from the - signature of the function. - :param method: HTTP method (`GET`, `POST`, `PUT`, ...) or a list of - methods to listen to. (default: `GET`) - :param callback: An optional shortcut to avoid the decorator - syntax. ``route(..., callback=func)`` equals ``route(...)(func)`` - :param name: The name for this route. (default: None) - :param apply: A decorator or plugin or a list of plugins. These are - applied to the route callback in addition to installed plugins. - :param skip: A list of plugins, plugin classes or names. Matching - plugins are not installed to this route. ``True`` skips all. - - Any additional keyword arguments are stored as route-specific - configuration and passed to plugins (see :meth:`Plugin.apply`). - """ - if callable(path): path, callback = None, path - plugins = makelist(apply) - skiplist = makelist(skip) - def decorator(callback): - # TODO: Documentation and tests - if isinstance(callback, basestring): callback = load(callback) - for rule in makelist(path) or yieldroutes(callback): - for verb in makelist(method): - verb = verb.upper() - route = Route(self, rule, verb, callback, name=name, - plugins=plugins, skiplist=skiplist, **config) - self.add_route(route) - return callback - return decorator(callback) if callback else decorator - - def get(self, path=None, method='GET', **options): - """ Equals :meth:`route`. """ - return self.route(path, method, **options) - - def post(self, path=None, method='POST', **options): - """ Equals :meth:`route` with a ``POST`` method parameter. """ - return self.route(path, method, **options) - - def put(self, path=None, method='PUT', **options): - """ Equals :meth:`route` with a ``PUT`` method parameter. """ - return self.route(path, method, **options) - - def delete(self, path=None, method='DELETE', **options): - """ Equals :meth:`route` with a ``DELETE`` method parameter. """ - return self.route(path, method, **options) - - def error(self, code=500): - """ Decorator: Register an output handler for a HTTP error code""" - def wrapper(handler): - self.error_handler[int(code)] = handler - return handler - return wrapper - - def hook(self, name): - """ Return a decorator that attaches a callback to a hook. Three hooks - are currently implemented: - - - before_request: Executed once before each request - - after_request: Executed once after each request - - app_reset: Called whenever :meth:`reset` is called. - """ - def wrapper(func): - self.hooks.add(name, func) - return func - return wrapper - - def handle(self, path, method='GET'): - """ (deprecated) Execute the first matching route callback and return - the result. :exc:`HTTPResponse` exceptions are caught and returned. - If :attr:`Bottle.catchall` is true, other exceptions are caught as - well and returned as :exc:`HTTPError` instances (500). - """ - depr("This method will change semantics in 0.10. Try to avoid it.") - if isinstance(path, dict): - return self._handle(path) - return self._handle({'PATH_INFO': path, 'REQUEST_METHOD': method.upper()}) - - def default_error_handler(self, res): - return tob(template(ERROR_PAGE_TEMPLATE, e=res)) - - def _handle(self, environ): - try: - environ['bottle.app'] = self - request.bind(environ) - response.bind() - route, args = self.router.match(environ) - environ['route.handle'] = route - environ['bottle.route'] = route - environ['route.url_args'] = args - return route.call(**args) - except HTTPResponse: - return _e() - except RouteReset: - route.reset() - return self._handle(environ) - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - stacktrace = format_exc() - environ['wsgi.errors'].write(stacktrace) - return HTTPError(500, "Internal Server Error", _e(), stacktrace) - - def _cast(self, out, peek=None): - """ Try to convert the parameter into something WSGI compatible and set - correct HTTP headers when possible. - Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like, - iterable of strings and iterable of unicodes - """ - - # Empty output is done here - if not out: - if 'Content-Length' not in response: - response['Content-Length'] = 0 - return [] - # Join lists of byte or unicode strings. Mixed lists are NOT supported - if isinstance(out, (tuple, list))\ - and isinstance(out[0], (bytes, unicode)): - out = out[0][0:0].join(out) # b'abc'[0:0] -> b'' - # Encode unicode strings - if isinstance(out, unicode): - out = out.encode(response.charset) - # Byte Strings are just returned - if isinstance(out, bytes): - if 'Content-Length' not in response: - response['Content-Length'] = len(out) - return [out] - # HTTPError or HTTPException (recursive, because they may wrap anything) - # TODO: Handle these explicitly in handle() or make them iterable. - if isinstance(out, HTTPError): - out.apply(response) - out = self.error_handler.get(out.status_code, self.default_error_handler)(out) - return self._cast(out) - if isinstance(out, HTTPResponse): - out.apply(response) - return self._cast(out.body) - - # File-like objects. - if hasattr(out, 'read'): - if 'wsgi.file_wrapper' in request.environ: - return request.environ['wsgi.file_wrapper'](out) - elif hasattr(out, 'close') or not hasattr(out, '__iter__'): - return WSGIFileWrapper(out) - - # Handle Iterables. We peek into them to detect their inner type. - try: - out = iter(out) - first = next(out) - while not first: - first = next(out) - except StopIteration: - return self._cast('') - except HTTPResponse: - first = _e() - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - first = HTTPError(500, 'Unhandled exception', _e(), format_exc()) - - # These are the inner types allowed in iterator or generator objects. - if isinstance(first, HTTPResponse): - return self._cast(first) - if isinstance(first, bytes): - return itertools.chain([first], out) - if isinstance(first, unicode): - return imap(lambda x: x.encode(response.charset), - itertools.chain([first], out)) - return self._cast(HTTPError(500, 'Unsupported response type: %s'\ - % type(first))) - - def wsgi(self, environ, start_response): - """ The bottle WSGI-interface. """ - try: - out = self._cast(self._handle(environ)) - # rfc2616 section 4.3 - if response._status_code in (100, 101, 204, 304)\ - or environ['REQUEST_METHOD'] == 'HEAD': - if hasattr(out, 'close'): out.close() - out = [] - start_response(response._status_line, response.headerlist) - return out - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception: - if not self.catchall: raise - err = '<h1>Critical error while processing request: %s</h1>' \ - % html_escape(environ.get('PATH_INFO', '/')) - if DEBUG: - err += '<h2>Error:</h2>\n<pre>\n%s\n</pre>\n' \ - '<h2>Traceback:</h2>\n<pre>\n%s\n</pre>\n' \ - % (html_escape(repr(_e())), html_escape(format_exc())) - environ['wsgi.errors'].write(err) - headers = [('Content-Type', 'text/html; charset=UTF-8')] - start_response('500 INTERNAL SERVER ERROR', headers) - return [tob(err)] - - def __call__(self, environ, start_response): - ''' Each instance of :class:'Bottle' is a WSGI application. ''' - return self.wsgi(environ, start_response) - - - - - - -############################################################################### -# HTTP and WSGI Tools ########################################################## -############################################################################### - - -class BaseRequest(object): - """ A wrapper for WSGI environment dictionaries that adds a lot of - convenient access methods and properties. Most of them are read-only. - - Adding new attributes to a request actually adds them to the environ - dictionary (as 'bottle.request.ext.<name>'). This is the recommended - way to store and access request-specific data. - """ - - __slots__ = ('environ') - - #: Maximum size of memory buffer for :attr:`body` in bytes. - MEMFILE_MAX = 102400 - #: Maximum number pr GET or POST parameters per request - MAX_PARAMS = 100 - - def __init__(self, environ=None): - """ Wrap a WSGI environ dictionary. """ - #: The wrapped WSGI environ dictionary. This is the only real attribute. - #: All other attributes actually are read-only properties. - self.environ = {} if environ is None else environ - self.environ['bottle.request'] = self - - @DictProperty('environ', 'bottle.app', read_only=True) - def app(self): - ''' Bottle application handling this request. ''' - raise RuntimeError('This request is not connected to an application.') - - @property - def path(self): - ''' The value of ``PATH_INFO`` with exactly one prefixed slash (to fix - broken clients and avoid the "empty path" edge case). ''' - return '/' + self.environ.get('PATH_INFO','').lstrip('/') - - @property - def method(self): - ''' The ``REQUEST_METHOD`` value as an uppercase string. ''' - return self.environ.get('REQUEST_METHOD', 'GET').upper() - - @DictProperty('environ', 'bottle.request.headers', read_only=True) - def headers(self): - ''' A :class:`WSGIHeaderDict` that provides case-insensitive access to - HTTP request headers. ''' - return WSGIHeaderDict(self.environ) - - def get_header(self, name, default=None): - ''' Return the value of a request header, or a given default value. ''' - return self.headers.get(name, default) - - @DictProperty('environ', 'bottle.request.cookies', read_only=True) - def cookies(self): - """ Cookies parsed into a :class:`FormsDict`. Signed cookies are NOT - decoded. Use :meth:`get_cookie` if you expect signed cookies. """ - cookies = SimpleCookie(self.environ.get('HTTP_COOKIE','')) - cookies = list(cookies.values())[:self.MAX_PARAMS] - return FormsDict((c.key, c.value) for c in cookies) - - def get_cookie(self, key, default=None, secret=None): - """ Return the content of a cookie. To read a `Signed Cookie`, the - `secret` must match the one used to create the cookie (see - :meth:`BaseResponse.set_cookie`). If anything goes wrong (missing - cookie or wrong signature), return a default value. """ - value = self.cookies.get(key) - if secret and value: - dec = cookie_decode(value, secret) # (key, value) tuple or None - return dec[1] if dec and dec[0] == key else default - return value or default - - @DictProperty('environ', 'bottle.request.query', read_only=True) - def query(self): - ''' The :attr:`query_string` parsed into a :class:`FormsDict`. These - values are sometimes called "URL arguments" or "GET parameters", but - not to be confused with "URL wildcards" as they are provided by the - :class:`Router`. ''' - get = self.environ['bottle.get'] = FormsDict() - pairs = _parse_qsl(self.environ.get('QUERY_STRING', '')) - for key, value in pairs[:self.MAX_PARAMS]: - get[key] = value - return get - - @DictProperty('environ', 'bottle.request.forms', read_only=True) - def forms(self): - """ Form values parsed from an `url-encoded` or `multipart/form-data` - encoded POST or PUT request body. The result is retuned as a - :class:`FormsDict`. All keys and values are strings. File uploads - are stored separately in :attr:`files`. """ - forms = FormsDict() - for name, item in self.POST.allitems(): - if not hasattr(item, 'filename'): - forms[name] = item - return forms - - @DictProperty('environ', 'bottle.request.params', read_only=True) - def params(self): - """ A :class:`FormsDict` with the combined values of :attr:`query` and - :attr:`forms`. File uploads are stored in :attr:`files`. """ - params = FormsDict() - for key, value in self.query.allitems(): - params[key] = value - for key, value in self.forms.allitems(): - params[key] = value - return params - - @DictProperty('environ', 'bottle.request.files', read_only=True) - def files(self): - """ File uploads parsed from an `url-encoded` or `multipart/form-data` - encoded POST or PUT request body. The values are instances of - :class:`cgi.FieldStorage`. The most important attributes are: - - filename - The filename, if specified; otherwise None; this is the client - side filename, *not* the file name on which it is stored (that's - a temporary file you don't deal with) - file - The file(-like) object from which you can read the data. - value - The value as a *string*; for file uploads, this transparently - reads the file every time you request the value. Do not do this - on big files. - """ - files = FormsDict() - for name, item in self.POST.allitems(): - if hasattr(item, 'filename'): - files[name] = item - return files - - @DictProperty('environ', 'bottle.request.json', read_only=True) - def json(self): - ''' If the ``Content-Type`` header is ``application/json``, this - property holds the parsed content of the request body. Only requests - smaller than :attr:`MEMFILE_MAX` are processed to avoid memory - exhaustion. ''' - if 'application/json' in self.environ.get('CONTENT_TYPE', '') \ - and 0 < self.content_length < self.MEMFILE_MAX: - return json_loads(self.body.read(self.MEMFILE_MAX)) - return None - - @DictProperty('environ', 'bottle.request.body', read_only=True) - def _body(self): - maxread = max(0, self.content_length) - stream = self.environ['wsgi.input'] - body = BytesIO() if maxread < self.MEMFILE_MAX else TemporaryFile(mode='w+b') - while maxread > 0: - part = stream.read(min(maxread, self.MEMFILE_MAX)) - if not part: break - body.write(part) - maxread -= len(part) - self.environ['wsgi.input'] = body - body.seek(0) - return body - - @property - def body(self): - """ The HTTP request body as a seek-able file-like object. Depending on - :attr:`MEMFILE_MAX`, this is either a temporary file or a - :class:`io.BytesIO` instance. Accessing this property for the first - time reads and replaces the ``wsgi.input`` environ variable. - Subsequent accesses just do a `seek(0)` on the file object. """ - self._body.seek(0) - return self._body - - #: An alias for :attr:`query`. - GET = query - - @DictProperty('environ', 'bottle.request.post', read_only=True) - def POST(self): - """ The values of :attr:`forms` and :attr:`files` combined into a single - :class:`FormsDict`. Values are either strings (form values) or - instances of :class:`cgi.FieldStorage` (file uploads). - """ - post = FormsDict() - # We default to application/x-www-form-urlencoded for everything that - # is not multipart and take the fast path (also: 3.1 workaround) - if not self.content_type.startswith('multipart/'): - maxlen = max(0, min(self.content_length, self.MEMFILE_MAX)) - pairs = _parse_qsl(tonat(self.body.read(maxlen), 'latin1')) - for key, value in pairs[:self.MAX_PARAMS]: - post[key] = value - return post - - safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi - for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'): - if key in self.environ: safe_env[key] = self.environ[key] - args = dict(fp=self.body, environ=safe_env, keep_blank_values=True) - if py31: - args['fp'] = NCTextIOWrapper(args['fp'], encoding='ISO-8859-1', - newline='\n') - elif py3k: - args['encoding'] = 'ISO-8859-1' - data = FieldStorage(**args) - for item in (data.list or [])[:self.MAX_PARAMS]: - post[item.name] = item if item.filename else item.value - return post - - @property - def COOKIES(self): - ''' Alias for :attr:`cookies` (deprecated). ''' - depr('BaseRequest.COOKIES was renamed to BaseRequest.cookies (lowercase).') - return self.cookies - - @property - def url(self): - """ The full request URI including hostname and scheme. If your app - lives behind a reverse proxy or load balancer and you get confusing - results, make sure that the ``X-Forwarded-Host`` header is set - correctly. """ - return self.urlparts.geturl() - - @DictProperty('environ', 'bottle.request.urlparts', read_only=True) - def urlparts(self): - ''' The :attr:`url` string as an :class:`urlparse.SplitResult` tuple. - The tuple contains (scheme, host, path, query_string and fragment), - but the fragment is always empty because it is not visible to the - server. ''' - env = self.environ - http = env.get('HTTP_X_FORWARDED_PROTO') or env.get('wsgi.url_scheme', 'http') - host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') - if not host: - # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. - host = env.get('SERVER_NAME', '127.0.0.1') - port = env.get('SERVER_PORT') - if port and port != ('80' if http == 'http' else '443'): - host += ':' + port - path = urlquote(self.fullpath) - return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') - - @property - def fullpath(self): - """ Request path including :attr:`script_name` (if present). """ - return urljoin(self.script_name, self.path.lstrip('/')) - - @property - def query_string(self): - """ The raw :attr:`query` part of the URL (everything in between ``?`` - and ``#``) as a string. """ - return self.environ.get('QUERY_STRING', '') - - @property - def script_name(self): - ''' The initial portion of the URL's `path` that was removed by a higher - level (server or routing middleware) before the application was - called. This script path is returned with leading and tailing - slashes. ''' - script_name = self.environ.get('SCRIPT_NAME', '').strip('/') - return '/' + script_name + '/' if script_name else '/' - - def path_shift(self, shift=1): - ''' Shift path segments from :attr:`path` to :attr:`script_name` and - vice versa. - - :param shift: The number of path segments to shift. May be negative - to change the shift direction. (default: 1) - ''' - script = self.environ.get('SCRIPT_NAME','/') - self['SCRIPT_NAME'], self['PATH_INFO'] = path_shift(script, self.path, shift) - - @property - def content_length(self): - ''' The request body length as an integer. The client is responsible to - set this header. Otherwise, the real length of the body is unknown - and -1 is returned. In this case, :attr:`body` will be empty. ''' - return int(self.environ.get('CONTENT_LENGTH') or -1) - - @property - def content_type(self): - ''' The Content-Type header as a lowercase-string (default: empty). ''' - return self.environ.get('CONTENT_TYPE', '').lower() - - @property - def is_xhr(self): - ''' True if the request was triggered by a XMLHttpRequest. This only - works with JavaScript libraries that support the `X-Requested-With` - header (most of the popular libraries do). ''' - requested_with = self.environ.get('HTTP_X_REQUESTED_WITH','') - return requested_with.lower() == 'xmlhttprequest' - - @property - def is_ajax(self): - ''' Alias for :attr:`is_xhr`. "Ajax" is not the right term. ''' - return self.is_xhr - - @property - def auth(self): - """ HTTP authentication data as a (user, password) tuple. This - implementation currently supports basic (not digest) authentication - only. If the authentication happened at a higher level (e.g. in the - front web-server or a middleware), the password field is None, but - the user field is looked up from the ``REMOTE_USER`` environ - variable. On any errors, None is returned. """ - basic = parse_auth(self.environ.get('HTTP_AUTHORIZATION','')) - if basic: return basic - ruser = self.environ.get('REMOTE_USER') - if ruser: return (ruser, None) - return None - - @property - def remote_route(self): - """ A list of all IPs that were involved in this request, starting with - the client IP and followed by zero or more proxies. This does only - work if all proxies support the ```X-Forwarded-For`` header. Note - that this information can be forged by malicious clients. """ - proxy = self.environ.get('HTTP_X_FORWARDED_FOR') - if proxy: return [ip.strip() for ip in proxy.split(',')] - remote = self.environ.get('REMOTE_ADDR') - return [remote] if remote else [] - - @property - def remote_addr(self): - """ The client IP as a string. Note that this information can be forged - by malicious clients. """ - route = self.remote_route - return route[0] if route else None - - def copy(self): - """ Return a new :class:`Request` with a shallow :attr:`environ` copy. """ - return Request(self.environ.copy()) - - def get(self, value, default=None): return self.environ.get(value, default) - def __getitem__(self, key): return self.environ[key] - def __delitem__(self, key): self[key] = ""; del(self.environ[key]) - def __iter__(self): return iter(self.environ) - def __len__(self): return len(self.environ) - def keys(self): return self.environ.keys() - def __setitem__(self, key, value): - """ Change an environ value and clear all caches that depend on it. """ - - if self.environ.get('bottle.request.readonly'): - raise KeyError('The environ dictionary is read-only.') - - self.environ[key] = value - todelete = () - - if key == 'wsgi.input': - todelete = ('body', 'forms', 'files', 'params', 'post', 'json') - elif key == 'QUERY_STRING': - todelete = ('query', 'params') - elif key.startswith('HTTP_'): - todelete = ('headers', 'cookies') - - for key in todelete: - self.environ.pop('bottle.request.'+key, None) - - def __repr__(self): - return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.url) - - def __getattr__(self, name): - ''' Search in self.environ for additional user defined attributes. ''' - try: - var = self.environ['bottle.request.ext.%s'%name] - return var.__get__(self) if hasattr(var, '__get__') else var - except KeyError: - raise AttributeError('Attribute %r not defined.' % name) - - def __setattr__(self, name, value): - if name == 'environ': return object.__setattr__(self, name, value) - self.environ['bottle.request.ext.%s'%name] = value - - - - -def _hkey(s): - return s.title().replace('_','-') - - -class HeaderProperty(object): - def __init__(self, name, reader=None, writer=str, default=''): - self.name, self.default = name, default - self.reader, self.writer = reader, writer - self.__doc__ = 'Current value of the %r header.' % name.title() - - def __get__(self, obj, cls): - if obj is None: return self - value = obj.headers.get(self.name, self.default) - return self.reader(value) if self.reader else value - - def __set__(self, obj, value): - obj.headers[self.name] = self.writer(value) - - def __delete__(self, obj): - del obj.headers[self.name] - - -class BaseResponse(object): - """ Storage class for a response body as well as headers and cookies. - - This class does support dict-like case-insensitive item-access to - headers, but is NOT a dict. Most notably, iterating over a response - yields parts of the body and not the headers. - """ - - default_status = 200 - default_content_type = 'text/html; charset=UTF-8' - - # Header blacklist for specific response codes - # (rfc2616 section 10.2.3 and 10.3.5) - bad_headers = { - 204: set(('Content-Type',)), - 304: set(('Allow', 'Content-Encoding', 'Content-Language', - 'Content-Length', 'Content-Range', 'Content-Type', - 'Content-Md5', 'Last-Modified'))} - - def __init__(self, body='', status=None, **headers): - self._cookies = None - self._headers = {'Content-Type': [self.default_content_type]} - self.body = body - self.status = status or self.default_status - if headers: - for name, value in headers.items(): - self[name] = value - - def copy(self): - ''' Returns a copy of self. ''' - copy = Response() - copy.status = self.status - copy._headers = dict((k, v[:]) for (k, v) in self._headers.items()) - return copy - - def __iter__(self): - return iter(self.body) - - def close(self): - if hasattr(self.body, 'close'): - self.body.close() - - @property - def status_line(self): - ''' The HTTP status line as a string (e.g. ``404 Not Found``).''' - return self._status_line - - @property - def status_code(self): - ''' The HTTP status code as an integer (e.g. 404).''' - return self._status_code - - def _set_status(self, status): - if isinstance(status, int): - code, status = status, _HTTP_STATUS_LINES.get(status) - elif ' ' in status: - status = status.strip() - code = int(status.split()[0]) - else: - raise ValueError('String status line without a reason phrase.') - if not 100 <= code <= 999: raise ValueError('Status code out of range.') - self._status_code = code - self._status_line = str(status or ('%d Unknown' % code)) - - def _get_status(self): - return self._status_line - - status = property(_get_status, _set_status, None, - ''' A writeable property to change the HTTP response status. It accepts - either a numeric code (100-999) or a string with a custom reason - phrase (e.g. "404 Brain not found"). Both :data:`status_line` and - :data:`status_code` are updated accordingly. The return value is - always a status string. ''') - del _get_status, _set_status - - @property - def headers(self): - ''' An instance of :class:`HeaderDict`, a case-insensitive dict-like - view on the response headers. ''' - hdict = HeaderDict() - hdict.dict = self._headers - return hdict - - def __contains__(self, name): return _hkey(name) in self._headers - def __delitem__(self, name): del self._headers[_hkey(name)] - def __getitem__(self, name): return self._headers[_hkey(name)][-1] - def __setitem__(self, name, value): self._headers[_hkey(name)] = [str(value)] - - def get_header(self, name, default=None): - ''' Return the value of a previously defined header. If there is no - header with that name, return a default value. ''' - return self._headers.get(_hkey(name), [default])[-1] - - def set_header(self, name, value): - ''' Create a new response header, replacing any previously defined - headers with the same name. ''' - self._headers[_hkey(name)] = [str(value)] - - def add_header(self, name, value): - ''' Add an additional response header, not removing duplicates. ''' - self._headers.setdefault(_hkey(name), []).append(str(value)) - - def iter_headers(self): - ''' Yield (header, value) tuples, skipping headers that are not - allowed with the current response status code. ''' - return self.headerlist - - def wsgiheader(self): - depr('The wsgiheader method is deprecated. See headerlist.') #0.10 - return self.headerlist - - @property - def headerlist(self): - ''' WSGI conform list of (header, value) tuples. ''' - out = [] - headers = self._headers.items() - if self._status_code in self.bad_headers: - bad_headers = self.bad_headers[self._status_code] - headers = [h for h in headers if h[0] not in bad_headers] - out += [(name, val) for name, vals in headers for val in vals] - if self._cookies: - for c in self._cookies.values(): - out.append(('Set-Cookie', c.OutputString())) - return out - - content_type = HeaderProperty('Content-Type') - content_length = HeaderProperty('Content-Length', reader=int) - - @property - def charset(self): - """ Return the charset specified in the content-type header (default: utf8). """ - if 'charset=' in self.content_type: - return self.content_type.split('charset=')[-1].split(';')[0].strip() - return 'UTF-8' - - @property - def COOKIES(self): - """ A dict-like SimpleCookie instance. This should not be used directly. - See :meth:`set_cookie`. """ - depr('The COOKIES dict is deprecated. Use `set_cookie()` instead.') # 0.10 - if not self._cookies: - self._cookies = SimpleCookie() - return self._cookies - - def set_cookie(self, name, value, secret=None, **options): - ''' Create a new cookie or replace an old one. If the `secret` parameter is - set, create a `Signed Cookie` (described below). - - :param name: the name of the cookie. - :param value: the value of the cookie. - :param secret: a signature key required for signed cookies. - - Additionally, this method accepts all RFC 2109 attributes that are - supported by :class:`cookie.Morsel`, including: - - :param max_age: maximum age in seconds. (default: None) - :param expires: a datetime object or UNIX timestamp. (default: None) - :param domain: the domain that is allowed to read the cookie. - (default: current domain) - :param path: limits the cookie to a given path (default: current path) - :param secure: limit the cookie to HTTPS connections (default: off). - :param httponly: prevents client-side javascript to read this cookie - (default: off, requires Python 2.6 or newer). - - If neither `expires` nor `max_age` is set (default), the cookie will - expire at the end of the browser session (as soon as the browser - window is closed). - - Signed cookies may store any pickle-able object and are - cryptographically signed to prevent manipulation. Keep in mind that - cookies are limited to 4kb in most browsers. - - Warning: Signed cookies are not encrypted (the client can still see - the content) and not copy-protected (the client can restore an old - cookie). The main intention is to make pickling and unpickling - save, not to store secret information at client side. - ''' - if not self._cookies: - self._cookies = SimpleCookie() - - if secret: - value = touni(cookie_encode((name, value), secret)) - elif not isinstance(value, basestring): - raise TypeError('Secret key missing for non-string Cookie.') - - if len(value) > 4096: raise ValueError('Cookie value to long.') - self._cookies[name] = value - - for key, value in options.items(): - if key == 'max_age': - if isinstance(value, timedelta): - value = value.seconds + value.days * 24 * 3600 - if key == 'expires': - if isinstance(value, (datedate, datetime)): - value = value.timetuple() - elif isinstance(value, (int, float)): - value = time.gmtime(value) - value = time.strftime("%a, %d %b %Y %H:%M:%S GMT", value) - self._cookies[name][key.replace('_', '-')] = value - - def delete_cookie(self, key, **kwargs): - ''' Delete a cookie. Be sure to use the same `domain` and `path` - settings as used to create the cookie. ''' - kwargs['max_age'] = -1 - kwargs['expires'] = 0 - self.set_cookie(key, '', **kwargs) - - def __repr__(self): - out = '' - for name, value in self.headerlist: - out += '%s: %s\n' % (name.title(), value.strip()) - return out - -#: Thread-local storage for :class:`LocalRequest` and :class:`LocalResponse` -#: attributes. -_lctx = threading.local() - -def local_property(name): - def fget(self): - try: - return getattr(_lctx, name) - except AttributeError: - raise RuntimeError("Request context not initialized.") - def fset(self, value): setattr(_lctx, name, value) - def fdel(self): delattr(_lctx, name) - return property(fget, fset, fdel, - 'Thread-local property stored in :data:`_lctx.%s`' % name) - - -class LocalRequest(BaseRequest): - ''' A thread-local subclass of :class:`BaseRequest` with a different - set of attribues for each thread. There is usually only one global - instance of this class (:data:`request`). If accessed during a - request/response cycle, this instance always refers to the *current* - request (even on a multithreaded server). ''' - bind = BaseRequest.__init__ - environ = local_property('request_environ') - - -class LocalResponse(BaseResponse): - ''' A thread-local subclass of :class:`BaseResponse` with a different - set of attribues for each thread. There is usually only one global - instance of this class (:data:`response`). Its attributes are used - to build the HTTP response at the end of the request/response cycle. - ''' - bind = BaseResponse.__init__ - _status_line = local_property('response_status_line') - _status_code = local_property('response_status_code') - _cookies = local_property('response_cookies') - _headers = local_property('response_headers') - body = local_property('response_body') - -Request = BaseRequest -Response = BaseResponse - -class HTTPResponse(Response, BottleException): - def __init__(self, body='', status=None, header=None, **headers): - if header or 'output' in headers: - depr('Call signature changed (for the better)') - if header: headers.update(header) - if 'output' in headers: body = headers.pop('output') - super(HTTPResponse, self).__init__(body, status, **headers) - - def apply(self, response): - response._status_code = self._status_code - response._status_line = self._status_line - response._headers = self._headers - response._cookies = self._cookies - response.body = self.body - - def _output(self, value=None): - depr('Use HTTPResponse.body instead of HTTPResponse.output') - if value is None: return self.body - self.body = value - - output = property(_output, _output, doc='Alias for .body') - -class HTTPError(HTTPResponse): - default_status = 500 - def __init__(self, status=None, body=None, exception=None, traceback=None, header=None, **headers): - self.exception = exception - self.traceback = traceback - super(HTTPError, self).__init__(body, status, header, **headers) - - - - - -############################################################################### -# Plugins ###################################################################### -############################################################################### - -class PluginError(BottleException): pass - -class JSONPlugin(object): - name = 'json' - api = 2 - - def __init__(self, json_dumps=json_dumps): - self.json_dumps = json_dumps - - def apply(self, callback, route): - dumps = self.json_dumps - if not dumps: return callback - def wrapper(*a, **ka): - rv = callback(*a, **ka) - if isinstance(rv, dict): - #Attempt to serialize, raises exception on failure - json_response = dumps(rv) - #Set content type only if serialization succesful - response.content_type = 'application/json' - return json_response - return rv - return wrapper - - -class HooksPlugin(object): - name = 'hooks' - api = 2 - - _names = 'before_request', 'after_request', 'app_reset' - - def __init__(self): - self.hooks = dict((name, []) for name in self._names) - self.app = None - - def _empty(self): - return not (self.hooks['before_request'] or self.hooks['after_request']) - - def setup(self, app): - self.app = app - - def add(self, name, func): - ''' Attach a callback to a hook. ''' - was_empty = self._empty() - self.hooks.setdefault(name, []).append(func) - if self.app and was_empty and not self._empty(): self.app.reset() - - def remove(self, name, func): - ''' Remove a callback from a hook. ''' - was_empty = self._empty() - if name in self.hooks and func in self.hooks[name]: - self.hooks[name].remove(func) - if self.app and not was_empty and self._empty(): self.app.reset() - - def trigger(self, name, *a, **ka): - ''' Trigger a hook and return a list of results. ''' - hooks = self.hooks[name] - if ka.pop('reversed', False): hooks = hooks[::-1] - return [hook(*a, **ka) for hook in hooks] - - def apply(self, callback, route): - if self._empty(): return callback - def wrapper(*a, **ka): - self.trigger('before_request') - rv = callback(*a, **ka) - self.trigger('after_request', reversed=True) - return rv - return wrapper - - -class TemplatePlugin(object): - ''' This plugin applies the :func:`view` decorator to all routes with a - `template` config parameter. If the parameter is a tuple, the second - element must be a dict with additional options (e.g. `template_engine`) - or default variables for the template. ''' - name = 'template' - api = 2 - - def apply(self, callback, route): - conf = route.config.get('template') - if isinstance(conf, (tuple, list)) and len(conf) == 2: - return view(conf[0], **conf[1])(callback) - elif isinstance(conf, str) and 'template_opts' in route.config: - depr('The `template_opts` parameter is deprecated.') #0.9 - return view(conf, **route.config['template_opts'])(callback) - elif isinstance(conf, str): - return view(conf)(callback) - else: - return callback - - -#: Not a plugin, but part of the plugin API. TODO: Find a better place. -class _ImportRedirect(object): - def __init__(self, name, impmask): - ''' Create a virtual package that redirects imports (see PEP 302). ''' - self.name = name - self.impmask = impmask - self.module = sys.modules.setdefault(name, imp.new_module(name)) - self.module.__dict__.update({'__file__': __file__, '__path__': [], - '__all__': [], '__loader__': self}) - sys.meta_path.append(self) - - def find_module(self, fullname, path=None): - if '.' not in fullname: return - packname, modname = fullname.rsplit('.', 1) - if packname != self.name: return - return self - - def load_module(self, fullname): - if fullname in sys.modules: return sys.modules[fullname] - packname, modname = fullname.rsplit('.', 1) - realname = self.impmask % modname - __import__(realname) - module = sys.modules[fullname] = sys.modules[realname] - setattr(self.module, modname, module) - module.__loader__ = self - return module - - - - - - -############################################################################### -# Common Utilities ############################################################# -############################################################################### - - -class MultiDict(DictMixin): - """ This dict stores multiple values per key, but behaves exactly like a - normal dict in that it returns only the newest value for any given key. - There are special methods available to access the full list of values. - """ - - def __init__(self, *a, **k): - self.dict = dict((k, [v]) for (k, v) in dict(*a, **k).items()) - - def __len__(self): return len(self.dict) - def __iter__(self): return iter(self.dict) - def __contains__(self, key): return key in self.dict - def __delitem__(self, key): del self.dict[key] - def __getitem__(self, key): return self.dict[key][-1] - def __setitem__(self, key, value): self.append(key, value) - def keys(self): return self.dict.keys() - - if py3k: - def values(self): return (v[-1] for v in self.dict.values()) - def items(self): return ((k, v[-1]) for k, v in self.dict.items()) - def allitems(self): - return ((k, v) for k, vl in self.dict.items() for v in vl) - iterkeys = keys - itervalues = values - iteritems = items - iterallitems = allitems - - else: - def values(self): return [v[-1] for v in self.dict.values()] - def items(self): return [(k, v[-1]) for k, v in self.dict.items()] - def iterkeys(self): return self.dict.iterkeys() - def itervalues(self): return (v[-1] for v in self.dict.itervalues()) - def iteritems(self): - return ((k, v[-1]) for k, v in self.dict.iteritems()) - def iterallitems(self): - return ((k, v) for k, vl in self.dict.iteritems() for v in vl) - def allitems(self): - return [(k, v) for k, vl in self.dict.iteritems() for v in vl] - - def get(self, key, default=None, index=-1, type=None): - ''' Return the most recent value for a key. - - :param default: The default value to be returned if the key is not - present or the type conversion fails. - :param index: An index for the list of available values. - :param type: If defined, this callable is used to cast the value - into a specific type. Exception are suppressed and result in - the default value to be returned. - ''' - try: - val = self.dict[key][index] - return type(val) if type else val - except Exception: - pass - return default - - def append(self, key, value): - ''' Add a new value to the list of values for this key. ''' - self.dict.setdefault(key, []).append(value) - - def replace(self, key, value): - ''' Replace the list of values with a single value. ''' - self.dict[key] = [value] - - def getall(self, key): - ''' Return a (possibly empty) list of values for a key. ''' - return self.dict.get(key) or [] - - #: Aliases for WTForms to mimic other multi-dict APIs (Django) - getone = get - getlist = getall - - - -class FormsDict(MultiDict): - ''' This :class:`MultiDict` subclass is used to store request form data. - Additionally to the normal dict-like item access methods (which return - unmodified data as native strings), this container also supports - attribute-like access to its values. Attributes are automatically de- - or recoded to match :attr:`input_encoding` (default: 'utf8'). Missing - attributes default to an empty string. ''' - - #: Encoding used for attribute values. - input_encoding = 'utf8' - #: If true (default), unicode strings are first encoded with `latin1` - #: and then decoded to match :attr:`input_encoding`. - recode_unicode = True - - def _fix(self, s, encoding=None): - if isinstance(s, unicode) and self.recode_unicode: # Python 3 WSGI - s = s.encode('latin1') - if isinstance(s, bytes): # Python 2 WSGI - return s.decode(encoding or self.input_encoding) - return s - - def decode(self, encoding=None): - ''' Returns a copy with all keys and values de- or recoded to match - :attr:`input_encoding`. Some libraries (e.g. WTForms) want a - unicode dictionary. ''' - copy = FormsDict() - enc = copy.input_encoding = encoding or self.input_encoding - copy.recode_unicode = False - for key, value in self.allitems(): - copy.append(self._fix(key, enc), self._fix(value, enc)) - return copy - - def getunicode(self, name, default=None, encoding=None): - try: - return self._fix(self[name], encoding) - except (UnicodeError, KeyError): - return default - - def __getattr__(self, name, default=unicode()): - # Without this guard, pickle generates a cryptic TypeError: - if name.startswith('__') and name.endswith('__'): - return super(FormsDict, self).__getattr__(name) - return self.getunicode(name, default=default) - - -class HeaderDict(MultiDict): - """ A case-insensitive version of :class:`MultiDict` that defaults to - replace the old value instead of appending it. """ - - def __init__(self, *a, **ka): - self.dict = {} - if a or ka: self.update(*a, **ka) - - def __contains__(self, key): return _hkey(key) in self.dict - def __delitem__(self, key): del self.dict[_hkey(key)] - def __getitem__(self, key): return self.dict[_hkey(key)][-1] - def __setitem__(self, key, value): self.dict[_hkey(key)] = [str(value)] - def append(self, key, value): - self.dict.setdefault(_hkey(key), []).append(str(value)) - def replace(self, key, value): self.dict[_hkey(key)] = [str(value)] - def getall(self, key): return self.dict.get(_hkey(key)) or [] - def get(self, key, default=None, index=-1): - return MultiDict.get(self, _hkey(key), default, index) - def filter(self, names): - for name in [_hkey(n) for n in names]: - if name in self.dict: - del self.dict[name] - - -class WSGIHeaderDict(DictMixin): - ''' This dict-like class wraps a WSGI environ dict and provides convenient - access to HTTP_* fields. Keys and values are native strings - (2.x bytes or 3.x unicode) and keys are case-insensitive. If the WSGI - environment contains non-native string values, these are de- or encoded - using a lossless 'latin1' character set. - - The API will remain stable even on changes to the relevant PEPs. - Currently PEP 333, 444 and 3333 are supported. (PEP 444 is the only one - that uses non-native strings.) - ''' - #: List of keys that do not have a ``HTTP_`` prefix. - cgikeys = ('CONTENT_TYPE', 'CONTENT_LENGTH') - - def __init__(self, environ): - self.environ = environ - - def _ekey(self, key): - ''' Translate header field name to CGI/WSGI environ key. ''' - key = key.replace('-','_').upper() - if key in self.cgikeys: - return key - return 'HTTP_' + key - - def raw(self, key, default=None): - ''' Return the header value as is (may be bytes or unicode). ''' - return self.environ.get(self._ekey(key), default) - - def __getitem__(self, key): - return tonat(self.environ[self._ekey(key)], 'latin1') - - def __setitem__(self, key, value): - raise TypeError("%s is read-only." % self.__class__) - - def __delitem__(self, key): - raise TypeError("%s is read-only." % self.__class__) - - def __iter__(self): - for key in self.environ: - if key[:5] == 'HTTP_': - yield key[5:].replace('_', '-').title() - elif key in self.cgikeys: - yield key.replace('_', '-').title() - - def keys(self): return [x for x in self] - def __len__(self): return len(self.keys()) - def __contains__(self, key): return self._ekey(key) in self.environ - - -class ConfigDict(dict): - ''' A dict-subclass with some extras: You can access keys like attributes. - Uppercase attributes create new ConfigDicts and act as name-spaces. - Other missing attributes return None. Calling a ConfigDict updates its - values and returns itself. - - >>> cfg = ConfigDict() - >>> cfg.Namespace.value = 5 - >>> cfg.OtherNamespace(a=1, b=2) - >>> cfg - {'Namespace': {'value': 5}, 'OtherNamespace': {'a': 1, 'b': 2}} - ''' - - def __getattr__(self, key): - if key not in self and key[0].isupper(): - self[key] = ConfigDict() - return self.get(key) - - def __setattr__(self, key, value): - if hasattr(dict, key): - raise AttributeError('Read-only attribute.') - if key in self and self[key] and isinstance(self[key], ConfigDict): - raise AttributeError('Non-empty namespace attribute.') - self[key] = value - - def __delattr__(self, key): - if key in self: del self[key] - - def __call__(self, *a, **ka): - for key, value in dict(*a, **ka).items(): setattr(self, key, value) - return self - - -class AppStack(list): - """ A stack-like list. Calling it returns the head of the stack. """ - - def __call__(self): - """ Return the current default application. """ - return self[-1] - - def push(self, value=None): - """ Add a new :class:`Bottle` instance to the stack """ - if not isinstance(value, Bottle): - value = Bottle() - self.append(value) - return value - - -class WSGIFileWrapper(object): - - def __init__(self, fp, buffer_size=1024*64): - self.fp, self.buffer_size = fp, buffer_size - for attr in ('fileno', 'close', 'read', 'readlines', 'tell', 'seek'): - if hasattr(fp, attr): setattr(self, attr, getattr(fp, attr)) - - def __iter__(self): - buff, read = self.buffer_size, self.read - while True: - part = read(buff) - if not part: return - yield part - - -class ResourceManager(object): - ''' This class manages a list of search paths and helps to find and open - application-bound resources (files). - - :param base: default value for :meth:`add_path` calls. - :param opener: callable used to open resources. - :param cachemode: controls which lookups are cached. One of 'all', - 'found' or 'none'. - ''' - - def __init__(self, base='./', opener=open, cachemode='all'): - self.opener = open - self.base = base - self.cachemode = cachemode - - #: A list of search paths. See :meth:`add_path` for details. - self.path = [] - #: A cache for resolved paths. ``res.cache.clear()`` clears the cache. - self.cache = {} - - def add_path(self, path, base=None, index=None, create=False): - ''' Add a new path to the list of search paths. Return False if the - path does not exist. - - :param path: The new search path. Relative paths are turned into - an absolute and normalized form. If the path looks like a file - (not ending in `/`), the filename is stripped off. - :param base: Path used to absolutize relative search paths. - Defaults to :attr:`base` which defaults to ``os.getcwd()``. - :param index: Position within the list of search paths. Defaults - to last index (appends to the list). - - The `base` parameter makes it easy to reference files installed - along with a python module or package:: - - res.add_path('./resources/', __file__) - ''' - base = os.path.abspath(os.path.dirname(base or self.base)) - path = os.path.abspath(os.path.join(base, os.path.dirname(path))) - path += os.sep - if path in self.path: - self.path.remove(path) - if create and not os.path.isdir(path): - os.makedirs(path) - if index is None: - self.path.append(path) - else: - self.path.insert(index, path) - self.cache.clear() - return os.path.exists(path) - - def __iter__(self): - ''' Iterate over all existing files in all registered paths. ''' - search = self.path[:] - while search: - path = search.pop() - if not os.path.isdir(path): continue - for name in os.listdir(path): - full = os.path.join(path, name) - if os.path.isdir(full): search.append(full) - else: yield full - - def lookup(self, name): - ''' Search for a resource and return an absolute file path, or `None`. - - The :attr:`path` list is searched in order. The first match is - returend. Symlinks are followed. The result is cached to speed up - future lookups. ''' - if name not in self.cache or DEBUG: - for path in self.path: - fpath = os.path.join(path, name) - if os.path.isfile(fpath): - if self.cachemode in ('all', 'found'): - self.cache[name] = fpath - return fpath - if self.cachemode == 'all': - self.cache[name] = None - return self.cache[name] - - def open(self, name, mode='r', *args, **kwargs): - ''' Find a resource and return a file object, or raise IOError. ''' - fname = self.lookup(name) - if not fname: raise IOError("Resource %r not found." % name) - return self.opener(name, mode=mode, *args, **kwargs) - - - - - - -############################################################################### -# Application Helper ########################################################### -############################################################################### - - -def abort(code=500, text='Unknown Error: Application stopped.'): - """ Aborts execution and causes a HTTP error. """ - raise HTTPError(code, text) - - -def redirect(url, code=None): - """ Aborts execution and causes a 303 or 302 redirect, depending on - the HTTP protocol version. """ - if code is None: - code = 303 if request.get('SERVER_PROTOCOL') == "HTTP/1.1" else 302 - location = urljoin(request.url, url) - res = HTTPResponse("", status=code, Location=location) - if response._cookies: - res._cookies = response._cookies - raise res - - -def _file_iter_range(fp, offset, bytes, maxread=1024*1024): - ''' Yield chunks from a range in a file. No chunk is bigger than maxread.''' - fp.seek(offset) - while bytes > 0: - part = fp.read(min(bytes, maxread)) - if not part: break - bytes -= len(part) - yield part - - -def static_file(filename, root, mimetype='auto', download=False): - """ Open a file in a safe way and return :exc:`HTTPResponse` with status - code 200, 305, 401 or 404. Set Content-Type, Content-Encoding, - Content-Length and Last-Modified header. Obey If-Modified-Since header - and HEAD requests. - """ - root = os.path.abspath(root) + os.sep - filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) - headers = dict() - - if not filename.startswith(root): - return HTTPError(403, "Access denied.") - if not os.path.exists(filename) or not os.path.isfile(filename): - return HTTPError(404, "File does not exist.") - if not os.access(filename, os.R_OK): - return HTTPError(403, "You do not have permission to access this file.") - - if mimetype == 'auto': - mimetype, encoding = mimetypes.guess_type(filename) - if mimetype: headers['Content-Type'] = mimetype - if encoding: headers['Content-Encoding'] = encoding - elif mimetype: - headers['Content-Type'] = mimetype - - if download: - download = os.path.basename(filename if download == True else download) - headers['Content-Disposition'] = 'attachment; filename="%s"' % download - - stats = os.stat(filename) - headers['Content-Length'] = clen = stats.st_size - lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime)) - headers['Last-Modified'] = lm - - ims = request.environ.get('HTTP_IF_MODIFIED_SINCE') - if ims: - ims = parse_date(ims.split(";")[0].strip()) - if ims is not None and ims >= int(stats.st_mtime): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - return HTTPResponse(status=304, **headers) - - body = '' if request.method == 'HEAD' else open(filename, 'rb') - - headers["Accept-Ranges"] = "bytes" - ranges = request.environ.get('HTTP_RANGE') - if 'HTTP_RANGE' in request.environ: - ranges = list(parse_range_header(request.environ['HTTP_RANGE'], clen)) - if not ranges: - return HTTPError(416, "Requested Range Not Satisfiable") - offset, end = ranges[0] - headers["Content-Range"] = "bytes %d-%d/%d" % (offset, end-1, clen) - headers["Content-Length"] = str(end-offset) - if body: body = _file_iter_range(body, offset, end-offset) - return HTTPResponse(body, status=206, **headers) - return HTTPResponse(body, **headers) - - - - - - -############################################################################### -# HTTP Utilities and MISC (TODO) ############################################### -############################################################################### - - -def debug(mode=True): - """ Change the debug level. - There is only one debug level supported at the moment.""" - global DEBUG - DEBUG = bool(mode) - - -def parse_date(ims): - """ Parse rfc1123, rfc850 and asctime timestamps and return UTC epoch. """ - try: - ts = email.utils.parsedate_tz(ims) - return time.mktime(ts[:8] + (0,)) - (ts[9] or 0) - time.timezone - except (TypeError, ValueError, IndexError, OverflowError): - return None - - -def parse_auth(header): - """ Parse rfc2617 HTTP authentication header string (basic) and return (user,pass) tuple or None""" - try: - method, data = header.split(None, 1) - if method.lower() == 'basic': - user, pwd = touni(base64.b64decode(tob(data))).split(':',1) - return user, pwd - except (KeyError, ValueError): - return None - -def parse_range_header(header, maxlen=0): - ''' Yield (start, end) ranges parsed from a HTTP Range header. Skip - unsatisfiable ranges. The end index is non-inclusive.''' - if not header or header[:6] != 'bytes=': return - ranges = [r.split('-', 1) for r in header[6:].split(',') if '-' in r] - for start, end in ranges: - try: - if not start: # bytes=-100 -> last 100 bytes - start, end = max(0, maxlen-int(end)), maxlen - elif not end: # bytes=100- -> all but the first 99 bytes - start, end = int(start), maxlen - else: # bytes=100-200 -> bytes 100-200 (inclusive) - start, end = int(start), min(int(end)+1, maxlen) - if 0 <= start < end <= maxlen: - yield start, end - except ValueError: - pass - -def _parse_qsl(qs): - r = [] - for pair in qs.replace(';','&').split('&'): - if not pair: continue - nv = pair.split('=', 1) - if len(nv) != 2: nv.append('') - key = urlunquote(nv[0].replace('+', ' ')) - value = urlunquote(nv[1].replace('+', ' ')) - r.append((key, value)) - return r - -def _lscmp(a, b): - ''' Compares two strings in a cryptographically safe way: - Runtime is not affected by length of common prefix. ''' - return not sum(0 if x==y else 1 for x, y in zip(a, b)) and len(a) == len(b) - - -def cookie_encode(data, key): - ''' Encode and sign a pickle-able object. Return a (byte) string ''' - msg = base64.b64encode(pickle.dumps(data, -1)) - sig = base64.b64encode(hmac.new(tob(key), msg).digest()) - return tob('!') + sig + tob('?') + msg - - -def cookie_decode(data, key): - ''' Verify and decode an encoded string. Return an object or None.''' - data = tob(data) - if cookie_is_encoded(data): - sig, msg = data.split(tob('?'), 1) - if _lscmp(sig[1:], base64.b64encode(hmac.new(tob(key), msg).digest())): - return pickle.loads(base64.b64decode(msg)) - return None - - -def cookie_is_encoded(data): - ''' Return True if the argument looks like a encoded cookie.''' - return bool(data.startswith(tob('!')) and tob('?') in data) - - -def html_escape(string): - ''' Escape HTML special characters ``&<>`` and quotes ``'"``. ''' - return string.replace('&','&').replace('<','<').replace('>','>')\ - .replace('"','"').replace("'",''') - - -def html_quote(string): - ''' Escape and quote a string to be used as an HTTP attribute.''' - return '"%s"' % html_escape(string).replace('\n','%#10;')\ - .replace('\r',' ').replace('\t','	') - - -def yieldroutes(func): - """ Return a generator for routes that match the signature (name, args) - of the func parameter. This may yield more than one route if the function - takes optional keyword arguments. The output is best described by example:: - - a() -> '/a' - b(x, y) -> '/b/:x/:y' - c(x, y=5) -> '/c/:x' and '/c/:x/:y' - d(x=5, y=6) -> '/d' and '/d/:x' and '/d/:x/:y' - """ - import inspect # Expensive module. Only import if necessary. - path = '/' + func.__name__.replace('__','/').lstrip('/') - spec = inspect.getargspec(func) - argc = len(spec[0]) - len(spec[3] or []) - path += ('/:%s' * argc) % tuple(spec[0][:argc]) - yield path - for arg in spec[0][argc:]: - path += '/:%s' % arg - yield path - - -def path_shift(script_name, path_info, shift=1): - ''' Shift path fragments from PATH_INFO to SCRIPT_NAME and vice versa. - - :return: The modified paths. - :param script_name: The SCRIPT_NAME path. - :param script_name: The PATH_INFO path. - :param shift: The number of path fragments to shift. May be negative to - change the shift direction. (default: 1) - ''' - if shift == 0: return script_name, path_info - pathlist = path_info.strip('/').split('/') - scriptlist = script_name.strip('/').split('/') - if pathlist and pathlist[0] == '': pathlist = [] - if scriptlist and scriptlist[0] == '': scriptlist = [] - if shift > 0 and shift <= len(pathlist): - moved = pathlist[:shift] - scriptlist = scriptlist + moved - pathlist = pathlist[shift:] - elif shift < 0 and shift >= -len(scriptlist): - moved = scriptlist[shift:] - pathlist = moved + pathlist - scriptlist = scriptlist[:shift] - else: - empty = 'SCRIPT_NAME' if shift < 0 else 'PATH_INFO' - raise AssertionError("Cannot shift. Nothing left from %s" % empty) - new_script_name = '/' + '/'.join(scriptlist) - new_path_info = '/' + '/'.join(pathlist) - if path_info.endswith('/') and pathlist: new_path_info += '/' - return new_script_name, new_path_info - - -def validate(**vkargs): - """ - Validates and manipulates keyword arguments by user defined callables. - Handles ValueError and missing arguments by raising HTTPError(403). - """ - depr('Use route wildcard filters instead.') - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kargs): - for key, value in vkargs.items(): - if key not in kargs: - abort(403, 'Missing parameter: %s' % key) - try: - kargs[key] = value(kargs[key]) - except ValueError: - abort(403, 'Wrong parameter format for: %s' % key) - return func(*args, **kargs) - return wrapper - return decorator - - -def auth_basic(check, realm="private", text="Access denied"): - ''' Callback decorator to require HTTP auth (basic). - TODO: Add route(check_auth=...) parameter. ''' - def decorator(func): - def wrapper(*a, **ka): - user, password = request.auth or (None, None) - if user is None or not check(user, password): - response.headers['WWW-Authenticate'] = 'Basic realm="%s"' % realm - return HTTPError(401, text) - return func(*a, **ka) - return wrapper - return decorator - - -# Shortcuts for common Bottle methods. -# They all refer to the current default application. - -def make_default_app_wrapper(name): - ''' Return a callable that relays calls to the current default app. ''' - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - return wrapper - -route = make_default_app_wrapper('route') -get = make_default_app_wrapper('get') -post = make_default_app_wrapper('post') -put = make_default_app_wrapper('put') -delete = make_default_app_wrapper('delete') -error = make_default_app_wrapper('error') -mount = make_default_app_wrapper('mount') -hook = make_default_app_wrapper('hook') -install = make_default_app_wrapper('install') -uninstall = make_default_app_wrapper('uninstall') -url = make_default_app_wrapper('get_url') - - - - - - - -############################################################################### -# Server Adapter ############################################################### -############################################################################### - - -class ServerAdapter(object): - quiet = False - def __init__(self, host='127.0.0.1', port=8080, **config): - self.options = config - self.host = host - self.port = int(port) - - def run(self, handler): # pragma: no cover - pass - - def __repr__(self): - args = ', '.join(['%s=%s'%(k,repr(v)) for k, v in self.options.items()]) - return "%s(%s)" % (self.__class__.__name__, args) - - -class CGIServer(ServerAdapter): - quiet = True - def run(self, handler): # pragma: no cover - from wsgiref.handlers import CGIHandler - def fixed_environ(environ, start_response): - environ.setdefault('PATH_INFO', '') - return handler(environ, start_response) - CGIHandler().run(fixed_environ) - - -class FlupFCGIServer(ServerAdapter): - def run(self, handler): # pragma: no cover - import flup.server.fcgi - self.options.setdefault('bindAddress', (self.host, self.port)) - flup.server.fcgi.WSGIServer(handler, **self.options).run() - - -class WSGIRefServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from wsgiref.simple_server import make_server, WSGIRequestHandler - if self.quiet: - class QuietHandler(WSGIRequestHandler): - def log_request(*args, **kw): pass - self.options['handler_class'] = QuietHandler - srv = make_server(self.host, self.port, handler, **self.options) - srv.serve_forever() - - -class CherryPyServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from cherrypy import wsgiserver - server = wsgiserver.CherryPyWSGIServer((self.host, self.port), handler) - try: - server.start() - finally: - server.stop() - - -class WaitressServer(ServerAdapter): - def run(self, handler): - from waitress import serve - serve(handler, host=self.host, port=self.port) - - -class PasteServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from paste import httpserver - if not self.quiet: - from paste.translogger import TransLogger - handler = TransLogger(handler) - httpserver.serve(handler, host=self.host, port=str(self.port), - **self.options) - - -class MeinheldServer(ServerAdapter): - def run(self, handler): - from meinheld import server - server.listen((self.host, self.port)) - server.run(handler) - - -class FapwsServer(ServerAdapter): - """ Extremely fast webserver using libev. See http://www.fapws.org/ """ - def run(self, handler): # pragma: no cover - import fapws._evwsgi as evwsgi - from fapws import base, config - port = self.port - if float(config.SERVER_IDENT[-2:]) > 0.4: - # fapws3 silently changed its API in 0.5 - port = str(port) - evwsgi.start(self.host, port) - # fapws3 never releases the GIL. Complain upstream. I tried. No luck. - if 'BOTTLE_CHILD' in os.environ and not self.quiet: - _stderr("WARNING: Auto-reloading does not work with Fapws3.\n") - _stderr(" (Fapws3 breaks python thread support)\n") - evwsgi.set_base_module(base) - def app(environ, start_response): - environ['wsgi.multiprocess'] = False - return handler(environ, start_response) - evwsgi.wsgi_cb(('', app)) - evwsgi.run() - - -class TornadoServer(ServerAdapter): - """ The super hyped asynchronous server by facebook. Untested. """ - def run(self, handler): # pragma: no cover - import tornado.wsgi, tornado.httpserver, tornado.ioloop - container = tornado.wsgi.WSGIContainer(handler) - server = tornado.httpserver.HTTPServer(container) - server.listen(port=self.port) - tornado.ioloop.IOLoop.instance().start() - - -class AppEngineServer(ServerAdapter): - """ Adapter for Google App Engine. """ - quiet = True - def run(self, handler): - from google.appengine.ext.webapp import util - # A main() function in the handler script enables 'App Caching'. - # Lets makes sure it is there. This _really_ improves performance. - module = sys.modules.get('__main__') - if module and not hasattr(module, 'main'): - module.main = lambda: util.run_wsgi_app(handler) - util.run_wsgi_app(handler) - - -class TwistedServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from twisted.web import server, wsgi - from twisted.python.threadpool import ThreadPool - from twisted.internet import reactor - thread_pool = ThreadPool() - thread_pool.start() - reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) - factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) - reactor.listenTCP(self.port, factory, interface=self.host) - reactor.run() - - -class DieselServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from diesel.protocols.wsgi import WSGIApplication - app = WSGIApplication(handler, port=self.port) - app.run() - - -class GeventServer(ServerAdapter): - """ Untested. Options: - - * `fast` (default: False) uses libevent's http server, but has some - issues: No streaming, no pipelining, no SSL. - """ - def run(self, handler): - from gevent import wsgi, pywsgi, local - if not isinstance(_lctx, local.local): - msg = "Bottle requires gevent.monkey.patch_all() (before import)" - raise RuntimeError(msg) - if not self.options.get('fast'): wsgi = pywsgi - log = None if self.quiet else 'default' - wsgi.WSGIServer((self.host, self.port), handler, log=log).serve_forever() - - -class GunicornServer(ServerAdapter): - """ Untested. See http://gunicorn.org/configure.html for options. """ - def run(self, handler): - from gunicorn.app.base import Application - - config = {'bind': "%s:%d" % (self.host, int(self.port))} - config.update(self.options) - - class GunicornApplication(Application): - def init(self, parser, opts, args): - return config - - def load(self): - return handler - - GunicornApplication().run() - - -class EventletServer(ServerAdapter): - """ Untested """ - def run(self, handler): - from eventlet import wsgi, listen - try: - wsgi.server(listen((self.host, self.port)), handler, - log_output=(not self.quiet)) - except TypeError: - # Fallback, if we have old version of eventlet - wsgi.server(listen((self.host, self.port)), handler) - - -class RocketServer(ServerAdapter): - """ Untested. """ - def run(self, handler): - from rocket import Rocket - server = Rocket((self.host, self.port), 'wsgi', { 'wsgi_app' : handler }) - server.start() - - -class BjoernServer(ServerAdapter): - """ Fast server written in C: https://github.com/jonashaag/bjoern """ - def run(self, handler): - from bjoern import run - run(handler, self.host, self.port) - - -class AutoServer(ServerAdapter): - """ Untested. """ - adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, WSGIRefServer] - def run(self, handler): - for sa in self.adapters: - try: - return sa(self.host, self.port, **self.options).run(handler) - except ImportError: - pass - -server_names = { - 'cgi': CGIServer, - 'flup': FlupFCGIServer, - 'wsgiref': WSGIRefServer, - 'waitress': WaitressServer, - 'cherrypy': CherryPyServer, - 'paste': PasteServer, - 'fapws3': FapwsServer, - 'tornado': TornadoServer, - 'gae': AppEngineServer, - 'twisted': TwistedServer, - 'diesel': DieselServer, - 'meinheld': MeinheldServer, - 'gunicorn': GunicornServer, - 'eventlet': EventletServer, - 'gevent': GeventServer, - 'rocket': RocketServer, - 'bjoern' : BjoernServer, - 'auto': AutoServer, -} - - - - - - -############################################################################### -# Application Control ########################################################## -############################################################################### - - -def load(target, **namespace): - """ Import a module or fetch an object from a module. - - * ``package.module`` returns `module` as a module object. - * ``pack.mod:name`` returns the module variable `name` from `pack.mod`. - * ``pack.mod:func()`` calls `pack.mod.func()` and returns the result. - - The last form accepts not only function calls, but any type of - expression. Keyword arguments passed to this function are available as - local variables. Example: ``import_string('re:compile(x)', x='[a-z]')`` - """ - module, target = target.split(":", 1) if ':' in target else (target, None) - if module not in sys.modules: __import__(module) - if not target: return sys.modules[module] - if target.isalnum(): return getattr(sys.modules[module], target) - package_name = module.split('.')[0] - namespace[package_name] = sys.modules[package_name] - return eval('%s.%s' % (module, target), namespace) - - -def load_app(target): - """ Load a bottle application from a module and make sure that the import - does not affect the current default application, but returns a separate - application object. See :func:`load` for the target parameter. """ - global NORUN; NORUN, nr_old = True, NORUN - try: - tmp = default_app.push() # Create a new "default application" - rv = load(target) # Import the target module - return rv if callable(rv) else tmp - finally: - default_app.remove(tmp) # Remove the temporary added default application - NORUN = nr_old - -_debug = debug -def run(app=None, server='wsgiref', host='127.0.0.1', port=8080, - interval=1, reloader=False, quiet=False, plugins=None, - debug=False, **kargs): - """ Start a server instance. This method blocks until the server terminates. - - :param app: WSGI application or target string supported by - :func:`load_app`. (default: :func:`default_app`) - :param server: Server adapter to use. See :data:`server_names` keys - for valid names or pass a :class:`ServerAdapter` subclass. - (default: `wsgiref`) - :param host: Server address to bind to. Pass ``0.0.0.0`` to listens on - all interfaces including the external one. (default: 127.0.0.1) - :param port: Server port to bind to. Values below 1024 require root - privileges. (default: 8080) - :param reloader: Start auto-reloading server? (default: False) - :param interval: Auto-reloader interval in seconds (default: 1) - :param quiet: Suppress output to stdout and stderr? (default: False) - :param options: Options passed to the server adapter. - """ - if NORUN: return - if reloader and not os.environ.get('BOTTLE_CHILD'): - try: - lockfile = None - fd, lockfile = tempfile.mkstemp(prefix='bottle.', suffix='.lock') - os.close(fd) # We only need this file to exist. We never write to it - while os.path.exists(lockfile): - args = [sys.executable] + sys.argv - environ = os.environ.copy() - environ['BOTTLE_CHILD'] = 'true' - environ['BOTTLE_LOCKFILE'] = lockfile - p = subprocess.Popen(args, env=environ) - while p.poll() is None: # Busy wait... - os.utime(lockfile, None) # I am alive! - time.sleep(interval) - if p.poll() != 3: - if os.path.exists(lockfile): os.unlink(lockfile) - sys.exit(p.poll()) - except KeyboardInterrupt: - pass - finally: - if os.path.exists(lockfile): - os.unlink(lockfile) - return - - try: - _debug(debug) - app = app or default_app() - if isinstance(app, basestring): - app = load_app(app) - if not callable(app): - raise ValueError("Application is not callable: %r" % app) - - for plugin in plugins or []: - app.install(plugin) - - if server in server_names: - server = server_names.get(server) - if isinstance(server, basestring): - server = load(server) - if isinstance(server, type): - server = server(host=host, port=port, **kargs) - if not isinstance(server, ServerAdapter): - raise ValueError("Unknown or unsupported server: %r" % server) - - server.quiet = server.quiet or quiet - if not server.quiet: - _stderr("Bottle v%s server starting up (using %s)...\n" % (__version__, repr(server))) - _stderr("Listening on http://%s:%d/\n" % (server.host, server.port)) - _stderr("Hit Ctrl-C to quit.\n\n") - - if reloader: - lockfile = os.environ.get('BOTTLE_LOCKFILE') - bgcheck = FileCheckerThread(lockfile, interval) - with bgcheck: - server.run(app) - if bgcheck.status == 'reload': - sys.exit(3) - else: - server.run(app) - except KeyboardInterrupt: - pass - except (SystemExit, MemoryError): - raise - except: - if not reloader: raise - if not getattr(server, 'quiet', quiet): - print_exc() - time.sleep(interval) - sys.exit(3) - - - -class FileCheckerThread(threading.Thread): - ''' Interrupt main-thread as soon as a changed module file is detected, - the lockfile gets deleted or gets to old. ''' - - def __init__(self, lockfile, interval): - threading.Thread.__init__(self) - self.lockfile, self.interval = lockfile, interval - #: Is one of 'reload', 'error' or 'exit' - self.status = None - - def run(self): - exists = os.path.exists - mtime = lambda path: os.stat(path).st_mtime - files = dict() - - for module in list(sys.modules.values()): - path = getattr(module, '__file__', '') - if path[-4:] in ('.pyo', '.pyc'): path = path[:-1] - if path and exists(path): files[path] = mtime(path) - - while not self.status: - if not exists(self.lockfile)\ - or mtime(self.lockfile) < time.time() - self.interval - 5: - self.status = 'error' - thread.interrupt_main() - for path, lmtime in list(files.items()): - if not exists(path) or mtime(path) > lmtime: - self.status = 'reload' - thread.interrupt_main() - break - time.sleep(self.interval) - - def __enter__(self): - self.start() - - def __exit__(self, exc_type, exc_val, exc_tb): - if not self.status: self.status = 'exit' # silent exit - self.join() - return exc_type is not None and issubclass(exc_type, KeyboardInterrupt) - - - - - -############################################################################### -# Template Adapters ############################################################ -############################################################################### - - -class TemplateError(HTTPError): - def __init__(self, message): - HTTPError.__init__(self, 500, message) - - -class BaseTemplate(object): - """ Base class and minimal API for template adapters """ - extensions = ['tpl','html','thtml','stpl'] - settings = {} #used in prepare() - defaults = {} #used in render() - - def __init__(self, source=None, name=None, lookup=[], encoding='utf8', **settings): - """ Create a new template. - If the source parameter (str or buffer) is missing, the name argument - is used to guess a template filename. Subclasses can assume that - self.source and/or self.filename are set. Both are strings. - The lookup, encoding and settings parameters are stored as instance - variables. - The lookup parameter stores a list containing directory paths. - The encoding parameter should be used to decode byte strings or files. - The settings parameter contains a dict for engine-specific settings. - """ - self.name = name - self.source = source.read() if hasattr(source, 'read') else source - self.filename = source.filename if hasattr(source, 'filename') else None - self.lookup = [os.path.abspath(x) for x in lookup] - self.encoding = encoding - self.settings = self.settings.copy() # Copy from class variable - self.settings.update(settings) # Apply - if not self.source and self.name: - self.filename = self.search(self.name, self.lookup) - if not self.filename: - raise TemplateError('Template %s not found.' % repr(name)) - if not self.source and not self.filename: - raise TemplateError('No template specified.') - self.prepare(**self.settings) - - @classmethod - def search(cls, name, lookup=[]): - """ Search name in all directories specified in lookup. - First without, then with common extensions. Return first hit. """ - if not lookup: - depr('The template lookup path list should not be empty.') - lookup = ['.'] - - if os.path.isabs(name) and os.path.isfile(name): - depr('Absolute template path names are deprecated.') - return os.path.abspath(name) - - for spath in lookup: - spath = os.path.abspath(spath) + os.sep - fname = os.path.abspath(os.path.join(spath, name)) - if not fname.startswith(spath): continue - if os.path.isfile(fname): return fname - for ext in cls.extensions: - if os.path.isfile('%s.%s' % (fname, ext)): - return '%s.%s' % (fname, ext) - - @classmethod - def global_config(cls, key, *args): - ''' This reads or sets the global settings stored in class.settings. ''' - if args: - cls.settings = cls.settings.copy() # Make settings local to class - cls.settings[key] = args[0] - else: - return cls.settings[key] - - def prepare(self, **options): - """ Run preparations (parsing, caching, ...). - It should be possible to call this again to refresh a template or to - update settings. - """ - raise NotImplementedError - - def render(self, *args, **kwargs): - """ Render the template with the specified local variables and return - a single byte or unicode string. If it is a byte string, the encoding - must match self.encoding. This method must be thread-safe! - Local variables may be provided in dictionaries (*args) - or directly, as keywords (**kwargs). - """ - raise NotImplementedError - - -class MakoTemplate(BaseTemplate): - def prepare(self, **options): - from mako.template import Template - from mako.lookup import TemplateLookup - options.update({'input_encoding':self.encoding}) - options.setdefault('format_exceptions', bool(DEBUG)) - lookup = TemplateLookup(directories=self.lookup, **options) - if self.source: - self.tpl = Template(self.source, lookup=lookup, **options) - else: - self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - -class CheetahTemplate(BaseTemplate): - def prepare(self, **options): - from Cheetah.Template import Template - self.context = threading.local() - self.context.vars = {} - options['searchList'] = [self.context.vars] - if self.source: - self.tpl = Template(source=self.source, **options) - else: - self.tpl = Template(file=self.filename, **options) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - self.context.vars.update(self.defaults) - self.context.vars.update(kwargs) - out = str(self.tpl) - self.context.vars.clear() - return out - - -class Jinja2Template(BaseTemplate): - def prepare(self, filters=None, tests=None, **kwargs): - from jinja2 import Environment, FunctionLoader - if 'prefix' in kwargs: # TODO: to be removed after a while - raise RuntimeError('The keyword argument `prefix` has been removed. ' - 'Use the full jinja2 environment name line_statement_prefix instead.') - self.env = Environment(loader=FunctionLoader(self.loader), **kwargs) - if filters: self.env.filters.update(filters) - if tests: self.env.tests.update(tests) - if self.source: - self.tpl = self.env.from_string(self.source) - else: - self.tpl = self.env.get_template(self.filename) - - def render(self, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - def loader(self, name): - fname = self.search(name, self.lookup) - if not fname: return - with open(fname, "rb") as f: - return f.read().decode(self.encoding) - - -class SimpleTALTemplate(BaseTemplate): - ''' Deprecated, do not use. ''' - def prepare(self, **options): - depr('The SimpleTAL template handler is deprecated'\ - ' and will be removed in 0.12') - from simpletal import simpleTAL - if self.source: - self.tpl = simpleTAL.compileHTMLTemplate(self.source) - else: - with open(self.filename, 'rb') as fp: - self.tpl = simpleTAL.compileHTMLTemplate(tonat(fp.read())) - - def render(self, *args, **kwargs): - from simpletal import simpleTALES - for dictarg in args: kwargs.update(dictarg) - context = simpleTALES.Context() - for k,v in self.defaults.items(): - context.addGlobal(k, v) - for k,v in kwargs.items(): - context.addGlobal(k, v) - output = StringIO() - self.tpl.expand(context, output) - return output.getvalue() - - -class SimpleTemplate(BaseTemplate): - blocks = ('if', 'elif', 'else', 'try', 'except', 'finally', 'for', 'while', - 'with', 'def', 'class') - dedent_blocks = ('elif', 'else', 'except', 'finally') - - @lazy_attribute - def re_pytokens(cls): - ''' This matches comments and all kinds of quoted strings but does - NOT match comments (#...) within quoted strings. (trust me) ''' - return re.compile(r''' - (''(?!')|""(?!")|'{6}|"{6} # Empty strings (all 4 types) - |'(?:[^\\']|\\.)+?' # Single quotes (') - |"(?:[^\\"]|\\.)+?" # Double quotes (") - |'{3}(?:[^\\]|\\.|\n)+?'{3} # Triple-quoted strings (') - |"{3}(?:[^\\]|\\.|\n)+?"{3} # Triple-quoted strings (") - |\#.* # Comments - )''', re.VERBOSE) - - def prepare(self, escape_func=html_escape, noescape=False, **kwargs): - self.cache = {} - enc = self.encoding - self._str = lambda x: touni(x, enc) - self._escape = lambda x: escape_func(touni(x, enc)) - if noescape: - self._str, self._escape = self._escape, self._str - - @classmethod - def split_comment(cls, code): - """ Removes comments (#...) from python code. """ - if '#' not in code: return code - #: Remove comments only (leave quoted strings as they are) - subf = lambda m: '' if m.group(0)[0]=='#' else m.group(0) - return re.sub(cls.re_pytokens, subf, code) - - @cached_property - def co(self): - return compile(self.code, self.filename or '<string>', 'exec') - - @cached_property - def code(self): - stack = [] # Current Code indentation - lineno = 0 # Current line of code - ptrbuffer = [] # Buffer for printable strings and token tuple instances - codebuffer = [] # Buffer for generated python code - multiline = dedent = oneline = False - template = self.source or open(self.filename, 'rb').read() - - def yield_tokens(line): - for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)): - if i % 2: - if part.startswith('!'): yield 'RAW', part[1:] - else: yield 'CMD', part - else: yield 'TXT', part - - def flush(): # Flush the ptrbuffer - if not ptrbuffer: return - cline = '' - for line in ptrbuffer: - for token, value in line: - if token == 'TXT': cline += repr(value) - elif token == 'RAW': cline += '_str(%s)' % value - elif token == 'CMD': cline += '_escape(%s)' % value - cline += ', ' - cline = cline[:-2] + '\\\n' - cline = cline[:-2] - if cline[:-1].endswith('\\\\\\\\\\n'): - cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr' - cline = '_printlist([' + cline + '])' - del ptrbuffer[:] # Do this before calling code() again - code(cline) - - def code(stmt): - for line in stmt.splitlines(): - codebuffer.append(' ' * len(stack) + line.strip()) - - for line in template.splitlines(True): - lineno += 1 - line = touni(line, self.encoding) - sline = line.lstrip() - if lineno <= 2: - m = re.match(r"%\s*#.*coding[:=]\s*([-\w.]+)", sline) - if m: self.encoding = m.group(1) - if m: line = line.replace('coding','coding (removed)') - if sline and sline[0] == '%' and sline[:2] != '%%': - line = line.split('%',1)[1].lstrip() # Full line following the % - cline = self.split_comment(line).strip() - cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0] - flush() # You are actually reading this? Good luck, it's a mess :) - if cmd in self.blocks or multiline: - cmd = multiline or cmd - dedent = cmd in self.dedent_blocks # "else:" - if dedent and not oneline and not multiline: - cmd = stack.pop() - code(line) - oneline = not cline.endswith(':') # "if 1: pass" - multiline = cmd if cline.endswith('\\') else False - if not oneline and not multiline: - stack.append(cmd) - elif cmd == 'end' and stack: - code('#end(%s) %s' % (stack.pop(), line.strip()[3:])) - elif cmd == 'include': - p = cline.split(None, 2)[1:] - if len(p) == 2: - code("_=_include(%s, _stdout, %s)" % (repr(p[0]), p[1])) - elif p: - code("_=_include(%s, _stdout)" % repr(p[0])) - else: # Empty %include -> reverse of %rebase - code("_printlist(_base)") - elif cmd == 'rebase': - p = cline.split(None, 2)[1:] - if len(p) == 2: - code("globals()['_rebase']=(%s, dict(%s))" % (repr(p[0]), p[1])) - elif p: - code("globals()['_rebase']=(%s, {})" % repr(p[0])) - else: - code(line) - else: # Line starting with text (not '%') or '%%' (escaped) - if line.strip().startswith('%%'): - line = line.replace('%%', '%', 1) - ptrbuffer.append(yield_tokens(line)) - flush() - return '\n'.join(codebuffer) + '\n' - - def subtemplate(self, _name, _stdout, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - if _name not in self.cache: - self.cache[_name] = self.__class__(name=_name, lookup=self.lookup) - return self.cache[_name].execute(_stdout, kwargs) - - def execute(self, _stdout, *args, **kwargs): - for dictarg in args: kwargs.update(dictarg) - env = self.defaults.copy() - env.update({'_stdout': _stdout, '_printlist': _stdout.extend, - '_include': self.subtemplate, '_str': self._str, - '_escape': self._escape, 'get': env.get, - 'setdefault': env.setdefault, 'defined': env.__contains__}) - env.update(kwargs) - eval(self.co, env) - if '_rebase' in env: - subtpl, rargs = env['_rebase'] - rargs['_base'] = _stdout[:] #copy stdout - del _stdout[:] # clear stdout - return self.subtemplate(subtpl,_stdout,rargs) - return env - - def render(self, *args, **kwargs): - """ Render the template using keyword arguments as local variables. """ - for dictarg in args: kwargs.update(dictarg) - stdout = [] - self.execute(stdout, kwargs) - return ''.join(stdout) - - -def template(*args, **kwargs): - ''' - Get a rendered template as a string iterator. - You can use a name, a filename or a template string as first parameter. - Template rendering arguments can be passed as dictionaries - or directly (as keyword arguments). - ''' - tpl = args[0] if args else None - adapter = kwargs.pop('template_adapter', SimpleTemplate) - lookup = kwargs.pop('template_lookup', TEMPLATE_PATH) - tplid = (id(lookup), tpl) - if tplid not in TEMPLATES or DEBUG: - settings = kwargs.pop('template_settings', {}) - if isinstance(tpl, adapter): - TEMPLATES[tplid] = tpl - if settings: TEMPLATES[tplid].prepare(**settings) - elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl: - TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings) - else: - TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings) - if not TEMPLATES[tplid]: - abort(500, 'Template (%s) not found' % tpl) - for dictarg in args[1:]: kwargs.update(dictarg) - return TEMPLATES[tplid].render(kwargs) - -mako_template = functools.partial(template, template_adapter=MakoTemplate) -cheetah_template = functools.partial(template, template_adapter=CheetahTemplate) -jinja2_template = functools.partial(template, template_adapter=Jinja2Template) -simpletal_template = functools.partial(template, template_adapter=SimpleTALTemplate) - - -def view(tpl_name, **defaults): - ''' Decorator: renders a template for a handler. - The handler can control its behavior like that: - - - return a dict of template vars to fill out the template - - return something other than a dict and the view decorator will not - process the template, but return the handler result as is. - This includes returning a HTTPResponse(dict) to get, - for instance, JSON with autojson or other castfilters. - ''' - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - if isinstance(result, (dict, DictMixin)): - tplvars = defaults.copy() - tplvars.update(result) - return template(tpl_name, **tplvars) - return result - return wrapper - return decorator - -mako_view = functools.partial(view, template_adapter=MakoTemplate) -cheetah_view = functools.partial(view, template_adapter=CheetahTemplate) -jinja2_view = functools.partial(view, template_adapter=Jinja2Template) -simpletal_view = functools.partial(view, template_adapter=SimpleTALTemplate) - - - - - - -############################################################################### -# Constants and Globals ######################################################## -############################################################################### - - -TEMPLATE_PATH = ['./', './views/'] -TEMPLATES = {} -DEBUG = False -NORUN = False # If set, run() does nothing. Used by load_app() - -#: A dict to map HTTP status codes (e.g. 404) to phrases (e.g. 'Not Found') -HTTP_CODES = httplib.responses -HTTP_CODES[418] = "I'm a teapot" # RFC 2324 -HTTP_CODES[428] = "Precondition Required" -HTTP_CODES[429] = "Too Many Requests" -HTTP_CODES[431] = "Request Header Fields Too Large" -HTTP_CODES[511] = "Network Authentication Required" -_HTTP_STATUS_LINES = dict((k, '%d %s'%(k,v)) for (k,v) in HTTP_CODES.items()) - -#: The default template used for error pages. Override with @error() -ERROR_PAGE_TEMPLATE = """ -%%try: - %%from %s import DEBUG, HTTP_CODES, request, touni - <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> - <html> - <head> - <title>Error: {{e.status}}</title> - <style type="text/css"> - html {background-color: #eee; font-family: sans;} - body {background-color: #fff; border: 1px solid #ddd; - padding: 15px; margin: 15px;} - pre {background-color: #eee; border: 1px solid #ddd; padding: 5px;} - </style> - </head> - <body> - <h1>Error: {{e.status}}</h1> - <p>Sorry, the requested URL <tt>{{repr(request.url)}}</tt> - caused an error:</p> - <pre>{{e.body}}</pre> - %%if DEBUG and e.exception: - <h2>Exception:</h2> - <pre>{{repr(e.exception)}}</pre> - %%end - %%if DEBUG and e.traceback: - <h2>Traceback:</h2> - <pre>{{e.traceback}}</pre> - %%end - </body> - </html> -%%except ImportError: - <b>ImportError:</b> Could not generate the error page. Please add bottle to - the import path. -%%end -""" % __name__ - -#: A thread-safe instance of :class:`LocalRequest`. If accessed from within a -#: request callback, this instance always refers to the *current* request -#: (even on a multithreaded server). -request = LocalRequest() - -#: A thread-safe instance of :class:`LocalResponse`. It is used to change the -#: HTTP response for the *current* request. -response = LocalResponse() - -#: A thread-safe namespace. Not used by Bottle. -local = threading.local() - -# Initialize app stack (create first empty Bottle app) -# BC: 0.6.4 and needed for run() -app = default_app = AppStack() -app.push() - -#: A virtual package that redirects import statements. -#: Example: ``import bottle.ext.sqlite`` actually imports `bottle_sqlite`. -ext = _ImportRedirect('bottle.ext' if __name__ == '__main__' else __name__+".ext", 'bottle_%s').module - -if __name__ == '__main__': - opt, args, parser = _cmd_options, _cmd_args, _cmd_parser - if opt.version: - _stdout('Bottle %s\n'%__version__) - sys.exit(0) - if not args: - parser.print_help() - _stderr('\nError: No application specified.\n') - sys.exit(1) - - sys.path.insert(0, '.') - sys.modules.setdefault('bottle', sys.modules['__main__']) - - host, port = (opt.bind or 'localhost'), 8080 - if ':' in host: - host, port = host.rsplit(':', 1) - - run(args[0], host=host, port=port, server=opt.server, - reloader=opt.reload, plugins=opt.plugin, debug=opt.debug) - - - - -# THE END diff --git a/module/lib/forwarder.py b/module/lib/forwarder.py deleted file mode 100644 index eacb33c2b..000000000 --- a/module/lib/forwarder.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from sys import argv -from sys import exit - -import socket -import thread - -from traceback import print_exc - -class Forwarder(): - - def __init__(self, extip,extport=9666): - print "Start portforwarding to %s:%s" % (extip, extport) - proxy(extip, extport, 9666) - - -def proxy(*settings): - while True: - server(*settings) - -def server(*settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind(("127.0.0.1", settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect((settings[0], settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except Exception: - print_exc() - - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) - -if __name__ == "__main__": - args = argv[1:] - if not args: - print "Usage: forwarder.py <remote ip> <remote port>" - exit() - if len(args) == 1: - args.append(9666) - - f = Forwarder(args[0], int(args[1])) -
\ No newline at end of file diff --git a/module/lib/hg_tool.py b/module/lib/hg_tool.py deleted file mode 100644 index cd97833df..000000000 --- a/module/lib/hg_tool.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from subprocess import Popen, PIPE -from time import time, gmtime, strftime - -aliases = {"zoidber": "zoidberg", "zoidberg10": "zoidberg", "webmaster": "dhmh", "mast3rranan": "ranan", - "ranan2": "ranan"} -exclude = ["locale/*", "module/lib/*"] -date_format = "%Y-%m-%d" -line_re = re.compile(r" (\d+) \**", re.I) - -def add_exclude_flags(args): - for dir in exclude: - args.extend(["-X", dir]) - -# remove small percentages -def wipe(data, perc=1): - s = (sum(data.values()) * perc) / 100 - for k, v in data.items(): - if v < s: del data[k] - - return data - -# remove aliases -def de_alias(data): - for k, v in aliases.iteritems(): - if k not in data: continue - alias = aliases[k] - - if alias in data: data[alias] += data[k] - else: data[alias] = data[k] - - del data[k] - - return data - - -def output(data): - s = float(sum(data.values())) - print "Total Lines: %d" % s - for k, v in data.iteritems(): - print "%15s: %.1f%% | %d" % (k, (v * 100) / s, v) - print - - -def file_list(): - args = ["hg", "status", "-A"] - add_exclude_flags(args) - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - return [x.split()[1] for x in out.splitlines() if x.split()[0] in "CMA"] - - -def hg_annotate(path): - args = ["hg", "annotate", "-u", path] - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - - data = {} - - for line in out.splitlines(): - author, non, line = line.partition(":") - - # probably binary file - if author == path: return {} - - author = author.strip().lower() - if not line.strip(): continue # don't count blank lines - - if author in data: data[author] += 1 - else: data[author] = 1 - - return de_alias(data) - - -def hg_churn(days=None): - args = ["hg", "churn"] - if days: - args.append("-d") - t = time() - 60 * 60 * 24 * days - args.append("%s to %s" % (strftime(date_format, gmtime(t)), strftime(date_format))) - - add_exclude_flags(args) - p = Popen(args, stdout=PIPE) - out, err = p.communicate() - - data = {} - - for line in out.splitlines(): - m = line_re.search(line) - author = line.split()[0] - lines = int(m.group(1)) - - if "@" in author: - author, n, email = author.partition("@") - - author = author.strip().lower() - - if author in data: data[author] += lines - else: data[author] = lines - - return de_alias(data) - - -def complete_annotate(): - files = file_list() - data = {} - for f in files: - tmp = hg_annotate(f) - for k, v in tmp.iteritems(): - if k in data: data[k] += v - else: data[k] = v - - return data - - -if __name__ == "__main__": - for d in (30, 90, 180): - c = wipe(hg_churn(d)) - print "Changes in %d days:" % d - output(c) - - c = wipe(hg_churn()) - print "Total changes:" - output(c) - - print "Current source code version:" - data = wipe(complete_annotate()) - output(data) - - diff --git a/module/lib/mod_pywebsocket/COPYING b/module/lib/mod_pywebsocket/COPYING deleted file mode 100644 index 989d02e4c..000000000 --- a/module/lib/mod_pywebsocket/COPYING +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2012, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/module/lib/mod_pywebsocket/__init__.py b/module/lib/mod_pywebsocket/__init__.py deleted file mode 100644 index 454ae0c45..000000000 --- a/module/lib/mod_pywebsocket/__init__.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket extension for Apache HTTP Server. - -mod_pywebsocket is a WebSocket extension for Apache HTTP Server -intended for testing or experimental purposes. mod_python is required. - - -Installation -============ - -0. Prepare an Apache HTTP Server for which mod_python is enabled. - -1. Specify the following Apache HTTP Server directives to suit your - configuration. - - If mod_pywebsocket is not in the Python path, specify the following. - <websock_lib> is the directory where mod_pywebsocket is installed. - - PythonPath "sys.path+['<websock_lib>']" - - Always specify the following. <websock_handlers> is the directory where - user-written WebSocket handlers are placed. - - PythonOption mod_pywebsocket.handler_root <websock_handlers> - PythonHeaderParserHandler mod_pywebsocket.headerparserhandler - - To limit the search for WebSocket handlers to a directory <scan_dir> - under <websock_handlers>, configure as follows: - - PythonOption mod_pywebsocket.handler_scan <scan_dir> - - <scan_dir> is useful in saving scan time when <websock_handlers> - contains many non-WebSocket handler files. - - If you want to allow handlers whose canonical path is not under the root - directory (i.e. symbolic link is in root directory but its target is not), - configure as follows: - - PythonOption mod_pywebsocket.allow_handlers_outside_root_dir On - - Example snippet of httpd.conf: - (mod_pywebsocket is in /websock_lib, WebSocket handlers are in - /websock_handlers, port is 80 for ws, 443 for wss.) - - <IfModule python_module> - PythonPath "sys.path+['/websock_lib']" - PythonOption mod_pywebsocket.handler_root /websock_handlers - PythonHeaderParserHandler mod_pywebsocket.headerparserhandler - </IfModule> - -2. Tune Apache parameters for serving WebSocket. We'd like to note that at - least TimeOut directive from core features and RequestReadTimeout - directive from mod_reqtimeout should be modified not to kill connections - in only a few seconds of idle time. - -3. Verify installation. You can use example/console.html to poke the server. - - -Writing WebSocket handlers -========================== - -When a WebSocket request comes in, the resource name -specified in the handshake is considered as if it is a file path under -<websock_handlers> and the handler defined in -<websock_handlers>/<resource_name>_wsh.py is invoked. - -For example, if the resource name is /example/chat, the handler defined in -<websock_handlers>/example/chat_wsh.py is invoked. - -A WebSocket handler is composed of the following three functions: - - web_socket_do_extra_handshake(request) - web_socket_transfer_data(request) - web_socket_passive_closing_handshake(request) - -where: - request: mod_python request. - -web_socket_do_extra_handshake is called during the handshake after the -headers are successfully parsed and WebSocket properties (ws_location, -ws_origin, and ws_resource) are added to request. A handler -can reject the request by raising an exception. - -A request object has the following properties that you can use during the -extra handshake (web_socket_do_extra_handshake): -- ws_resource -- ws_origin -- ws_version -- ws_location (HyBi 00 only) -- ws_extensions (HyBi 06 and later) -- ws_deflate (HyBi 06 and later) -- ws_protocol -- ws_requested_protocols (HyBi 06 and later) - -The last two are a bit tricky. See the next subsection. - - -Subprotocol Negotiation ------------------------ - -For HyBi 06 and later, ws_protocol is always set to None when -web_socket_do_extra_handshake is called. If ws_requested_protocols is not -None, you must choose one subprotocol from this list and set it to -ws_protocol. - -For HyBi 00, when web_socket_do_extra_handshake is called, -ws_protocol is set to the value given by the client in -Sec-WebSocket-Protocol header or None if -such header was not found in the opening handshake request. Finish extra -handshake with ws_protocol untouched to accept the request subprotocol. -Then, Sec-WebSocket-Protocol header will be sent to -the client in response with the same value as requested. Raise an exception -in web_socket_do_extra_handshake to reject the requested subprotocol. - - -Data Transfer -------------- - -web_socket_transfer_data is called after the handshake completed -successfully. A handler can receive/send messages from/to the client -using request. mod_pywebsocket.msgutil module provides utilities -for data transfer. - -You can receive a message by the following statement. - - message = request.ws_stream.receive_message() - -This call blocks until any complete text frame arrives, and the payload data -of the incoming frame will be stored into message. When you're using IETF -HyBi 00 or later protocol, receive_message() will return None on receiving -client-initiated closing handshake. When any error occurs, receive_message() -will raise some exception. - -You can send a message by the following statement. - - request.ws_stream.send_message(message) - - -Closing Connection ------------------- - -Executing the following statement or just return-ing from -web_socket_transfer_data cause connection close. - - request.ws_stream.close_connection() - -close_connection will wait -for closing handshake acknowledgement coming from the client. When it -couldn't receive a valid acknowledgement, raises an exception. - -web_socket_passive_closing_handshake is called after the server receives -incoming closing frame from the client peer immediately. You can specify -code and reason by return values. They are sent as a outgoing closing frame -from the server. A request object has the following properties that you can -use in web_socket_passive_closing_handshake. -- ws_close_code -- ws_close_reason - - -Threading ---------- - -A WebSocket handler must be thread-safe if the server (Apache or -standalone.py) is configured to use threads. -""" - - -# vi:sts=4 sw=4 et tw=72 diff --git a/module/lib/mod_pywebsocket/_stream_base.py b/module/lib/mod_pywebsocket/_stream_base.py deleted file mode 100644 index 60fb33d2c..000000000 --- a/module/lib/mod_pywebsocket/_stream_base.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Base stream class. -""" - - -# Note: request.connection.write/read are used in this module, even though -# mod_python document says that they should be used only in connection -# handlers. Unfortunately, we have no other options. For example, -# request.write/read are not suitable because they don't allow direct raw bytes -# writing/reading. - - -from mod_pywebsocket import util - - -# Exceptions - - -class ConnectionTerminatedException(Exception): - """This exception will be raised when a connection is terminated - unexpectedly. - """ - - pass - - -class InvalidFrameException(ConnectionTerminatedException): - """This exception will be raised when we received an invalid frame we - cannot parse. - """ - - pass - - -class BadOperationException(Exception): - """This exception will be raised when send_message() is called on - server-terminated connection or receive_message() is called on - client-terminated connection. - """ - - pass - - -class UnsupportedFrameException(Exception): - """This exception will be raised when we receive a frame with flag, opcode - we cannot handle. Handlers can just catch and ignore this exception and - call receive_message() again to continue processing the next frame. - """ - - pass - - -class InvalidUTF8Exception(Exception): - """This exception will be raised when we receive a text frame which - contains invalid UTF-8 strings. - """ - - pass - - -class StreamBase(object): - """Base stream class.""" - - def __init__(self, request): - """Construct an instance. - - Args: - request: mod_python request. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - - def _read(self, length): - """Reads length bytes from connection. In case we catch any exception, - prepends remote address to the exception message and raise again. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = self._request.connection.read(length) - if not bytes: - raise ConnectionTerminatedException( - 'Receiving %d byte failed. Peer (%r) closed connection' % - (length, (self._request.connection.remote_addr,))) - return bytes - - def _write(self, bytes): - """Writes given bytes to connection. In case we catch any exception, - prepends remote address to the exception message and raise again. - """ - - try: - self._request.connection.write(bytes) - except Exception, e: - util.prepend_message_to_exception( - 'Failed to send message to %r: ' % - (self._request.connection.remote_addr,), - e) - raise - - def receive_bytes(self, length): - """Receives multiple bytes. Retries read when we couldn't receive the - specified amount. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = [] - while length > 0: - new_bytes = self._read(length) - bytes.append(new_bytes) - length -= len(new_bytes) - return ''.join(bytes) - - def _read_until(self, delim_char): - """Reads bytes until we encounter delim_char. The result will not - contain delim_char. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - bytes = [] - while True: - ch = self._read(1) - if ch == delim_char: - break - bytes.append(ch) - return ''.join(bytes) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/_stream_hixie75.py b/module/lib/mod_pywebsocket/_stream_hixie75.py deleted file mode 100644 index 94cf5b31b..000000000 --- a/module/lib/mod_pywebsocket/_stream_hixie75.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides a class for parsing/building frames of the WebSocket -protocol version HyBi 00 and Hixie 75. - -Specification: -- HyBi 00 http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 -- Hixie 75 http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 -""" - - -from mod_pywebsocket import common -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import StreamBase -from mod_pywebsocket._stream_base import UnsupportedFrameException -from mod_pywebsocket import util - - -class StreamHixie75(StreamBase): - """A class for parsing/building frames of the WebSocket protocol version - HyBi 00 and Hixie 75. - """ - - def __init__(self, request, enable_closing_handshake=False): - """Construct an instance. - - Args: - request: mod_python request. - enable_closing_handshake: to let StreamHixie75 perform closing - handshake as specified in HyBi 00, set - this option to True. - """ - - StreamBase.__init__(self, request) - - self._logger = util.get_class_logger(self) - - self._enable_closing_handshake = enable_closing_handshake - - self._request.client_terminated = False - self._request.server_terminated = False - - def send_message(self, message, end=True, binary=False): - """Send message. - - Args: - message: unicode string to send. - binary: not used in hixie75. - - Raises: - BadOperationException: when called on a server-terminated - connection. - """ - - if not end: - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_message with end=False') - - if binary: - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_message with binary=True') - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - self._write(''.join(['\x00', message.encode('utf-8'), '\xff'])) - - def _read_payload_length_hixie75(self): - """Reads a length header in a Hixie75 version frame with length. - - Raises: - ConnectionTerminatedException: when read returns empty string. - """ - - length = 0 - while True: - b_str = self._read(1) - b = ord(b_str) - length = length * 128 + (b & 0x7f) - if (b & 0x80) == 0: - break - return length - - def receive_message(self): - """Receive a WebSocket frame and return its payload an unicode string. - - Returns: - payload unicode string in a WebSocket frame. - - Raises: - ConnectionTerminatedException: when read returns empty - string. - BadOperationException: when called on a client-terminated - connection. - """ - - if self._request.client_terminated: - raise BadOperationException( - 'Requested receive_message after receiving a closing ' - 'handshake') - - while True: - # Read 1 byte. - # mp_conn.read will block if no bytes are available. - # Timeout is controlled by TimeOut directive of Apache. - frame_type_str = self.receive_bytes(1) - frame_type = ord(frame_type_str) - if (frame_type & 0x80) == 0x80: - # The payload length is specified in the frame. - # Read and discard. - length = self._read_payload_length_hixie75() - if length > 0: - _ = self.receive_bytes(length) - # 5.3 3. 12. if /type/ is 0xFF and /length/ is 0, then set the - # /client terminated/ flag and abort these steps. - if not self._enable_closing_handshake: - continue - - if frame_type == 0xFF and length == 0: - self._request.client_terminated = True - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing ' - 'handshake') - return None - - self._logger.debug( - 'Received client-initiated closing handshake') - - self._send_closing_handshake() - self._logger.debug( - 'Sent ack for client-initiated closing handshake') - return None - else: - # The payload is delimited with \xff. - bytes = self._read_until('\xff') - # The WebSocket protocol section 4.4 specifies that invalid - # characters must be replaced with U+fffd REPLACEMENT - # CHARACTER. - message = bytes.decode('utf-8', 'replace') - if frame_type == 0x00: - return message - # Discard data of other types. - - def _send_closing_handshake(self): - if not self._enable_closing_handshake: - raise BadOperationException( - 'Closing handshake is not supported in Hixie 75 protocol') - - self._request.server_terminated = True - - # 5.3 the server may decide to terminate the WebSocket connection by - # running through the following steps: - # 1. send a 0xFF byte and a 0x00 byte to the client to indicate the - # start of the closing handshake. - self._write('\xff\x00') - - def close_connection(self, unused_code='', unused_reason=''): - """Closes a WebSocket connection. - - Raises: - ConnectionTerminatedException: when closing handshake was - not successfull. - """ - - if self._request.server_terminated: - self._logger.debug( - 'Requested close_connection but server is already terminated') - return - - if not self._enable_closing_handshake: - self._request.server_terminated = True - self._logger.debug('Connection closed') - return - - self._send_closing_handshake() - self._logger.debug('Sent server-initiated closing handshake') - - # TODO(ukai): 2. wait until the /client terminated/ flag has been set, - # or until a server-defined timeout expires. - # - # For now, we expect receiving closing handshake right after sending - # out closing handshake, and if we couldn't receive non-handshake - # frame, we take it as ConnectionTerminatedException. - message = self.receive_message() - if message is not None: - raise ConnectionTerminatedException( - 'Didn\'t receive valid ack for closing handshake') - # TODO: 3. close the WebSocket connection. - # note: mod_python Connection (mp_conn) doesn't have close method. - - def send_ping(self, body): - raise BadOperationException( - 'StreamHixie75 doesn\'t support send_ping') - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/_stream_hybi.py b/module/lib/mod_pywebsocket/_stream_hybi.py deleted file mode 100644 index bd158fa6b..000000000 --- a/module/lib/mod_pywebsocket/_stream_hybi.py +++ /dev/null @@ -1,915 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides classes and helper functions for parsing/building frames -of the WebSocket protocol (RFC 6455). - -Specification: -http://tools.ietf.org/html/rfc6455 -""" - - -from collections import deque -import logging -import os -import struct -import time - -from mod_pywebsocket import common -from mod_pywebsocket import util -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import InvalidUTF8Exception -from mod_pywebsocket._stream_base import StreamBase -from mod_pywebsocket._stream_base import UnsupportedFrameException - - -_NOOP_MASKER = util.NoopMasker() - - -class Frame(object): - - def __init__(self, fin=1, rsv1=0, rsv2=0, rsv3=0, - opcode=None, payload=''): - self.fin = fin - self.rsv1 = rsv1 - self.rsv2 = rsv2 - self.rsv3 = rsv3 - self.opcode = opcode - self.payload = payload - - -# Helper functions made public to be used for writing unittests for WebSocket -# clients. - - -def create_length_header(length, mask): - """Creates a length header. - - Args: - length: Frame length. Must be less than 2^63. - mask: Mask bit. Must be boolean. - - Raises: - ValueError: when bad data is given. - """ - - if mask: - mask_bit = 1 << 7 - else: - mask_bit = 0 - - if length < 0: - raise ValueError('length must be non negative integer') - elif length <= 125: - return chr(mask_bit | length) - elif length < (1 << 16): - return chr(mask_bit | 126) + struct.pack('!H', length) - elif length < (1 << 63): - return chr(mask_bit | 127) + struct.pack('!Q', length) - else: - raise ValueError('Payload is too big for one frame') - - -def create_header(opcode, payload_length, fin, rsv1, rsv2, rsv3, mask): - """Creates a frame header. - - Raises: - Exception: when bad data is given. - """ - - if opcode < 0 or 0xf < opcode: - raise ValueError('Opcode out of range') - - if payload_length < 0 or (1 << 63) <= payload_length: - raise ValueError('payload_length out of range') - - if (fin | rsv1 | rsv2 | rsv3) & ~1: - raise ValueError('FIN bit and Reserved bit parameter must be 0 or 1') - - header = '' - - first_byte = ((fin << 7) - | (rsv1 << 6) | (rsv2 << 5) | (rsv3 << 4) - | opcode) - header += chr(first_byte) - header += create_length_header(payload_length, mask) - - return header - - -def _build_frame(header, body, mask): - if not mask: - return header + body - - masking_nonce = os.urandom(4) - masker = util.RepeatedXorMasker(masking_nonce) - - return header + masking_nonce + masker.mask(body) - - -def _filter_and_format_frame_object(frame, mask, frame_filters): - for frame_filter in frame_filters: - frame_filter.filter(frame) - - header = create_header( - frame.opcode, len(frame.payload), frame.fin, - frame.rsv1, frame.rsv2, frame.rsv3, mask) - return _build_frame(header, frame.payload, mask) - - -def create_binary_frame( - message, opcode=common.OPCODE_BINARY, fin=1, mask=False, frame_filters=[]): - """Creates a simple binary frame with no extension, reserved bit.""" - - frame = Frame(fin=fin, opcode=opcode, payload=message) - return _filter_and_format_frame_object(frame, mask, frame_filters) - - -def create_text_frame( - message, opcode=common.OPCODE_TEXT, fin=1, mask=False, frame_filters=[]): - """Creates a simple text frame with no extension, reserved bit.""" - - encoded_message = message.encode('utf-8') - return create_binary_frame(encoded_message, opcode, fin, mask, - frame_filters) - - -def parse_frame(receive_bytes, logger=None, - ws_version=common.VERSION_HYBI_LATEST, - unmask_receive=True): - """Parses a frame. Returns a tuple containing each header field and - payload. - - Args: - receive_bytes: a function that reads frame data from a stream or - something similar. The function takes length of the bytes to be - read. The function must raise ConnectionTerminatedException if - there is not enough data to be read. - logger: a logging object. - ws_version: the version of WebSocket protocol. - unmask_receive: unmask received frames. When received unmasked - frame, raises InvalidFrameException. - - Raises: - ConnectionTerminatedException: when receive_bytes raises it. - InvalidFrameException: when the frame contains invalid data. - """ - - if not logger: - logger = logging.getLogger() - - logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') - - received = receive_bytes(2) - - first_byte = ord(received[0]) - fin = (first_byte >> 7) & 1 - rsv1 = (first_byte >> 6) & 1 - rsv2 = (first_byte >> 5) & 1 - rsv3 = (first_byte >> 4) & 1 - opcode = first_byte & 0xf - - second_byte = ord(received[1]) - mask = (second_byte >> 7) & 1 - payload_length = second_byte & 0x7f - - logger.log(common.LOGLEVEL_FINE, - 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' - 'Mask=%s, Payload_length=%s', - fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) - - if (mask == 1) != unmask_receive: - raise InvalidFrameException( - 'Mask bit on the received frame did\'nt match masking ' - 'configuration for received frames') - - # The HyBi and later specs disallow putting a value in 0x0-0xFFFF - # into the 8-octet extended payload length field (or 0x0-0xFD in - # 2-octet field). - valid_length_encoding = True - length_encoding_bytes = 1 - if payload_length == 127: - logger.log(common.LOGLEVEL_FINE, - 'Receive 8-octet extended payload length') - - extended_payload_length = receive_bytes(8) - payload_length = struct.unpack( - '!Q', extended_payload_length)[0] - if payload_length > 0x7FFFFFFFFFFFFFFF: - raise InvalidFrameException( - 'Extended payload length >= 2^63') - if ws_version >= 13 and payload_length < 0x10000: - valid_length_encoding = False - length_encoding_bytes = 8 - - logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - elif payload_length == 126: - logger.log(common.LOGLEVEL_FINE, - 'Receive 2-octet extended payload length') - - extended_payload_length = receive_bytes(2) - payload_length = struct.unpack( - '!H', extended_payload_length)[0] - if ws_version >= 13 and payload_length < 126: - valid_length_encoding = False - length_encoding_bytes = 2 - - logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - - if not valid_length_encoding: - logger.warning( - 'Payload length is not encoded using the minimal number of ' - 'bytes (%d is encoded using %d bytes)', - payload_length, - length_encoding_bytes) - - if mask == 1: - logger.log(common.LOGLEVEL_FINE, 'Receive mask') - - masking_nonce = receive_bytes(4) - masker = util.RepeatedXorMasker(masking_nonce) - - logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) - else: - masker = _NOOP_MASKER - - logger.log(common.LOGLEVEL_FINE, 'Receive payload data') - if logger.isEnabledFor(common.LOGLEVEL_FINE): - receive_start = time.time() - - raw_payload_bytes = receive_bytes(payload_length) - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - logger.log( - common.LOGLEVEL_FINE, - 'Done receiving payload data at %s MB/s', - payload_length / (time.time() - receive_start) / 1000 / 1000) - logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - unmask_start = time.time() - - bytes = masker.mask(raw_payload_bytes) - - if logger.isEnabledFor(common.LOGLEVEL_FINE): - logger.log( - common.LOGLEVEL_FINE, - 'Done unmasking payload data at %s MB/s', - payload_length / (time.time() - unmask_start) / 1000 / 1000) - - return opcode, bytes, fin, rsv1, rsv2, rsv3 - - -class FragmentedFrameBuilder(object): - """A stateful class to send a message as fragments.""" - - def __init__(self, mask, frame_filters=[], encode_utf8=True): - """Constructs an instance.""" - - self._mask = mask - self._frame_filters = frame_filters - # This is for skipping UTF-8 encoding when building text type frames - # from compressed data. - self._encode_utf8 = encode_utf8 - - self._started = False - - # Hold opcode of the first frame in messages to verify types of other - # frames in the message are all the same. - self._opcode = common.OPCODE_TEXT - - def build(self, payload_data, end, binary): - if binary: - frame_type = common.OPCODE_BINARY - else: - frame_type = common.OPCODE_TEXT - if self._started: - if self._opcode != frame_type: - raise ValueError('Message types are different in frames for ' - 'the same message') - opcode = common.OPCODE_CONTINUATION - else: - opcode = frame_type - self._opcode = frame_type - - if end: - self._started = False - fin = 1 - else: - self._started = True - fin = 0 - - if binary or not self._encode_utf8: - return create_binary_frame( - payload_data, opcode, fin, self._mask, self._frame_filters) - else: - return create_text_frame( - payload_data, opcode, fin, self._mask, self._frame_filters) - - -def _create_control_frame(opcode, body, mask, frame_filters): - frame = Frame(opcode=opcode, payload=body) - - for frame_filter in frame_filters: - frame_filter.filter(frame) - - if len(frame.payload) > 125: - raise BadOperationException( - 'Payload data size of control frames must be 125 bytes or less') - - header = create_header( - frame.opcode, len(frame.payload), frame.fin, - frame.rsv1, frame.rsv2, frame.rsv3, mask) - return _build_frame(header, frame.payload, mask) - - -def create_ping_frame(body, mask=False, frame_filters=[]): - return _create_control_frame(common.OPCODE_PING, body, mask, frame_filters) - - -def create_pong_frame(body, mask=False, frame_filters=[]): - return _create_control_frame(common.OPCODE_PONG, body, mask, frame_filters) - - -def create_close_frame(body, mask=False, frame_filters=[]): - return _create_control_frame( - common.OPCODE_CLOSE, body, mask, frame_filters) - - -def create_closing_handshake_body(code, reason): - body = '' - if code is not None: - if (code > common.STATUS_USER_PRIVATE_MAX or - code < common.STATUS_NORMAL_CLOSURE): - raise BadOperationException('Status code is out of range') - if (code == common.STATUS_NO_STATUS_RECEIVED or - code == common.STATUS_ABNORMAL_CLOSURE or - code == common.STATUS_TLS_HANDSHAKE): - raise BadOperationException('Status code is reserved pseudo ' - 'code') - encoded_reason = reason.encode('utf-8') - body = struct.pack('!H', code) + encoded_reason - return body - - -class StreamOptions(object): - """Holds option values to configure Stream objects.""" - - def __init__(self): - """Constructs StreamOptions.""" - - # Enables deflate-stream extension. - self.deflate_stream = False - - # Filters applied to frames. - self.outgoing_frame_filters = [] - self.incoming_frame_filters = [] - - # Filters applied to messages. Control frames are not affected by them. - self.outgoing_message_filters = [] - self.incoming_message_filters = [] - - self.encode_text_message_to_utf8 = True - self.mask_send = False - self.unmask_receive = True - # RFC6455 disallows fragmented control frames, but mux extension - # relaxes the restriction. - self.allow_fragmented_control_frame = False - - -class Stream(StreamBase): - """A class for parsing/building frames of the WebSocket protocol - (RFC 6455). - """ - - def __init__(self, request, options): - """Constructs an instance. - - Args: - request: mod_python request. - """ - - StreamBase.__init__(self, request) - - self._logger = util.get_class_logger(self) - - self._options = options - - if self._options.deflate_stream: - self._logger.debug('Setup filter for deflate-stream') - self._request = util.DeflateRequest(self._request) - - self._request.client_terminated = False - self._request.server_terminated = False - - # Holds body of received fragments. - self._received_fragments = [] - # Holds the opcode of the first fragment. - self._original_opcode = None - - self._writer = FragmentedFrameBuilder( - self._options.mask_send, self._options.outgoing_frame_filters, - self._options.encode_text_message_to_utf8) - - self._ping_queue = deque() - - def _receive_frame(self): - """Receives a frame and return data in the frame as a tuple containing - each header field and payload separately. - - Raises: - ConnectionTerminatedException: when read returns empty - string. - InvalidFrameException: when the frame contains invalid data. - """ - - def _receive_bytes(length): - return self.receive_bytes(length) - - return parse_frame(receive_bytes=_receive_bytes, - logger=self._logger, - ws_version=self._request.ws_version, - unmask_receive=self._options.unmask_receive) - - def _receive_frame_as_frame_object(self): - opcode, bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() - - return Frame(fin=fin, rsv1=rsv1, rsv2=rsv2, rsv3=rsv3, - opcode=opcode, payload=bytes) - - def receive_filtered_frame(self): - """Receives a frame and applies frame filters and message filters. - The frame to be received must satisfy following conditions: - - The frame is not fragmented. - - The opcode of the frame is TEXT or BINARY. - - DO NOT USE this method except for testing purpose. - """ - - frame = self._receive_frame_as_frame_object() - if not frame.fin: - raise InvalidFrameException( - 'Segmented frames must not be received via ' - 'receive_filtered_frame()') - if (frame.opcode != common.OPCODE_TEXT and - frame.opcode != common.OPCODE_BINARY): - raise InvalidFrameException( - 'Control frames must not be received via ' - 'receive_filtered_frame()') - - for frame_filter in self._options.incoming_frame_filters: - frame_filter.filter(frame) - for message_filter in self._options.incoming_message_filters: - frame.payload = message_filter.filter(frame.payload) - return frame - - def send_message(self, message, end=True, binary=False): - """Send message. - - Args: - message: text in unicode or binary in str to send. - binary: send message as binary frame. - - Raises: - BadOperationException: when called on a server-terminated - connection or called with inconsistent message type or - binary parameter. - """ - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - if binary and isinstance(message, unicode): - raise BadOperationException( - 'Message for binary frame must be instance of str') - - for message_filter in self._options.outgoing_message_filters: - message = message_filter.filter(message, end, binary) - - try: - # Set this to any positive integer to limit maximum size of data in - # payload data of each frame. - MAX_PAYLOAD_DATA_SIZE = -1 - - if MAX_PAYLOAD_DATA_SIZE <= 0: - self._write(self._writer.build(message, end, binary)) - return - - bytes_written = 0 - while True: - end_for_this_frame = end - bytes_to_write = len(message) - bytes_written - if (MAX_PAYLOAD_DATA_SIZE > 0 and - bytes_to_write > MAX_PAYLOAD_DATA_SIZE): - end_for_this_frame = False - bytes_to_write = MAX_PAYLOAD_DATA_SIZE - - frame = self._writer.build( - message[bytes_written:bytes_written + bytes_to_write], - end_for_this_frame, - binary) - self._write(frame) - - bytes_written += bytes_to_write - - # This if must be placed here (the end of while block) so that - # at least one frame is sent. - if len(message) <= bytes_written: - break - except ValueError, e: - raise BadOperationException(e) - - def _get_message_from_frame(self, frame): - """Gets a message from frame. If the message is composed of fragmented - frames and the frame is not the last fragmented frame, this method - returns None. The whole message will be returned when the last - fragmented frame is passed to this method. - - Raises: - InvalidFrameException: when the frame doesn't match defragmentation - context, or the frame contains invalid data. - """ - - if frame.opcode == common.OPCODE_CONTINUATION: - if not self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received a termination frame but fragmentation ' - 'not started') - else: - raise InvalidFrameException( - 'Received an intermediate frame but ' - 'fragmentation not started') - - if frame.fin: - # End of fragmentation frame - self._received_fragments.append(frame.payload) - message = ''.join(self._received_fragments) - self._received_fragments = [] - return message - else: - # Intermediate frame - self._received_fragments.append(frame.payload) - return None - else: - if self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received an unfragmented frame without ' - 'terminating existing fragmentation') - else: - raise InvalidFrameException( - 'New fragmentation started without terminating ' - 'existing fragmentation') - - if frame.fin: - # Unfragmented frame - - self._original_opcode = frame.opcode - return frame.payload - else: - # Start of fragmentation frame - - if (not self._options.allow_fragmented_control_frame and - common.is_control_opcode(frame.opcode)): - raise InvalidFrameException( - 'Control frames must not be fragmented') - - self._original_opcode = frame.opcode - self._received_fragments.append(frame.payload) - return None - - def _process_close_message(self, message): - """Processes close message. - - Args: - message: close message. - - Raises: - InvalidFrameException: when the message is invalid. - """ - - self._request.client_terminated = True - - # Status code is optional. We can have status reason only if we - # have status code. Status reason can be empty string. So, - # allowed cases are - # - no application data: no code no reason - # - 2 octet of application data: has code but no reason - # - 3 or more octet of application data: both code and reason - if len(message) == 0: - self._logger.debug('Received close frame (empty body)') - self._request.ws_close_code = ( - common.STATUS_NO_STATUS_RECEIVED) - elif len(message) == 1: - raise InvalidFrameException( - 'If a close frame has status code, the length of ' - 'status code must be 2 octet') - elif len(message) >= 2: - self._request.ws_close_code = struct.unpack( - '!H', message[0:2])[0] - self._request.ws_close_reason = message[2:].decode( - 'utf-8', 'replace') - self._logger.debug( - 'Received close frame (code=%d, reason=%r)', - self._request.ws_close_code, - self._request.ws_close_reason) - - # Drain junk data after the close frame if necessary. - self._drain_received_data() - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing handshake') - return - - self._logger.debug( - 'Received client-initiated closing handshake') - - code = common.STATUS_NORMAL_CLOSURE - reason = '' - if hasattr(self._request, '_dispatcher'): - dispatcher = self._request._dispatcher - code, reason = dispatcher.passive_closing_handshake( - self._request) - if code is None and reason is not None and len(reason) > 0: - self._logger.warning( - 'Handler specified reason despite code being None') - reason = '' - if reason is None: - reason = '' - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent ack for client-initiated closing handshake ' - '(code=%r, reason=%r)', code, reason) - - def _process_ping_message(self, message): - """Processes ping message. - - Args: - message: ping message. - """ - - try: - handler = self._request.on_ping_handler - if handler: - handler(self._request, message) - return - except AttributeError, e: - pass - self._send_pong(message) - - def _process_pong_message(self, message): - """Processes pong message. - - Args: - message: pong message. - """ - - # TODO(tyoshino): Add ping timeout handling. - - inflight_pings = deque() - - while True: - try: - expected_body = self._ping_queue.popleft() - if expected_body == message: - # inflight_pings contains pings ignored by the - # other peer. Just forget them. - self._logger.debug( - 'Ping %r is acked (%d pings were ignored)', - expected_body, len(inflight_pings)) - break - else: - inflight_pings.append(expected_body) - except IndexError, e: - # The received pong was unsolicited pong. Keep the - # ping queue as is. - self._ping_queue = inflight_pings - self._logger.debug('Received a unsolicited pong') - break - - try: - handler = self._request.on_pong_handler - if handler: - handler(self._request, message) - except AttributeError, e: - pass - - def receive_message(self): - """Receive a WebSocket frame and return its payload as a text in - unicode or a binary in str. - - Returns: - payload data of the frame - - as unicode instance if received text frame - - as str instance if received binary frame - or None iff received closing handshake. - Raises: - BadOperationException: when called on a client-terminated - connection. - ConnectionTerminatedException: when read returns empty - string. - InvalidFrameException: when the frame contains invalid - data. - UnsupportedFrameException: when the received frame has - flags, opcode we cannot handle. You can ignore this - exception and continue receiving the next frame. - """ - - if self._request.client_terminated: - raise BadOperationException( - 'Requested receive_message after receiving a closing ' - 'handshake') - - while True: - # mp_conn.read will block if no bytes are available. - # Timeout is controlled by TimeOut directive of Apache. - - frame = self._receive_frame_as_frame_object() - - # Check the constraint on the payload size for control frames - # before extension processes the frame. - # See also http://tools.ietf.org/html/rfc6455#section-5.5 - if (common.is_control_opcode(frame.opcode) and - len(frame.payload) > 125): - raise InvalidFrameException( - 'Payload data size of control frames must be 125 bytes or ' - 'less') - - for frame_filter in self._options.incoming_frame_filters: - frame_filter.filter(frame) - - if frame.rsv1 or frame.rsv2 or frame.rsv3: - raise UnsupportedFrameException( - 'Unsupported flag is set (rsv = %d%d%d)' % - (frame.rsv1, frame.rsv2, frame.rsv3)) - - message = self._get_message_from_frame(frame) - if message is None: - continue - - for message_filter in self._options.incoming_message_filters: - message = message_filter.filter(message) - - if self._original_opcode == common.OPCODE_TEXT: - # The WebSocket protocol section 4.4 specifies that invalid - # characters must be replaced with U+fffd REPLACEMENT - # CHARACTER. - try: - return message.decode('utf-8') - except UnicodeDecodeError, e: - raise InvalidUTF8Exception(e) - elif self._original_opcode == common.OPCODE_BINARY: - return message - elif self._original_opcode == common.OPCODE_CLOSE: - self._process_close_message(message) - return None - elif self._original_opcode == common.OPCODE_PING: - self._process_ping_message(message) - elif self._original_opcode == common.OPCODE_PONG: - self._process_pong_message(message) - else: - raise UnsupportedFrameException( - 'Opcode %d is not supported' % self._original_opcode) - - def _send_closing_handshake(self, code, reason): - body = create_closing_handshake_body(code, reason) - frame = create_close_frame( - body, mask=self._options.mask_send, - frame_filters=self._options.outgoing_frame_filters) - - self._request.server_terminated = True - - self._write(frame) - - def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): - """Closes a WebSocket connection. - - Args: - code: Status code for close frame. If code is None, a close - frame with empty body will be sent. - reason: string representing close reason. - Raises: - BadOperationException: when reason is specified with code None - or reason is not an instance of both str and unicode. - """ - - if self._request.server_terminated: - self._logger.debug( - 'Requested close_connection but server is already terminated') - return - - if code is None: - if reason is not None and len(reason) > 0: - raise BadOperationException( - 'close reason must not be specified if code is None') - reason = '' - else: - if not isinstance(reason, str) and not isinstance(reason, unicode): - raise BadOperationException( - 'close reason must be an instance of str or unicode') - - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent server-initiated closing handshake (code=%r, reason=%r)', - code, reason) - - if (code == common.STATUS_GOING_AWAY or - code == common.STATUS_PROTOCOL_ERROR): - # It doesn't make sense to wait for a close frame if the reason is - # protocol error or that the server is going away. For some of - # other reasons, it might not make sense to wait for a close frame, - # but it's not clear, yet. - return - - # TODO(ukai): 2. wait until the /client terminated/ flag has been set, - # or until a server-defined timeout expires. - # - # For now, we expect receiving closing handshake right after sending - # out closing handshake. - message = self.receive_message() - if message is not None: - raise ConnectionTerminatedException( - 'Didn\'t receive valid ack for closing handshake') - # TODO: 3. close the WebSocket connection. - # note: mod_python Connection (mp_conn) doesn't have close method. - - def send_ping(self, body=''): - frame = create_ping_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) - self._write(frame) - - self._ping_queue.append(body) - - def _send_pong(self, body): - frame = create_pong_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) - self._write(frame) - - def get_last_received_opcode(self): - """Returns the opcode of the WebSocket message which the last received - frame belongs to. The return value is valid iff immediately after - receive_message call. - """ - - return self._original_opcode - - def _drain_received_data(self): - """Drains unread data in the receive buffer to avoid sending out TCP - RST packet. This is because when deflate-stream is enabled, some - DEFLATE block for flushing data may follow a close frame. If any data - remains in the receive buffer of a socket when the socket is closed, - it sends out TCP RST packet to the other peer. - - Since mod_python's mp_conn object doesn't support non-blocking read, - we perform this only when pywebsocket is running in standalone mode. - """ - - # If self._options.deflate_stream is true, self._request is - # DeflateRequest, so we can get wrapped request object by - # self._request._request. - # - # Only _StandaloneRequest has _drain_received_data method. - if (self._options.deflate_stream and - ('_drain_received_data' in dir(self._request._request))): - self._request._request._drain_received_data() - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/common.py b/module/lib/mod_pywebsocket/common.py deleted file mode 100644 index 2388379c0..000000000 --- a/module/lib/mod_pywebsocket/common.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file must not depend on any module specific to the WebSocket protocol. -""" - - -from mod_pywebsocket import http_header_util - - -# Additional log level definitions. -LOGLEVEL_FINE = 9 - -# Constants indicating WebSocket protocol version. -VERSION_HIXIE75 = -1 -VERSION_HYBI00 = 0 -VERSION_HYBI01 = 1 -VERSION_HYBI02 = 2 -VERSION_HYBI03 = 2 -VERSION_HYBI04 = 4 -VERSION_HYBI05 = 5 -VERSION_HYBI06 = 6 -VERSION_HYBI07 = 7 -VERSION_HYBI08 = 8 -VERSION_HYBI09 = 8 -VERSION_HYBI10 = 8 -VERSION_HYBI11 = 8 -VERSION_HYBI12 = 8 -VERSION_HYBI13 = 13 -VERSION_HYBI14 = 13 -VERSION_HYBI15 = 13 -VERSION_HYBI16 = 13 -VERSION_HYBI17 = 13 - -# Constants indicating WebSocket protocol latest version. -VERSION_HYBI_LATEST = VERSION_HYBI13 - -# Port numbers -DEFAULT_WEB_SOCKET_PORT = 80 -DEFAULT_WEB_SOCKET_SECURE_PORT = 443 - -# Schemes -WEB_SOCKET_SCHEME = 'ws' -WEB_SOCKET_SECURE_SCHEME = 'wss' - -# Frame opcodes defined in the spec. -OPCODE_CONTINUATION = 0x0 -OPCODE_TEXT = 0x1 -OPCODE_BINARY = 0x2 -OPCODE_CLOSE = 0x8 -OPCODE_PING = 0x9 -OPCODE_PONG = 0xa - -# UUIDs used by HyBi 04 and later opening handshake and frame masking. -WEBSOCKET_ACCEPT_UUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' - -# Opening handshake header names and expected values. -UPGRADE_HEADER = 'Upgrade' -WEBSOCKET_UPGRADE_TYPE = 'websocket' -WEBSOCKET_UPGRADE_TYPE_HIXIE75 = 'WebSocket' -CONNECTION_HEADER = 'Connection' -UPGRADE_CONNECTION_TYPE = 'Upgrade' -HOST_HEADER = 'Host' -ORIGIN_HEADER = 'Origin' -SEC_WEBSOCKET_ORIGIN_HEADER = 'Sec-WebSocket-Origin' -SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key' -SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept' -SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version' -SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol' -SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions' -SEC_WEBSOCKET_DRAFT_HEADER = 'Sec-WebSocket-Draft' -SEC_WEBSOCKET_KEY1_HEADER = 'Sec-WebSocket-Key1' -SEC_WEBSOCKET_KEY2_HEADER = 'Sec-WebSocket-Key2' -SEC_WEBSOCKET_LOCATION_HEADER = 'Sec-WebSocket-Location' - -# Extensions -DEFLATE_STREAM_EXTENSION = 'deflate-stream' -DEFLATE_FRAME_EXTENSION = 'deflate-frame' -PERFRAME_COMPRESSION_EXTENSION = 'perframe-compress' -PERMESSAGE_COMPRESSION_EXTENSION = 'permessage-compress' -X_WEBKIT_DEFLATE_FRAME_EXTENSION = 'x-webkit-deflate-frame' -X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION = 'x-webkit-permessage-compress' -MUX_EXTENSION = 'mux_DO_NOT_USE' - -# Status codes -# Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and -# STATUS_TLS_HANDSHAKE are pseudo codes to indicate specific error cases. -# Could not be used for codes in actual closing frames. -# Application level errors must use codes in the range -# STATUS_USER_REGISTERED_BASE to STATUS_USER_PRIVATE_MAX. The codes in the -# range STATUS_USER_REGISTERED_BASE to STATUS_USER_REGISTERED_MAX are managed -# by IANA. Usually application must define user protocol level errors in the -# range STATUS_USER_PRIVATE_BASE to STATUS_USER_PRIVATE_MAX. -STATUS_NORMAL_CLOSURE = 1000 -STATUS_GOING_AWAY = 1001 -STATUS_PROTOCOL_ERROR = 1002 -STATUS_UNSUPPORTED_DATA = 1003 -STATUS_NO_STATUS_RECEIVED = 1005 -STATUS_ABNORMAL_CLOSURE = 1006 -STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 -STATUS_POLICY_VIOLATION = 1008 -STATUS_MESSAGE_TOO_BIG = 1009 -STATUS_MANDATORY_EXTENSION = 1010 -STATUS_INTERNAL_ENDPOINT_ERROR = 1011 -STATUS_TLS_HANDSHAKE = 1015 -STATUS_USER_REGISTERED_BASE = 3000 -STATUS_USER_REGISTERED_MAX = 3999 -STATUS_USER_PRIVATE_BASE = 4000 -STATUS_USER_PRIVATE_MAX = 4999 -# Following definitions are aliases to keep compatibility. Applications must -# not use these obsoleted definitions anymore. -STATUS_NORMAL = STATUS_NORMAL_CLOSURE -STATUS_UNSUPPORTED = STATUS_UNSUPPORTED_DATA -STATUS_CODE_NOT_AVAILABLE = STATUS_NO_STATUS_RECEIVED -STATUS_ABNORMAL_CLOSE = STATUS_ABNORMAL_CLOSURE -STATUS_INVALID_FRAME_PAYLOAD = STATUS_INVALID_FRAME_PAYLOAD_DATA -STATUS_MANDATORY_EXT = STATUS_MANDATORY_EXTENSION - -# HTTP status codes -HTTP_STATUS_BAD_REQUEST = 400 -HTTP_STATUS_FORBIDDEN = 403 -HTTP_STATUS_NOT_FOUND = 404 - - -def is_control_opcode(opcode): - return (opcode >> 3) == 1 - - -class ExtensionParameter(object): - """Holds information about an extension which is exchanged on extension - negotiation in opening handshake. - """ - - def __init__(self, name): - self._name = name - # TODO(tyoshino): Change the data structure to more efficient one such - # as dict when the spec changes to say like - # - Parameter names must be unique - # - The order of parameters is not significant - self._parameters = [] - - def name(self): - return self._name - - def add_parameter(self, name, value): - self._parameters.append((name, value)) - - def get_parameters(self): - return self._parameters - - def get_parameter_names(self): - return [name for name, unused_value in self._parameters] - - def has_parameter(self, name): - for param_name, param_value in self._parameters: - if param_name == name: - return True - return False - - def get_parameter_value(self, name): - for param_name, param_value in self._parameters: - if param_name == name: - return param_value - - -class ExtensionParsingException(Exception): - def __init__(self, name): - super(ExtensionParsingException, self).__init__(name) - - -def _parse_extension_param(state, definition, allow_quoted_string): - param_name = http_header_util.consume_token(state) - - if param_name is None: - raise ExtensionParsingException('No valid parameter name found') - - http_header_util.consume_lwses(state) - - if not http_header_util.consume_string(state, '='): - definition.add_parameter(param_name, None) - return - - http_header_util.consume_lwses(state) - - if allow_quoted_string: - # TODO(toyoshim): Add code to validate that parsed param_value is token - param_value = http_header_util.consume_token_or_quoted_string(state) - else: - param_value = http_header_util.consume_token(state) - if param_value is None: - raise ExtensionParsingException( - 'No valid parameter value found on the right-hand side of ' - 'parameter %r' % param_name) - - definition.add_parameter(param_name, param_value) - - -def _parse_extension(state, allow_quoted_string): - extension_token = http_header_util.consume_token(state) - if extension_token is None: - return None - - extension = ExtensionParameter(extension_token) - - while True: - http_header_util.consume_lwses(state) - - if not http_header_util.consume_string(state, ';'): - break - - http_header_util.consume_lwses(state) - - try: - _parse_extension_param(state, extension, allow_quoted_string) - except ExtensionParsingException, e: - raise ExtensionParsingException( - 'Failed to parse parameter for %r (%r)' % - (extension_token, e)) - - return extension - - -def parse_extensions(data, allow_quoted_string=False): - """Parses Sec-WebSocket-Extensions header value returns a list of - ExtensionParameter objects. - - Leading LWSes must be trimmed. - """ - - state = http_header_util.ParsingState(data) - - extension_list = [] - while True: - extension = _parse_extension(state, allow_quoted_string) - if extension is not None: - extension_list.append(extension) - - http_header_util.consume_lwses(state) - - if http_header_util.peek(state) is None: - break - - if not http_header_util.consume_string(state, ','): - raise ExtensionParsingException( - 'Failed to parse Sec-WebSocket-Extensions header: ' - 'Expected a comma but found %r' % - http_header_util.peek(state)) - - http_header_util.consume_lwses(state) - - if len(extension_list) == 0: - raise ExtensionParsingException( - 'No valid extension entry found') - - return extension_list - - -def format_extension(extension): - """Formats an ExtensionParameter object.""" - - formatted_params = [extension.name()] - for param_name, param_value in extension.get_parameters(): - if param_value is None: - formatted_params.append(param_name) - else: - quoted_value = http_header_util.quote_if_necessary(param_value) - formatted_params.append('%s=%s' % (param_name, quoted_value)) - return '; '.join(formatted_params) - - -def format_extensions(extension_list): - """Formats a list of ExtensionParameter objects.""" - - formatted_extension_list = [] - for extension in extension_list: - formatted_extension_list.append(format_extension(extension)) - return ', '.join(formatted_extension_list) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/dispatch.py b/module/lib/mod_pywebsocket/dispatch.py deleted file mode 100644 index 25905f180..000000000 --- a/module/lib/mod_pywebsocket/dispatch.py +++ /dev/null @@ -1,387 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Dispatch WebSocket request. -""" - - -import logging -import os -import re - -from mod_pywebsocket import common -from mod_pywebsocket import handshake -from mod_pywebsocket import msgutil -from mod_pywebsocket import mux -from mod_pywebsocket import stream -from mod_pywebsocket import util - - -_SOURCE_PATH_PATTERN = re.compile(r'(?i)_wsh\.py$') -_SOURCE_SUFFIX = '_wsh.py' -_DO_EXTRA_HANDSHAKE_HANDLER_NAME = 'web_socket_do_extra_handshake' -_TRANSFER_DATA_HANDLER_NAME = 'web_socket_transfer_data' -_PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME = ( - 'web_socket_passive_closing_handshake') - - -class DispatchException(Exception): - """Exception in dispatching WebSocket request.""" - - def __init__(self, name, status=common.HTTP_STATUS_NOT_FOUND): - super(DispatchException, self).__init__(name) - self.status = status - - -def _default_passive_closing_handshake_handler(request): - """Default web_socket_passive_closing_handshake handler.""" - - return common.STATUS_NORMAL_CLOSURE, '' - - -def _normalize_path(path): - """Normalize path. - - Args: - path: the path to normalize. - - Path is converted to the absolute path. - The input path can use either '\\' or '/' as the separator. - The normalized path always uses '/' regardless of the platform. - """ - - path = path.replace('\\', os.path.sep) - path = os.path.realpath(path) - path = path.replace('\\', '/') - return path - - -def _create_path_to_resource_converter(base_dir): - """Returns a function that converts the path of a WebSocket handler source - file to a resource string by removing the path to the base directory from - its head, removing _SOURCE_SUFFIX from its tail, and replacing path - separators in it with '/'. - - Args: - base_dir: the path to the base directory. - """ - - base_dir = _normalize_path(base_dir) - - base_len = len(base_dir) - suffix_len = len(_SOURCE_SUFFIX) - - def converter(path): - if not path.endswith(_SOURCE_SUFFIX): - return None - # _normalize_path must not be used because resolving symlink breaks - # following path check. - path = path.replace('\\', '/') - if not path.startswith(base_dir): - return None - return path[base_len:-suffix_len] - - return converter - - -def _enumerate_handler_file_paths(directory): - """Returns a generator that enumerates WebSocket Handler source file names - in the given directory. - """ - - for root, unused_dirs, files in os.walk(directory): - for base in files: - path = os.path.join(root, base) - if _SOURCE_PATH_PATTERN.search(path): - yield path - - -class _HandlerSuite(object): - """A handler suite holder class.""" - - def __init__(self, do_extra_handshake, transfer_data, - passive_closing_handshake): - self.do_extra_handshake = do_extra_handshake - self.transfer_data = transfer_data - self.passive_closing_handshake = passive_closing_handshake - - -def _source_handler_file(handler_definition): - """Source a handler definition string. - - Args: - handler_definition: a string containing Python statements that define - handler functions. - """ - - global_dic = {} - try: - exec handler_definition in global_dic - except Exception: - raise DispatchException('Error in sourcing handler:' + - util.get_stack_trace()) - passive_closing_handshake_handler = None - try: - passive_closing_handshake_handler = _extract_handler( - global_dic, _PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME) - except Exception: - passive_closing_handshake_handler = ( - _default_passive_closing_handshake_handler) - return _HandlerSuite( - _extract_handler(global_dic, _DO_EXTRA_HANDSHAKE_HANDLER_NAME), - _extract_handler(global_dic, _TRANSFER_DATA_HANDLER_NAME), - passive_closing_handshake_handler) - - -def _extract_handler(dic, name): - """Extracts a callable with the specified name from the given dictionary - dic. - """ - - if name not in dic: - raise DispatchException('%s is not defined.' % name) - handler = dic[name] - if not callable(handler): - raise DispatchException('%s is not callable.' % name) - return handler - - -class Dispatcher(object): - """Dispatches WebSocket requests. - - This class maintains a map from resource name to handlers. - """ - - def __init__( - self, root_dir, scan_dir=None, - allow_handlers_outside_root_dir=True): - """Construct an instance. - - Args: - root_dir: The directory where handler definition files are - placed. - scan_dir: The directory where handler definition files are - searched. scan_dir must be a directory under root_dir, - including root_dir itself. If scan_dir is None, - root_dir is used as scan_dir. scan_dir can be useful - in saving scan time when root_dir contains many - subdirectories. - allow_handlers_outside_root_dir: Scans handler files even if their - canonical path is not under root_dir. - """ - - self._logger = util.get_class_logger(self) - - self._handler_suite_map = {} - self._source_warnings = [] - if scan_dir is None: - scan_dir = root_dir - if not os.path.realpath(scan_dir).startswith( - os.path.realpath(root_dir)): - raise DispatchException('scan_dir:%s must be a directory under ' - 'root_dir:%s.' % (scan_dir, root_dir)) - self._source_handler_files_in_dir( - root_dir, scan_dir, allow_handlers_outside_root_dir) - - def add_resource_path_alias(self, - alias_resource_path, existing_resource_path): - """Add resource path alias. - - Once added, request to alias_resource_path would be handled by - handler registered for existing_resource_path. - - Args: - alias_resource_path: alias resource path - existing_resource_path: existing resource path - """ - try: - handler_suite = self._handler_suite_map[existing_resource_path] - self._handler_suite_map[alias_resource_path] = handler_suite - except KeyError: - raise DispatchException('No handler for: %r' % - existing_resource_path) - - def source_warnings(self): - """Return warnings in sourcing handlers.""" - - return self._source_warnings - - def do_extra_handshake(self, request): - """Do extra checking in WebSocket handshake. - - Select a handler based on request.uri and call its - web_socket_do_extra_handshake function. - - Args: - request: mod_python request. - - Raises: - DispatchException: when handler was not found - AbortedByUserException: when user handler abort connection - HandshakeException: when opening handshake failed - """ - - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % request.ws_resource) - do_extra_handshake_ = handler_suite.do_extra_handshake - try: - do_extra_handshake_(request) - except handshake.AbortedByUserException, e: - raise - except Exception, e: - util.prepend_message_to_exception( - '%s raised exception for %s: ' % ( - _DO_EXTRA_HANDSHAKE_HANDLER_NAME, - request.ws_resource), - e) - raise handshake.HandshakeException(e, common.HTTP_STATUS_FORBIDDEN) - - def transfer_data(self, request): - """Let a handler transfer_data with a WebSocket client. - - Select a handler based on request.ws_resource and call its - web_socket_transfer_data function. - - Args: - request: mod_python request. - - Raises: - DispatchException: when handler was not found - AbortedByUserException: when user handler abort connection - """ - - # TODO(tyoshino): Terminate underlying TCP connection if possible. - try: - if mux.use_mux(request): - mux.start(request, self) - else: - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % - request.ws_resource) - transfer_data_ = handler_suite.transfer_data - transfer_data_(request) - - if not request.server_terminated: - request.ws_stream.close_connection() - # Catch non-critical exceptions the handler didn't handle. - except handshake.AbortedByUserException, e: - self._logger.debug('%s', e) - raise - except msgutil.BadOperationException, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_ABNORMAL_CLOSURE) - except msgutil.InvalidFrameException, e: - # InvalidFrameException must be caught before - # ConnectionTerminatedException that catches InvalidFrameException. - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_PROTOCOL_ERROR) - except msgutil.UnsupportedFrameException, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection(common.STATUS_UNSUPPORTED_DATA) - except stream.InvalidUTF8Exception, e: - self._logger.debug('%s', e) - request.ws_stream.close_connection( - common.STATUS_INVALID_FRAME_PAYLOAD_DATA) - except msgutil.ConnectionTerminatedException, e: - self._logger.debug('%s', e) - except Exception, e: - util.prepend_message_to_exception( - '%s raised exception for %s: ' % ( - _TRANSFER_DATA_HANDLER_NAME, request.ws_resource), - e) - raise - - def passive_closing_handshake(self, request): - """Prepare code and reason for responding client initiated closing - handshake. - """ - - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - return _default_passive_closing_handshake_handler(request) - return handler_suite.passive_closing_handshake(request) - - def get_handler_suite(self, resource): - """Retrieves two handlers (one for extra handshake processing, and one - for data transfer) for the given request as a HandlerSuite object. - """ - - fragment = None - if '#' in resource: - resource, fragment = resource.split('#', 1) - if '?' in resource: - resource = resource.split('?', 1)[0] - handler_suite = self._handler_suite_map.get(resource) - if handler_suite and fragment: - raise DispatchException('Fragment identifiers MUST NOT be used on ' - 'WebSocket URIs', - common.HTTP_STATUS_BAD_REQUEST) - return handler_suite - - def _source_handler_files_in_dir( - self, root_dir, scan_dir, allow_handlers_outside_root_dir): - """Source all the handler source files in the scan_dir directory. - - The resource path is determined relative to root_dir. - """ - - # We build a map from resource to handler code assuming that there's - # only one path from root_dir to scan_dir and it can be obtained by - # comparing realpath of them. - - # Here we cannot use abspath. See - # https://bugs.webkit.org/show_bug.cgi?id=31603 - - convert = _create_path_to_resource_converter(root_dir) - scan_realpath = os.path.realpath(scan_dir) - root_realpath = os.path.realpath(root_dir) - for path in _enumerate_handler_file_paths(scan_realpath): - if (not allow_handlers_outside_root_dir and - (not os.path.realpath(path).startswith(root_realpath))): - self._logger.debug( - 'Canonical path of %s is not under root directory' % - path) - continue - try: - handler_suite = _source_handler_file(open(path).read()) - except DispatchException, e: - self._source_warnings.append('%s: %s' % (path, e)) - continue - resource = convert(path) - if resource is None: - self._logger.debug( - 'Path to resource conversion on %s failed' % path) - else: - self._handler_suite_map[convert(path)] = handler_suite - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/extensions.py b/module/lib/mod_pywebsocket/extensions.py deleted file mode 100644 index 03dbf9ee1..000000000 --- a/module/lib/mod_pywebsocket/extensions.py +++ /dev/null @@ -1,727 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -from mod_pywebsocket import common -from mod_pywebsocket import util -from mod_pywebsocket.http_header_util import quote_if_necessary - - -_available_processors = {} - - -class ExtensionProcessorInterface(object): - - def name(self): - return None - - def get_extension_response(self): - return None - - def setup_stream_options(self, stream_options): - pass - - -class DeflateStreamExtensionProcessor(ExtensionProcessorInterface): - """WebSocket DEFLATE stream extension processor. - - Specification: - Section 9.2.1 in - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 - """ - - def __init__(self, request): - self._logger = util.get_class_logger(self) - - self._request = request - - def name(self): - return common.DEFLATE_STREAM_EXTENSION - - def get_extension_response(self): - if len(self._request.get_parameter_names()) != 0: - return None - - self._logger.debug( - 'Enable %s extension', common.DEFLATE_STREAM_EXTENSION) - - return common.ExtensionParameter(common.DEFLATE_STREAM_EXTENSION) - - def setup_stream_options(self, stream_options): - stream_options.deflate_stream = True - - -_available_processors[common.DEFLATE_STREAM_EXTENSION] = ( - DeflateStreamExtensionProcessor) - - -def _log_compression_ratio(logger, original_bytes, total_original_bytes, - filtered_bytes, total_filtered_bytes): - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if original_bytes != 0: - ratio = float(filtered_bytes) / original_bytes - if total_original_bytes != 0: - average_ratio = ( - float(total_filtered_bytes) / total_original_bytes) - logger.debug('Outgoing compress ratio: %f (average: %f)' % - (ratio, average_ratio)) - - -def _log_decompression_ratio(logger, received_bytes, total_received_bytes, - filtered_bytes, total_filtered_bytes): - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if received_bytes != 0: - ratio = float(received_bytes) / filtered_bytes - if total_filtered_bytes != 0: - average_ratio = ( - float(total_received_bytes) / total_filtered_bytes) - logger.debug('Incoming compress ratio: %f (average: %f)' % - (ratio, average_ratio)) - - -class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame DEFLATE extension processor. - - Specification: - http://tools.ietf.org/html/draft-tyoshino-hybi-websocket-perframe-deflate - """ - - _WINDOW_BITS_PARAM = 'max_window_bits' - _NO_CONTEXT_TAKEOVER_PARAM = 'no_context_takeover' - - def __init__(self, request): - self._logger = util.get_class_logger(self) - - self._request = request - - self._response_window_bits = None - self._response_no_context_takeover = False - self._bfinal = False - - # Counters for statistics. - - # Total number of outgoing bytes supplied to this filter. - self._total_outgoing_payload_bytes = 0 - # Total number of bytes sent to the network after applying this filter. - self._total_filtered_outgoing_payload_bytes = 0 - - # Total number of bytes received from the network. - self._total_incoming_payload_bytes = 0 - # Total number of incoming bytes obtained after applying this filter. - self._total_filtered_incoming_payload_bytes = 0 - - def name(self): - return common.DEFLATE_FRAME_EXTENSION - - def get_extension_response(self): - # Any unknown parameter will be just ignored. - - window_bits = self._request.get_parameter_value( - self._WINDOW_BITS_PARAM) - no_context_takeover = self._request.has_parameter( - self._NO_CONTEXT_TAKEOVER_PARAM) - if (no_context_takeover and - self._request.get_parameter_value( - self._NO_CONTEXT_TAKEOVER_PARAM) is not None): - return None - - if window_bits is not None: - try: - window_bits = int(window_bits) - except ValueError, e: - return None - if window_bits < 8 or window_bits > 15: - return None - - self._deflater = util._RFC1979Deflater( - window_bits, no_context_takeover) - - self._inflater = util._RFC1979Inflater() - - self._compress_outgoing = True - - response = common.ExtensionParameter(self._request.name()) - - if self._response_window_bits is not None: - response.add_parameter( - self._WINDOW_BITS_PARAM, str(self._response_window_bits)) - if self._response_no_context_takeover: - response.add_parameter( - self._NO_CONTEXT_TAKEOVER_PARAM, None) - - self._logger.debug( - 'Enable %s extension (' - 'request: window_bits=%s; no_context_takeover=%r, ' - 'response: window_wbits=%s; no_context_takeover=%r)' % - (self._request.name(), - window_bits, - no_context_takeover, - self._response_window_bits, - self._response_no_context_takeover)) - - return response - - def setup_stream_options(self, stream_options): - - class _OutgoingFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._outgoing_filter(frame) - - class _IncomingFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._incoming_filter(frame) - - stream_options.outgoing_frame_filters.append( - _OutgoingFilter(self)) - stream_options.incoming_frame_filters.insert( - 0, _IncomingFilter(self)) - - def set_response_window_bits(self, value): - self._response_window_bits = value - - def set_response_no_context_takeover(self, value): - self._response_no_context_takeover = value - - def set_bfinal(self, value): - self._bfinal = value - - def enable_outgoing_compression(self): - self._compress_outgoing = True - - def disable_outgoing_compression(self): - self._compress_outgoing = False - - def _outgoing_filter(self, frame): - """Transform outgoing frames. This method is called only by - an _OutgoingFilter instance. - """ - - original_payload_size = len(frame.payload) - self._total_outgoing_payload_bytes += original_payload_size - - if (not self._compress_outgoing or - common.is_control_opcode(frame.opcode)): - self._total_filtered_outgoing_payload_bytes += ( - original_payload_size) - return - - frame.payload = self._deflater.filter( - frame.payload, bfinal=self._bfinal) - frame.rsv1 = 1 - - filtered_payload_size = len(frame.payload) - self._total_filtered_outgoing_payload_bytes += filtered_payload_size - - _log_compression_ratio(self._logger, original_payload_size, - self._total_outgoing_payload_bytes, - filtered_payload_size, - self._total_filtered_outgoing_payload_bytes) - - def _incoming_filter(self, frame): - """Transform incoming frames. This method is called only by - an _IncomingFilter instance. - """ - - received_payload_size = len(frame.payload) - self._total_incoming_payload_bytes += received_payload_size - - if frame.rsv1 != 1 or common.is_control_opcode(frame.opcode): - self._total_filtered_incoming_payload_bytes += ( - received_payload_size) - return - - frame.payload = self._inflater.filter(frame.payload) - frame.rsv1 = 0 - - filtered_payload_size = len(frame.payload) - self._total_filtered_incoming_payload_bytes += filtered_payload_size - - _log_decompression_ratio(self._logger, received_payload_size, - self._total_incoming_payload_bytes, - filtered_payload_size, - self._total_filtered_incoming_payload_bytes) - - -_available_processors[common.DEFLATE_FRAME_EXTENSION] = ( - DeflateFrameExtensionProcessor) - - -# Adding vendor-prefixed deflate-frame extension. -# TODO(bashi): Remove this after WebKit stops using vendor prefix. -_available_processors[common.X_WEBKIT_DEFLATE_FRAME_EXTENSION] = ( - DeflateFrameExtensionProcessor) - - -def _parse_compression_method(data): - """Parses the value of "method" extension parameter.""" - - return common.parse_extensions(data, allow_quoted_string=True) - - -def _create_accepted_method_desc(method_name, method_params): - """Creates accepted-method-desc from given method name and parameters""" - - extension = common.ExtensionParameter(method_name) - for name, value in method_params: - extension.add_parameter(name, value) - return common.format_extension(extension) - - -class CompressionExtensionProcessorBase(ExtensionProcessorInterface): - """Base class for Per-frame and Per-message compression extension.""" - - _METHOD_PARAM = 'method' - - def __init__(self, request): - self._logger = util.get_class_logger(self) - self._request = request - self._compression_method_name = None - self._compression_processor = None - self._compression_processor_hook = None - - def name(self): - return '' - - def _lookup_compression_processor(self, method_desc): - return None - - def _get_compression_processor_response(self): - """Looks up the compression processor based on the self._request and - returns the compression processor's response. - """ - - method_list = self._request.get_parameter_value(self._METHOD_PARAM) - if method_list is None: - return None - methods = _parse_compression_method(method_list) - if methods is None: - return None - comression_processor = None - # The current implementation tries only the first method that matches - # supported algorithm. Following methods aren't tried even if the - # first one is rejected. - # TODO(bashi): Need to clarify this behavior. - for method_desc in methods: - compression_processor = self._lookup_compression_processor( - method_desc) - if compression_processor is not None: - self._compression_method_name = method_desc.name() - break - if compression_processor is None: - return None - - if self._compression_processor_hook: - self._compression_processor_hook(compression_processor) - - processor_response = compression_processor.get_extension_response() - if processor_response is None: - return None - self._compression_processor = compression_processor - return processor_response - - def get_extension_response(self): - processor_response = self._get_compression_processor_response() - if processor_response is None: - return None - - response = common.ExtensionParameter(self._request.name()) - accepted_method_desc = _create_accepted_method_desc( - self._compression_method_name, - processor_response.get_parameters()) - response.add_parameter(self._METHOD_PARAM, accepted_method_desc) - self._logger.debug( - 'Enable %s extension (method: %s)' % - (self._request.name(), self._compression_method_name)) - return response - - def setup_stream_options(self, stream_options): - if self._compression_processor is None: - return - self._compression_processor.setup_stream_options(stream_options) - - def set_compression_processor_hook(self, hook): - self._compression_processor_hook = hook - - def get_compression_processor(self): - return self._compression_processor - - -class PerFrameCompressionExtensionProcessor(CompressionExtensionProcessorBase): - """WebSocket Per-frame compression extension processor. - - Specification: - http://tools.ietf.org/html/draft-ietf-hybi-websocket-perframe-compression - """ - - _DEFLATE_METHOD = 'deflate' - - def __init__(self, request): - CompressionExtensionProcessorBase.__init__(self, request) - - def name(self): - return common.PERFRAME_COMPRESSION_EXTENSION - - def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateFrameExtensionProcessor(method_desc) - return None - - -_available_processors[common.PERFRAME_COMPRESSION_EXTENSION] = ( - PerFrameCompressionExtensionProcessor) - - -class DeflateMessageProcessor(ExtensionProcessorInterface): - """Per-message deflate processor.""" - - _S2C_MAX_WINDOW_BITS_PARAM = 's2c_max_window_bits' - _S2C_NO_CONTEXT_TAKEOVER_PARAM = 's2c_no_context_takeover' - _C2S_MAX_WINDOW_BITS_PARAM = 'c2s_max_window_bits' - _C2S_NO_CONTEXT_TAKEOVER_PARAM = 'c2s_no_context_takeover' - - def __init__(self, request): - self._request = request - self._logger = util.get_class_logger(self) - - self._c2s_max_window_bits = None - self._c2s_no_context_takeover = False - self._bfinal = False - - self._compress_outgoing_enabled = False - - # True if a message is fragmented and compression is ongoing. - self._compress_ongoing = False - - # Counters for statistics. - - # Total number of outgoing bytes supplied to this filter. - self._total_outgoing_payload_bytes = 0 - # Total number of bytes sent to the network after applying this filter. - self._total_filtered_outgoing_payload_bytes = 0 - - # Total number of bytes received from the network. - self._total_incoming_payload_bytes = 0 - # Total number of incoming bytes obtained after applying this filter. - self._total_filtered_incoming_payload_bytes = 0 - - def name(self): - return 'deflate' - - def get_extension_response(self): - # Any unknown parameter will be just ignored. - - s2c_max_window_bits = self._request.get_parameter_value( - self._S2C_MAX_WINDOW_BITS_PARAM) - if s2c_max_window_bits is not None: - try: - s2c_max_window_bits = int(s2c_max_window_bits) - except ValueError, e: - return None - if s2c_max_window_bits < 8 or s2c_max_window_bits > 15: - return None - - s2c_no_context_takeover = self._request.has_parameter( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM) - if (s2c_no_context_takeover and - self._request.get_parameter_value( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM) is not None): - return None - - self._deflater = util._RFC1979Deflater( - s2c_max_window_bits, s2c_no_context_takeover) - - self._inflater = util._RFC1979Inflater() - - self._compress_outgoing_enabled = True - - response = common.ExtensionParameter(self._request.name()) - - if s2c_max_window_bits is not None: - response.add_parameter( - self._S2C_MAX_WINDOW_BITS_PARAM, str(s2c_max_window_bits)) - - if s2c_no_context_takeover: - response.add_parameter( - self._S2C_NO_CONTEXT_TAKEOVER_PARAM, None) - - if self._c2s_max_window_bits is not None: - response.add_parameter( - self._C2S_MAX_WINDOW_BITS_PARAM, - str(self._c2s_max_window_bits)) - if self._c2s_no_context_takeover: - response.add_parameter( - self._C2S_NO_CONTEXT_TAKEOVER_PARAM, None) - - self._logger.debug( - 'Enable %s extension (' - 'request: s2c_max_window_bits=%s; s2c_no_context_takeover=%r, ' - 'response: c2s_max_window_bits=%s; c2s_no_context_takeover=%r)' % - (self._request.name(), - s2c_max_window_bits, - s2c_no_context_takeover, - self._c2s_max_window_bits, - self._c2s_no_context_takeover)) - - return response - - def setup_stream_options(self, stream_options): - class _OutgoingMessageFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, message, end=True, binary=False): - return self._parent._process_outgoing_message( - message, end, binary) - - class _IncomingMessageFilter(object): - - def __init__(self, parent): - self._parent = parent - self._decompress_next_message = False - - def decompress_next_message(self): - self._decompress_next_message = True - - def filter(self, message): - message = self._parent._process_incoming_message( - message, self._decompress_next_message) - self._decompress_next_message = False - return message - - self._outgoing_message_filter = _OutgoingMessageFilter(self) - self._incoming_message_filter = _IncomingMessageFilter(self) - stream_options.outgoing_message_filters.append( - self._outgoing_message_filter) - stream_options.incoming_message_filters.append( - self._incoming_message_filter) - - class _OutgoingFrameFilter(object): - - def __init__(self, parent): - self._parent = parent - self._set_compression_bit = False - - def set_compression_bit(self): - self._set_compression_bit = True - - def filter(self, frame): - self._parent._process_outgoing_frame( - frame, self._set_compression_bit) - self._set_compression_bit = False - - class _IncomingFrameFilter(object): - - def __init__(self, parent): - self._parent = parent - - def filter(self, frame): - self._parent._process_incoming_frame(frame) - - self._outgoing_frame_filter = _OutgoingFrameFilter(self) - self._incoming_frame_filter = _IncomingFrameFilter(self) - stream_options.outgoing_frame_filters.append( - self._outgoing_frame_filter) - stream_options.incoming_frame_filters.append( - self._incoming_frame_filter) - - stream_options.encode_text_message_to_utf8 = False - - def set_c2s_max_window_bits(self, value): - self._c2s_max_window_bits = value - - def set_c2s_no_context_takeover(self, value): - self._c2s_no_context_takeover = value - - def set_bfinal(self, value): - self._bfinal = value - - def enable_outgoing_compression(self): - self._compress_outgoing_enabled = True - - def disable_outgoing_compression(self): - self._compress_outgoing_enabled = False - - def _process_incoming_message(self, message, decompress): - if not decompress: - return message - - received_payload_size = len(message) - self._total_incoming_payload_bytes += received_payload_size - - message = self._inflater.filter(message) - - filtered_payload_size = len(message) - self._total_filtered_incoming_payload_bytes += filtered_payload_size - - _log_decompression_ratio(self._logger, received_payload_size, - self._total_incoming_payload_bytes, - filtered_payload_size, - self._total_filtered_incoming_payload_bytes) - - return message - - def _process_outgoing_message(self, message, end, binary): - if not binary: - message = message.encode('utf-8') - - if not self._compress_outgoing_enabled: - return message - - original_payload_size = len(message) - self._total_outgoing_payload_bytes += original_payload_size - - message = self._deflater.filter( - message, flush=end, bfinal=self._bfinal) - - filtered_payload_size = len(message) - self._total_filtered_outgoing_payload_bytes += filtered_payload_size - - _log_compression_ratio(self._logger, original_payload_size, - self._total_outgoing_payload_bytes, - filtered_payload_size, - self._total_filtered_outgoing_payload_bytes) - - if not self._compress_ongoing: - self._outgoing_frame_filter.set_compression_bit() - self._compress_ongoing = not end - return message - - def _process_incoming_frame(self, frame): - if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): - self._incoming_message_filter.decompress_next_message() - frame.rsv1 = 0 - - def _process_outgoing_frame(self, frame, compression_bit): - if (not compression_bit or - common.is_control_opcode(frame.opcode)): - return - - frame.rsv1 = 1 - - -class PerMessageCompressionExtensionProcessor( - CompressionExtensionProcessorBase): - """WebSocket Per-message compression extension processor. - - Specification: - http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression - """ - - _DEFLATE_METHOD = 'deflate' - - def __init__(self, request): - CompressionExtensionProcessorBase.__init__(self, request) - - def name(self): - return common.PERMESSAGE_COMPRESSION_EXTENSION - - def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateMessageProcessor(method_desc) - return None - - -_available_processors[common.PERMESSAGE_COMPRESSION_EXTENSION] = ( - PerMessageCompressionExtensionProcessor) - - -# Adding vendor-prefixed permessage-compress extension. -# TODO(bashi): Remove this after WebKit stops using vendor prefix. -_available_processors[common.X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION] = ( - PerMessageCompressionExtensionProcessor) - - -class MuxExtensionProcessor(ExtensionProcessorInterface): - """WebSocket multiplexing extension processor.""" - - _QUOTA_PARAM = 'quota' - - def __init__(self, request): - self._request = request - - def name(self): - return common.MUX_EXTENSION - - def get_extension_response(self, ws_request, - logical_channel_extensions): - # Mux extension cannot be used after extensions that depend on - # frame boundary, extension data field, or any reserved bits - # which are attributed to each frame. - for extension in logical_channel_extensions: - name = extension.name() - if (name == common.PERFRAME_COMPRESSION_EXTENSION or - name == common.DEFLATE_FRAME_EXTENSION or - name == common.X_WEBKIT_DEFLATE_FRAME_EXTENSION): - return None - - quota = self._request.get_parameter_value(self._QUOTA_PARAM) - if quota is None: - ws_request.mux_quota = 0 - else: - try: - quota = int(quota) - except ValueError, e: - return None - if quota < 0 or quota >= 2 ** 32: - return None - ws_request.mux_quota = quota - - ws_request.mux = True - ws_request.mux_extensions = logical_channel_extensions - return common.ExtensionParameter(common.MUX_EXTENSION) - - def setup_stream_options(self, stream_options): - pass - - -_available_processors[common.MUX_EXTENSION] = MuxExtensionProcessor - - -def get_extension_processor(extension_request): - global _available_processors - processor_class = _available_processors.get(extension_request.name()) - if processor_class is None: - return None - return processor_class(extension_request) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/__init__.py b/module/lib/mod_pywebsocket/handshake/__init__.py deleted file mode 100644 index 194f6b395..000000000 --- a/module/lib/mod_pywebsocket/handshake/__init__.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket opening handshake processor. This class try to apply available -opening handshake processors for each protocol version until a connection is -successfully established. -""" - - -import logging - -from mod_pywebsocket import common -from mod_pywebsocket.handshake import hybi00 -from mod_pywebsocket.handshake import hybi -# Export AbortedByUserException, HandshakeException, and VersionException -# symbol from this module. -from mod_pywebsocket.handshake._base import AbortedByUserException -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import VersionException - - -_LOGGER = logging.getLogger(__name__) - - -def do_handshake(request, dispatcher, allowDraft75=False, strict=False): - """Performs WebSocket handshake. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - allowDraft75: obsolete argument. ignored. - strict: obsolete argument. ignored. - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri) - # To print mimetools.Message as escaped one-line string, we converts - # headers_in to dict object. Without conversion, if we use %r, it just - # prints the type and address, and if we use %s, it prints the original - # header string as multiple lines. - # - # Both mimetools.Message and MpTable_Type of mod_python can be - # converted to dict. - # - # mimetools.Message.__str__ returns the original header string. - # dict(mimetools.Message object) returns the map from header names to - # header values. While MpTable_Type doesn't have such __str__ but just - # __repr__ which formats itself as well as dictionary object. - _LOGGER.debug( - 'Client\'s opening handshake headers: %r', dict(request.headers_in)) - - handshakers = [] - handshakers.append( - ('RFC 6455', hybi.Handshaker(request, dispatcher))) - handshakers.append( - ('HyBi 00', hybi00.Handshaker(request, dispatcher))) - - for name, handshaker in handshakers: - _LOGGER.debug('Trying protocol version %s', name) - try: - handshaker.do_handshake() - _LOGGER.info('Established (%s protocol)', name) - return - except HandshakeException, e: - _LOGGER.debug( - 'Failed to complete opening handshake as %s protocol: %r', - name, e) - if e.status: - raise e - except AbortedByUserException, e: - raise - except VersionException, e: - raise - - # TODO(toyoshim): Add a test to cover the case all handshakers fail. - raise HandshakeException( - 'Failed to complete opening handshake for all available protocols', - status=common.HTTP_STATUS_BAD_REQUEST) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/_base.py b/module/lib/mod_pywebsocket/handshake/_base.py deleted file mode 100644 index e5c94ca90..000000000 --- a/module/lib/mod_pywebsocket/handshake/_base.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Common functions and exceptions used by WebSocket opening handshake -processors. -""" - - -from mod_pywebsocket import common -from mod_pywebsocket import http_header_util - - -class AbortedByUserException(Exception): - """Exception for aborting a connection intentionally. - - If this exception is raised in do_extra_handshake handler, the connection - will be abandoned. No other WebSocket or HTTP(S) handler will be invoked. - - If this exception is raised in transfer_data_handler, the connection will - be closed without closing handshake. No other WebSocket or HTTP(S) handler - will be invoked. - """ - - pass - - -class HandshakeException(Exception): - """This exception will be raised when an error occurred while processing - WebSocket initial handshake. - """ - - def __init__(self, name, status=None): - super(HandshakeException, self).__init__(name) - self.status = status - - -class VersionException(Exception): - """This exception will be raised when a version of client request does not - match with version the server supports. - """ - - def __init__(self, name, supported_versions=''): - """Construct an instance. - - Args: - supported_version: a str object to show supported hybi versions. - (e.g. '8, 13') - """ - super(VersionException, self).__init__(name) - self.supported_versions = supported_versions - - -def get_default_port(is_secure): - if is_secure: - return common.DEFAULT_WEB_SOCKET_SECURE_PORT - else: - return common.DEFAULT_WEB_SOCKET_PORT - - -def validate_subprotocol(subprotocol, hixie): - """Validate a value in the Sec-WebSocket-Protocol field. - - See - - RFC 6455: Section 4.1., 4.2.2., and 4.3. - - HyBi 00: Section 4.1. Opening handshake - - Args: - hixie: if True, checks if characters in subprotocol are in range - between U+0020 and U+007E. It's required by HyBi 00 but not by - RFC 6455. - """ - - if not subprotocol: - raise HandshakeException('Invalid subprotocol name: empty') - if hixie: - # Parameter should be in the range U+0020 to U+007E. - for c in subprotocol: - if not 0x20 <= ord(c) <= 0x7e: - raise HandshakeException( - 'Illegal character in subprotocol name: %r' % c) - else: - # Parameter should be encoded HTTP token. - state = http_header_util.ParsingState(subprotocol) - token = http_header_util.consume_token(state) - rest = http_header_util.peek(state) - # If |rest| is not None, |subprotocol| is not one token or invalid. If - # |rest| is None, |token| must not be None because |subprotocol| is - # concatenation of |token| and |rest| and is not None. - if rest is not None: - raise HandshakeException('Invalid non-token string in subprotocol ' - 'name: %r' % rest) - - -def parse_host_header(request): - fields = request.headers_in['Host'].split(':', 1) - if len(fields) == 1: - return fields[0], get_default_port(request.is_https()) - try: - return fields[0], int(fields[1]) - except ValueError, e: - raise HandshakeException('Invalid port number format: %r' % e) - - -def format_header(name, value): - return '%s: %s\r\n' % (name, value) - - -def build_location(request): - """Build WebSocket location for request.""" - location_parts = [] - if request.is_https(): - location_parts.append(common.WEB_SOCKET_SECURE_SCHEME) - else: - location_parts.append(common.WEB_SOCKET_SCHEME) - location_parts.append('://') - host, port = parse_host_header(request) - connection_port = request.connection.local_addr[1] - if port != connection_port: - raise HandshakeException('Header/connection port mismatch: %d/%d' % - (port, connection_port)) - location_parts.append(host) - if (port != get_default_port(request.is_https())): - location_parts.append(':') - location_parts.append(str(port)) - location_parts.append(request.uri) - return ''.join(location_parts) - - -def get_mandatory_header(request, key): - value = request.headers_in.get(key) - if value is None: - raise HandshakeException('Header %s is not defined' % key) - return value - - -def validate_mandatory_header(request, key, expected_value, fail_status=None): - value = get_mandatory_header(request, key) - - if value.lower() != expected_value.lower(): - raise HandshakeException( - 'Expected %r for header %s but found %r (case-insensitive)' % - (expected_value, key, value), status=fail_status) - - -def check_request_line(request): - # 5.1 1. The three character UTF-8 string "GET". - # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). - if request.method != 'GET': - raise HandshakeException('Method is not GET: %r' % request.method) - - if request.protocol != 'HTTP/1.1': - raise HandshakeException('Version is not HTTP/1.1: %r' % - request.protocol) - - -def check_header_lines(request, mandatory_headers): - check_request_line(request) - - # The expected field names, and the meaning of their corresponding - # values, are as follows. - # |Upgrade| and |Connection| - for key, expected_value in mandatory_headers: - validate_mandatory_header(request, key, expected_value) - - -def parse_token_list(data): - """Parses a header value which follows 1#token and returns parsed elements - as a list of strings. - - Leading LWSes must be trimmed. - """ - - state = http_header_util.ParsingState(data) - - token_list = [] - - while True: - token = http_header_util.consume_token(state) - if token is not None: - token_list.append(token) - - http_header_util.consume_lwses(state) - - if http_header_util.peek(state) is None: - break - - if not http_header_util.consume_string(state, ','): - raise HandshakeException( - 'Expected a comma but found %r' % http_header_util.peek(state)) - - http_header_util.consume_lwses(state) - - if len(token_list) == 0: - raise HandshakeException('No valid token found') - - return token_list - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/hybi.py b/module/lib/mod_pywebsocket/handshake/hybi.py deleted file mode 100644 index fc0e2a096..000000000 --- a/module/lib/mod_pywebsocket/handshake/hybi.py +++ /dev/null @@ -1,404 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides the opening handshake processor for the WebSocket -protocol (RFC 6455). - -Specification: -http://tools.ietf.org/html/rfc6455 -""" - - -# Note: request.connection.write is used in this module, even though mod_python -# document says that it should be used only in connection handlers. -# Unfortunately, we have no other options. For example, request.write is not -# suitable because it doesn't allow direct raw bytes writing. - - -import base64 -import logging -import os -import re - -from mod_pywebsocket import common -from mod_pywebsocket.extensions import get_extension_processor -from mod_pywebsocket.handshake._base import check_request_line -from mod_pywebsocket.handshake._base import format_header -from mod_pywebsocket.handshake._base import get_mandatory_header -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import parse_token_list -from mod_pywebsocket.handshake._base import validate_mandatory_header -from mod_pywebsocket.handshake._base import validate_subprotocol -from mod_pywebsocket.handshake._base import VersionException -from mod_pywebsocket.stream import Stream -from mod_pywebsocket.stream import StreamOptions -from mod_pywebsocket import util - - -# Used to validate the value in the Sec-WebSocket-Key header strictly. RFC 4648 -# disallows non-zero padding, so the character right before == must be any of -# A, Q, g and w. -_SEC_WEBSOCKET_KEY_REGEX = re.compile('^[+/0-9A-Za-z]{21}[AQgw]==$') - -# Defining aliases for values used frequently. -_VERSION_HYBI08 = common.VERSION_HYBI08 -_VERSION_HYBI08_STRING = str(_VERSION_HYBI08) -_VERSION_LATEST = common.VERSION_HYBI_LATEST -_VERSION_LATEST_STRING = str(_VERSION_LATEST) -_SUPPORTED_VERSIONS = [ - _VERSION_LATEST, - _VERSION_HYBI08, -] - - -def compute_accept(key): - """Computes value for the Sec-WebSocket-Accept header from value of the - Sec-WebSocket-Key header. - """ - - accept_binary = util.sha1_hash( - key + common.WEBSOCKET_ACCEPT_UUID).digest() - accept = base64.b64encode(accept_binary) - - return (accept, accept_binary) - - -class Handshaker(object): - """Opening handshake processor for the WebSocket protocol (RFC 6455).""" - - def __init__(self, request, dispatcher): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - - Handshaker will add attributes such as ws_resource during handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - - def _validate_connection_header(self): - connection = get_mandatory_header( - self._request, common.CONNECTION_HEADER) - - try: - connection_tokens = parse_token_list(connection) - except HandshakeException, e: - raise HandshakeException( - 'Failed to parse %s: %s' % (common.CONNECTION_HEADER, e)) - - connection_is_valid = False - for token in connection_tokens: - if token.lower() == common.UPGRADE_CONNECTION_TYPE.lower(): - connection_is_valid = True - break - if not connection_is_valid: - raise HandshakeException( - '%s header doesn\'t contain "%s"' % - (common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - - def do_handshake(self): - self._request.ws_close_code = None - self._request.ws_close_reason = None - - # Parsing. - - check_request_line(self._request) - - validate_mandatory_header( - self._request, - common.UPGRADE_HEADER, - common.WEBSOCKET_UPGRADE_TYPE) - - self._validate_connection_header() - - self._request.ws_resource = self._request.uri - - unused_host = get_mandatory_header(self._request, common.HOST_HEADER) - - self._request.ws_version = self._check_version() - - # This handshake must be based on latest hybi. We are responsible to - # fallback to HTTP on handshake failure as latest hybi handshake - # specifies. - try: - self._get_origin() - self._set_protocol() - self._parse_extensions() - - # Key validation, response generation. - - key = self._get_key() - (accept, accept_binary) = compute_accept(key) - self._logger.debug( - '%s: %r (%s)', - common.SEC_WEBSOCKET_ACCEPT_HEADER, - accept, - util.hexify(accept_binary)) - - self._logger.debug('Protocol version is RFC 6455') - - # Setup extension processors. - - processors = [] - if self._request.ws_requested_extensions is not None: - for extension_request in self._request.ws_requested_extensions: - processor = get_extension_processor(extension_request) - # Unknown extension requests are just ignored. - if processor is not None: - processors.append(processor) - self._request.ws_extension_processors = processors - - # Extra handshake handler may modify/remove processors. - self._dispatcher.do_extra_handshake(self._request) - processors = filter(lambda processor: processor is not None, - self._request.ws_extension_processors) - - accepted_extensions = [] - - # We need to take care of mux extension here. Extensions that - # are placed before mux should be applied to logical channels. - mux_index = -1 - for i, processor in enumerate(processors): - if processor.name() == common.MUX_EXTENSION: - mux_index = i - break - if mux_index >= 0: - mux_processor = processors[mux_index] - logical_channel_processors = processors[:mux_index] - processors = processors[mux_index+1:] - - for processor in logical_channel_processors: - extension_response = processor.get_extension_response() - if extension_response is None: - # Rejected. - continue - accepted_extensions.append(extension_response) - # Pass a shallow copy of accepted_extensions as extensions for - # logical channels. - mux_response = mux_processor.get_extension_response( - self._request, accepted_extensions[:]) - if mux_response is not None: - accepted_extensions.append(mux_response) - - stream_options = StreamOptions() - - # When there is mux extension, here, |processors| contain only - # prosessors for extensions placed after mux. - for processor in processors: - - extension_response = processor.get_extension_response() - if extension_response is None: - # Rejected. - continue - - accepted_extensions.append(extension_response) - - processor.setup_stream_options(stream_options) - - if len(accepted_extensions) > 0: - self._request.ws_extensions = accepted_extensions - self._logger.debug( - 'Extensions accepted: %r', - map(common.ExtensionParameter.name, accepted_extensions)) - else: - self._request.ws_extensions = None - - self._request.ws_stream = self._create_stream(stream_options) - - if self._request.ws_requested_protocols is not None: - if self._request.ws_protocol is None: - raise HandshakeException( - 'do_extra_handshake must choose one subprotocol from ' - 'ws_requested_protocols and set it to ws_protocol') - validate_subprotocol(self._request.ws_protocol, hixie=False) - - self._logger.debug( - 'Subprotocol accepted: %r', - self._request.ws_protocol) - else: - if self._request.ws_protocol is not None: - raise HandshakeException( - 'ws_protocol must be None when the client didn\'t ' - 'request any subprotocol') - - self._send_handshake(accept) - except HandshakeException, e: - if not e.status: - # Fallback to 400 bad request by default. - e.status = common.HTTP_STATUS_BAD_REQUEST - raise e - - def _get_origin(self): - if self._request.ws_version is _VERSION_HYBI08: - origin_header = common.SEC_WEBSOCKET_ORIGIN_HEADER - else: - origin_header = common.ORIGIN_HEADER - origin = self._request.headers_in.get(origin_header) - if origin is None: - self._logger.debug('Client request does not have origin header') - self._request.ws_origin = origin - - def _check_version(self): - version = get_mandatory_header(self._request, - common.SEC_WEBSOCKET_VERSION_HEADER) - if version == _VERSION_HYBI08_STRING: - return _VERSION_HYBI08 - if version == _VERSION_LATEST_STRING: - return _VERSION_LATEST - - if version.find(',') >= 0: - raise HandshakeException( - 'Multiple versions (%r) are not allowed for header %s' % - (version, common.SEC_WEBSOCKET_VERSION_HEADER), - status=common.HTTP_STATUS_BAD_REQUEST) - raise VersionException( - 'Unsupported version %r for header %s' % - (version, common.SEC_WEBSOCKET_VERSION_HEADER), - supported_versions=', '.join(map(str, _SUPPORTED_VERSIONS))) - - def _set_protocol(self): - self._request.ws_protocol = None - - protocol_header = self._request.headers_in.get( - common.SEC_WEBSOCKET_PROTOCOL_HEADER) - - if protocol_header is None: - self._request.ws_requested_protocols = None - return - - self._request.ws_requested_protocols = parse_token_list( - protocol_header) - self._logger.debug('Subprotocols requested: %r', - self._request.ws_requested_protocols) - - def _parse_extensions(self): - extensions_header = self._request.headers_in.get( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER) - if not extensions_header: - self._request.ws_requested_extensions = None - return - - if self._request.ws_version is common.VERSION_HYBI08: - allow_quoted_string=False - else: - allow_quoted_string=True - try: - self._request.ws_requested_extensions = common.parse_extensions( - extensions_header, allow_quoted_string=allow_quoted_string) - except common.ExtensionParsingException, e: - raise HandshakeException( - 'Failed to parse Sec-WebSocket-Extensions header: %r' % e) - - self._logger.debug( - 'Extensions requested: %r', - map(common.ExtensionParameter.name, - self._request.ws_requested_extensions)) - - def _validate_key(self, key): - if key.find(',') >= 0: - raise HandshakeException('Request has multiple %s header lines or ' - 'contains illegal character \',\': %r' % - (common.SEC_WEBSOCKET_KEY_HEADER, key)) - - # Validate - key_is_valid = False - try: - # Validate key by quick regex match before parsing by base64 - # module. Because base64 module skips invalid characters, we have - # to do this in advance to make this server strictly reject illegal - # keys. - if _SEC_WEBSOCKET_KEY_REGEX.match(key): - decoded_key = base64.b64decode(key) - if len(decoded_key) == 16: - key_is_valid = True - except TypeError, e: - pass - - if not key_is_valid: - raise HandshakeException( - 'Illegal value for header %s: %r' % - (common.SEC_WEBSOCKET_KEY_HEADER, key)) - - return decoded_key - - def _get_key(self): - key = get_mandatory_header( - self._request, common.SEC_WEBSOCKET_KEY_HEADER) - - decoded_key = self._validate_key(key) - - self._logger.debug( - '%s: %r (%s)', - common.SEC_WEBSOCKET_KEY_HEADER, - key, - util.hexify(decoded_key)) - - return key - - def _create_stream(self, stream_options): - return Stream(self._request, stream_options) - - def _create_handshake_response(self, accept): - response = [] - - response.append('HTTP/1.1 101 Switching Protocols\r\n') - - response.append(format_header( - common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE)) - response.append(format_header( - common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - response.append(format_header( - common.SEC_WEBSOCKET_ACCEPT_HEADER, accept)) - if self._request.ws_protocol is not None: - response.append(format_header( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - if (self._request.ws_extensions is not None and - len(self._request.ws_extensions) != 0): - response.append(format_header( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER, - common.format_extensions(self._request.ws_extensions))) - response.append('\r\n') - - return ''.join(response) - - def _send_handshake(self, accept): - raw_response = self._create_handshake_response(accept) - self._request.connection.write(raw_response) - self._logger.debug('Sent server\'s opening handshake: %r', - raw_response) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/handshake/hybi00.py b/module/lib/mod_pywebsocket/handshake/hybi00.py deleted file mode 100644 index cc6f8dc43..000000000 --- a/module/lib/mod_pywebsocket/handshake/hybi00.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides the opening handshake processor for the WebSocket -protocol version HyBi 00. - -Specification: -http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 -""" - - -# Note: request.connection.write/read are used in this module, even though -# mod_python document says that they should be used only in connection -# handlers. Unfortunately, we have no other options. For example, -# request.write/read are not suitable because they don't allow direct raw bytes -# writing/reading. - - -import logging -import re -import struct - -from mod_pywebsocket import common -from mod_pywebsocket.stream import StreamHixie75 -from mod_pywebsocket import util -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import build_location -from mod_pywebsocket.handshake._base import check_header_lines -from mod_pywebsocket.handshake._base import format_header -from mod_pywebsocket.handshake._base import get_mandatory_header -from mod_pywebsocket.handshake._base import validate_subprotocol - - -_MANDATORY_HEADERS = [ - # key, expected value or None - [common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75], - [common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE], -] - - -class Handshaker(object): - """Opening handshake processor for the WebSocket protocol version HyBi 00. - """ - - def __init__(self, request, dispatcher): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - - def do_handshake(self): - """Perform WebSocket Handshake. - - On _request, we set - ws_resource, ws_protocol, ws_location, ws_origin, ws_challenge, - ws_challenge_md5: WebSocket handshake information. - ws_stream: Frame generation/parsing class. - ws_version: Protocol version. - - Raises: - HandshakeException: when any error happened in parsing the opening - handshake request. - """ - - # 5.1 Reading the client's opening handshake. - # dispatcher sets it in self._request. - check_header_lines(self._request, _MANDATORY_HEADERS) - self._set_resource() - self._set_subprotocol() - self._set_location() - self._set_origin() - self._set_challenge_response() - self._set_protocol_version() - - self._dispatcher.do_extra_handshake(self._request) - - self._send_handshake() - - def _set_resource(self): - self._request.ws_resource = self._request.uri - - def _set_subprotocol(self): - # |Sec-WebSocket-Protocol| - subprotocol = self._request.headers_in.get( - common.SEC_WEBSOCKET_PROTOCOL_HEADER) - if subprotocol is not None: - validate_subprotocol(subprotocol, hixie=True) - self._request.ws_protocol = subprotocol - - def _set_location(self): - # |Host| - host = self._request.headers_in.get(common.HOST_HEADER) - if host is not None: - self._request.ws_location = build_location(self._request) - # TODO(ukai): check host is this host. - - def _set_origin(self): - # |Origin| - origin = self._request.headers_in.get(common.ORIGIN_HEADER) - if origin is not None: - self._request.ws_origin = origin - - def _set_protocol_version(self): - # |Sec-WebSocket-Draft| - draft = self._request.headers_in.get(common.SEC_WEBSOCKET_DRAFT_HEADER) - if draft is not None and draft != '0': - raise HandshakeException('Illegal value for %s: %s' % - (common.SEC_WEBSOCKET_DRAFT_HEADER, - draft)) - - self._logger.debug('Protocol version is HyBi 00') - self._request.ws_version = common.VERSION_HYBI00 - self._request.ws_stream = StreamHixie75(self._request, True) - - def _set_challenge_response(self): - # 5.2 4-8. - self._request.ws_challenge = self._get_challenge() - # 5.2 9. let /response/ be the MD5 finterprint of /challenge/ - self._request.ws_challenge_md5 = util.md5_hash( - self._request.ws_challenge).digest() - self._logger.debug( - 'Challenge: %r (%s)', - self._request.ws_challenge, - util.hexify(self._request.ws_challenge)) - self._logger.debug( - 'Challenge response: %r (%s)', - self._request.ws_challenge_md5, - util.hexify(self._request.ws_challenge_md5)) - - def _get_key_value(self, key_field): - key_value = get_mandatory_header(self._request, key_field) - - self._logger.debug('%s: %r', key_field, key_value) - - # 5.2 4. let /key-number_n/ be the digits (characters in the range - # U+0030 DIGIT ZERO (0) to U+0039 DIGIT NINE (9)) in /key_n/, - # interpreted as a base ten integer, ignoring all other characters - # in /key_n/. - try: - key_number = int(re.sub("\\D", "", key_value)) - except: - raise HandshakeException('%s field contains no digit' % key_field) - # 5.2 5. let /spaces_n/ be the number of U+0020 SPACE characters - # in /key_n/. - spaces = re.subn(" ", "", key_value)[1] - if spaces == 0: - raise HandshakeException('%s field contains no space' % key_field) - - self._logger.debug( - '%s: Key-number is %d and number of spaces is %d', - key_field, key_number, spaces) - - # 5.2 6. if /key-number_n/ is not an integral multiple of /spaces_n/ - # then abort the WebSocket connection. - if key_number % spaces != 0: - raise HandshakeException( - '%s: Key-number (%d) is not an integral multiple of spaces ' - '(%d)' % (key_field, key_number, spaces)) - # 5.2 7. let /part_n/ be /key-number_n/ divided by /spaces_n/. - part = key_number / spaces - self._logger.debug('%s: Part is %d', key_field, part) - return part - - def _get_challenge(self): - # 5.2 4-7. - key1 = self._get_key_value(common.SEC_WEBSOCKET_KEY1_HEADER) - key2 = self._get_key_value(common.SEC_WEBSOCKET_KEY2_HEADER) - # 5.2 8. let /challenge/ be the concatenation of /part_1/, - challenge = '' - challenge += struct.pack('!I', key1) # network byteorder int - challenge += struct.pack('!I', key2) # network byteorder int - challenge += self._request.connection.read(8) - return challenge - - def _send_handshake(self): - response = [] - - # 5.2 10. send the following line. - response.append('HTTP/1.1 101 WebSocket Protocol Handshake\r\n') - - # 5.2 11. send the following fields to the client. - response.append(format_header( - common.UPGRADE_HEADER, common.WEBSOCKET_UPGRADE_TYPE_HIXIE75)) - response.append(format_header( - common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) - response.append(format_header( - common.SEC_WEBSOCKET_LOCATION_HEADER, self._request.ws_location)) - response.append(format_header( - common.SEC_WEBSOCKET_ORIGIN_HEADER, self._request.ws_origin)) - if self._request.ws_protocol: - response.append(format_header( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - # 5.2 12. send two bytes 0x0D 0x0A. - response.append('\r\n') - # 5.2 13. send /response/ - response.append(self._request.ws_challenge_md5) - - raw_response = ''.join(response) - self._request.connection.write(raw_response) - self._logger.debug('Sent server\'s opening handshake: %r', - raw_response) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/headerparserhandler.py b/module/lib/mod_pywebsocket/headerparserhandler.py deleted file mode 100644 index 2cc62de04..000000000 --- a/module/lib/mod_pywebsocket/headerparserhandler.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""PythonHeaderParserHandler for mod_pywebsocket. - -Apache HTTP Server and mod_python must be configured such that this -function is called to handle WebSocket request. -""" - - -import logging - -from mod_python import apache - -from mod_pywebsocket import common -from mod_pywebsocket import dispatch -from mod_pywebsocket import handshake -from mod_pywebsocket import util - - -# PythonOption to specify the handler root directory. -_PYOPT_HANDLER_ROOT = 'mod_pywebsocket.handler_root' - -# PythonOption to specify the handler scan directory. -# This must be a directory under the root directory. -# The default is the root directory. -_PYOPT_HANDLER_SCAN = 'mod_pywebsocket.handler_scan' - -# PythonOption to allow handlers whose canonical path is -# not under the root directory. It's disallowed by default. -# Set this option with value of 'yes' to allow. -_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT = ( - 'mod_pywebsocket.allow_handlers_outside_root_dir') -# Map from values to their meanings. 'Yes' and 'No' are allowed just for -# compatibility. -_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION = { - 'off': False, 'no': False, 'on': True, 'yes': True} - -# (Obsolete option. Ignored.) -# PythonOption to specify to allow handshake defined in Hixie 75 version -# protocol. The default is None (Off) -_PYOPT_ALLOW_DRAFT75 = 'mod_pywebsocket.allow_draft75' -# Map from values to their meanings. -_PYOPT_ALLOW_DRAFT75_DEFINITION = {'off': False, 'on': True} - - -class ApacheLogHandler(logging.Handler): - """Wrapper logging.Handler to emit log message to apache's error.log.""" - - _LEVELS = { - logging.DEBUG: apache.APLOG_DEBUG, - logging.INFO: apache.APLOG_INFO, - logging.WARNING: apache.APLOG_WARNING, - logging.ERROR: apache.APLOG_ERR, - logging.CRITICAL: apache.APLOG_CRIT, - } - - def __init__(self, request=None): - logging.Handler.__init__(self) - self._log_error = apache.log_error - if request is not None: - self._log_error = request.log_error - - # Time and level will be printed by Apache. - self._formatter = logging.Formatter('%(name)s: %(message)s') - - def emit(self, record): - apache_level = apache.APLOG_DEBUG - if record.levelno in ApacheLogHandler._LEVELS: - apache_level = ApacheLogHandler._LEVELS[record.levelno] - - msg = self._formatter.format(record) - - # "server" parameter must be passed to have "level" parameter work. - # If only "level" parameter is passed, nothing shows up on Apache's - # log. However, at this point, we cannot get the server object of the - # virtual host which will process WebSocket requests. The only server - # object we can get here is apache.main_server. But Wherever (server - # configuration context or virtual host context) we put - # PythonHeaderParserHandler directive, apache.main_server just points - # the main server instance (not any of virtual server instance). Then, - # Apache follows LogLevel directive in the server configuration context - # to filter logs. So, we need to specify LogLevel in the server - # configuration context. Even if we specify "LogLevel debug" in the - # virtual host context which actually handles WebSocket connections, - # DEBUG level logs never show up unless "LogLevel debug" is specified - # in the server configuration context. - # - # TODO(tyoshino): Provide logging methods on request object. When - # request is mp_request object (when used together with Apache), the - # methods call request.log_error indirectly. When request is - # _StandaloneRequest, the methods call Python's logging facility which - # we create in standalone.py. - self._log_error(msg, apache_level, apache.main_server) - - -def _configure_logging(): - logger = logging.getLogger() - # Logs are filtered by Apache based on LogLevel directive in Apache - # configuration file. We must just pass logs for all levels to - # ApacheLogHandler. - logger.setLevel(logging.DEBUG) - logger.addHandler(ApacheLogHandler()) - - -_configure_logging() - -_LOGGER = logging.getLogger(__name__) - - -def _parse_option(name, value, definition): - if value is None: - return False - - meaning = definition.get(value.lower()) - if meaning is None: - raise Exception('Invalid value for PythonOption %s: %r' % - (name, value)) - return meaning - - -def _create_dispatcher(): - _LOGGER.info('Initializing Dispatcher') - - options = apache.main_server.get_options() - - handler_root = options.get(_PYOPT_HANDLER_ROOT, None) - if not handler_root: - raise Exception('PythonOption %s is not defined' % _PYOPT_HANDLER_ROOT, - apache.APLOG_ERR) - - handler_scan = options.get(_PYOPT_HANDLER_SCAN, handler_root) - - allow_handlers_outside_root = _parse_option( - _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT, - options.get(_PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT), - _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION) - - dispatcher = dispatch.Dispatcher( - handler_root, handler_scan, allow_handlers_outside_root) - - for warning in dispatcher.source_warnings(): - apache.log_error('mod_pywebsocket: %s' % warning, apache.APLOG_WARNING) - - return dispatcher - - -# Initialize -_dispatcher = _create_dispatcher() - - -def headerparserhandler(request): - """Handle request. - - Args: - request: mod_python request. - - This function is named headerparserhandler because it is the default - name for a PythonHeaderParserHandler. - """ - - handshake_is_done = False - try: - # Fallback to default http handler for request paths for which - # we don't have request handlers. - if not _dispatcher.get_handler_suite(request.uri): - request.log_error('No handler for resource: %r' % request.uri, - apache.APLOG_INFO) - request.log_error('Fallback to Apache', apache.APLOG_INFO) - return apache.DECLINED - except dispatch.DispatchException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - if not handshake_is_done: - return e.status - - try: - allow_draft75 = _parse_option( - _PYOPT_ALLOW_DRAFT75, - apache.main_server.get_options().get(_PYOPT_ALLOW_DRAFT75), - _PYOPT_ALLOW_DRAFT75_DEFINITION) - - try: - handshake.do_handshake( - request, _dispatcher, allowDraft75=allow_draft75) - except handshake.VersionException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - request.err_headers_out.add(common.SEC_WEBSOCKET_VERSION_HEADER, - e.supported_versions) - return apache.HTTP_BAD_REQUEST - except handshake.HandshakeException, e: - # Handshake for ws/wss failed. - # Send http response with error status. - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - return e.status - - handshake_is_done = True - request._dispatcher = _dispatcher - _dispatcher.transfer_data(request) - except handshake.AbortedByUserException, e: - request.log_error('mod_pywebsocket: %s' % e, apache.APLOG_INFO) - except Exception, e: - # DispatchException can also be thrown if something is wrong in - # pywebsocket code. It's caught here, then. - - request.log_error('mod_pywebsocket: %s\n%s' % - (e, util.get_stack_trace()), - apache.APLOG_ERR) - # Unknown exceptions before handshake mean Apache must handle its - # request with another handler. - if not handshake_is_done: - return apache.DECLINED - # Set assbackwards to suppress response header generation by Apache. - request.assbackwards = 1 - return apache.DONE # Return DONE such that no other handlers are invoked. - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/http_header_util.py b/module/lib/mod_pywebsocket/http_header_util.py deleted file mode 100644 index b77465393..000000000 --- a/module/lib/mod_pywebsocket/http_header_util.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Utilities for parsing and formatting headers that follow the grammar defined -in HTTP RFC http://www.ietf.org/rfc/rfc2616.txt. -""" - - -import urlparse - - -_SEPARATORS = '()<>@,;:\\"/[]?={} \t' - - -def _is_char(c): - """Returns true iff c is in CHAR as specified in HTTP RFC.""" - - return ord(c) <= 127 - - -def _is_ctl(c): - """Returns true iff c is in CTL as specified in HTTP RFC.""" - - return ord(c) <= 31 or ord(c) == 127 - - -class ParsingState(object): - - def __init__(self, data): - self.data = data - self.head = 0 - - -def peek(state, pos=0): - """Peeks the character at pos from the head of data.""" - - if state.head + pos >= len(state.data): - return None - - return state.data[state.head + pos] - - -def consume(state, amount=1): - """Consumes specified amount of bytes from the head and returns the - consumed bytes. If there's not enough bytes to consume, returns None. - """ - - if state.head + amount > len(state.data): - return None - - result = state.data[state.head:state.head + amount] - state.head = state.head + amount - return result - - -def consume_string(state, expected): - """Given a parsing state and a expected string, consumes the string from - the head. Returns True if consumed successfully. Otherwise, returns - False. - """ - - pos = 0 - - for c in expected: - if c != peek(state, pos): - return False - pos += 1 - - consume(state, pos) - return True - - -def consume_lws(state): - """Consumes a LWS from the head. Returns True if any LWS is consumed. - Otherwise, returns False. - - LWS = [CRLF] 1*( SP | HT ) - """ - - original_head = state.head - - consume_string(state, '\r\n') - - pos = 0 - - while True: - c = peek(state, pos) - if c == ' ' or c == '\t': - pos += 1 - else: - if pos == 0: - state.head = original_head - return False - else: - consume(state, pos) - return True - - -def consume_lwses(state): - """Consumes *LWS from the head.""" - - while consume_lws(state): - pass - - -def consume_token(state): - """Consumes a token from the head. Returns the token or None if no token - was found. - """ - - pos = 0 - - while True: - c = peek(state, pos) - if c is None or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): - if pos == 0: - return None - - return consume(state, pos) - else: - pos += 1 - - -def consume_token_or_quoted_string(state): - """Consumes a token or a quoted-string, and returns the token or unquoted - string. If no token or quoted-string was found, returns None. - """ - - original_head = state.head - - if not consume_string(state, '"'): - return consume_token(state) - - result = [] - - expect_quoted_pair = False - - while True: - if not expect_quoted_pair and consume_lws(state): - result.append(' ') - continue - - c = consume(state) - if c is None: - # quoted-string is not enclosed with double quotation - state.head = original_head - return None - elif expect_quoted_pair: - expect_quoted_pair = False - if _is_char(c): - result.append(c) - else: - # Non CHAR character found in quoted-pair - state.head = original_head - return None - elif c == '\\': - expect_quoted_pair = True - elif c == '"': - return ''.join(result) - elif _is_ctl(c): - # Invalid character %r found in qdtext - state.head = original_head - return None - else: - result.append(c) - - -def quote_if_necessary(s): - """Quotes arbitrary string into quoted-string.""" - - quote = False - if s == '': - return '""' - - result = [] - for c in s: - if c == '"' or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): - quote = True - - if c == '"' or _is_ctl(c): - result.append('\\' + c) - else: - result.append(c) - - if quote: - return '"' + ''.join(result) + '"' - else: - return ''.join(result) - - -def parse_uri(uri): - """Parse absolute URI then return host, port and resource.""" - - parsed = urlparse.urlsplit(uri) - if parsed.scheme != 'wss' and parsed.scheme != 'ws': - # |uri| must be a relative URI. - # TODO(toyoshim): Should validate |uri|. - return None, None, uri - - if parsed.hostname is None: - return None, None, None - - port = None - try: - port = parsed.port - except ValueError, e: - # port property cause ValueError on invalid null port description like - # 'ws://host:/path'. - return None, None, None - - if port is None: - if parsed.scheme == 'ws': - port = 80 - else: - port = 443 - - path = parsed.path - if not path: - path += '/' - if parsed.query: - path += '?' + parsed.query - if parsed.fragment: - path += '#' + parsed.fragment - - return parsed.hostname, port, path - - -try: - urlparse.uses_netloc.index('ws') -except ValueError, e: - # urlparse in Python2.5.1 doesn't have 'ws' and 'wss' entries. - urlparse.uses_netloc.append('ws') - urlparse.uses_netloc.append('wss') - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/memorizingfile.py b/module/lib/mod_pywebsocket/memorizingfile.py deleted file mode 100644 index 4d4cd9585..000000000 --- a/module/lib/mod_pywebsocket/memorizingfile.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Memorizing file. - -A memorizing file wraps a file and memorizes lines read by readline. -""" - - -import sys - - -class MemorizingFile(object): - """MemorizingFile wraps a file and memorizes lines read by readline. - - Note that data read by other methods are not memorized. This behavior - is good enough for memorizing lines SimpleHTTPServer reads before - the control reaches WebSocketRequestHandler. - """ - - def __init__(self, file_, max_memorized_lines=sys.maxint): - """Construct an instance. - - Args: - file_: the file object to wrap. - max_memorized_lines: the maximum number of lines to memorize. - Only the first max_memorized_lines are memorized. - Default: sys.maxint. - """ - - self._file = file_ - self._memorized_lines = [] - self._max_memorized_lines = max_memorized_lines - self._buffered = False - self._buffered_line = None - - def __getattribute__(self, name): - if name in ('_file', '_memorized_lines', '_max_memorized_lines', - '_buffered', '_buffered_line', 'readline', - 'get_memorized_lines'): - return object.__getattribute__(self, name) - return self._file.__getattribute__(name) - - def readline(self, size=-1): - """Override file.readline and memorize the line read. - - Note that even if size is specified and smaller than actual size, - the whole line will be read out from underlying file object by - subsequent readline calls. - """ - - if self._buffered: - line = self._buffered_line - self._buffered = False - else: - line = self._file.readline() - if line and len(self._memorized_lines) < self._max_memorized_lines: - self._memorized_lines.append(line) - if size >= 0 and size < len(line): - self._buffered = True - self._buffered_line = line[size:] - return line[:size] - return line - - def get_memorized_lines(self): - """Get lines memorized so far.""" - return self._memorized_lines - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/msgutil.py b/module/lib/mod_pywebsocket/msgutil.py deleted file mode 100644 index 4c1a0114b..000000000 --- a/module/lib/mod_pywebsocket/msgutil.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Message related utilities. - -Note: request.connection.write/read are used in this module, even though -mod_python document says that they should be used only in connection -handlers. Unfortunately, we have no other options. For example, -request.write/read are not suitable because they don't allow direct raw -bytes writing/reading. -""" - - -import Queue -import threading - - -# Export Exception symbols from msgutil for backward compatibility -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import UnsupportedFrameException - - -# An API for handler to send/receive WebSocket messages. -def close_connection(request): - """Close connection. - - Args: - request: mod_python request. - """ - request.ws_stream.close_connection() - - -def send_message(request, payload_data, end=True, binary=False): - """Send a message (or part of a message). - - Args: - request: mod_python request. - payload_data: unicode text or str binary to send. - end: True to terminate a message. - False to send payload_data as part of a message that is to be - terminated by next or later send_message call with end=True. - binary: send payload_data as binary frame(s). - Raises: - BadOperationException: when server already terminated. - """ - request.ws_stream.send_message(payload_data, end, binary) - - -def receive_message(request): - """Receive a WebSocket frame and return its payload as a text in - unicode or a binary in str. - - Args: - request: mod_python request. - Raises: - InvalidFrameException: when client send invalid frame. - UnsupportedFrameException: when client send unsupported frame e.g. some - of reserved bit is set but no extension can - recognize it. - InvalidUTF8Exception: when client send a text frame containing any - invalid UTF-8 string. - ConnectionTerminatedException: when the connection is closed - unexpectedly. - BadOperationException: when client already terminated. - """ - return request.ws_stream.receive_message() - - -def send_ping(request, body=''): - request.ws_stream.send_ping(body) - - -class MessageReceiver(threading.Thread): - """This class receives messages from the client. - - This class provides three ways to receive messages: blocking, - non-blocking, and via callback. Callback has the highest precedence. - - Note: This class should not be used with the standalone server for wss - because pyOpenSSL used by the server raises a fatal error if the socket - is accessed from multiple threads. - """ - - def __init__(self, request, onmessage=None): - """Construct an instance. - - Args: - request: mod_python request. - onmessage: a function to be called when a message is received. - May be None. If not None, the function is called on - another thread. In that case, MessageReceiver.receive - and MessageReceiver.receive_nowait are useless - because they will never return any messages. - """ - - threading.Thread.__init__(self) - self._request = request - self._queue = Queue.Queue() - self._onmessage = onmessage - self._stop_requested = False - self.setDaemon(True) - self.start() - - def run(self): - try: - while not self._stop_requested: - message = receive_message(self._request) - if self._onmessage: - self._onmessage(message) - else: - self._queue.put(message) - finally: - close_connection(self._request) - - def receive(self): - """ Receive a message from the channel, blocking. - - Returns: - message as a unicode string. - """ - return self._queue.get() - - def receive_nowait(self): - """ Receive a message from the channel, non-blocking. - - Returns: - message as a unicode string if available. None otherwise. - """ - try: - message = self._queue.get_nowait() - except Queue.Empty: - message = None - return message - - def stop(self): - """Request to stop this instance. - - The instance will be stopped after receiving the next message. - This method may not be very useful, but there is no clean way - in Python to forcefully stop a running thread. - """ - self._stop_requested = True - - -class MessageSender(threading.Thread): - """This class sends messages to the client. - - This class provides both synchronous and asynchronous ways to send - messages. - - Note: This class should not be used with the standalone server for wss - because pyOpenSSL used by the server raises a fatal error if the socket - is accessed from multiple threads. - """ - - def __init__(self, request): - """Construct an instance. - - Args: - request: mod_python request. - """ - threading.Thread.__init__(self) - self._request = request - self._queue = Queue.Queue() - self.setDaemon(True) - self.start() - - def run(self): - while True: - message, condition = self._queue.get() - condition.acquire() - send_message(self._request, message) - condition.notify() - condition.release() - - def send(self, message): - """Send a message, blocking.""" - - condition = threading.Condition() - condition.acquire() - self._queue.put((message, condition)) - condition.wait() - - def send_nowait(self, message): - """Send a message, non-blocking.""" - - self._queue.put((message, threading.Condition())) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/mux.py b/module/lib/mod_pywebsocket/mux.py deleted file mode 100644 index f0bdd2461..000000000 --- a/module/lib/mod_pywebsocket/mux.py +++ /dev/null @@ -1,1636 +0,0 @@ -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file provides classes and helper functions for multiplexing extension. - -Specification: -http://tools.ietf.org/html/draft-ietf-hybi-websocket-multiplexing-06 -""" - - -import collections -import copy -import email -import email.parser -import logging -import math -import struct -import threading -import traceback - -from mod_pywebsocket import common -from mod_pywebsocket import handshake -from mod_pywebsocket import util -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_hybi import Frame -from mod_pywebsocket._stream_hybi import Stream -from mod_pywebsocket._stream_hybi import StreamOptions -from mod_pywebsocket._stream_hybi import create_binary_frame -from mod_pywebsocket._stream_hybi import create_closing_handshake_body -from mod_pywebsocket._stream_hybi import create_header -from mod_pywebsocket._stream_hybi import create_length_header -from mod_pywebsocket._stream_hybi import parse_frame -from mod_pywebsocket.handshake import hybi - - -_CONTROL_CHANNEL_ID = 0 -_DEFAULT_CHANNEL_ID = 1 - -_MUX_OPCODE_ADD_CHANNEL_REQUEST = 0 -_MUX_OPCODE_ADD_CHANNEL_RESPONSE = 1 -_MUX_OPCODE_FLOW_CONTROL = 2 -_MUX_OPCODE_DROP_CHANNEL = 3 -_MUX_OPCODE_NEW_CHANNEL_SLOT = 4 - -_MAX_CHANNEL_ID = 2 ** 29 - 1 - -_INITIAL_NUMBER_OF_CHANNEL_SLOTS = 64 -_INITIAL_QUOTA_FOR_CLIENT = 8 * 1024 - -_HANDSHAKE_ENCODING_IDENTITY = 0 -_HANDSHAKE_ENCODING_DELTA = 1 - -# We need only these status code for now. -_HTTP_BAD_RESPONSE_MESSAGES = { - common.HTTP_STATUS_BAD_REQUEST: 'Bad Request', -} - -# DropChannel reason code -# TODO(bashi): Define all reason code defined in -05 draft. -_DROP_CODE_NORMAL_CLOSURE = 1000 - -_DROP_CODE_INVALID_ENCAPSULATING_MESSAGE = 2001 -_DROP_CODE_CHANNEL_ID_TRUNCATED = 2002 -_DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED = 2003 -_DROP_CODE_UNKNOWN_MUX_OPCODE = 2004 -_DROP_CODE_INVALID_MUX_CONTROL_BLOCK = 2005 -_DROP_CODE_CHANNEL_ALREADY_EXISTS = 2006 -_DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION = 2007 - -_DROP_CODE_UNKNOWN_REQUEST_ENCODING = 3002 -_DROP_CODE_SEND_QUOTA_VIOLATION = 3005 -_DROP_CODE_ACKNOWLEDGED = 3008 - - -class MuxUnexpectedException(Exception): - """Exception in handling multiplexing extension.""" - pass - - -# Temporary -class MuxNotImplementedException(Exception): - """Raised when a flow enters unimplemented code path.""" - pass - - -class LogicalConnectionClosedException(Exception): - """Raised when logical connection is gracefully closed.""" - pass - - -class PhysicalConnectionError(Exception): - """Raised when there is a physical connection error.""" - def __init__(self, drop_code, message=''): - super(PhysicalConnectionError, self).__init__( - 'code=%d, message=%r' % (drop_code, message)) - self.drop_code = drop_code - self.message = message - - -class LogicalChannelError(Exception): - """Raised when there is a logical channel error.""" - def __init__(self, channel_id, drop_code, message=''): - super(LogicalChannelError, self).__init__( - 'channel_id=%d, code=%d, message=%r' % ( - channel_id, drop_code, message)) - self.channel_id = channel_id - self.drop_code = drop_code - self.message = message - - -def _encode_channel_id(channel_id): - if channel_id < 0: - raise ValueError('Channel id %d must not be negative' % channel_id) - - if channel_id < 2 ** 7: - return chr(channel_id) - if channel_id < 2 ** 14: - return struct.pack('!H', 0x8000 + channel_id) - if channel_id < 2 ** 21: - first = chr(0xc0 + (channel_id >> 16)) - return first + struct.pack('!H', channel_id & 0xffff) - if channel_id < 2 ** 29: - return struct.pack('!L', 0xe0000000 + channel_id) - - raise ValueError('Channel id %d is too large' % channel_id) - - -def _encode_number(number): - return create_length_header(number, False) - - -def _create_add_channel_response(channel_id, encoded_handshake, - encoding=0, rejected=False, - outer_frame_mask=False): - if encoding != 0 and encoding != 1: - raise ValueError('Invalid encoding %d' % encoding) - - first_byte = ((_MUX_OPCODE_ADD_CHANNEL_RESPONSE << 5) | - (rejected << 4) | encoding) - block = (chr(first_byte) + - _encode_channel_id(channel_id) + - _encode_number(len(encoded_handshake)) + - encoded_handshake) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_drop_channel(channel_id, code=None, message='', - outer_frame_mask=False): - if len(message) > 0 and code is None: - raise ValueError('Code must be specified if message is specified') - - first_byte = _MUX_OPCODE_DROP_CHANNEL << 5 - block = chr(first_byte) + _encode_channel_id(channel_id) - if code is None: - block += _encode_number(0) # Reason size - else: - reason = struct.pack('!H', code) + message - reason_size = _encode_number(len(reason)) - block += reason_size + reason - - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_flow_control(channel_id, replenished_quota, - outer_frame_mask=False): - first_byte = _MUX_OPCODE_FLOW_CONTROL << 5 - block = (chr(first_byte) + - _encode_channel_id(channel_id) + - _encode_number(replenished_quota)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_new_channel_slot(slots, send_quota, outer_frame_mask=False): - if slots < 0 or send_quota < 0: - raise ValueError('slots and send_quota must be non-negative.') - first_byte = _MUX_OPCODE_NEW_CHANNEL_SLOT << 5 - block = (chr(first_byte) + - _encode_number(slots) + - _encode_number(send_quota)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _create_fallback_new_channel_slot(outer_frame_mask=False): - first_byte = (_MUX_OPCODE_NEW_CHANNEL_SLOT << 5) | 1 # Set the F flag - block = (chr(first_byte) + _encode_number(0) + _encode_number(0)) - payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block - return create_binary_frame(payload, mask=outer_frame_mask) - - -def _parse_request_text(request_text): - request_line, header_lines = request_text.split('\r\n', 1) - - words = request_line.split(' ') - if len(words) != 3: - raise ValueError('Bad Request-Line syntax %r' % request_line) - [command, path, version] = words - if version != 'HTTP/1.1': - raise ValueError('Bad request version %r' % version) - - # email.parser.Parser() parses RFC 2822 (RFC 822) style headers. - # RFC 6455 refers RFC 2616 for handshake parsing, and RFC 2616 refers - # RFC 822. - headers = email.parser.Parser().parsestr(header_lines) - return command, path, version, headers - - -class _ControlBlock(object): - """A structure that holds parsing result of multiplexing control block. - Control block specific attributes will be added by _MuxFramePayloadParser. - (e.g. encoded_handshake will be added for AddChannelRequest and - AddChannelResponse) - """ - - def __init__(self, opcode): - self.opcode = opcode - - -class _MuxFramePayloadParser(object): - """A class that parses multiplexed frame payload.""" - - def __init__(self, payload): - self._data = payload - self._read_position = 0 - self._logger = util.get_class_logger(self) - - def read_channel_id(self): - """Reads channel id. - - Raises: - ValueError: when the payload doesn't contain - valid channel id. - """ - - remaining_length = len(self._data) - self._read_position - pos = self._read_position - if remaining_length == 0: - raise ValueError('Invalid channel id format') - - channel_id = ord(self._data[pos]) - channel_id_length = 1 - if channel_id & 0xe0 == 0xe0: - if remaining_length < 4: - raise ValueError('Invalid channel id format') - channel_id = struct.unpack('!L', - self._data[pos:pos+4])[0] & 0x1fffffff - channel_id_length = 4 - elif channel_id & 0xc0 == 0xc0: - if remaining_length < 3: - raise ValueError('Invalid channel id format') - channel_id = (((channel_id & 0x1f) << 16) + - struct.unpack('!H', self._data[pos+1:pos+3])[0]) - channel_id_length = 3 - elif channel_id & 0x80 == 0x80: - if remaining_length < 2: - raise ValueError('Invalid channel id format') - channel_id = struct.unpack('!H', - self._data[pos:pos+2])[0] & 0x3fff - channel_id_length = 2 - self._read_position += channel_id_length - - return channel_id - - def read_inner_frame(self): - """Reads an inner frame. - - Raises: - PhysicalConnectionError: when the inner frame is invalid. - """ - - if len(self._data) == self._read_position: - raise PhysicalConnectionError( - _DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED) - - bits = ord(self._data[self._read_position]) - self._read_position += 1 - fin = (bits & 0x80) == 0x80 - rsv1 = (bits & 0x40) == 0x40 - rsv2 = (bits & 0x20) == 0x20 - rsv3 = (bits & 0x10) == 0x10 - opcode = bits & 0xf - payload = self.remaining_data() - # Consume rest of the message which is payload data of the original - # frame. - self._read_position = len(self._data) - return fin, rsv1, rsv2, rsv3, opcode, payload - - def _read_number(self): - if self._read_position + 1 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Cannot read the first byte of number field') - - number = ord(self._data[self._read_position]) - if number & 0x80 == 0x80: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'The most significant bit of the first byte of number should ' - 'be unset') - self._read_position += 1 - pos = self._read_position - if number == 127: - if pos + 8 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Invalid number field') - self._read_position += 8 - number = struct.unpack('!Q', self._data[pos:pos+8])[0] - if number > 0x7FFFFFFFFFFFFFFF: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Encoded number >= 2^63') - if number <= 0xFFFF: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - '%d should not be encoded by 9 bytes encoding' % number) - return number - if number == 126: - if pos + 2 > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Invalid number field') - self._read_position += 2 - number = struct.unpack('!H', self._data[pos:pos+2])[0] - if number <= 125: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - '%d should not be encoded by 3 bytes encoding' % number) - return number - - def _read_size_and_contents(self): - """Reads data that consists of followings: - - the size of the contents encoded the same way as payload length - of the WebSocket Protocol with 1 bit padding at the head. - - the contents. - """ - - size = self._read_number() - pos = self._read_position - if pos + size > len(self._data): - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Cannot read %d bytes data' % size) - - self._read_position += size - return self._data[pos:pos+size] - - def _read_add_channel_request(self, first_byte, control_block): - reserved = (first_byte >> 2) & 0x7 - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - # Invalid encoding will be handled by MuxHandler. - encoding = first_byte & 0x3 - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.encoding = encoding - encoded_handshake = self._read_size_and_contents() - control_block.encoded_handshake = encoded_handshake - return control_block - - def _read_add_channel_response(self, first_byte, control_block): - reserved = (first_byte >> 2) & 0x3 - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - control_block.accepted = (first_byte >> 4) & 1 - control_block.encoding = first_byte & 0x3 - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.encoded_handshake = self._read_size_and_contents() - return control_block - - def _read_flow_control(self, first_byte, control_block): - reserved = first_byte & 0x1f - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - control_block.send_quota = self._read_number() - return control_block - - def _read_drop_channel(self, first_byte, control_block): - reserved = first_byte & 0x1f - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - - try: - control_block.channel_id = self.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) - reason = self._read_size_and_contents() - if len(reason) == 0: - control_block.drop_code = None - control_block.drop_message = '' - elif len(reason) >= 2: - control_block.drop_code = struct.unpack('!H', reason[:2])[0] - control_block.drop_message = reason[2:] - else: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received DropChannel that conains only 1-byte reason') - return control_block - - def _read_new_channel_slot(self, first_byte, control_block): - reserved = first_byte & 0x1e - if reserved != 0: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Reserved bits must be unset') - control_block.fallback = first_byte & 1 - control_block.slots = self._read_number() - control_block.send_quota = self._read_number() - return control_block - - def read_control_blocks(self): - """Reads control block(s). - - Raises: - PhysicalConnectionError: when the payload contains invalid control - block(s). - StopIteration: when no control blocks left. - """ - - while self._read_position < len(self._data): - first_byte = ord(self._data[self._read_position]) - self._read_position += 1 - opcode = (first_byte >> 5) & 0x7 - control_block = _ControlBlock(opcode=opcode) - if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: - yield self._read_add_channel_request(first_byte, control_block) - elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: - yield self._read_add_channel_response( - first_byte, control_block) - elif opcode == _MUX_OPCODE_FLOW_CONTROL: - yield self._read_flow_control(first_byte, control_block) - elif opcode == _MUX_OPCODE_DROP_CHANNEL: - yield self._read_drop_channel(first_byte, control_block) - elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: - yield self._read_new_channel_slot(first_byte, control_block) - else: - raise PhysicalConnectionError( - _DROP_CODE_UNKNOWN_MUX_OPCODE, - 'Invalid opcode %d' % opcode) - - assert self._read_position == len(self._data) - raise StopIteration - - def remaining_data(self): - """Returns remaining data.""" - - return self._data[self._read_position:] - - -class _LogicalRequest(object): - """Mimics mod_python request.""" - - def __init__(self, channel_id, command, path, protocol, headers, - connection): - """Constructs an instance. - - Args: - channel_id: the channel id of the logical channel. - command: HTTP request command. - path: HTTP request path. - headers: HTTP headers. - connection: _LogicalConnection instance. - """ - - self.channel_id = channel_id - self.method = command - self.uri = path - self.protocol = protocol - self.headers_in = headers - self.connection = connection - self.server_terminated = False - self.client_terminated = False - - def is_https(self): - """Mimics request.is_https(). Returns False because this method is - used only by old protocols (hixie and hybi00). - """ - - return False - - -class _LogicalConnection(object): - """Mimics mod_python mp_conn.""" - - # For details, see the comment of set_read_state(). - STATE_ACTIVE = 1 - STATE_GRACEFULLY_CLOSED = 2 - STATE_TERMINATED = 3 - - def __init__(self, mux_handler, channel_id): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - channel_id: channel id of this connection. - """ - - self._mux_handler = mux_handler - self._channel_id = channel_id - self._incoming_data = '' - self._write_condition = threading.Condition() - self._waiting_write_completion = False - self._read_condition = threading.Condition() - self._read_state = self.STATE_ACTIVE - - def get_local_addr(self): - """Getter to mimic mp_conn.local_addr.""" - - return self._mux_handler.physical_connection.get_local_addr() - local_addr = property(get_local_addr) - - def get_remote_addr(self): - """Getter to mimic mp_conn.remote_addr.""" - - return self._mux_handler.physical_connection.get_remote_addr() - remote_addr = property(get_remote_addr) - - def get_memorized_lines(self): - """Gets memorized lines. Not supported.""" - - raise MuxUnexpectedException('_LogicalConnection does not support ' - 'get_memorized_lines') - - def write(self, data): - """Writes data. mux_handler sends data asynchronously. The caller will - be suspended until write done. - - Args: - data: data to be written. - - Raises: - MuxUnexpectedException: when called before finishing the previous - write. - """ - - try: - self._write_condition.acquire() - if self._waiting_write_completion: - raise MuxUnexpectedException( - 'Logical connection %d is already waiting the completion ' - 'of write' % self._channel_id) - - self._waiting_write_completion = True - self._mux_handler.send_data(self._channel_id, data) - self._write_condition.wait() - finally: - self._write_condition.release() - - def write_control_data(self, data): - """Writes data via the control channel. Don't wait finishing write - because this method can be called by mux dispatcher. - - Args: - data: data to be written. - """ - - self._mux_handler.send_control_data(data) - - def notify_write_done(self): - """Called when sending data is completed.""" - - try: - self._write_condition.acquire() - if not self._waiting_write_completion: - raise MuxUnexpectedException( - 'Invalid call of notify_write_done for logical connection' - ' %d' % self._channel_id) - self._waiting_write_completion = False - self._write_condition.notify() - finally: - self._write_condition.release() - - def append_frame_data(self, frame_data): - """Appends incoming frame data. Called when mux_handler dispatches - frame data to the corresponding application. - - Args: - frame_data: incoming frame data. - """ - - self._read_condition.acquire() - self._incoming_data += frame_data - self._read_condition.notify() - self._read_condition.release() - - def read(self, length): - """Reads data. Blocks until enough data has arrived via physical - connection. - - Args: - length: length of data to be read. - Raises: - LogicalConnectionClosedException: when closing handshake for this - logical channel has been received. - ConnectionTerminatedException: when the physical connection has - closed, or an error is caused on the reader thread. - """ - - self._read_condition.acquire() - while (self._read_state == self.STATE_ACTIVE and - len(self._incoming_data) < length): - self._read_condition.wait() - - try: - if self._read_state == self.STATE_GRACEFULLY_CLOSED: - raise LogicalConnectionClosedException( - 'Logical channel %d has closed.' % self._channel_id) - elif self._read_state == self.STATE_TERMINATED: - raise ConnectionTerminatedException( - 'Receiving %d byte failed. Logical channel (%d) closed' % - (length, self._channel_id)) - - value = self._incoming_data[:length] - self._incoming_data = self._incoming_data[length:] - finally: - self._read_condition.release() - - return value - - def set_read_state(self, new_state): - """Sets the state of this connection. Called when an event for this - connection has occurred. - - Args: - new_state: state to be set. new_state must be one of followings: - - STATE_GRACEFULLY_CLOSED: when closing handshake for this - connection has been received. - - STATE_TERMINATED: when the physical connection has closed or - DropChannel of this connection has received. - """ - - self._read_condition.acquire() - self._read_state = new_state - self._read_condition.notify() - self._read_condition.release() - - -class _LogicalStream(Stream): - """Mimics the Stream class. This class interprets multiplexed WebSocket - frames. - """ - - def __init__(self, request, send_quota, receive_quota): - """Constructs an instance. - - Args: - request: _LogicalRequest instance. - send_quota: Initial send quota. - receive_quota: Initial receive quota. - """ - - # TODO(bashi): Support frame filters. - stream_options = StreamOptions() - # Physical stream is responsible for masking. - stream_options.unmask_receive = False - # Control frames can be fragmented on logical channel. - stream_options.allow_fragmented_control_frame = True - Stream.__init__(self, request, stream_options) - self._send_quota = send_quota - self._send_quota_condition = threading.Condition() - self._receive_quota = receive_quota - self._write_inner_frame_semaphore = threading.Semaphore() - - def _create_inner_frame(self, opcode, payload, end=True): - # TODO(bashi): Support extensions that use reserved bits. - first_byte = (end << 7) | opcode - return (_encode_channel_id(self._request.channel_id) + - chr(first_byte) + payload) - - def _write_inner_frame(self, opcode, payload, end=True): - payload_length = len(payload) - write_position = 0 - - try: - # An inner frame will be fragmented if there is no enough send - # quota. This semaphore ensures that fragmented inner frames are - # sent in order on the logical channel. - # Note that frames that come from other logical channels or - # multiplexing control blocks can be inserted between fragmented - # inner frames on the physical channel. - self._write_inner_frame_semaphore.acquire() - while write_position < payload_length: - try: - self._send_quota_condition.acquire() - while self._send_quota == 0: - self._logger.debug( - 'No quota. Waiting FlowControl message for %d.' % - self._request.channel_id) - self._send_quota_condition.wait() - - remaining = payload_length - write_position - write_length = min(self._send_quota, remaining) - inner_frame_end = ( - end and - (write_position + write_length == payload_length)) - - inner_frame = self._create_inner_frame( - opcode, - payload[write_position:write_position+write_length], - inner_frame_end) - frame_data = self._writer.build( - inner_frame, end=True, binary=True) - self._send_quota -= write_length - self._logger.debug('Consumed quota=%d, remaining=%d' % - (write_length, self._send_quota)) - finally: - self._send_quota_condition.release() - - # Writing data will block the worker so we need to release - # _send_quota_condition before writing. - self._logger.debug('Sending inner frame: %r' % frame_data) - self._request.connection.write(frame_data) - write_position += write_length - - opcode = common.OPCODE_CONTINUATION - - except ValueError, e: - raise BadOperationException(e) - finally: - self._write_inner_frame_semaphore.release() - - def replenish_send_quota(self, send_quota): - """Replenish send quota.""" - - self._send_quota_condition.acquire() - self._send_quota += send_quota - self._logger.debug('Replenished send quota for channel id %d: %d' % - (self._request.channel_id, self._send_quota)) - self._send_quota_condition.notify() - self._send_quota_condition.release() - - def consume_receive_quota(self, amount): - """Consumes receive quota. Returns False on failure.""" - - if self._receive_quota < amount: - self._logger.debug('Violate quota on channel id %d: %d < %d' % - (self._request.channel_id, - self._receive_quota, amount)) - return False - self._receive_quota -= amount - return True - - def send_message(self, message, end=True, binary=False): - """Override Stream.send_message.""" - - if self._request.server_terminated: - raise BadOperationException( - 'Requested send_message after sending out a closing handshake') - - if binary and isinstance(message, unicode): - raise BadOperationException( - 'Message for binary frame must be instance of str') - - if binary: - opcode = common.OPCODE_BINARY - else: - opcode = common.OPCODE_TEXT - message = message.encode('utf-8') - - self._write_inner_frame(opcode, message, end) - - def _receive_frame(self): - """Overrides Stream._receive_frame. - - In addition to call Stream._receive_frame, this method adds the amount - of payload to receiving quota and sends FlowControl to the client. - We need to do it here because Stream.receive_message() handles - control frames internally. - """ - - opcode, payload, fin, rsv1, rsv2, rsv3 = Stream._receive_frame(self) - amount = len(payload) - self._receive_quota += amount - frame_data = _create_flow_control(self._request.channel_id, - amount) - self._logger.debug('Sending flow control for %d, replenished=%d' % - (self._request.channel_id, amount)) - self._request.connection.write_control_data(frame_data) - return opcode, payload, fin, rsv1, rsv2, rsv3 - - def receive_message(self): - """Overrides Stream.receive_message.""" - - # Just call Stream.receive_message(), but catch - # LogicalConnectionClosedException, which is raised when the logical - # connection has closed gracefully. - try: - return Stream.receive_message(self) - except LogicalConnectionClosedException, e: - self._logger.debug('%s', e) - return None - - def _send_closing_handshake(self, code, reason): - """Overrides Stream._send_closing_handshake.""" - - body = create_closing_handshake_body(code, reason) - self._logger.debug('Sending closing handshake for %d: (%r, %r)' % - (self._request.channel_id, code, reason)) - self._write_inner_frame(common.OPCODE_CLOSE, body, end=True) - - self._request.server_terminated = True - - def send_ping(self, body=''): - """Overrides Stream.send_ping""" - - self._logger.debug('Sending ping on logical channel %d: %r' % - (self._request.channel_id, body)) - self._write_inner_frame(common.OPCODE_PING, body, end=True) - - self._ping_queue.append(body) - - def _send_pong(self, body): - """Overrides Stream._send_pong""" - - self._logger.debug('Sending pong on logical channel %d: %r' % - (self._request.channel_id, body)) - self._write_inner_frame(common.OPCODE_PONG, body, end=True) - - def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): - """Overrides Stream.close_connection.""" - - # TODO(bashi): Implement - self._logger.debug('Closing logical connection %d' % - self._request.channel_id) - self._request.server_terminated = True - - def _drain_received_data(self): - """Overrides Stream._drain_received_data. Nothing need to be done for - logical channel. - """ - - pass - - -class _OutgoingData(object): - """A structure that holds data to be sent via physical connection and - origin of the data. - """ - - def __init__(self, channel_id, data): - self.channel_id = channel_id - self.data = data - - -class _PhysicalConnectionWriter(threading.Thread): - """A thread that is responsible for writing data to physical connection. - - TODO(bashi): Make sure there is no thread-safety problem when the reader - thread reads data from the same socket at a time. - """ - - def __init__(self, mux_handler): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self.setDaemon(True) - self._stop_requested = False - self._deque = collections.deque() - self._deque_condition = threading.Condition() - - def put_outgoing_data(self, data): - """Puts outgoing data. - - Args: - data: _OutgoingData instance. - - Raises: - BadOperationException: when the thread has been requested to - terminate. - """ - - try: - self._deque_condition.acquire() - if self._stop_requested: - raise BadOperationException('Cannot write data anymore') - - self._deque.append(data) - self._deque_condition.notify() - finally: - self._deque_condition.release() - - def _write_data(self, outgoing_data): - try: - self._mux_handler.physical_connection.write(outgoing_data.data) - except Exception, e: - util.prepend_message_to_exception( - 'Failed to send message to %r: ' % - (self._mux_handler.physical_connection.remote_addr,), e) - raise - - # TODO(bashi): It would be better to block the thread that sends - # control data as well. - if outgoing_data.channel_id != _CONTROL_CHANNEL_ID: - self._mux_handler.notify_write_done(outgoing_data.channel_id) - - def run(self): - self._deque_condition.acquire() - while not self._stop_requested: - if len(self._deque) == 0: - self._deque_condition.wait() - continue - - outgoing_data = self._deque.popleft() - self._deque_condition.release() - self._write_data(outgoing_data) - self._deque_condition.acquire() - - # Flush deque - try: - while len(self._deque) > 0: - outgoing_data = self._deque.popleft() - self._write_data(outgoing_data) - finally: - self._deque_condition.release() - - def stop(self): - """Stops the writer thread.""" - - self._deque_condition.acquire() - self._stop_requested = True - self._deque_condition.notify() - self._deque_condition.release() - - -class _PhysicalConnectionReader(threading.Thread): - """A thread that is responsible for reading data from physical connection. - """ - - def __init__(self, mux_handler): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self.setDaemon(True) - - def run(self): - while True: - try: - physical_stream = self._mux_handler.physical_stream - message = physical_stream.receive_message() - if message is None: - break - # Below happens only when a data message is received. - opcode = physical_stream.get_last_received_opcode() - if opcode != common.OPCODE_BINARY: - self._mux_handler.fail_physical_connection( - _DROP_CODE_INVALID_ENCAPSULATING_MESSAGE, - 'Received a text message on physical connection') - break - - except ConnectionTerminatedException, e: - self._logger.debug('%s', e) - break - - try: - self._mux_handler.dispatch_message(message) - except PhysicalConnectionError, e: - self._mux_handler.fail_physical_connection( - e.drop_code, e.message) - break - except LogicalChannelError, e: - self._mux_handler.fail_logical_channel( - e.channel_id, e.drop_code, e.message) - except Exception, e: - self._logger.debug(traceback.format_exc()) - break - - self._mux_handler.notify_reader_done() - - -class _Worker(threading.Thread): - """A thread that is responsible for running the corresponding application - handler. - """ - - def __init__(self, mux_handler, request): - """Constructs an instance. - - Args: - mux_handler: _MuxHandler instance. - request: _LogicalRequest instance. - """ - - threading.Thread.__init__(self) - self._logger = util.get_class_logger(self) - self._mux_handler = mux_handler - self._request = request - self.setDaemon(True) - - def run(self): - self._logger.debug('Logical channel worker started. (id=%d)' % - self._request.channel_id) - try: - # Non-critical exceptions will be handled by dispatcher. - self._mux_handler.dispatcher.transfer_data(self._request) - finally: - self._mux_handler.notify_worker_done(self._request.channel_id) - - -class _MuxHandshaker(hybi.Handshaker): - """Opening handshake processor for multiplexing.""" - - _DUMMY_WEBSOCKET_KEY = 'dGhlIHNhbXBsZSBub25jZQ==' - - def __init__(self, request, dispatcher, send_quota, receive_quota): - """Constructs an instance. - Args: - request: _LogicalRequest instance. - dispatcher: Dispatcher instance (dispatch.Dispatcher). - send_quota: Initial send quota. - receive_quota: Initial receive quota. - """ - - hybi.Handshaker.__init__(self, request, dispatcher) - self._send_quota = send_quota - self._receive_quota = receive_quota - - # Append headers which should not be included in handshake field of - # AddChannelRequest. - # TODO(bashi): Make sure whether we should raise exception when - # these headers are included already. - request.headers_in[common.UPGRADE_HEADER] = ( - common.WEBSOCKET_UPGRADE_TYPE) - request.headers_in[common.CONNECTION_HEADER] = ( - common.UPGRADE_CONNECTION_TYPE) - request.headers_in[common.SEC_WEBSOCKET_VERSION_HEADER] = ( - str(common.VERSION_HYBI_LATEST)) - request.headers_in[common.SEC_WEBSOCKET_KEY_HEADER] = ( - self._DUMMY_WEBSOCKET_KEY) - - def _create_stream(self, stream_options): - """Override hybi.Handshaker._create_stream.""" - - self._logger.debug('Creating logical stream for %d' % - self._request.channel_id) - return _LogicalStream(self._request, self._send_quota, - self._receive_quota) - - def _create_handshake_response(self, accept): - """Override hybi._create_handshake_response.""" - - response = [] - - response.append('HTTP/1.1 101 Switching Protocols\r\n') - - # Upgrade, Connection and Sec-WebSocket-Accept should be excluded. - if self._request.ws_protocol is not None: - response.append('%s: %s\r\n' % ( - common.SEC_WEBSOCKET_PROTOCOL_HEADER, - self._request.ws_protocol)) - if (self._request.ws_extensions is not None and - len(self._request.ws_extensions) != 0): - response.append('%s: %s\r\n' % ( - common.SEC_WEBSOCKET_EXTENSIONS_HEADER, - common.format_extensions(self._request.ws_extensions))) - response.append('\r\n') - - return ''.join(response) - - def _send_handshake(self, accept): - """Override hybi.Handshaker._send_handshake.""" - - # Don't send handshake response for the default channel - if self._request.channel_id == _DEFAULT_CHANNEL_ID: - return - - handshake_response = self._create_handshake_response(accept) - frame_data = _create_add_channel_response( - self._request.channel_id, - handshake_response) - self._logger.debug('Sending handshake response for %d: %r' % - (self._request.channel_id, frame_data)) - self._request.connection.write_control_data(frame_data) - - -class _LogicalChannelData(object): - """A structure that holds information about logical channel. - """ - - def __init__(self, request, worker): - self.request = request - self.worker = worker - self.drop_code = _DROP_CODE_NORMAL_CLOSURE - self.drop_message = '' - - -class _HandshakeDeltaBase(object): - """A class that holds information for delta-encoded handshake.""" - - def __init__(self, headers): - self._headers = headers - - def create_headers(self, delta=None): - """Creates request headers for an AddChannelRequest that has - delta-encoded handshake. - - Args: - delta: headers should be overridden. - """ - - headers = copy.copy(self._headers) - if delta: - for key, value in delta.items(): - # The spec requires that a header with an empty value is - # removed from the delta base. - if len(value) == 0 and headers.has_key(key): - del headers[key] - else: - headers[key] = value - # TODO(bashi): Support extensions - headers['Sec-WebSocket-Extensions'] = '' - return headers - - -class _MuxHandler(object): - """Multiplexing handler. When a handler starts, it launches three - threads; the reader thread, the writer thread, and a worker thread. - - The reader thread reads data from the physical stream, i.e., the - ws_stream object of the underlying websocket connection. The reader - thread interprets multiplexed frames and dispatches them to logical - channels. Methods of this class are mostly called by the reader thread. - - The writer thread sends multiplexed frames which are created by - logical channels via the physical connection. - - The worker thread launched at the starting point handles the - "Implicitly Opened Connection". If multiplexing handler receives - an AddChannelRequest and accepts it, the handler will launch a new worker - thread and dispatch the request to it. - """ - - def __init__(self, request, dispatcher): - """Constructs an instance. - - Args: - request: mod_python request of the physical connection. - dispatcher: Dispatcher instance (dispatch.Dispatcher). - """ - - self.original_request = request - self.dispatcher = dispatcher - self.physical_connection = request.connection - self.physical_stream = request.ws_stream - self._logger = util.get_class_logger(self) - self._logical_channels = {} - self._logical_channels_condition = threading.Condition() - # Holds client's initial quota - self._channel_slots = collections.deque() - self._handshake_base = None - self._worker_done_notify_received = False - self._reader = None - self._writer = None - - def start(self): - """Starts the handler. - - Raises: - MuxUnexpectedException: when the handler already started, or when - opening handshake of the default channel fails. - """ - - if self._reader or self._writer: - raise MuxUnexpectedException('MuxHandler already started') - - self._reader = _PhysicalConnectionReader(self) - self._writer = _PhysicalConnectionWriter(self) - self._reader.start() - self._writer.start() - - # Create "Implicitly Opened Connection". - logical_connection = _LogicalConnection(self, _DEFAULT_CHANNEL_ID) - self._handshake_base = _HandshakeDeltaBase( - self.original_request.headers_in) - logical_request = _LogicalRequest( - _DEFAULT_CHANNEL_ID, - self.original_request.method, - self.original_request.uri, - self.original_request.protocol, - self._handshake_base.create_headers(), - logical_connection) - # Client's send quota for the implicitly opened connection is zero, - # but we will send FlowControl later so set the initial quota to - # _INITIAL_QUOTA_FOR_CLIENT. - self._channel_slots.append(_INITIAL_QUOTA_FOR_CLIENT) - if not self._do_handshake_for_logical_request( - logical_request, send_quota=self.original_request.mux_quota): - raise MuxUnexpectedException( - 'Failed handshake on the default channel id') - self._add_logical_channel(logical_request) - - # Send FlowControl for the implicitly opened connection. - frame_data = _create_flow_control(_DEFAULT_CHANNEL_ID, - _INITIAL_QUOTA_FOR_CLIENT) - logical_request.connection.write_control_data(frame_data) - - def add_channel_slots(self, slots, send_quota): - """Adds channel slots. - - Args: - slots: number of slots to be added. - send_quota: initial send quota for slots. - """ - - self._channel_slots.extend([send_quota] * slots) - # Send NewChannelSlot to client. - frame_data = _create_new_channel_slot(slots, send_quota) - self.send_control_data(frame_data) - - def wait_until_done(self, timeout=None): - """Waits until all workers are done. Returns False when timeout has - occurred. Returns True on success. - - Args: - timeout: timeout in sec. - """ - - self._logical_channels_condition.acquire() - try: - while len(self._logical_channels) > 0: - self._logger.debug('Waiting workers(%d)...' % - len(self._logical_channels)) - self._worker_done_notify_received = False - self._logical_channels_condition.wait(timeout) - if not self._worker_done_notify_received: - self._logger.debug('Waiting worker(s) timed out') - return False - - finally: - self._logical_channels_condition.release() - - # Flush pending outgoing data - self._writer.stop() - self._writer.join() - - return True - - def notify_write_done(self, channel_id): - """Called by the writer thread when a write operation has done. - - Args: - channel_id: objective channel id. - """ - - try: - self._logical_channels_condition.acquire() - if channel_id in self._logical_channels: - channel_data = self._logical_channels[channel_id] - channel_data.request.connection.notify_write_done() - else: - self._logger.debug('Seems that logical channel for %d has gone' - % channel_id) - finally: - self._logical_channels_condition.release() - - def send_control_data(self, data): - """Sends data via the control channel. - - Args: - data: data to be sent. - """ - - self._writer.put_outgoing_data(_OutgoingData( - channel_id=_CONTROL_CHANNEL_ID, data=data)) - - def send_data(self, channel_id, data): - """Sends data via given logical channel. This method is called by - worker threads. - - Args: - channel_id: objective channel id. - data: data to be sent. - """ - - self._writer.put_outgoing_data(_OutgoingData( - channel_id=channel_id, data=data)) - - def _send_drop_channel(self, channel_id, code=None, message=''): - frame_data = _create_drop_channel(channel_id, code, message) - self._logger.debug( - 'Sending drop channel for channel id %d' % channel_id) - self.send_control_data(frame_data) - - def _send_error_add_channel_response(self, channel_id, status=None): - if status is None: - status = common.HTTP_STATUS_BAD_REQUEST - - if status in _HTTP_BAD_RESPONSE_MESSAGES: - message = _HTTP_BAD_RESPONSE_MESSAGES[status] - else: - self._logger.debug('Response message for %d is not found' % status) - message = '???' - - response = 'HTTP/1.1 %d %s\r\n\r\n' % (status, message) - frame_data = _create_add_channel_response(channel_id, - encoded_handshake=response, - encoding=0, rejected=True) - self.send_control_data(frame_data) - - def _create_logical_request(self, block): - if block.channel_id == _CONTROL_CHANNEL_ID: - # TODO(bashi): Raise PhysicalConnectionError with code 2006 - # instead of MuxUnexpectedException. - raise MuxUnexpectedException( - 'Received the control channel id (0) as objective channel ' - 'id for AddChannel') - - if block.encoding > _HANDSHAKE_ENCODING_DELTA: - raise PhysicalConnectionError( - _DROP_CODE_UNKNOWN_REQUEST_ENCODING) - - method, path, version, headers = _parse_request_text( - block.encoded_handshake) - if block.encoding == _HANDSHAKE_ENCODING_DELTA: - headers = self._handshake_base.create_headers(headers) - - connection = _LogicalConnection(self, block.channel_id) - request = _LogicalRequest(block.channel_id, method, path, version, - headers, connection) - return request - - def _do_handshake_for_logical_request(self, request, send_quota=0): - try: - receive_quota = self._channel_slots.popleft() - except IndexError: - raise LogicalChannelError( - request.channel_id, _DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION) - - handshaker = _MuxHandshaker(request, self.dispatcher, - send_quota, receive_quota) - try: - handshaker.do_handshake() - except handshake.VersionException, e: - self._logger.info('%s', e) - self._send_error_add_channel_response( - request.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - return False - except handshake.HandshakeException, e: - # TODO(bashi): Should we _Fail the Logical Channel_ with 3001 - # instead? - self._logger.info('%s', e) - self._send_error_add_channel_response(request.channel_id, - status=e.status) - return False - except handshake.AbortedByUserException, e: - self._logger.info('%s', e) - self._send_error_add_channel_response(request.channel_id) - return False - - return True - - def _add_logical_channel(self, logical_request): - try: - self._logical_channels_condition.acquire() - if logical_request.channel_id in self._logical_channels: - self._logger.debug('Channel id %d already exists' % - logical_request.channel_id) - raise PhysicalConnectionError( - _DROP_CODE_CHANNEL_ALREADY_EXISTS, - 'Channel id %d already exists' % - logical_request.channel_id) - worker = _Worker(self, logical_request) - channel_data = _LogicalChannelData(logical_request, worker) - self._logical_channels[logical_request.channel_id] = channel_data - worker.start() - finally: - self._logical_channels_condition.release() - - def _process_add_channel_request(self, block): - try: - logical_request = self._create_logical_request(block) - except ValueError, e: - self._logger.debug('Failed to create logical request: %r' % e) - self._send_error_add_channel_response( - block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - return - if self._do_handshake_for_logical_request(logical_request): - if block.encoding == _HANDSHAKE_ENCODING_IDENTITY: - # Update handshake base. - # TODO(bashi): Make sure this is the right place to update - # handshake base. - self._handshake_base = _HandshakeDeltaBase( - logical_request.headers_in) - self._add_logical_channel(logical_request) - else: - self._send_error_add_channel_response( - block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) - - def _process_flow_control(self, block): - try: - self._logical_channels_condition.acquire() - if not block.channel_id in self._logical_channels: - return - channel_data = self._logical_channels[block.channel_id] - channel_data.request.ws_stream.replenish_send_quota( - block.send_quota) - finally: - self._logical_channels_condition.release() - - def _process_drop_channel(self, block): - self._logger.debug( - 'DropChannel received for %d: code=%r, reason=%r' % - (block.channel_id, block.drop_code, block.drop_message)) - try: - self._logical_channels_condition.acquire() - if not block.channel_id in self._logical_channels: - return - channel_data = self._logical_channels[block.channel_id] - channel_data.drop_code = _DROP_CODE_ACKNOWLEDGED - # Close the logical channel - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - finally: - self._logical_channels_condition.release() - - def _process_control_blocks(self, parser): - for control_block in parser.read_control_blocks(): - opcode = control_block.opcode - self._logger.debug('control block received, opcode: %d' % opcode) - if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: - self._process_add_channel_request(control_block) - elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received AddChannelResponse') - elif opcode == _MUX_OPCODE_FLOW_CONTROL: - self._process_flow_control(control_block) - elif opcode == _MUX_OPCODE_DROP_CHANNEL: - self._process_drop_channel(control_block) - elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: - raise PhysicalConnectionError( - _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, - 'Received NewChannelSlot') - else: - raise MuxUnexpectedException( - 'Unexpected opcode %r' % opcode) - - def _process_logical_frame(self, channel_id, parser): - self._logger.debug('Received a frame. channel id=%d' % channel_id) - try: - self._logical_channels_condition.acquire() - if not channel_id in self._logical_channels: - # We must ignore the message for an inactive channel. - return - channel_data = self._logical_channels[channel_id] - fin, rsv1, rsv2, rsv3, opcode, payload = parser.read_inner_frame() - if not channel_data.request.ws_stream.consume_receive_quota( - len(payload)): - # The client violates quota. Close logical channel. - raise LogicalChannelError( - channel_id, _DROP_CODE_SEND_QUOTA_VIOLATION) - header = create_header(opcode, len(payload), fin, rsv1, rsv2, rsv3, - mask=False) - frame_data = header + payload - channel_data.request.connection.append_frame_data(frame_data) - finally: - self._logical_channels_condition.release() - - def dispatch_message(self, message): - """Dispatches message. The reader thread calls this method. - - Args: - message: a message that contains encapsulated frame. - Raises: - PhysicalConnectionError: if the message contains physical - connection level errors. - LogicalChannelError: if the message contains logical channel - level errors. - """ - - parser = _MuxFramePayloadParser(message) - try: - channel_id = parser.read_channel_id() - except ValueError, e: - raise PhysicalConnectionError(_DROP_CODE_CHANNEL_ID_TRUNCATED) - if channel_id == _CONTROL_CHANNEL_ID: - self._process_control_blocks(parser) - else: - self._process_logical_frame(channel_id, parser) - - def notify_worker_done(self, channel_id): - """Called when a worker has finished. - - Args: - channel_id: channel id corresponded with the worker. - """ - - self._logger.debug('Worker for channel id %d terminated' % channel_id) - try: - self._logical_channels_condition.acquire() - if not channel_id in self._logical_channels: - raise MuxUnexpectedException( - 'Channel id %d not found' % channel_id) - channel_data = self._logical_channels.pop(channel_id) - finally: - self._worker_done_notify_received = True - self._logical_channels_condition.notify() - self._logical_channels_condition.release() - - if not channel_data.request.server_terminated: - self._send_drop_channel( - channel_id, code=channel_data.drop_code, - message=channel_data.drop_message) - - def notify_reader_done(self): - """This method is called by the reader thread when the reader has - finished. - """ - - # Terminate all logical connections - self._logger.debug('termiating all logical connections...') - self._logical_channels_condition.acquire() - for channel_data in self._logical_channels.values(): - try: - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - except Exception: - pass - self._logical_channels_condition.release() - - def fail_physical_connection(self, code, message): - """Fail the physical connection. - - Args: - code: drop reason code. - message: drop message. - """ - - self._logger.debug('Failing the physical connection...') - self._send_drop_channel(_CONTROL_CHANNEL_ID, code, message) - self.physical_stream.close_connection( - common.STATUS_INTERNAL_ENDPOINT_ERROR) - - def fail_logical_channel(self, channel_id, code, message): - """Fail a logical channel. - - Args: - channel_id: channel id. - code: drop reason code. - message: drop message. - """ - - self._logger.debug('Failing logical channel %d...' % channel_id) - try: - self._logical_channels_condition.acquire() - if channel_id in self._logical_channels: - channel_data = self._logical_channels[channel_id] - # Close the logical channel. notify_worker_done() will be - # called later and it will send DropChannel. - channel_data.drop_code = code - channel_data.drop_message = message - channel_data.request.connection.set_read_state( - _LogicalConnection.STATE_TERMINATED) - else: - self._send_drop_channel(channel_id, code, message) - finally: - self._logical_channels_condition.release() - - -def use_mux(request): - return hasattr(request, 'mux') and request.mux - - -def start(request, dispatcher): - mux_handler = _MuxHandler(request, dispatcher) - mux_handler.start() - - mux_handler.add_channel_slots(_INITIAL_NUMBER_OF_CHANNEL_SLOTS, - _INITIAL_QUOTA_FOR_CLIENT) - - mux_handler.wait_until_done() - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/standalone.py b/module/lib/mod_pywebsocket/standalone.py deleted file mode 100755 index 07a33d9c9..000000000 --- a/module/lib/mod_pywebsocket/standalone.py +++ /dev/null @@ -1,998 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""Standalone WebSocket server. - -Use this file to launch pywebsocket without Apache HTTP Server. - - -BASIC USAGE - -Go to the src directory and run - - $ python mod_pywebsocket/standalone.py [-p <ws_port>] - [-w <websock_handlers>] - [-d <document_root>] - -<ws_port> is the port number to use for ws:// connection. - -<document_root> is the path to the root directory of HTML files. - -<websock_handlers> is the path to the root directory of WebSocket handlers. -If not specified, <document_root> will be used. See __init__.py (or -run $ pydoc mod_pywebsocket) for how to write WebSocket handlers. - -For more detail and other options, run - - $ python mod_pywebsocket/standalone.py --help - -or see _build_option_parser method below. - -For trouble shooting, adding "--log_level debug" might help you. - - -TRY DEMO - -Go to the src directory and run - - $ python standalone.py -d example - -to launch pywebsocket with the sample handler and html on port 80. Open -http://localhost/console.html, click the connect button, type something into -the text box next to the send button and click the send button. If everything -is working, you'll see the message you typed echoed by the server. - - -SUPPORTING TLS - -To support TLS, run standalone.py with -t, -k, and -c options. - - -SUPPORTING CLIENT AUTHENTICATION - -To support client authentication with TLS, run standalone.py with -t, -k, -c, -and --tls-client-auth, and --tls-client-ca options. - -E.g., $./standalone.py -d ../example -p 10443 -t -c ../test/cert/cert.pem -k -../test/cert/key.pem --tls-client-auth --tls-client-ca=../test/cert/cacert.pem - - -CONFIGURATION FILE - -You can also write a configuration file and use it by specifying the path to -the configuration file by --config option. Please write a configuration file -following the documentation of the Python ConfigParser library. Name of each -entry must be the long version argument name. E.g. to set log level to debug, -add the following line: - -log_level=debug - -For options which doesn't take value, please add some fake value. E.g. for ---tls option, add the following line: - -tls=True - -Note that tls will be enabled even if you write tls=False as the value part is -fake. - -When both a command line argument and a configuration file entry are set for -the same configuration item, the command line value will override one in the -configuration file. - - -THREADING - -This server is derived from SocketServer.ThreadingMixIn. Hence a thread is -used for each request. - - -SECURITY WARNING - -This uses CGIHTTPServer and CGIHTTPServer is not secure. -It may execute arbitrary Python code or external programs. It should not be -used outside a firewall. -""" - -import BaseHTTPServer -import CGIHTTPServer -import SimpleHTTPServer -import SocketServer -import ConfigParser -import base64 -import httplib -import logging -import logging.handlers -import optparse -import os -import re -import select -import socket -import sys -import threading -import time - -_HAS_SSL = False -_HAS_OPEN_SSL = False -try: - import ssl - _HAS_SSL = True -except ImportError: - try: - import OpenSSL.SSL - _HAS_OPEN_SSL = True - except ImportError: - pass - -from mod_pywebsocket import common -from mod_pywebsocket import dispatch -from mod_pywebsocket import handshake -from mod_pywebsocket import http_header_util -from mod_pywebsocket import memorizingfile -from mod_pywebsocket import util - - -_DEFAULT_LOG_MAX_BYTES = 1024 * 256 -_DEFAULT_LOG_BACKUP_COUNT = 5 - -_DEFAULT_REQUEST_QUEUE_SIZE = 128 - -# 1024 is practically large enough to contain WebSocket handshake lines. -_MAX_MEMORIZED_LINES = 1024 - - -class _StandaloneConnection(object): - """Mimic mod_python mp_conn.""" - - def __init__(self, request_handler): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._request_handler = request_handler - - def get_local_addr(self): - """Getter to mimic mp_conn.local_addr.""" - - return (self._request_handler.server.server_name, - self._request_handler.server.server_port) - local_addr = property(get_local_addr) - - def get_remote_addr(self): - """Getter to mimic mp_conn.remote_addr. - - Setting the property in __init__ won't work because the request - handler is not initialized yet there.""" - - return self._request_handler.client_address - remote_addr = property(get_remote_addr) - - def write(self, data): - """Mimic mp_conn.write().""" - - return self._request_handler.wfile.write(data) - - def read(self, length): - """Mimic mp_conn.read().""" - - return self._request_handler.rfile.read(length) - - def get_memorized_lines(self): - """Get memorized lines.""" - - return self._request_handler.rfile.get_memorized_lines() - - -class _StandaloneRequest(object): - """Mimic mod_python request.""" - - def __init__(self, request_handler, use_tls): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._logger = util.get_class_logger(self) - - self._request_handler = request_handler - self.connection = _StandaloneConnection(request_handler) - self._use_tls = use_tls - self.headers_in = request_handler.headers - - def get_uri(self): - """Getter to mimic request.uri.""" - - return self._request_handler.path - uri = property(get_uri) - - def get_method(self): - """Getter to mimic request.method.""" - - return self._request_handler.command - method = property(get_method) - - def get_protocol(self): - """Getter to mimic request.protocol.""" - - return self._request_handler.request_version - protocol = property(get_protocol) - - def is_https(self): - """Mimic request.is_https().""" - - return self._use_tls - - def _drain_received_data(self): - """Don't use this method from WebSocket handler. Drains unread data - in the receive buffer. - """ - - raw_socket = self._request_handler.connection - drained_data = util.drain_received_data(raw_socket) - - if drained_data: - self._logger.debug( - 'Drained data following close frame: %r', drained_data) - - -class _StandaloneSSLConnection(object): - """A wrapper class for OpenSSL.SSL.Connection to provide makefile method - which is not supported by the class. - """ - - def __init__(self, connection): - self._connection = connection - - def __getattribute__(self, name): - if name in ('_connection', 'makefile'): - return object.__getattribute__(self, name) - return self._connection.__getattribute__(name) - - def __setattr__(self, name, value): - if name in ('_connection', 'makefile'): - return object.__setattr__(self, name, value) - return self._connection.__setattr__(name, value) - - def makefile(self, mode='r', bufsize=-1): - return socket._fileobject(self._connection, mode, bufsize) - - -def _alias_handlers(dispatcher, websock_handlers_map_file): - """Set aliases specified in websock_handler_map_file in dispatcher. - - Args: - dispatcher: dispatch.Dispatcher instance - websock_handler_map_file: alias map file - """ - - fp = open(websock_handlers_map_file) - try: - for line in fp: - if line[0] == '#' or line.isspace(): - continue - m = re.match('(\S+)\s+(\S+)', line) - if not m: - logging.warning('Wrong format in map file:' + line) - continue - try: - dispatcher.add_resource_path_alias( - m.group(1), m.group(2)) - except dispatch.DispatchException, e: - logging.error(str(e)) - finally: - fp.close() - - -class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): - """HTTPServer specialized for WebSocket.""" - - # Overrides SocketServer.ThreadingMixIn.daemon_threads - daemon_threads = True - # Overrides BaseHTTPServer.HTTPServer.allow_reuse_address - allow_reuse_address = True - - def __init__(self, options): - """Override SocketServer.TCPServer.__init__ to set SSL enabled - socket object to self.socket before server_bind and server_activate, - if necessary. - """ - - # Share a Dispatcher among request handlers to save time for - # instantiation. Dispatcher can be shared because it is thread-safe. - options.dispatcher = dispatch.Dispatcher( - options.websock_handlers, - options.scan_dir, - options.allow_handlers_outside_root_dir) - if options.websock_handlers_map_file: - _alias_handlers(options.dispatcher, - options.websock_handlers_map_file) - warnings = options.dispatcher.source_warnings() - if warnings: - for warning in warnings: - logging.warning('mod_pywebsocket: %s' % warning) - - self._logger = util.get_class_logger(self) - - self.request_queue_size = options.request_queue_size - self.__ws_is_shut_down = threading.Event() - self.__ws_serving = False - - SocketServer.BaseServer.__init__( - self, (options.server_host, options.port), WebSocketRequestHandler) - - # Expose the options object to allow handler objects access it. We name - # it with websocket_ prefix to avoid conflict. - self.websocket_server_options = options - - self._create_sockets() - self.server_bind() - self.server_activate() - - def _create_sockets(self): - self.server_name, self.server_port = self.server_address - self._sockets = [] - if not self.server_name: - # On platforms that doesn't support IPv6, the first bind fails. - # On platforms that supports IPv6 - # - If it binds both IPv4 and IPv6 on call with AF_INET6, the - # first bind succeeds and the second fails (we'll see 'Address - # already in use' error). - # - If it binds only IPv6 on call with AF_INET6, both call are - # expected to succeed to listen both protocol. - addrinfo_array = [ - (socket.AF_INET6, socket.SOCK_STREAM, '', '', ''), - (socket.AF_INET, socket.SOCK_STREAM, '', '', '')] - else: - addrinfo_array = socket.getaddrinfo(self.server_name, - self.server_port, - socket.AF_UNSPEC, - socket.SOCK_STREAM, - socket.IPPROTO_TCP) - for addrinfo in addrinfo_array: - self._logger.info('Create socket on: %r', addrinfo) - family, socktype, proto, canonname, sockaddr = addrinfo - try: - socket_ = socket.socket(family, socktype) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - continue - if self.websocket_server_options.use_tls: - if _HAS_SSL: - if self.websocket_server_options.tls_client_auth: - client_cert_ = ssl.CERT_REQUIRED - else: - client_cert_ = ssl.CERT_NONE - socket_ = ssl.wrap_socket(socket_, - keyfile=self.websocket_server_options.private_key, - certfile=self.websocket_server_options.certificate, - ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.websocket_server_options.tls_client_ca, - cert_reqs=client_cert_) - if _HAS_OPEN_SSL: - ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - ctx.use_privatekey_file( - self.websocket_server_options.private_key) - ctx.use_certificate_file( - self.websocket_server_options.certificate) - socket_ = OpenSSL.SSL.Connection(ctx, socket_) - self._sockets.append((socket_, addrinfo)) - - def server_bind(self): - """Override SocketServer.TCPServer.server_bind to enable multiple - sockets bind. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Bind on: %r', addrinfo) - if self.allow_reuse_address: - socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - socket_.bind(self.server_address) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - if self.server_address[1] == 0: - # The operating system assigns the actual port number for port - # number 0. This case, the second and later sockets should use - # the same port number. Also self.server_port is rewritten - # because it is exported, and will be used by external code. - self.server_address = ( - self.server_name, socket_.getsockname()[1]) - self.server_port = self.server_address[1] - self._logger.info('Port %r is assigned', self.server_port) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - def server_activate(self): - """Override SocketServer.TCPServer.server_activate to enable multiple - sockets listen. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Listen on: %r', addrinfo) - try: - socket_.listen(self.request_queue_size) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - if len(self._sockets) == 0: - self._logger.critical( - 'No sockets activated. Use info log level to see the reason.') - - def server_close(self): - """Override SocketServer.TCPServer.server_close to enable multiple - sockets close. - """ - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Close on: %r', addrinfo) - socket_.close() - - def fileno(self): - """Override SocketServer.TCPServer.fileno.""" - - self._logger.critical('Not supported: fileno') - return self._sockets[0][0].fileno() - - def handle_error(self, rquest, client_address): - """Override SocketServer.handle_error.""" - - self._logger.error( - 'Exception in processing request from: %r\n%s', - client_address, - util.get_stack_trace()) - # Note: client_address is a tuple. - - def get_request(self): - """Override TCPServer.get_request to wrap OpenSSL.SSL.Connection - object with _StandaloneSSLConnection to provide makefile method. We - cannot substitute OpenSSL.SSL.Connection.makefile since it's readonly - attribute. - """ - - accepted_socket, client_address = self.socket.accept() - if self.websocket_server_options.use_tls and _HAS_OPEN_SSL: - accepted_socket = _StandaloneSSLConnection(accepted_socket) - return accepted_socket, client_address - - def serve_forever(self, poll_interval=0.5): - """Override SocketServer.BaseServer.serve_forever.""" - - self.__ws_serving = True - self.__ws_is_shut_down.clear() - handle_request = self.handle_request - if hasattr(self, '_handle_request_noblock'): - handle_request = self._handle_request_noblock - else: - self._logger.warning('Fallback to blocking request handler') - try: - while self.__ws_serving: - r, w, e = select.select( - [socket_[0] for socket_ in self._sockets], - [], [], poll_interval) - for socket_ in r: - self.socket = socket_ - handle_request() - self.socket = None - finally: - self.__ws_is_shut_down.set() - - def shutdown(self): - """Override SocketServer.BaseServer.shutdown.""" - - self.__ws_serving = False - self.__ws_is_shut_down.wait() - - -class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): - """CGIHTTPRequestHandler specialized for WebSocket.""" - - # Use httplib.HTTPMessage instead of mimetools.Message. - MessageClass = httplib.HTTPMessage - - def setup(self): - """Override SocketServer.StreamRequestHandler.setup to wrap rfile - with MemorizingFile. - - This method will be called by BaseRequestHandler's constructor - before calling BaseHTTPRequestHandler.handle. - BaseHTTPRequestHandler.handle will call - BaseHTTPRequestHandler.handle_one_request and it will call - WebSocketRequestHandler.parse_request. - """ - - # Call superclass's setup to prepare rfile, wfile, etc. See setup - # definition on the root class SocketServer.StreamRequestHandler to - # understand what this does. - CGIHTTPServer.CGIHTTPRequestHandler.setup(self) - - self.rfile = memorizingfile.MemorizingFile( - self.rfile, - max_memorized_lines=_MAX_MEMORIZED_LINES) - - def __init__(self, request, client_address, server): - self._logger = util.get_class_logger(self) - - self._options = server.websocket_server_options - - # Overrides CGIHTTPServerRequestHandler.cgi_directories. - self.cgi_directories = self._options.cgi_directories - # Replace CGIHTTPRequestHandler.is_executable method. - if self._options.is_executable_method is not None: - self.is_executable = self._options.is_executable_method - - # This actually calls BaseRequestHandler.__init__. - CGIHTTPServer.CGIHTTPRequestHandler.__init__( - self, request, client_address, server) - - def parse_request(self): - """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. - - Return True to continue processing for HTTP(S), False otherwise. - - See BaseHTTPRequestHandler.handle_one_request method which calls - this method to understand how the return value will be handled. - """ - - # We hook parse_request method, but also call the original - # CGIHTTPRequestHandler.parse_request since when we return False, - # CGIHTTPRequestHandler.handle_one_request continues processing and - # it needs variables set by CGIHTTPRequestHandler.parse_request. - # - # Variables set by this method will be also used by WebSocket request - # handling (self.path, self.command, self.requestline, etc. See also - # how _StandaloneRequest's members are implemented using these - # attributes). - if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): - return False - - if self._options.use_basic_auth: - auth = self.headers.getheader('Authorization') - if auth != self._options.basic_auth_credential: - self.send_response(401) - self.send_header('WWW-Authenticate', - 'Basic realm="Pywebsocket"') - self.end_headers() - self._logger.info('Request basic authentication') - return True - - host, port, resource = http_header_util.parse_uri(self.path) - if resource is None: - self._logger.info('Invalid URI: %r', self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - server_options = self.server.websocket_server_options - if host is not None: - validation_host = server_options.validation_host - if validation_host is not None and host != validation_host: - self._logger.info('Invalid host: %r (expected: %r)', - host, - validation_host) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - if port is not None: - validation_port = server_options.validation_port - if validation_port is not None and port != validation_port: - self._logger.info('Invalid port: %r (expected: %r)', - port, - validation_port) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - self.path = resource - - request = _StandaloneRequest(self, self._options.use_tls) - - try: - # Fallback to default http handler for request paths for which - # we don't have request handlers. - if not self._options.dispatcher.get_handler_suite(self.path): - self._logger.info('No handler for resource: %r', - self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - except dispatch.DispatchException, e: - self._logger.info('%s', e) - self.send_error(e.status) - return False - - # If any Exceptions without except clause setup (including - # DispatchException) is raised below this point, it will be caught - # and logged by WebSocketServer. - - try: - try: - handshake.do_handshake( - request, - self._options.dispatcher, - allowDraft75=self._options.allow_draft75, - strict=self._options.strict) - except handshake.VersionException, e: - self._logger.info('%s', e) - self.send_response(common.HTTP_STATUS_BAD_REQUEST) - self.send_header(common.SEC_WEBSOCKET_VERSION_HEADER, - e.supported_versions) - self.end_headers() - return False - except handshake.HandshakeException, e: - # Handshake for ws(s) failed. - self._logger.info('%s', e) - self.send_error(e.status) - return False - - request._dispatcher = self._options.dispatcher - self._options.dispatcher.transfer_data(request) - except handshake.AbortedByUserException, e: - self._logger.info('%s', e) - return False - - def log_request(self, code='-', size='-'): - """Override BaseHTTPServer.log_request.""" - - self._logger.info('"%s" %s %s', - self.requestline, str(code), str(size)) - - def log_error(self, *args): - """Override BaseHTTPServer.log_error.""" - - # Despite the name, this method is for warnings than for errors. - # For example, HTTP status code is logged by this method. - self._logger.warning('%s - %s', - self.address_string(), - args[0] % args[1:]) - - def is_cgi(self): - """Test whether self.path corresponds to a CGI script. - - Add extra check that self.path doesn't contains .. - Also check if the file is a executable file or not. - If the file is not executable, it is handled as static file or dir - rather than a CGI script. - """ - - if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): - if '..' in self.path: - return False - # strip query parameter from request path - resource_name = self.path.split('?', 2)[0] - # convert resource_name into real path name in filesystem. - scriptfile = self.translate_path(resource_name) - if not os.path.isfile(scriptfile): - return False - if not self.is_executable(scriptfile): - return False - return True - return False - - -def _get_logger_from_class(c): - return logging.getLogger('%s.%s' % (c.__module__, c.__name__)) - - -def _configure_logging(options): - logging.addLevelName(common.LOGLEVEL_FINE, 'FINE') - - logger = logging.getLogger() - logger.setLevel(logging.getLevelName(options.log_level.upper())) - if options.log_file: - handler = logging.handlers.RotatingFileHandler( - options.log_file, 'a', options.log_max, options.log_count) - else: - handler = logging.StreamHandler() - formatter = logging.Formatter( - '[%(asctime)s] [%(levelname)s] %(name)s: %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - - deflate_log_level_name = logging.getLevelName( - options.deflate_log_level.upper()) - _get_logger_from_class(util._Deflater).setLevel( - deflate_log_level_name) - _get_logger_from_class(util._Inflater).setLevel( - deflate_log_level_name) - - -def _build_option_parser(): - parser = optparse.OptionParser() - - parser.add_option('--config', dest='config_file', type='string', - default=None, - help=('Path to configuration file. See the file comment ' - 'at the top of this file for the configuration ' - 'file format')) - parser.add_option('-H', '--server-host', '--server_host', - dest='server_host', - default='', - help='server hostname to listen to') - parser.add_option('-V', '--validation-host', '--validation_host', - dest='validation_host', - default=None, - help='server hostname to validate in absolute path.') - parser.add_option('-p', '--port', dest='port', type='int', - default=common.DEFAULT_WEB_SOCKET_PORT, - help='port to listen to') - parser.add_option('-P', '--validation-port', '--validation_port', - dest='validation_port', type='int', - default=None, - help='server port to validate in absolute path.') - parser.add_option('-w', '--websock-handlers', '--websock_handlers', - dest='websock_handlers', - default='.', - help=('The root directory of WebSocket handler files. ' - 'If the path is relative, --document-root is used ' - 'as the base.')) - parser.add_option('-m', '--websock-handlers-map-file', - '--websock_handlers_map_file', - dest='websock_handlers_map_file', - default=None, - help=('WebSocket handlers map file. ' - 'Each line consists of alias_resource_path and ' - 'existing_resource_path, separated by spaces.')) - parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', - default=None, - help=('Must be a directory under --websock-handlers. ' - 'Only handlers under this directory are scanned ' - 'and registered to the server. ' - 'Useful for saving scan time when the handler ' - 'root directory contains lots of files that are ' - 'not handler file or are handler files but you ' - 'don\'t want them to be registered. ')) - parser.add_option('--allow-handlers-outside-root-dir', - '--allow_handlers_outside_root_dir', - dest='allow_handlers_outside_root_dir', - action='store_true', - default=False, - help=('Scans WebSocket handlers even if their canonical ' - 'path is not under --websock-handlers.')) - parser.add_option('-d', '--document-root', '--document_root', - dest='document_root', default='.', - help='Document root directory.') - parser.add_option('-x', '--cgi-paths', '--cgi_paths', dest='cgi_paths', - default=None, - help=('CGI paths relative to document_root.' - 'Comma-separated. (e.g -x /cgi,/htbin) ' - 'Files under document_root/cgi_path are handled ' - 'as CGI programs. Must be executable.')) - parser.add_option('-t', '--tls', dest='use_tls', action='store_true', - default=False, help='use TLS (wss://)') - parser.add_option('-k', '--private-key', '--private_key', - dest='private_key', - default='', help='TLS private key file.') - parser.add_option('-c', '--certificate', dest='certificate', - default='', help='TLS certificate file.') - parser.add_option('--tls-client-auth', dest='tls_client_auth', - action='store_true', default=False, - help='Requires TLS client auth on every connection.') - parser.add_option('--tls-client-ca', dest='tls_client_ca', default='', - help=('Specifies a pem file which contains a set of ' - 'concatenated CA certificates which are used to ' - 'validate certificates passed from clients')) - parser.add_option('--basic-auth', dest='use_basic_auth', - action='store_true', default=False, - help='Requires Basic authentication.') - parser.add_option('--basic-auth-credential', - dest='basic_auth_credential', default='test:test', - help='Specifies the credential of basic authentication ' - 'by username:password pair (e.g. test:test).') - parser.add_option('-l', '--log-file', '--log_file', dest='log_file', - default='', help='Log file.') - # Custom log level: - # - FINE: Prints status of each frame processing step - parser.add_option('--log-level', '--log_level', type='choice', - dest='log_level', default='warn', - choices=['fine', - 'debug', 'info', 'warning', 'warn', 'error', - 'critical'], - help='Log level.') - parser.add_option('--deflate-log-level', '--deflate_log_level', - type='choice', - dest='deflate_log_level', default='warn', - choices=['debug', 'info', 'warning', 'warn', 'error', - 'critical'], - help='Log level for _Deflater and _Inflater.') - parser.add_option('--thread-monitor-interval-in-sec', - '--thread_monitor_interval_in_sec', - dest='thread_monitor_interval_in_sec', - type='int', default=-1, - help=('If positive integer is specified, run a thread ' - 'monitor to show the status of server threads ' - 'periodically in the specified inteval in ' - 'second. If non-positive integer is specified, ' - 'disable the thread monitor.')) - parser.add_option('--log-max', '--log_max', dest='log_max', type='int', - default=_DEFAULT_LOG_MAX_BYTES, - help='Log maximum bytes') - parser.add_option('--log-count', '--log_count', dest='log_count', - type='int', default=_DEFAULT_LOG_BACKUP_COUNT, - help='Log backup count') - parser.add_option('--allow-draft75', dest='allow_draft75', - action='store_true', default=False, - help='Obsolete option. Ignored.') - parser.add_option('--strict', dest='strict', action='store_true', - default=False, help='Obsolete option. Ignored.') - parser.add_option('-q', '--queue', dest='request_queue_size', type='int', - default=_DEFAULT_REQUEST_QUEUE_SIZE, - help='request queue size') - - return parser - - -class ThreadMonitor(threading.Thread): - daemon = True - - def __init__(self, interval_in_sec): - threading.Thread.__init__(self, name='ThreadMonitor') - - self._logger = util.get_class_logger(self) - - self._interval_in_sec = interval_in_sec - - def run(self): - while True: - thread_name_list = [] - for thread in threading.enumerate(): - thread_name_list.append(thread.name) - self._logger.info( - "%d active threads: %s", - threading.active_count(), - ', '.join(thread_name_list)) - time.sleep(self._interval_in_sec) - - -def _parse_args_and_config(args): - parser = _build_option_parser() - - # First, parse options without configuration file. - temporary_options, temporary_args = parser.parse_args(args=args) - if temporary_args: - logging.critical( - 'Unrecognized positional arguments: %r', temporary_args) - sys.exit(1) - - if temporary_options.config_file: - try: - config_fp = open(temporary_options.config_file, 'r') - except IOError, e: - logging.critical( - 'Failed to open configuration file %r: %r', - temporary_options.config_file, - e) - sys.exit(1) - - config_parser = ConfigParser.SafeConfigParser() - config_parser.readfp(config_fp) - config_fp.close() - - args_from_config = [] - for name, value in config_parser.items('pywebsocket'): - args_from_config.append('--' + name) - args_from_config.append(value) - if args is None: - args = args_from_config - else: - args = args_from_config + args - return parser.parse_args(args=args) - else: - return temporary_options, temporary_args - - -def _main(args=None): - """You can call this function from your own program, but please note that - this function has some side-effects that might affect your program. For - example, util.wrap_popen3_for_win use in this method replaces implementation - of os.popen3. - """ - - options, args = _parse_args_and_config(args=args) - - os.chdir(options.document_root) - - _configure_logging(options) - - # TODO(tyoshino): Clean up initialization of CGI related values. Move some - # of code here to WebSocketRequestHandler class if it's better. - options.cgi_directories = [] - options.is_executable_method = None - if options.cgi_paths: - options.cgi_directories = options.cgi_paths.split(',') - if sys.platform in ('cygwin', 'win32'): - cygwin_path = None - # For Win32 Python, it is expected that CYGWIN_PATH - # is set to a directory of cygwin binaries. - # For example, websocket_server.py in Chromium sets CYGWIN_PATH to - # full path of third_party/cygwin/bin. - if 'CYGWIN_PATH' in os.environ: - cygwin_path = os.environ['CYGWIN_PATH'] - util.wrap_popen3_for_win(cygwin_path) - - def __check_script(scriptpath): - return util.get_script_interp(scriptpath, cygwin_path) - - options.is_executable_method = __check_script - - if options.use_tls: - if not (_HAS_SSL or _HAS_OPEN_SSL): - logging.critical('TLS support requires ssl or pyOpenSSL module.') - sys.exit(1) - if not options.private_key or not options.certificate: - logging.critical( - 'To use TLS, specify private_key and certificate.') - sys.exit(1) - - if options.tls_client_auth: - if not options.use_tls: - logging.critical('TLS must be enabled for client authentication.') - sys.exit(1) - if not _HAS_SSL: - logging.critical('Client authentication requires ssl module.') - - if not options.scan_dir: - options.scan_dir = options.websock_handlers - - if options.use_basic_auth: - options.basic_auth_credential = 'Basic ' + base64.b64encode( - options.basic_auth_credential) - - try: - if options.thread_monitor_interval_in_sec > 0: - # Run a thread monitor to show the status of server threads for - # debugging. - ThreadMonitor(options.thread_monitor_interval_in_sec).start() - - server = WebSocketServer(options) - server.serve_forever() - except Exception, e: - logging.critical('mod_pywebsocket: %s' % e) - logging.critical('mod_pywebsocket: %s' % util.get_stack_trace()) - sys.exit(1) - - -if __name__ == '__main__': - _main(sys.argv[1:]) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/stream.py b/module/lib/mod_pywebsocket/stream.py deleted file mode 100644 index edc533279..000000000 --- a/module/lib/mod_pywebsocket/stream.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""This file exports public symbols. -""" - - -from mod_pywebsocket._stream_base import BadOperationException -from mod_pywebsocket._stream_base import ConnectionTerminatedException -from mod_pywebsocket._stream_base import InvalidFrameException -from mod_pywebsocket._stream_base import InvalidUTF8Exception -from mod_pywebsocket._stream_base import UnsupportedFrameException -from mod_pywebsocket._stream_hixie75 import StreamHixie75 -from mod_pywebsocket._stream_hybi import Frame -from mod_pywebsocket._stream_hybi import Stream -from mod_pywebsocket._stream_hybi import StreamOptions - -# These methods are intended to be used by WebSocket client developers to have -# their implementations receive broken data in tests. -from mod_pywebsocket._stream_hybi import create_close_frame -from mod_pywebsocket._stream_hybi import create_header -from mod_pywebsocket._stream_hybi import create_length_header -from mod_pywebsocket._stream_hybi import create_ping_frame -from mod_pywebsocket._stream_hybi import create_pong_frame -from mod_pywebsocket._stream_hybi import create_binary_frame -from mod_pywebsocket._stream_hybi import create_text_frame -from mod_pywebsocket._stream_hybi import create_closing_handshake_body - - -# vi:sts=4 sw=4 et diff --git a/module/lib/mod_pywebsocket/util.py b/module/lib/mod_pywebsocket/util.py deleted file mode 100644 index 7bb0b5d9e..000000000 --- a/module/lib/mod_pywebsocket/util.py +++ /dev/null @@ -1,515 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket utilities. -""" - - -import array -import errno - -# Import hash classes from a module available and recommended for each Python -# version and re-export those symbol. Use sha and md5 module in Python 2.4, and -# hashlib module in Python 2.6. -try: - import hashlib - md5_hash = hashlib.md5 - sha1_hash = hashlib.sha1 -except ImportError: - import md5 - import sha - md5_hash = md5.md5 - sha1_hash = sha.sha - -import StringIO -import logging -import os -import re -import socket -import traceback -import zlib - - -def get_stack_trace(): - """Get the current stack trace as string. - - This is needed to support Python 2.3. - TODO: Remove this when we only support Python 2.4 and above. - Use traceback.format_exc instead. - """ - - out = StringIO.StringIO() - traceback.print_exc(file=out) - return out.getvalue() - - -def prepend_message_to_exception(message, exc): - """Prepend message to the exception.""" - - exc.args = (message + str(exc),) - return - - -def __translate_interp(interp, cygwin_path): - """Translate interp program path for Win32 python to run cygwin program - (e.g. perl). Note that it doesn't support path that contains space, - which is typically true for Unix, where #!-script is written. - For Win32 python, cygwin_path is a directory of cygwin binaries. - - Args: - interp: interp command line - cygwin_path: directory name of cygwin binary, or None - Returns: - translated interp command line. - """ - if not cygwin_path: - return interp - m = re.match('^[^ ]*/([^ ]+)( .*)?', interp) - if m: - cmd = os.path.join(cygwin_path, m.group(1)) - return cmd + m.group(2) - return interp - - -def get_script_interp(script_path, cygwin_path=None): - """Gets #!-interpreter command line from the script. - - It also fixes command path. When Cygwin Python is used, e.g. in WebKit, - it could run "/usr/bin/perl -wT hello.pl". - When Win32 Python is used, e.g. in Chromium, it couldn't. So, fix - "/usr/bin/perl" to "<cygwin_path>\perl.exe". - - Args: - script_path: pathname of the script - cygwin_path: directory name of cygwin binary, or None - Returns: - #!-interpreter command line, or None if it is not #!-script. - """ - fp = open(script_path) - line = fp.readline() - fp.close() - m = re.match('^#!(.*)', line) - if m: - return __translate_interp(m.group(1), cygwin_path) - return None - - -def wrap_popen3_for_win(cygwin_path): - """Wrap popen3 to support #!-script on Windows. - - Args: - cygwin_path: path for cygwin binary if command path is needed to be - translated. None if no translation required. - """ - - __orig_popen3 = os.popen3 - - def __wrap_popen3(cmd, mode='t', bufsize=-1): - cmdline = cmd.split(' ') - interp = get_script_interp(cmdline[0], cygwin_path) - if interp: - cmd = interp + ' ' + cmd - return __orig_popen3(cmd, mode, bufsize) - - os.popen3 = __wrap_popen3 - - -def hexify(s): - return ' '.join(map(lambda x: '%02x' % ord(x), s)) - - -def get_class_logger(o): - return logging.getLogger( - '%s.%s' % (o.__class__.__module__, o.__class__.__name__)) - - -class NoopMasker(object): - """A masking object that has the same interface as RepeatedXorMasker but - just returns the string passed in without making any change. - """ - - def __init__(self): - pass - - def mask(self, s): - return s - - -class RepeatedXorMasker(object): - """A masking object that applies XOR on the string given to mask method - with the masking bytes given to the constructor repeatedly. This object - remembers the position in the masking bytes the last mask method call - ended and resumes from that point on the next mask method call. - """ - - def __init__(self, mask): - self._mask = map(ord, mask) - self._mask_size = len(self._mask) - self._count = 0 - - def mask(self, s): - result = array.array('B') - result.fromstring(s) - # Use temporary local variables to eliminate the cost to access - # attributes - count = self._count - mask = self._mask - mask_size = self._mask_size - for i in xrange(len(result)): - result[i] ^= mask[count] - count = (count + 1) % mask_size - self._count = count - - return result.tostring() - - -class DeflateRequest(object): - """A wrapper class for request object to intercept send and recv to perform - deflate compression and decompression transparently. - """ - - def __init__(self, request): - self._request = request - self.connection = DeflateConnection(request.connection) - - def __getattribute__(self, name): - if name in ('_request', 'connection'): - return object.__getattribute__(self, name) - return self._request.__getattribute__(name) - - def __setattr__(self, name, value): - if name in ('_request', 'connection'): - return object.__setattr__(self, name, value) - return self._request.__setattr__(name, value) - - -# By making wbits option negative, we can suppress CMF/FLG (2 octet) and -# ADLER32 (4 octet) fields of zlib so that we can use zlib module just as -# deflate library. DICTID won't be added as far as we don't set dictionary. -# LZ77 window of 32K will be used for both compression and decompression. -# For decompression, we can just use 32K to cover any windows size. For -# compression, we use 32K so receivers must use 32K. -# -# Compression level is Z_DEFAULT_COMPRESSION. We don't have to match level -# to decode. -# -# See zconf.h, deflate.cc, inflate.cc of zlib library, and zlibmodule.c of -# Python. See also RFC1950 (ZLIB 3.3). - - -class _Deflater(object): - - def __init__(self, window_bits): - self._logger = get_class_logger(self) - - self._compress = zlib.compressobj( - zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) - - def compress(self, bytes): - compressed_bytes = self._compress.compress(bytes) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - - def compress_and_flush(self, bytes): - compressed_bytes = self._compress.compress(bytes) - compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - - def compress_and_finish(self, bytes): - compressed_bytes = self._compress.compress(bytes) - compressed_bytes += self._compress.flush(zlib.Z_FINISH) - self._logger.debug('Compress input %r', bytes) - self._logger.debug('Compress result %r', compressed_bytes) - return compressed_bytes - -class _Inflater(object): - - def __init__(self): - self._logger = get_class_logger(self) - - self._unconsumed = '' - - self.reset() - - def decompress(self, size): - if not (size == -1 or size > 0): - raise Exception('size must be -1 or positive') - - data = '' - - while True: - if size == -1: - data += self._decompress.decompress(self._unconsumed) - # See Python bug http://bugs.python.org/issue12050 to - # understand why the same code cannot be used for updating - # self._unconsumed for here and else block. - self._unconsumed = '' - else: - data += self._decompress.decompress( - self._unconsumed, size - len(data)) - self._unconsumed = self._decompress.unconsumed_tail - if self._decompress.unused_data: - # Encountered a last block (i.e. a block with BFINAL = 1) and - # found a new stream (unused_data). We cannot use the same - # zlib.Decompress object for the new stream. Create a new - # Decompress object to decompress the new one. - # - # It's fine to ignore unconsumed_tail if unused_data is not - # empty. - self._unconsumed = self._decompress.unused_data - self.reset() - if size >= 0 and len(data) == size: - # data is filled. Don't call decompress again. - break - else: - # Re-invoke Decompress.decompress to try to decompress all - # available bytes before invoking read which blocks until - # any new byte is available. - continue - else: - # Here, since unused_data is empty, even if unconsumed_tail is - # not empty, bytes of requested length are already in data. We - # don't have to "continue" here. - break - - if data: - self._logger.debug('Decompressed %r', data) - return data - - def append(self, data): - self._logger.debug('Appended %r', data) - self._unconsumed += data - - def reset(self): - self._logger.debug('Reset') - self._decompress = zlib.decompressobj(-zlib.MAX_WBITS) - - -# Compresses/decompresses given octets using the method introduced in RFC1979. - - -class _RFC1979Deflater(object): - """A compressor class that applies DEFLATE to given byte sequence and - flushes using the algorithm described in the RFC1979 section 2.1. - """ - - def __init__(self, window_bits, no_context_takeover): - self._deflater = None - if window_bits is None: - window_bits = zlib.MAX_WBITS - self._window_bits = window_bits - self._no_context_takeover = no_context_takeover - - def filter(self, bytes, flush=True, bfinal=False): - if self._deflater is None or (self._no_context_takeover and flush): - self._deflater = _Deflater(self._window_bits) - - if bfinal: - result = self._deflater.compress_and_finish(bytes) - # Add a padding block with BFINAL = 0 and BTYPE = 0. - result = result + chr(0) - self._deflater = None - return result - if flush: - # Strip last 4 octets which is LEN and NLEN field of a - # non-compressed block added for Z_SYNC_FLUSH. - return self._deflater.compress_and_flush(bytes)[:-4] - return self._deflater.compress(bytes) - -class _RFC1979Inflater(object): - """A decompressor class for byte sequence compressed and flushed following - the algorithm described in the RFC1979 section 2.1. - """ - - def __init__(self): - self._inflater = _Inflater() - - def filter(self, bytes): - # Restore stripped LEN and NLEN field of a non-compressed block added - # for Z_SYNC_FLUSH. - self._inflater.append(bytes + '\x00\x00\xff\xff') - return self._inflater.decompress(-1) - - -class DeflateSocket(object): - """A wrapper class for socket object to intercept send and recv to perform - deflate compression and decompression transparently. - """ - - # Size of the buffer passed to recv to receive compressed data. - _RECV_SIZE = 4096 - - def __init__(self, socket): - self._socket = socket - - self._logger = get_class_logger(self) - - self._deflater = _Deflater(zlib.MAX_WBITS) - self._inflater = _Inflater() - - def recv(self, size): - """Receives data from the socket specified on the construction up - to the specified size. Once any data is available, returns it even - if it's smaller than the specified size. - """ - - # TODO(tyoshino): Allow call with size=0. It should block until any - # decompressed data is available. - if size <= 0: - raise Exception('Non-positive size passed') - while True: - data = self._inflater.decompress(size) - if len(data) != 0: - return data - - read_data = self._socket.recv(DeflateSocket._RECV_SIZE) - if not read_data: - return '' - self._inflater.append(read_data) - - def sendall(self, bytes): - self.send(bytes) - - def send(self, bytes): - self._socket.sendall(self._deflater.compress_and_flush(bytes)) - return len(bytes) - - -class DeflateConnection(object): - """A wrapper class for request object to intercept write and read to - perform deflate compression and decompression transparently. - """ - - def __init__(self, connection): - self._connection = connection - - self._logger = get_class_logger(self) - - self._deflater = _Deflater(zlib.MAX_WBITS) - self._inflater = _Inflater() - - def get_remote_addr(self): - return self._connection.remote_addr - remote_addr = property(get_remote_addr) - - def put_bytes(self, bytes): - self.write(bytes) - - def read(self, size=-1): - """Reads at most size bytes. Blocks until there's at least one byte - available. - """ - - # TODO(tyoshino): Allow call with size=0. - if not (size == -1 or size > 0): - raise Exception('size must be -1 or positive') - - data = '' - while True: - if size == -1: - data += self._inflater.decompress(-1) - else: - data += self._inflater.decompress(size - len(data)) - - if size >= 0 and len(data) != 0: - break - - # TODO(tyoshino): Make this read efficient by some workaround. - # - # In 3.0.3 and prior of mod_python, read blocks until length bytes - # was read. We don't know the exact size to read while using - # deflate, so read byte-by-byte. - # - # _StandaloneRequest.read that ultimately performs - # socket._fileobject.read also blocks until length bytes was read - read_data = self._connection.read(1) - if not read_data: - break - self._inflater.append(read_data) - return data - - def write(self, bytes): - self._connection.write(self._deflater.compress_and_flush(bytes)) - - -def _is_ewouldblock_errno(error_number): - """Returns True iff error_number indicates that receive operation would - block. To make this portable, we check availability of errno and then - compare them. - """ - - for error_name in ['WSAEWOULDBLOCK', 'EWOULDBLOCK', 'EAGAIN']: - if (error_name in dir(errno) and - error_number == getattr(errno, error_name)): - return True - return False - - -def drain_received_data(raw_socket): - # Set the socket non-blocking. - original_timeout = raw_socket.gettimeout() - raw_socket.settimeout(0.0) - - drained_data = [] - - # Drain until the socket is closed or no data is immediately - # available for read. - while True: - try: - data = raw_socket.recv(1) - if not data: - break - drained_data.append(data) - except socket.error, e: - # e can be either a pair (errno, string) or just a string (or - # something else) telling what went wrong. We suppress only - # the errors that indicates that the socket blocks. Those - # exceptions can be parsed as a pair (errno, string). - try: - error_number, message = e - except: - # Failed to parse socket.error. - raise e - - if _is_ewouldblock_errno(error_number): - break - else: - raise e - - # Rollback timeout value. - raw_socket.settimeout(original_timeout) - - return ''.join(drained_data) - - -# vi:sts=4 sw=4 et diff --git a/module/lib/new_collections.py b/module/lib/new_collections.py deleted file mode 100644 index 12d05b4b9..000000000 --- a/module/lib/new_collections.py +++ /dev/null @@ -1,375 +0,0 @@ -## {{{ http://code.activestate.com/recipes/576693/ (r9) -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. - -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) -## end of http://code.activestate.com/recipes/576693/ }}} - -## {{{ http://code.activestate.com/recipes/500261/ (r15) -from operator import itemgetter as _itemgetter -from keyword import iskeyword as _iskeyword -import sys as _sys - -def namedtuple(typename, field_names, verbose=False, rename=False): - """Returns a new subclass of tuple with named fields. - - >>> Point = namedtuple('Point', 'x y') - >>> Point.__doc__ # docstring for the new class - 'Point(x, y)' - >>> p = Point(11, y=22) # instantiate with positional args or keywords - >>> p[0] + p[1] # indexable like a plain tuple - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessable by name - 33 - >>> d = p._asdict() # convert to a dictionary - >>> d['x'] - 11 - >>> Point(**d) # convert from a dictionary - Point(x=11, y=22) - >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields - Point(x=100, y=22) - - """ - - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. - if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) - if rename: - names = list(field_names) - seen = set() - for i, name in enumerate(names): - if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) - or not name or name[0].isdigit() or name.startswith('_') - or name in seen): - names[i] = '_%d' % i - seen.add(name) - field_names = tuple(names) - for name in (typename,) + field_names: - if not min(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) - if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) - if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() - for name in field_names: - if name.startswith('_') and not rename: - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: - raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) - - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(self): - 'Return a new dict which maps field names to their values' - return dict(zip(self._fields, self)) \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += ' %s = _property(_itemgetter(%d))\n' % (name, i) - if verbose: - print template - - # Execute the template string in a temporary namespace - namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, - _property=property, _tuple=tuple) - try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) - result = namespace[typename] - - # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where - # sys._getframe is not defined (Jython for example) or sys._getframe is not - # defined for arguments greater than 0 (IronPython). - try: - result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass - - return result -## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/module/lib/rename_process.py b/module/lib/rename_process.py deleted file mode 100644 index 2527cef39..000000000 --- a/module/lib/rename_process.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -def renameProcess(new_name): - """ Renames the process calling the function to the given name. """ - if sys.platform != 'linux2': - return False - try: - from ctypes import CDLL - libc = CDLL('libc.so.6') - libc.prctl(15, new_name, 0, 0, 0) - return True - except Exception, e: - #print "Rename process failed", e - return False diff --git a/module/lib/simplejson/__init__.py b/module/lib/simplejson/__init__.py deleted file mode 100644 index ef5c0db48..000000000 --- a/module/lib/simplejson/__init__.py +++ /dev/null @@ -1,466 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.2.1' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', -] - -__author__ = 'Bob Ippolito <bob@redivi.com>' - -from decimal import Decimal - -from decoder import JSONDecoder, JSONDecodeError -from encoder import JSONEncoder -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from simplejson._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - use_decimal=True, - namedtuple_as_object=True, - tuple_as_array=True, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, tuple_as_array=True, - **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=True, - namedtuple_as_object=True, - tuple_as_array=True, - **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``True``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - If *namedtuple_as_object* is true (default: ``True``), - :class:`tuple` subclasses with ``_asdict()`` methods will be encoded - as JSON objects. - - If *tuple_as_array* is true (default: ``True``), - :class:`tuple` (and subclasses) will be encoded as JSON arrays. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, - namedtuple_as_object=namedtuple_as_object, - tuple_as_array=tuple_as_array, - **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None, - object_pairs_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, namedtuple_as_object=True, tuple_as_array=True, - **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - import simplejson.decoder as dec - import simplejson.encoder as enc - import simplejson.scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder( - encoding=None, - object_hook=None, - object_pairs_hook=None, - ) - global _default_encoder - _default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - ) diff --git a/module/lib/simplejson/decoder.py b/module/lib/simplejson/decoder.py deleted file mode 100644 index e5496d6e7..000000000 --- a/module/lib/simplejson/decoder.py +++ /dev/null @@ -1,421 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -def _import_c_scanstring(): - try: - from simplejson._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - # The struct module in Python 2.4 would get frexp() out of range here - # when an endian is specified in the format string. Fixed in Python 2.5+ - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise JSONDecodeError(msg, s, end) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise JSONDecodeError(msg, s, end) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise JSONDecodeError(msg, s, end) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting : delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise JSONDecodeError("No JSON object could be decoded", s, idx) - return obj, end diff --git a/module/lib/simplejson/encoder.py b/module/lib/simplejson/encoder.py deleted file mode 100644 index 5ec7440f1..000000000 --- a/module/lib/simplejson/encoder.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re -from decimal import Decimal - -def _import_speedups(): - try: - from simplejson import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from simplejson.decoder import PosInf - -ESCAPE = re.compile(ur'[\x00-\x1f\\"\b\f\n\r\t\u2028\u2029]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', - u'\u2028': '\\u2028', - u'\u2029': '\\u2029', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - return ESCAPE_DCT[match.group(0)] - return u'"' + ESCAPE.sub(replace, s) + u'"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict, namedtuple | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=True, namedtuple_as_object=True, - tuple_as_array=True): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (not the default), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - If namedtuple_as_object is true (the default), tuple subclasses with - ``_asdict()`` methods will be encoded as JSON objects. - - If tuple_as_array is true (the default), tuple (and subclasses) will - be encoded as JSON arrays. - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - self.namedtuple_as_object = namedtuple_as_object - self.tuple_as_array = tuple_as_array - if isinstance(indent, (int, long)): - indent = ' ' * indent - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - key_memo = {} - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o, True) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - _use_decimal, _namedtuple_as_object, _tuple_as_array, - ## HACK: hand-optimized bytecode; turn globals into locals - False=False, - True=True, - ValueError=ValueError, - basestring=basestring, - Decimal=Decimal, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - long=long, - str=str, - tuple=tuple, - ): - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, basestring): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, (int, long)): - yield buf + str(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - if isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - elif (_namedtuple_as_object and isinstance(value, tuple) and - hasattr(value, '_asdict')): - chunks = _iterencode_dict(value._asdict(), - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _sort_keys: - items = dct.items() - items.sort(key=lambda kv: kv[0]) - else: - items = dct.iteritems() - for key, value in items: - if isinstance(key, basestring): - pass - # JavaScript is weakly typed for these, so it makes sense to - # also allow them. Many encoders seem to do something like this. - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, (int, long)): - key = str(key) - elif _skipkeys: - continue - else: - raise TypeError("key " + repr(key) + " is not a string") - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, basestring): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, (int, long)): - yield str(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - if isinstance(value, list): - chunks = _iterencode_list(value, _current_indent_level) - elif (_namedtuple_as_object and isinstance(value, tuple) and - hasattr(value, '_asdict')): - chunks = _iterencode_dict(value._asdict(), - _current_indent_level) - elif _tuple_as_array and isinstance(value, tuple): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, basestring): - yield _encoder(o) - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, (int, long)): - yield str(o) - elif isinstance(o, float): - yield _floatstr(o) - elif isinstance(o, list): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif (_namedtuple_as_object and isinstance(o, tuple) and - hasattr(o, '_asdict')): - for chunk in _iterencode_dict(o._asdict(), _current_indent_level): - yield chunk - elif (_tuple_as_array and isinstance(o, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/module/lib/simplejson/ordered_dict.py b/module/lib/simplejson/ordered_dict.py deleted file mode 100644 index 87ad88824..000000000 --- a/module/lib/simplejson/ordered_dict.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -# Modified from original to support Python 2.4, see -# http://code.google.com/p/simplejson/issues/detail?id=53 -try: - all -except NameError: - def all(seq): - for elem in seq: - if not elem: - return False - return True - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - # Modified from original to support Python 2.4, see - # http://code.google.com/p/simplejson/issues/detail?id=53 - if last: - key = reversed(self).next() - else: - key = iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/module/lib/simplejson/scanner.py b/module/lib/simplejson/scanner.py deleted file mode 100644 index 54593a371..000000000 --- a/module/lib/simplejson/scanner.py +++ /dev/null @@ -1,77 +0,0 @@ -"""JSON token scanner -""" -import re -def _import_c_make_scanner(): - try: - from simplejson._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - try: - nextchar = string[idx] - except IndexError: - raise StopIteration - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise StopIteration - - def scan_once(string, idx): - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/module/lib/simplejson/tool.py b/module/lib/simplejson/tool.py deleted file mode 100644 index 73370db55..000000000 --- a/module/lib/simplejson/tool.py +++ /dev/null @@ -1,39 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'rb') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'rb') - outfile = open(sys.argv[2], 'wb') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError, e: - raise SystemExit(e) - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/module/lib/wsgiserver/LICENSE.txt b/module/lib/wsgiserver/LICENSE.txt deleted file mode 100644 index a15165ee2..000000000 --- a/module/lib/wsgiserver/LICENSE.txt +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2004-2007, CherryPy Team (team@cherrypy.org) -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the CherryPy Team nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/module/lib/wsgiserver/__init__.py b/module/lib/wsgiserver/__init__.py deleted file mode 100644 index c380e18b0..000000000 --- a/module/lib/wsgiserver/__init__.py +++ /dev/null @@ -1,1794 +0,0 @@ -"""A high-speed, production ready, thread pooled, generic WSGI server. - -Simplest example on how to use this module directly -(without using CherryPy's application machinery): - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!\n'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set these attributes: - - server.ssl_certificate = <filename> - server.ssl_private_key = <filename> - - if __name__ == '__main__': - try: - server.start() - except KeyboardInterrupt: - server.stop() - -This won't call the CherryPy engine (application side) at all, only the -WSGI server, which is independant from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - req.read_headers() - req.respond() - -> response = wsgi_app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - - -import base64 -import os -import Queue -import re -quoted_slash = re.compile("(?i)%2F") -import rfc822 -import socket -try: - import cStringIO as StringIO -except ImportError: - import StringIO - -_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) - -import sys -import threading -import time -import traceback -from urllib import unquote -from urlparse import urlparse -import warnings - -try: - from OpenSSL import SSL - from OpenSSL import crypto -except ImportError: - SSL = None - -import errno - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return dict.fromkeys(nums).keys() - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", - ) -socket_errors_to_ignore.append("timed out") - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = ['ACCEPT', 'ACCEPT-CHARSET', 'ACCEPT-ENCODING', - 'ACCEPT-LANGUAGE', 'ACCEPT-RANGES', 'ALLOW', 'CACHE-CONTROL', - 'CONNECTION', 'CONTENT-ENCODING', 'CONTENT-LANGUAGE', 'EXPECT', - 'IF-MATCH', 'IF-NONE-MATCH', 'PRAGMA', 'PROXY-AUTHENTICATE', 'TE', - 'TRAILER', 'TRANSFER-ENCODING', 'UPGRADE', 'VARY', 'VIA', 'WARNING', - 'WWW-AUTHENTICATE'] - - -class WSGIPathInfoDispatcher(object): - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = apps.items() - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort() - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return [''] - - -class MaxSizeExceeded(Exception): - pass - -class SizeCheckWrapper(object): - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See http://www.cherrypy.org/ticket/421 - if len(data) < 256 or data[-1:] == "\n": - return ''.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class HTTPRequest(object): - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - - send: the 'send' method from the connection's socket object. - wsgi_app: the WSGI application to call. - environ: a partial WSGI environ (server and connection entries). - The caller MUST set the following entries: - * All wsgi.* entries, including .input - * SERVER_NAME and SERVER_PORT - * Any SSL_* entries - * Any custom entries like REMOTE_ADDR and REMOTE_PORT - * SERVER_SOFTWARE: the value to write in the "Server" response header. - * ACTUAL_SERVER_PROTOCOL: the value to write in the Status-Line of - the response. From RFC 2145: "An HTTP server SHOULD send a - response version equal to the highest version for which the - server is at least conditionally compliant, and whose major - version is less than or equal to the one received in the - request. An HTTP server MUST NOT send a version for which - it is not at least conditionally compliant." - - outheaders: a list of header tuples to write in the response. - ready: when True, the request has been parsed and is ready to begin - generating the response. When False, signals the calling Connection - that the response should not be generated and the connection should - close. - close_connection: signals the calling Connection that the request - should close. This does not imply an error! The client and/or - server may each request that the connection be closed. - chunked_write: if True, output will be encoded with the "chunked" - transfer-coding. This value is set automatically inside - send_headers. - """ - - max_request_header_size = 0 - max_request_body_size = 0 - - def __init__(self, wfile, environ, wsgi_app): - self.rfile = environ['wsgi.input'] - self.wfile = wfile - self.environ = environ.copy() - self.wsgi_app = wsgi_app - - self.ready = False - self.started_response = False - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = False - self.chunked_write = False - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile.maxlen = self.max_request_header_size - self.rfile.bytes_read = 0 - - try: - self._parse_request() - except MaxSizeExceeded: - self.simple_response("413 Request Entity Too Large") - return - - def _parse_request(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - if not request_line: - # Force self.ready = False so the connection will close. - self.ready = False - return - - if request_line == "\r\n": - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - self.ready = False - return - - environ = self.environ - - try: - method, path, req_protocol = request_line.strip().split(" ", 2) - except ValueError: - self.simple_response(400, "Malformed Request-Line") - return - - environ["REQUEST_METHOD"] = method - - # path may be an abs_path (including "http://host.domain.tld"); - scheme, location, path, params, qs, frag = urlparse(path) - - if frag: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return - - if scheme: - environ["wsgi.url_scheme"] = scheme - if params: - path = path + ";" + params - - environ["SCRIPT_NAME"] = "" - - # Unquote the path+params (e.g. "/this%20path" -> "this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - atoms = [unquote(x) for x in quoted_slash.split(path)] - path = "%2F".join(atoms) - environ["PATH_INFO"] = path - - # Note that, like wsgiref and most other WSGI servers, - # we unquote the path but not the query string. - environ["QUERY_STRING"] = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - rp = int(req_protocol[5]), int(req_protocol[7]) - server_protocol = environ["ACTUAL_SERVER_PROTOCOL"] - sp = int(server_protocol[5]), int(server_protocol[7]) - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - environ["SERVER_PROTOCOL"] = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - - # If the Request-URI was an absoluteURI, use its location atom. - if location: - environ["SERVER_NAME"] = location - - # then all the http headers - try: - self.read_headers() - except ValueError, ex: - self.simple_response("400 Bad Request", repr(ex.args)) - return - - mrbs = self.max_request_body_size - if mrbs and int(environ.get("CONTENT_LENGTH", 0)) > mrbs: - self.simple_response("413 Request Entity Too Large") - return - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if environ.get("HTTP_CONNECTION", "") == "close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if environ.get("HTTP_CONNECTION", "") != "Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = environ.get("HTTP_TRANSFER_ENCODING") - if te: - te = [x.strip().lower() for x in te.split(",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == "chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if environ.get("HTTP_EXPECT", "") == "100-continue": - self.simple_response(100) - - self.ready = True - - def read_headers(self): - """Read header lines from the incoming stream.""" - environ = self.environ - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == '\r\n': - # Normal end of headers - break - - if line[0] in ' \t': - # It's a continuation line. - v = line.strip() - else: - k, v = line.split(":", 1) - k, v = k.strip().upper(), v.strip() - envname = "HTTP_" + k.replace("-", "_") - - if k in comma_separated_headers: - existing = environ.get(envname) - if existing: - v = ", ".join((existing, v)) - environ[envname] = v - - ct = environ.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - environ["CONTENT_TYPE"] = ct - cl = environ.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - environ["CONTENT_LENGTH"] = cl - - def decode_chunked(self): - """Decode the 'chunked' transfer coding.""" - cl = 0 - data = StringIO.StringIO() - while True: - line = self.rfile.readline().strip().split(";", 1) - chunk_size = int(line.pop(0), 16) - if chunk_size <= 0: - break -## if line: chunk_extension = line[0] - cl += chunk_size - data.write(self.rfile.read(chunk_size)) - crlf = self.rfile.read(2) - if crlf != "\r\n": - self.simple_response("400 Bad Request", - "Bad chunked transfer coding " - "(expected '\\r\\n', got %r)" % crlf) - return - - # Grab any trailer headers - self.read_headers() - - data.seek(0) - self.environ["wsgi.input"] = data - self.environ["CONTENT_LENGTH"] = str(cl) or "" - return True - - def respond(self): - """Call the appropriate WSGI app and write its iterable output.""" - # Set rfile.maxlen to ensure we don't read past Content-Length. - # This will also be used to read the entire request body if errors - # are raised before the app can read the body. - if self.chunked_read: - # If chunked, Content-Length will be 0. - self.rfile.maxlen = self.max_request_body_size - else: - cl = int(self.environ.get("CONTENT_LENGTH", 0)) - if self.max_request_body_size: - self.rfile.maxlen = min(cl, self.max_request_body_size) - else: - self.rfile.maxlen = cl - self.rfile.bytes_read = 0 - - try: - self._respond() - except MaxSizeExceeded: - if not self.sent_headers: - self.simple_response("413 Request Entity Too Large") - return - - def _respond(self): - if self.chunked_read: - if not self.decode_chunked(): - self.close_connection = True - return - - response = self.wsgi_app(self.environ, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.wfile.sendall("0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = ["%s %s\r\n" % (self.environ['ACTUAL_SERVER_PROTOCOL'], status), - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n"] - - if status[:3] == "413" and self.response_protocol == 'HTTP/1.1': - # Request Entity Too Large - self.close_connection = True - buf.append("Connection: close\r\n") - - buf.append("\r\n") - if msg: - buf.append(msg) - - try: - self.wfile.sendall("".join(buf)) - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - raise - - def start_response(self, status, headers, exc_info = None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.sent_headers: - try: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None - - self.started_response = True - self.status = status - self.outheaders.extend(headers) - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - if not self.sent_headers: - self.sent_headers = True - self.send_headers() - - if self.chunked_write and chunk: - buf = [hex(len(chunk))[2:], "\r\n", chunk, "\r\n"] - self.wfile.sendall("".join(buf)) - else: - self.wfile.sendall(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers.""" - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif "content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.environ["REQUEST_METHOD"] != 'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append(("Transfer-Encoding", "chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if "connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append(("Connection", "close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append(("Connection", "Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - size = self.rfile.maxlen - self.rfile.bytes_read - if size > 0: - self.rfile.read(size) - - if "date" not in hkeys: - self.outheaders.append(("Date", rfc822.formatdate())) - - if "server" not in hkeys: - self.outheaders.append(("Server", self.environ['SERVER_SOFTWARE'])) - - buf = [self.environ['ACTUAL_SERVER_PROTOCOL'], " ", self.status, "\r\n"] - try: - buf += [k + ": " + v + "\r\n" for k, v in self.outheaders] - except TypeError: - if not isinstance(k, str): - raise TypeError("WSGI response header key %r is not a string.") - if not isinstance(v, str): - raise TypeError("WSGI response header value %r is not a string.") - else: - raise - buf.append("\r\n") - self.wfile.sendall("".join(buf)) - - -class NoSSLError(Exception): - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -if not _fileobject_uses_str_type: - class CP_fileobject(socket._fileobject): - """Faux file object attached to a socket object.""" - - def sendall(self, data): - """Sendall for non-blocking sockets.""" - while data: - try: - bytes_sent = self.send(data) - data = data[bytes_sent:] - except socket.error, e: - if e.args[0] not in socket_errors_nonblocking: - raise - - def send(self, data): - return self._sock.send(data) - - def flush(self): - if self._wbuf: - buffer = "".join(self._wbuf) - self._wbuf = [] - self.sendall(buffer) - - def recv(self, size): - while True: - try: - return self._sock.recv(size) - except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): - raise - - def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - if size < 0: - # Read until EOF - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(rbufsize) - if not data: - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - data = self.recv(left) - if not data: - break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - def readline(self, size=-1): - buf = self._rbuf - buf.seek(0, 2) # seek end - if buf.tell() > 0: - # check if we already have it in our buffer - buf.seek(0) - bline = buf.readline(size) - if bline.endswith('\n') or len(bline) == size: - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return bline - del bline - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - buf.seek(0) - buffers = [buf.read()] - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - data = None - recv = self.recv - while data != "\n": - data = recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - - buf.seek(0, 2) # seek end - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - nl = data.find('\n') - if nl >= 0: - nl += 1 - buf.write(data[:nl]) - self._rbuf.write(data[nl:]) - del data - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or \n or EOF seen, whichever comes first - buf.seek(0, 2) # seek end - buf_len = buf.tell() - if buf_len >= size: - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - left = size - buf_len - # did we just receive a newline? - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - # save the excess data to _rbuf - self._rbuf.write(data[nl:]) - if buf_len: - buf.write(data[:nl]) - break - else: - # Shortcut. Avoid data copy through buf when returning - # a substring of our first recv(). - return data[:nl] - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid data copy through buf when - # returning exactly all of our first recv(). - return data - if n >= left: - buf.write(data[:left]) - self._rbuf.write(data[left:]) - break - buf.write(data) - buf_len += n - #assert buf_len == buf.tell() - return buf.getvalue() - -else: - class CP_fileobject(socket._fileobject): - """Faux file object attached to a socket object.""" - - def sendall(self, data): - """Sendall for non-blocking sockets.""" - while data: - try: - bytes_sent = self.send(data) - data = data[bytes_sent:] - except socket.error, e: - if e.args[0] not in socket_errors_nonblocking: - raise - - def send(self, data): - return self._sock.send(data) - - def flush(self): - if self._wbuf: - buffer = "".join(self._wbuf) - self._wbuf = [] - self.sendall(buffer) - - def recv(self, size): - while True: - try: - return self._sock.recv(size) - except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): - raise - - def read(self, size=-1): - if size < 0: - # Read until EOF - buffers = [self._rbuf] - self._rbuf = "" - if self._rbufsize <= 1: - recv_size = self.default_bufsize - else: - recv_size = self._rbufsize - - while True: - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - return "".join(buffers) - else: - # Read until size bytes or EOF seen, whichever comes first - data = self._rbuf - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - left = size - buf_len - recv_size = max(self._rbufsize, left) - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - def readline(self, size=-1): - data = self._rbuf - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - assert data == "" - buffers = [] - while data != "\n": - data = self.recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - return "".join(buffers) - else: - # Read until size bytes or \n or EOF seen, whichever comes first - nl = data.find('\n', 0, size) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - left = size - buf_len - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - -class SSL_fileobject(CP_fileobject): - """SSL file object attached to a socket object.""" - - ssl_timeout = 3 - ssl_retry = .01 - - def _safe_call(self, is_reader, call, *args, **kwargs): - """Wrap the given call with SSL error-trapping. - - is_reader: if False EOF errors will be raised. If True, EOF errors - will return "" (to emulate normal sockets). - """ - start = time.time() - while True: - try: - return call(*args, **kwargs) - except SSL.WantReadError: - # Sleep and try again. This is dangerous, because it means - # the rest of the stack has no way of differentiating - # between a "new handshake" error and "client dropped". - # Note this isn't an endless loop: there's a timeout below. - time.sleep(self.ssl_retry) - except SSL.WantWriteError: - time.sleep(self.ssl_retry) - except SSL.SysCallError, e: - if is_reader and e.args == (-1, 'Unexpected EOF'): - return "" - - errnum = e.args[0] - if is_reader and errnum in socket_errors_to_ignore: - return "" - raise socket.error(errnum) - except SSL.Error, e: - if is_reader and e.args == (-1, 'Unexpected EOF'): - return "" - - thirdarg = None - try: - thirdarg = e.args[0][0][2] - except IndexError: - pass - - if thirdarg == 'http request': - # The client is talking HTTP to an HTTPS server. - raise NoSSLError() - raise FatalSSLAlert(*e.args) - except: - raise - - if time.time() - start > self.ssl_timeout: - raise socket.timeout("timed out") - - def recv(self, *args, **kwargs): - buf = [] - r = super(SSL_fileobject, self).recv - while True: - data = self._safe_call(True, r, *args, **kwargs) - buf.append(data) - p = self._sock.pending() - if not p: - return "".join(buf) - - def sendall(self, *args, **kwargs): - return self._safe_call(False, super(SSL_fileobject, self).sendall, *args, **kwargs) - - def send(self, *args, **kwargs): - return self._safe_call(False, super(SSL_fileobject, self).send, *args, **kwargs) - - -class HTTPConnection(object): - """An HTTP connection (active socket). - - socket: the raw socket object (usually TCP) for this connection. - wsgi_app: the WSGI application for this server/connection. - environ: a WSGI environ template. This will be copied for each request. - - rfile: a fileobject for reading from the socket. - send: a function for writing (+ flush) to the socket. - """ - - rbufsize = -1 - RequestHandlerClass = HTTPRequest - environ = {"wsgi.version": (1, 0), - "wsgi.url_scheme": "http", - "wsgi.multithread": True, - "wsgi.multiprocess": False, - "wsgi.run_once": False, - "wsgi.errors": sys.stderr, - } - - def __init__(self, sock, wsgi_app, environ): - self.socket = sock - self.wsgi_app = wsgi_app - - # Copy the class environ into self. - self.environ = self.environ.copy() - self.environ.update(environ) - - if SSL and isinstance(sock, SSL.ConnectionType): - timeout = sock.gettimeout() - self.rfile = SSL_fileobject(sock, "rb", self.rbufsize) - self.rfile.ssl_timeout = timeout - self.wfile = SSL_fileobject(sock, "wb", -1) - self.wfile.ssl_timeout = timeout - else: - self.rfile = CP_fileobject(sock, "rb", self.rbufsize) - self.wfile = CP_fileobject(sock, "wb", -1) - - # Wrap wsgi.input but not HTTPConnection.rfile itself. - # We're also not setting maxlen yet; we'll do that separately - # for headers and body for each iteration of self.communicate - # (if maxlen is 0 the wrapper doesn't check length). - self.environ["wsgi.input"] = SizeCheckWrapper(self.rfile, 0) - - def communicate(self): - """Read each request and respond appropriately.""" - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.wfile, self.environ, - self.wsgi_app) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if not req.ready: - return - - req.respond() - if req.close_connection: - return - - except socket.error, e: - errnum = e.args[0] - if errnum == 'timed out': - if req and not req.sent_headers: - req.simple_response("408 Request Timeout") - elif errnum not in socket_errors_to_ignore: - if req and not req.sent_headers: - req.simple_response("500 Internal Server Error", - format_exc()) - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert, e: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - req.wfile = CP_fileobject(self.socket._sock, "wb", -1) - req.simple_response("400 Bad Request", - "The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - self.linger = True - except Exception, e: - if req and not req.sent_headers: - req.simple_response("500 Internal Server Error", format_exc()) - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel socket - # when you call socket.close(). We do so manually here because we - # want this server to send a FIN TCP segment immediately. Note this - # must be called *before* calling socket.close(), because the latter - # drops its reference to the kernel socket. - self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -def format_exc(limit=None): - """Like print_exc() but return a string. Backport for Python 2.3.""" - try: - etype, value, tb = sys.exc_info() - return ''.join(traceback.format_exception(etype, value, tb, limit)) - finally: - etype = value = tb = None - - -_SHUTDOWNREQUEST = None - -class WorkerThread(threading.Thread): - """Thread which continuously polls a Queue for Connection objects. - - server: the HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it. - ready: a simple flag for the calling server to know when this thread - has begun polling the Queue. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - - def __init__(self, server): - self.ready = False - self.server = server - threading.Thread.__init__(self) - - def run(self): - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - try: - conn.communicate() - finally: - conn.close() - self.conn = None - except (KeyboardInterrupt, SystemExit), exc: - self.server.interrupt = exc - - -class ThreadPool(object): - """A Request Queue for the CherryPyWSGIServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = Queue.Queue() - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in xrange(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP WSGIServer " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - for i in xrange(amount): - if self.max > 0 and len(self._threads) >= self.max: - break - worker = WorkerThread(self.server) - worker.setName("CP WSGIServer " + worker.getName()) - self._threads.append(worker) - worker.start() - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - if amount > 0: - for i in xrange(min(amount, len(self._threads) - self.min)): - # Put a number of shutdown requests on the queue equal - # to 'amount'. Once each of those is processed by a worker, - # that worker will terminate and be culled from our list - # in self.put. - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - worker.join(timeout) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - if SSL and isinstance(c.socket, SSL.ConnectionType): - # pyOpenSSL.socket.shutdown takes no args - c.socket.shutdown() - else: - c.socket.shutdown(socket.SHUT_RD) - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See http://www.cherrypy.org/ticket/691. - KeyboardInterrupt), exc1: - pass - - - -class SSLConnection: - """A thread-safe wrapper for an SSL.Connection. - - *args: the arguments to create the wrapped SSL.Connection(*args). - """ - - def __init__(self, *args): - self._ssl_conn = SSL.Connection(*args) - self._lock = threading.RLock() - - for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', - 'renegotiate', 'bind', 'listen', 'connect', 'accept', - 'setblocking', 'fileno', 'shutdown', 'close', 'get_cipher_list', - 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', - 'makefile', 'get_app_data', 'set_app_data', 'state_string', - 'sock_shutdown', 'get_peer_certificate', 'want_read', - 'want_write', 'set_connect_state', 'set_accept_state', - 'connect_ex', 'sendall', 'settimeout'): - exec """def %s(self, *args): - self._lock.acquire() - try: - return self._ssl_conn.%s(*args) - finally: - self._lock.release() -""" % (f, f) - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class CherryPyWSGIServer(object): - """An HTTP server for WSGI. - - bind_addr: The interface on which to listen for connections. - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string. - wsgi_app: the WSGI 'application callable'; multiple WSGI applications - may be passed as (path_prefix, app) pairs. - numthreads: the number of worker threads to create (default 10). - server_name: the string to set for WSGI's SERVER_NAME environ entry. - Defaults to socket.gethostname(). - max: the maximum number of queued requests (defaults to -1 = no limit). - request_queue_size: the 'backlog' argument to socket.listen(); - specifies the maximum number of queued connections (default 5). - timeout: the timeout in seconds for accepted connections (default 10). - - nodelay: if True (the default since 3.1), sets the TCP_NODELAY socket - option. - - protocol: the version string to write in the Status-Line of all - HTTP responses. For example, "HTTP/1.1" (the default). This - also limits the supported features used in the response. - - - SSL/HTTPS - --------- - The OpenSSL module must be importable for SSL functionality. - You can obtain it from http://pyopenssl.sourceforge.net/ - - ssl_certificate: the filename of the server SSL certificate. - ssl_privatekey: the filename of the server's private key file. - - If either of these is None (both are None by default), this server - will not use SSL. If both are given and are valid, they will be read - on server start and used in the SSL context for the listening socket. - """ - - protocol = "HTTP/1.1" - _bind_addr = "127.0.0.1" - version = "CherryPy/3.1.2" - ready = False - _interrupt = None - - nodelay = True - - ConnectionClass = HTTPConnection - environ = {} - - # Paths to certificate and private key files - ssl_certificate = None - ssl_private_key = None - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): - self.requests = ThreadPool(self, min=numthreads or 1, max=max) - - if callable(wsgi_app): - # We've been handed a single wsgi_app, in CP-2.1 style. - # Assume it's mounted at "". - self.wsgi_app = wsgi_app - else: - # We've been handed a list of (path_prefix, wsgi_app) tuples, - # so that the server can call different wsgi_apps, and also - # correctly set SCRIPT_NAME. - warnings.warn("The ability to pass multiple apps is deprecated " - "and will be removed in 3.2. You should explicitly " - "include a WSGIPathInfoDispatcher instead.", - DeprecationWarning) - self.wsgi_app = WSGIPathInfoDispatcher(wsgi_app) - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - - def _get_numthreads(self): - return self.requests.min - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property(_get_bind_addr, _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: os.unlink(self.bind_addr) - except: pass - - # So everyone can access the socket... - try: os.chmod(self.bind_addr, 0777) - except: pass - - info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) - except socket.gaierror: - # Probably a DNS issue. Assume IPv4. - info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error, msg: - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error, msg - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - while self.ready: - self.tick() - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay: - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - if self.ssl_certificate and self.ssl_private_key: - if SSL is None: - raise ImportError("You must install pyOpenSSL to use HTTPS.") - - # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 - ctx = SSL.Context(SSL.SSLv23_METHOD) - ctx.use_privatekey_file(self.ssl_private_key) - ctx.use_certificate_file(self.ssl_certificate) - self.socket = SSLConnection(ctx, self.socket) - self.populate_ssl_environ() - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See http://www.cherrypy.org/ticket/871. - if (not isinstance(self.bind_addr, basestring) - and self.bind_addr[0] == '::' and family == socket.AF_INET6): - try: - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - prevent_socket_inheritance(s) - if not self.ready: - return - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - environ = self.environ.copy() - # SERVER_SOFTWARE is common for IIS. It's also helpful for - # us to pass a default value for the "Server" response header. - if environ.get("SERVER_SOFTWARE") is None: - environ["SERVER_SOFTWARE"] = "%s WSGI Server" % self.version - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - environ["ACTUAL_SERVER_PROTOCOL"] = self.protocol - environ["SERVER_NAME"] = self.server_name - - if isinstance(self.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - environ["SERVER_PORT"] = "" - else: - environ["SERVER_PORT"] = str(self.bind_addr[1]) - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - environ["REMOTE_ADDR"] = addr[0] - environ["REMOTE_PORT"] = str(addr[1]) - - conn = self.ConnectionClass(s, self.wsgi_app, environ) - self.requests.put(conn) - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error, x: - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See http://www.cherrypy.org/ticket/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See http://www.cherrypy.org/ticket/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See http://www.cherrypy.org/ticket/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - def populate_ssl_environ(self): - """Create WSGI environ entries to be merged into each request.""" - cert = open(self.ssl_certificate, 'rb').read() - cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) - ssl_environ = { - "wsgi.url_scheme": "https", - "HTTPS": "on", - # pyOpenSSL doesn't provide access to any of these AFAICT -## 'SSL_PROTOCOL': 'SSLv2', -## SSL_CIPHER string The cipher specification name -## SSL_VERSION_INTERFACE string The mod_ssl program version -## SSL_VERSION_LIBRARY string The OpenSSL program version - } - - # Server certificate attributes - ssl_environ.update({ - 'SSL_SERVER_M_VERSION': cert.get_version(), - 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), -## 'SSL_SERVER_V_START': Validity of server's certificate (start time), -## 'SSL_SERVER_V_END': Validity of server's certificate (end time), - }) - - for prefix, dn in [("I", cert.get_issuer()), - ("S", cert.get_subject())]: - # X509Name objects don't seem to have a way to get the - # complete DN string. Use str() and slice it instead, - # because str(dn) == "<X509Name object '/C=US/ST=...'>" - dnstr = str(dn)[18:-2] - - wsgikey = 'SSL_SERVER_%s_DN' % prefix - ssl_environ[wsgikey] = dnstr - - # The DN should be of the form: /k1=v1/k2=v2, but we must allow - # for any value to contain slashes itself (in a URL). - while dnstr: - pos = dnstr.rfind("=") - dnstr, value = dnstr[:pos], dnstr[pos + 1:] - pos = dnstr.rfind("/") - dnstr, key = dnstr[:pos], dnstr[pos + 1:] - if key and value: - wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) - ssl_environ[wsgikey] = value - - self.environ.update(ssl_environ) - diff --git a/module/network/Browser.py b/module/network/Browser.py deleted file mode 100644 index 25cbf669b..000000000 --- a/module/network/Browser.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from logging import getLogger - -from HTTPRequest import HTTPRequest -from HTTPDownload import HTTPDownload - - -class Browser(object): - __slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl") - - def __init__(self, bucket=None, options={}): - self.log = getLogger("log") - - self.options = options #holds pycurl options - self.bucket = bucket - - self.cj = None # needs to be set later - self._size = 0 - - self.renewHTTPRequest() - self.dl = None - - - def renewHTTPRequest(self): - if hasattr(self, "http"): self.http.close() - self.http = HTTPRequest(self.cj, self.options) - - def setLastURL(self, val): - self.http.lastURL = val - - # tunnel some attributes from HTTP Request to Browser - lastEffectiveURL = property(lambda self: self.http.lastEffectiveURL) - lastURL = property(lambda self: self.http.lastURL, setLastURL) - code = property(lambda self: self.http.code) - cookieJar = property(lambda self: self.cj) - - def setCookieJar(self, cj): - self.cj = cj - self.http.cj = cj - - @property - def speed(self): - if self.dl: - return self.dl.speed - return 0 - - @property - def size(self): - if self._size: - return self._size - if self.dl: - return self.dl.size - return 0 - - @property - def name(self): - if self.dl: - return self.dl.name - else: - return "" - - @property - def arrived(self): - if self.dl: - return self.dl.arrived - return 0 - - @property - def percent(self): - if not self.size: return 0 - return (self.arrived * 100) / self.size - - def clearCookies(self): - if self.cj: - self.cj.clear() - self.http.clearCookies() - - def clearReferer(self): - self.http.lastURL = None - - def abortDownloads(self): - self.http.abort = True - if self.dl: - self._size = self.dl.size - self.dl.abort = True - - def httpDownload(self, url, filename, get={}, post={}, ref=True, cookies=True, chunks=1, resume=False, - disposition=False): - """ this can also download ftp """ - self._size = 0 - self.dl = HTTPDownload(url, filename, get, post, self.lastEffectiveURL if ref else None, - self.cj if cookies else None, self.bucket, self.options, disposition) - name = self.dl.download(chunks, resume) - self._size = self.dl.size - - self.dl = None - - return name - - def load(self, *args, **kwargs): - """ retrieves page """ - return self.http.load(*args, **kwargs) - - def putHeader(self, name, value): - """ add a header to the request """ - self.http.putHeader(name, value) - - def addAuth(self, pwd): - """Adds user and pw for http auth - - :param pwd: string, user:password - """ - self.options["auth"] = pwd - self.renewHTTPRequest() #we need a new request - - def removeAuth(self): - if "auth" in self.options: del self.options["auth"] - self.renewHTTPRequest() - - def setOption(self, name, value): - """Adds an option to the request, see HTTPRequest for existing ones""" - self.options[name] = value - - def deleteOption(self, name): - if name in self.options: del self.options[name] - - def clearHeaders(self): - self.http.clearHeaders() - - def close(self): - """ cleanup """ - if hasattr(self, "http"): - self.http.close() - del self.http - if hasattr(self, "dl"): - del self.dl - if hasattr(self, "cj"): - del self.cj - -if __name__ == "__main__": - browser = Browser()#proxies={"socks5": "localhost:5000"}) - ip = "http://www.whatismyip.com/automation/n09230945.asp" - #browser.getPage("http://google.com/search?q=bar") - #browser.getPage("https://encrypted.google.com/") - #print browser.getPage(ip) - #print browser.getRedirectLocation("http://google.com/") - #browser.getPage("https://encrypted.google.com/") - #browser.getPage("http://google.com/search?q=bar") - - browser.httpDownload("http://speedtest.netcologne.de/test_10mb.bin", "test_10mb.bin") - diff --git a/module/network/Bucket.py b/module/network/Bucket.py deleted file mode 100644 index db67faa4a..000000000 --- a/module/network/Bucket.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from time import time -from threading import Lock - -# 10kb minimum rate -MIN_RATE = 10240 - -class Bucket: - def __init__(self): - self.rate = 0 # bytes per second, maximum targeted throughput - self.tokens = 0 - self.timestamp = time() - self.lock = Lock() - - def __nonzero__(self): - return False if self.rate < MIN_RATE else True - - def setRate(self, rate): - self.lock.acquire() - self.rate = int(rate) - self.lock.release() - - def consumed(self, amount): - """ return the time the process has to sleep, after it consumed a specified amount """ - if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise - self.lock.acquire() - - self.calc_tokens() - self.tokens -= amount - - if self.tokens < 0: - time = -self.tokens/float(self.rate) - else: - time = 0 - - self.lock.release() - return time - - def calc_tokens(self): - if self.tokens < self.rate: - now = time() - delta = self.rate * (now - self.timestamp) - self.tokens = min(self.rate, self.tokens + delta) - self.timestamp = now - diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py deleted file mode 100644 index ea2c43a9e..000000000 --- a/module/network/CookieJar.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" - -from time import time - -# TODO: replace with simplecookie? -class CookieJar(): - def __init__(self, pluginname): - self.cookies = {} - self.pluginname = pluginname - - def __repr__(self): - return ("<CookieJar plugin=%s>\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) - - def addCookies(self, clist): - for c in clist: - name = c.split("\t")[5] - self.cookies[name] = c - - def getCookies(self): - return self.cookies.values() - - def getCookie(self, name): - if name in self.cookies: - return self.cookies[name].split("\t")[6] - else: - return None - - def setCookie(self, domain, name, value, path="/", exp=None): - if not exp: exp = time() + 3600 * 24 * 180 - - # dot makes it valid on all subdomains - s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) - self.cookies[name] = s - - def clear(self): - self.cookies = {}
\ No newline at end of file diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py deleted file mode 100644 index 84a2acc5f..000000000 --- a/module/network/HTTPChunk.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -from os import remove, stat, fsync -from os.path import exists -from time import sleep -from re import search - -import codecs -import pycurl - -from module.utils import remove_chars -from module.utils.fs import fs_encode - -from HTTPRequest import HTTPRequest - -class WrongFormat(Exception): - pass - - -class ChunkInfo(): - def __init__(self, name): - self.name = unicode(name) - self.size = 0 - self.resume = False - self.chunks = [] - - def __repr__(self): - ret = "ChunkInfo: %s, %s\n" % (self.name, self.size) - for i, c in enumerate(self.chunks): - ret += "%s# %s\n" % (i, c[1]) - - return ret - - def setSize(self, size): - self.size = int(size) - - def addChunk(self, name, range): - self.chunks.append((name, range)) - - def clear(self): - self.chunks = [] - - def createChunks(self, chunks): - self.clear() - chunk_size = self.size / chunks - - current = 0 - for i in range(chunks): - end = self.size - 1 if (i == chunks - 1) else current + chunk_size - self.addChunk("%s.chunk%s" % (self.name, i), (current, end)) - current += chunk_size + 1 - - - def save(self): - fs_name = fs_encode("%s.chunks" % self.name) - fh = codecs.open(fs_name, "w", "utf_8") - fh.write("name:%s\n" % self.name) - fh.write("size:%s\n" % self.size) - for i, c in enumerate(self.chunks): - fh.write("#%d:\n" % i) - fh.write("\tname:%s\n" % c[0]) - fh.write("\trange:%i-%i\n" % c[1]) - fh.close() - - @staticmethod - def load(name): - fs_name = fs_encode("%s.chunks" % name) - if not exists(fs_name): - raise IOError() - fh = codecs.open(fs_name, "r", "utf_8") - name = fh.readline()[:-1] - size = fh.readline()[:-1] - if name.startswith("name:") and size.startswith("size:"): - name = name[5:] - size = size[5:] - else: - fh.close() - raise WrongFormat() - ci = ChunkInfo(name) - ci.loaded = True - ci.setSize(size) - while True: - if not fh.readline(): #skip line - break - name = fh.readline()[1:-1] - range = fh.readline()[1:-1] - if name.startswith("name:") and range.startswith("range:"): - name = name[5:] - range = range[6:].split("-") - else: - raise WrongFormat() - - ci.addChunk(name, (long(range[0]), long(range[1]))) - fh.close() - return ci - - def remove(self): - fs_name = fs_encode("%s.chunks" % self.name) - if exists(fs_name): remove(fs_name) - - def getCount(self): - return len(self.chunks) - - def getChunkName(self, index): - return self.chunks[index][0] - - def getChunkRange(self, index): - return self.chunks[index][1] - - -class HTTPChunk(HTTPRequest): - def __init__(self, id, parent, range=None, resume=False): - self.id = id - self.p = parent # HTTPDownload instance - self.range = range # tuple (start, end) - self.resume = resume - self.log = parent.log - - self.size = range[1] - range[0] if range else -1 - self.arrived = 0 - self.lastURL = self.p.referer - - self.c = pycurl.Curl() - - self.header = "" - self.headerParsed = False #indicates if the header has been processed - - self.fp = None #file handle - - self.initHandle() - self.setInterface(self.p.options) - - self.BOMChecked = False # check and remove byte order mark - - self.rep = None - - self.sleep = 0.000 - self.lastSize = 0 - - def __repr__(self): - return "<HTTPChunk id=%d, size=%d, arrived=%d>" % (self.id, self.size, self.arrived) - - @property - def cj(self): - return self.p.cj - - def getHandle(self): - """ returns a Curl handle ready to use for perform/multiperform """ - - self.setRequestContext(self.p.url, self.p.get, self.p.post, self.p.referer, self.p.cj) - self.c.setopt(pycurl.WRITEFUNCTION, self.writeBody) - self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) - - # request all bytes, since some servers in russia seems to have a defect arihmetic unit - - fs_name = fs_encode(self.p.info.getChunkName(self.id)) - if self.resume: - self.fp = open(fs_name, "ab") - self.arrived = self.fp.tell() - if not self.arrived: - self.arrived = stat(fs_name).st_size - - if self.range: - #do nothing if chunk already finished - if self.arrived + self.range[0] >= self.range[1]: return None - - if self.id == len(self.p.info.chunks) - 1: #as last chunk dont set end range, so we get everything - range = "%i-" % (self.arrived + self.range[0]) - else: - range = "%i-%i" % (self.arrived + self.range[0], min(self.range[1] + 1, self.p.size - 1)) - - self.log.debug("Chunked resume with range %s" % range) - self.c.setopt(pycurl.RANGE, range) - else: - self.log.debug("Resume File from %i" % self.arrived) - self.c.setopt(pycurl.RESUME_FROM, self.arrived) - - else: - if self.range: - if self.id == len(self.p.info.chunks) - 1: # see above - range = "%i-" % self.range[0] - else: - range = "%i-%i" % (self.range[0], min(self.range[1] + 1, self.p.size - 1)) - - self.log.debug("Chunked with range %s" % range) - self.c.setopt(pycurl.RANGE, range) - - self.fp = open(fs_name, "wb") - - return self.c - - def writeHeader(self, buf): - self.header += buf - #@TODO forward headers?, this is possibly unneeded, when we just parse valid 200 headers - # as first chunk, we will parse the headers - if not self.range and self.header.endswith("\r\n\r\n"): - self.parseHeader() - elif not self.range and buf.startswith("150") and "data connection" in buf: #ftp file size parsing - size = search(r"(\d+) bytes", buf) - if size: - self.p.size = int(size.group(1)) - self.p.chunkSupport = True - - self.headerParsed = True - - def writeBody(self, buf): - #ignore BOM, it confuses unrar - if not self.BOMChecked: - if [ord(b) for b in buf[:3]] == [239, 187, 191]: - buf = buf[3:] - self.BOMChecked = True - - size = len(buf) - - self.arrived += size - - self.fp.write(buf) - - if self.p.bucket: - sleep(self.p.bucket.consumed(size)) - else: - # Avoid small buffers, increasing sleep time slowly if buffer size gets smaller - # otherwise reduce sleep time percentile (values are based on tests) - # So in general cpu time is saved without reducing bandwidth too much - - if size < self.lastSize: - self.sleep += 0.002 - else: - self.sleep *= 0.7 - - self.lastSize = size - - sleep(self.sleep) - - if self.range and self.arrived > self.size: - return 0 #close if we have enough data - - - def parseHeader(self): - """parse data from received header""" - for orgline in self.decodeResponse(self.header).splitlines(): - line = orgline.strip().lower() - if line.startswith("accept-ranges") and "bytes" in line: - self.p.chunkSupport = True - - if "content-disposition" in line: - - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", line) - if m: - name = remove_chars(m.groupdict()['name'], "\"';/").strip() - self.p._name = name - self.log.debug("Content-Disposition: %s" % name) - - if not self.resume and line.startswith("content-length"): - self.p.size = int(line.split(":")[1]) - - self.headerParsed = True - - def stop(self): - """The download will not proceed after next call of writeBody""" - self.range = [0,0] - self.size = 0 - - def resetRange(self): - """ Reset the range, so the download will load all data available """ - self.range = None - - def setRange(self, range): - self.range = range - self.size = range[1] - range[0] - - def flushFile(self): - """ flush and close file """ - self.fp.flush() - fsync(self.fp.fileno()) #make sure everything was written to disk - self.fp.close() #needs to be closed, or merging chunks will fail - - def close(self): - """ closes everything, unusable after this """ - if self.fp: self.fp.close() - self.c.close() - if hasattr(self, "p"): del self.p diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py deleted file mode 100644 index 498cd5979..000000000 --- a/module/network/HTTPDownload.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from os import remove -from os.path import dirname -from time import time -from shutil import move -from logging import getLogger - -import pycurl - -from HTTPChunk import ChunkInfo, HTTPChunk -from HTTPRequest import BadHeader - -from module.plugins.Base import Abort -from module.utils.fs import save_join, fs_encode - -# TODO: save content-disposition for resuming - -class HTTPDownload(): - """ loads an url, http + ftp supported """ - - def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, - options={}, disposition=False): - self.url = url - self.filename = filename #complete file destination, not only name - self.get = get - self.post = post - self.referer = referer - self.cj = cj #cookiejar if cookies are needed - self.bucket = bucket - self.options = options - self.disposition = disposition - # all arguments - - self.abort = False - self.size = 0 - self._name = ""# will be parsed from content disposition - - self.chunks = [] - - self.log = getLogger("log") - - try: - self.info = ChunkInfo.load(filename) - self.info.resume = True #resume is only possible with valid info file - self.size = self.info.size - self.infoSaved = True - except IOError: - self.info = ChunkInfo(filename) - - self.chunkSupport = None - self.m = pycurl.CurlMulti() - - #needed for speed calculation - self.lastArrived = [] - self.speeds = [] - self.lastSpeeds = [0, 0] - - @property - def speed(self): - last = [sum(x) for x in self.lastSpeeds if x] - return (sum(self.speeds) + sum(last)) / (1 + len(last)) - - @property - def arrived(self): - return sum([c.arrived for c in self.chunks]) - - @property - def percent(self): - if not self.size: return 0 - return (self.arrived * 100) / self.size - - @property - def name(self): - return self._name if self.disposition else "" - - def _copyChunks(self): - init = fs_encode(self.info.getChunkName(0)) #initial chunk name - - if self.info.getCount() > 1: - fo = open(init, "rb+") #first chunkfile - for i in range(1, self.info.getCount()): - #input file - fo.seek( - self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks - fname = fs_encode("%s.chunk%d" % (self.filename, i)) - fi = open(fname, "rb") - buf = 32 * 1024 - while True: #copy in chunks, consumes less memory - data = fi.read(buf) - if not data: - break - fo.write(data) - fi.close() - if fo.tell() < self.info.getChunkRange(i)[1]: - fo.close() - remove(init) - self.info.remove() #there are probably invalid chunks - raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.") - remove(fname) #remove chunk - fo.close() - - if self.name: - self.filename = save_join(dirname(self.filename), self.name) - - move(init, fs_encode(self.filename)) - self.info.remove() #remove info file - - def download(self, chunks=1, resume=False): - """ returns new filename or None """ - - chunks = max(1, chunks) - resume = self.info.resume and resume - - try: - self._download(chunks, resume) - except pycurl.error, e: - #code 33 - no resume - code = e.args[0] - if code == 33: - # try again without resume - self.log.debug("Errno 33 -> Restart without resume") - - #remove old handles - for chunk in self.chunks: - self.closeChunk(chunk) - - return self._download(chunks, False) - else: - raise - finally: - self.close() - - return self.name - - def _download(self, chunks, resume): - if not resume: - self.info.clear() - self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry - - self.chunks = [] - - init = HTTPChunk(0, self, None, resume) #initial chunk that will load complete file (if needed) - - self.chunks.append(init) - self.m.add_handle(init.getHandle()) - - lastFinishCheck = 0 - lastTimeCheck = 0 - chunksDone = set() # list of curl handles that are finished - chunksCreated = False - done = False - if self.info.getCount() > 1: # This is a resume, if we were chunked originally assume still can - self.chunkSupport = True - - while 1: - #need to create chunks - if not chunksCreated and self.chunkSupport and self.size: #will be set later by first chunk - - if not resume: - self.info.setSize(self.size) - self.info.createChunks(chunks) - self.info.save() - - chunks = self.info.getCount() - - init.setRange(self.info.getChunkRange(0)) - - for i in range(1, chunks): - c = HTTPChunk(i, self, self.info.getChunkRange(i), resume) - - handle = c.getHandle() - if handle: - self.chunks.append(c) - self.m.add_handle(handle) - else: - #close immediately - self.log.debug("Invalid curl handle -> closed") - c.close() - - chunksCreated = True - - while 1: - ret, num_handles = self.m.perform() - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - - t = time() - - # reduce these calls - # when num_q is 0, the loop is exited - while lastFinishCheck + 0.5 < t: - # list of failed curl handles - failed = [] - ex = None # save only last exception, we can only raise one anyway - - num_q, ok_list, err_list = self.m.info_read() - for c in ok_list: - chunk = self.findChunk(c) - try: # check if the header implies success, else add it to failed list - chunk.verifyHeader() - except BadHeader, e: - self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e))) - failed.append(chunk) - ex = e - else: - chunksDone.add(c) - - for c in err_list: - curl, errno, msg = c - chunk = self.findChunk(curl) - #test if chunk was finished - if errno != 23 or "0 !=" not in msg: - failed.append(chunk) - ex = pycurl.error(errno, msg) - self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(ex))) - continue - - try: # check if the header implies success, else add it to failed list - chunk.verifyHeader() - except BadHeader, e: - self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e))) - failed.append(chunk) - ex = e - else: - chunksDone.add(curl) - if not num_q: # no more info to get - - # check if init is not finished so we reset download connections - # note that other chunks are closed and everything downloaded with initial connection - if failed and init not in failed and init.c not in chunksDone: - self.log.error(_("Download chunks failed, fallback to single connection | %s" % (str(ex)))) - - #list of chunks to clean and remove - to_clean = filter(lambda x: x is not init, self.chunks) - for chunk in to_clean: - self.closeChunk(chunk) - self.chunks.remove(chunk) - remove(fs_encode(self.info.getChunkName(chunk.id))) - - #let first chunk load the rest and update the info file - init.resetRange() - self.info.clear() - self.info.addChunk("%s.chunk0" % self.filename, (0, self.size)) - self.info.save() - elif failed: - raise ex - - lastFinishCheck = t - - if len(chunksDone) >= len(self.chunks): - if len(chunksDone) > len(self.chunks): - self.log.warning("Finished download chunks size incorrect, please report bug.") - done = True #all chunks loaded - - break - - if done: - break #all chunks loaded - - # calc speed once per second, averaging over 3 seconds - if lastTimeCheck + 1 < t: - diff = [c.arrived - (self.lastArrived[i] if len(self.lastArrived) > i else 0) for i, c in - enumerate(self.chunks)] - - self.lastSpeeds[1] = self.lastSpeeds[0] - self.lastSpeeds[0] = self.speeds - self.speeds = [float(a) / (t - lastTimeCheck) for a in diff] - self.lastArrived = [c.arrived for c in self.chunks] - lastTimeCheck = t - - if self.abort: - raise Abort() - - self.m.select(1) - - for chunk in self.chunks: - chunk.flushFile() #make sure downloads are written to disk - - self._copyChunks() - - def findChunk(self, handle): - """ linear search to find a chunk (should be ok since chunk size is usually low) """ - for chunk in self.chunks: - if chunk.c == handle: return chunk - - def closeChunk(self, chunk): - try: - self.m.remove_handle(chunk.c) - except pycurl.error, e: - self.log.debug("Error removing chunk: %s" % str(e)) - finally: - chunk.close() - - def close(self): - """ cleanup """ - for chunk in self.chunks: - self.closeChunk(chunk) - - self.chunks = [] - if hasattr(self, "m"): - self.m.close() - del self.m - if hasattr(self, "cj"): - del self.cj - if hasattr(self, "info"): - del self.info - -if __name__ == "__main__": - url = "http://speedtest.netcologne.de/test_100mb.bin" - - from Bucket import Bucket - - bucket = Bucket() - bucket.setRate(200 * 1024) - bucket = None - - print "starting" - - dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket) - dwnld.download(chunks=3, resume=True) diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py deleted file mode 100644 index 874da368b..000000000 --- a/module/network/HTTPRequest.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import pycurl - -from codecs import getincrementaldecoder, lookup, BOM_UTF8 -from urllib import quote, urlencode -from httplib import responses -from logging import getLogger -from cStringIO import StringIO - -from module.plugins.Base import Abort - -def myquote(url): - return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - -def myurlencode(data): - data = dict(data) - return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ - y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) - -bad_headers = range(400, 404) + range(405, 418) + range(500, 506) - -class BadHeader(Exception): - def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) - self.code = code - self.content = content - - -class HTTPRequest(): - def __init__(self, cookies=None, options=None): - self.c = pycurl.Curl() - self.rep = StringIO() - - self.cj = cookies #cookiejar - - self.lastURL = None - self.lastEffectiveURL = None - self.abort = False - self.code = 0 # last http code - - self.header = "" - - self.headers = [] #temporary request header - - self.initHandle() - self.setInterface(options) - self.setOptions(options) - - self.c.setopt(pycurl.WRITEFUNCTION, self.write) - self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) - - self.log = getLogger("log") - - - def initHandle(self): - """ sets common options to curl handle """ - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.MAXREDIRS, 5) - self.c.setopt(pycurl.CONNECTTIMEOUT, 30) - self.c.setopt(pycurl.NOSIGNAL, 1) - self.c.setopt(pycurl.NOPROGRESS, 1) - if hasattr(pycurl, "AUTOREFERER"): - self.c.setopt(pycurl.AUTOREFERER, 1) - self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - # Interval for low speed, detects connection loss, but can abort dl if hoster stalls the download - self.c.setopt(pycurl.LOW_SPEED_TIME, 45) - self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) - - #self.c.setopt(pycurl.VERBOSE, 1) - - self.c.setopt(pycurl.USERAGENT, - "Mozilla/5.0 (Windows NT 6.1; Win64; x64;en; rv:5.0) Gecko/20110619 Firefox/5.0") - if pycurl.version_info()[7]: - self.c.setopt(pycurl.ENCODING, "gzip, deflate") - self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*", - "Accept-Language: en-US,en", - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Connection: keep-alive", - "Keep-Alive: 300", - "Expect:"]) - - def setInterface(self, options): - - interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] - - if interface and interface.lower() != "none": - self.c.setopt(pycurl.INTERFACE, str(interface)) - - if proxy: - if proxy["type"] == "socks4": - self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) - elif proxy["type"] == "socks5": - self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) - else: - self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) - - self.c.setopt(pycurl.PROXY, str(proxy["address"])) - self.c.setopt(pycurl.PROXYPORT, proxy["port"]) - - if proxy["username"]: - self.c.setopt(pycurl.PROXYUSERPWD, str("%s:%s" % (proxy["username"], proxy["password"]))) - - if ipv6: - self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) - else: - self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) - - if "auth" in options: - self.c.setopt(pycurl.USERPWD, str(options["auth"])) - - if "timeout" in options: - self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) - - def setOptions(self, options): - """ Sets same options as available in pycurl """ - for k, v in options.iteritems(): - if hasattr(pycurl, k): - self.c.setopt(getattr(pycurl, k), v) - - def addCookies(self): - """ put cookies from curl handle to cj """ - if self.cj: - self.cj.addCookies(self.c.getinfo(pycurl.INFO_COOKIELIST)) - - def getCookies(self): - """ add cookies from cj to curl handle """ - if self.cj: - for c in self.cj.getCookies(): - self.c.setopt(pycurl.COOKIELIST, c) - return - - def clearCookies(self): - self.c.setopt(pycurl.COOKIELIST, "") - - def setRequestContext(self, url, get, post, referer, cookies, multipart=False): - """ sets everything needed for the request """ - - url = myquote(url) - - if get: - get = urlencode(get) - url = "%s?%s" % (url, get) - - self.c.setopt(pycurl.URL, url) - self.lastURL = url - - if post: - self.c.setopt(pycurl.POST, 1) - if not multipart: - if type(post) == unicode: - post = str(post) #unicode not allowed - elif type(post) == str: - pass - else: - post = myurlencode(post) - - self.c.setopt(pycurl.POSTFIELDS, post) - else: - post = [(x, y.encode('utf8') if type(y) == unicode else y ) for x, y in post.iteritems()] - self.c.setopt(pycurl.HTTPPOST, post) - else: - self.c.setopt(pycurl.POST, 0) - - if referer and self.lastURL: - self.c.setopt(pycurl.REFERER, str(self.lastURL)) - - if cookies: - self.c.setopt(pycurl.COOKIEFILE, "") - self.c.setopt(pycurl.COOKIEJAR, "") - self.getCookies() - - - def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False): - """ load and returns a given page """ - - self.setRequestContext(url, get, post, referer, cookies, multipart) - - # TODO: use http/rfc message instead - self.header = "" - - self.c.setopt(pycurl.HTTPHEADER, self.headers) - - if just_header: - self.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.c.setopt(pycurl.NOBODY, 1) #TODO: nobody= no post? - - # overwrite HEAD request, we want a common request type - if post: - self.c.setopt(pycurl.CUSTOMREQUEST, "POST") - else: - self.c.setopt(pycurl.CUSTOMREQUEST, "GET") - - try: - self.c.perform() - rep = self.header - finally: - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) - self.c.unsetopt(pycurl.CUSTOMREQUEST) - - else: - self.c.perform() - rep = self.getResponse() - - self.c.setopt(pycurl.POSTFIELDS, "") - self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) - self.code = self.verifyHeader() - - self.addCookies() - - if decode: - rep = self.decodeResponse(rep) - - return rep - - def verifyHeader(self): - """ raise an exceptions on bad headers """ - code = int(self.c.getinfo(pycurl.RESPONSE_CODE)) - # TODO: raise anyway to be consistent, also rename exception - if code in bad_headers: - #404 will NOT raise an exception - raise BadHeader(code, self.getResponse()) - return code - - def checkHeader(self): - """ check if header indicates failure""" - return int(self.c.getinfo(pycurl.RESPONSE_CODE)) not in bad_headers - - def getResponse(self): - """ retrieve response from string io """ - if self.rep is None: return "" - value = self.rep.getvalue() - self.rep.close() - self.rep = StringIO() - return value - - def decodeResponse(self, rep): - """ decode with correct encoding, relies on header """ - header = self.header.splitlines() - encoding = "utf8" # default encoding - - for line in header: - line = line.lower().replace(" ", "") - if not line.startswith("content-type:") or\ - ("text" not in line and "application" not in line): - continue - - none, delemiter, charset = line.rpartition("charset=") - if delemiter: - charset = charset.split(";") - if charset: - encoding = charset[0] - - try: - #self.log.debug("Decoded %s" % encoding ) - if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): - encoding = 'utf-8-sig' - - decoder = getincrementaldecoder(encoding)("replace") - rep = decoder.decode(rep, True) - - #TODO: html_unescape as default - - except LookupError: - self.log.debug("No Decoder found for %s" % encoding) - except Exception: - self.log.debug("Error when decoding string from %s." % encoding) - - return rep - - def write(self, buf): - """ writes response """ - if self.rep.tell() > 1000000 or self.abort: - rep = self.getResponse() - if self.abort: raise Abort() - f = open("response.dump", "wb") - f.write(rep) - f.close() - raise Exception("Loaded Url exceeded limit") - - self.rep.write(buf) - - def writeHeader(self, buf): - """ writes header """ - self.header += buf - - def putHeader(self, name, value): - self.headers.append("%s: %s" % (name, value)) - - def clearHeaders(self): - self.headers = [] - - def close(self): - """ cleanup, unusable after this """ - self.rep.close() - if hasattr(self, "cj"): - del self.cj - if hasattr(self, "c"): - self.c.close() - del self.c - -if __name__ == "__main__": - url = "http://pyload.org" - c = HTTPRequest() - print c.load(url) - diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py deleted file mode 100644 index 1581be9fc..000000000 --- a/module/network/RequestFactory.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" - -from threading import Lock - -from Browser import Browser -from Bucket import Bucket -from HTTPRequest import HTTPRequest -from CookieJar import CookieJar - -class RequestFactory(): - def __init__(self, core): - self.lock = Lock() - self.core = core - self.bucket = Bucket() - self.updateBucket() - - @property - def iface(self): - return self.core.config["download"]["interface"] - - def getRequest(self, pluginName, cj=None): - # TODO: mostly obsolete, browser could be removed completely - req = Browser(self.bucket, self.getOptions()) - - if cj: - req.setCookieJar(cj) - else: - req.setCookieJar(CookieJar(pluginName)) - - return req - - def getHTTPRequest(self, **kwargs): - """ returns a http request, don't forget to close it ! """ - options = self.getOptions() - options.update(kwargs) # submit kwargs as additional options - return HTTPRequest(CookieJar(None), options) - - def getURL(self, *args, **kwargs): - """ see HTTPRequest for argument list """ - h = HTTPRequest(None, self.getOptions()) - try: - rep = h.load(*args, **kwargs) - finally: - h.close() - - return rep - - def openCookieJar(self, pluginname): - """Create new CookieJar""" - return CookieJar(pluginname) - - def getProxies(self): - """ returns a proxy list for the request classes """ - if not self.core.config["proxy"]["proxy"]: - return {} - else: - type = "http" - setting = self.core.config["proxy"]["type"].lower() - if setting == "socks4": type = "socks4" - elif setting == "socks5": type = "socks5" - - username = None - if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none": - username = self.core.config["proxy"]["username"] - - pw = None - if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none": - pw = self.core.config["proxy"]["password"] - - return { - "type": type, - "address": self.core.config["proxy"]["address"], - "port": self.core.config["proxy"]["port"], - "username": username, - "password": pw, - } - - def getOptions(self): - """returns options needed for pycurl""" - return {"interface": self.iface, - "proxies": self.getProxies(), - "ipv6": self.core.config["download"]["ipv6"]} - - def updateBucket(self): - """ set values in the bucket according to settings""" - if not self.core.config["download"]["limit_speed"]: - self.bucket.setRate(-1) - else: - self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024) - -# needs pyreq in global namespace -def getURL(*args, **kwargs): - return pyreq.getURL(*args, **kwargs) - - -def getRequest(*args, **kwargs): - return pyreq.getHTTPRequest() diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py deleted file mode 100644 index 7a1a98cb5..000000000 --- a/module/network/XDCCRequest.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: jeix -""" - -import socket -import re - -from os import remove -from os.path import exists - -from time import time - -import struct -from select import select - -from module.plugins.Plugin import Abort - - -class XDCCRequest(): - def __init__(self, timeout=30, proxies={}): - - self.proxies = proxies - self.timeout = timeout - - self.filesize = 0 - self.recv = 0 - self.speed = 0 - - self.abort = False - - - def createSocket(self): - # proxytype = None - # proxy = None - # if self.proxies.has_key("socks5"): - # proxytype = socks.PROXY_TYPE_SOCKS5 - # proxy = self.proxies["socks5"] - # elif self.proxies.has_key("socks4"): - # proxytype = socks.PROXY_TYPE_SOCKS4 - # proxy = self.proxies["socks4"] - # if proxytype: - # sock = socks.socksocket() - # t = _parse_proxy(proxy) - # sock.setproxy(proxytype, addr=t[3].split(":")[0], port=int(t[3].split(":")[1]), username=t[1], password=t[2]) - # else: - # sock = socket.socket() - # return sock - - return socket.socket() - - def download(self, ip, port, filename, irc, progressNotify=None): - - ircbuffer = "" - lastUpdate = time() - cumRecvLen = 0 - - dccsock = self.createSocket() - - dccsock.settimeout(self.timeout) - dccsock.connect((ip, port)) - - if exists(filename): - i = 0 - nameParts = filename.rpartition(".") - while True: - newfilename = "%s-%d%s%s" % (nameParts[0], i, nameParts[1], nameParts[2]) - i += 1 - - if not exists(newfilename): - filename = newfilename - break - - fh = open(filename, "wb") - - # recv loop for dcc socket - while True: - if self.abort: - dccsock.close() - fh.close() - remove(filename) - raise Abort() - - self._keepAlive(irc, ircbuffer) - - data = dccsock.recv(4096) - dataLen = len(data) - self.recv += dataLen - - cumRecvLen += dataLen - - now = time() - timespan = now - lastUpdate - if timespan > 1: - self.speed = cumRecvLen / timespan - cumRecvLen = 0 - lastUpdate = now - - if progressNotify: - progressNotify(self.percent) - - - if not data: - break - - fh.write(data) - - # acknowledge data by sending number of received bytes - dccsock.send(struct.pack('!I', self.recv)) - - dccsock.close() - fh.close() - - return filename - - def _keepAlive(self, sock, readbuffer): - fdset = select([sock], [], [], 0) - if sock not in fdset[0]: - return - - readbuffer += sock.recv(1024) - temp = readbuffer.split("\n") - readbuffer = temp.pop() - - for line in temp: - line = line.rstrip() - first = line.split() - if first[0] == "PING": - sock.send("PONG %s\r\n" % first[1]) - - def abortDownloads(self): - self.abort = True - - @property - def size(self): - return self.filesize - - @property - def arrived(self): - return self.recv - - @property - def percent(self): - if not self.filesize: return 0 - return (self.recv * 100) / self.filesize - - def close(self): - pass diff --git a/module/network/__init__.py b/module/network/__init__.py deleted file mode 100644 index 8b1378917..000000000 --- a/module/network/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/module/plugins/Account.py b/module/plugins/Account.py deleted file mode 100644 index 3ba819a6f..000000000 --- a/module/plugins/Account.py +++ /dev/null @@ -1,295 +0,0 @@ -# -*- coding: utf-8 -*- - -from time import time -from traceback import print_exc -from threading import RLock - -from module.utils import compare_time, format_size, parseFileSize, lock, from_string -from module.Api import AccountInfo -from module.network.CookieJar import CookieJar - -from Base import Base - - -class WrongPassword(Exception): - pass - -#noinspection PyUnresolvedReferences -class Account(Base): - """ - Base class for every account plugin. - Just overwrite `login` and cookies will be stored and the account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ - An instance of this class is created for every entered account, it holds all \ - fields of AccountInfo ttype, and can be set easily at runtime. - """ - - # constants for special values - UNKNOWN = -1 - UNLIMITED = -2 - - # Default values - owner = None - valid = True - validuntil = -1 - trafficleft = -1 - maxtraffic = -1 - premium = True - activated = True - shared = False - - #: after that time [in minutes] pyload will relogin the account - login_timeout = 600 - #: account data will be reloaded after this time - info_threshold = 600 - - # known options - known_opt = ("time", "limitDL") - - def __init__(self, manager, loginname, password, options): - Base.__init__(self, manager.core) - - if "activated" in options: - self.activated = from_string(options["activated"], "bool") - else: - self.activated = Account.activated - - for opt in self.known_opt: - if opt not in options: - options[opt] = "" - - for opt in options.keys(): - if opt not in self.known_opt: - del options[opt] - - self.loginname = loginname - self.options = options - - self.manager = manager - - self.lock = RLock() - self.timestamp = 0 - self.login_ts = 0 # timestamp for login - self.cj = CookieJar(self.__name__) - self.password = password - self.error = None - - self.init() - - def toInfoData(self): - return AccountInfo(self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft, - self.maxtraffic, - self.premium, self.activated, self.shared, self.options) - - def init(self): - pass - - def login(self, req): - """login into account, the cookies will be saved so the user can be recognized - - :param req: `Request` instance - """ - raise NotImplemented - - def relogin(self): - """ Force a login. """ - req = self.getAccountRequest() - try: - return self._login(req) - finally: - req.close() - - @lock - def _login(self, req): - # set timestamp for login - self.login_ts = time() - - try: - try: - self.login(req) - except TypeError: #TODO: temporary - self.logDebug("Deprecated .login(...) signature omit user, data") - self.login(self.loginname, {"password": self.password}, req) - - self.valid = True - except WrongPassword: - self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname - , "msg": _("Wrong Password")}) - self.valid = False - - except Exception, e: - self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname - , "msg": e}) - self.valid = False - if self.core.debug: - print_exc() - - return self.valid - - def restoreDefaults(self): - self.validuntil = Account.validuntil - self.trafficleft = Account.trafficleft - self.maxtraffic = Account.maxtraffic - self.premium = Account.premium - - def update(self, password=None, options=None): - """ updates the account and returns true if anything changed """ - - self.login_ts = 0 - self.valid = True #set valid, so the login will be retried - - if "activated" in options: - self.activated = from_string(options["avtivated"], "bool") - - if password: - self.password = password - self.relogin() - return True - if options: - # remove unknown options - for opt in options.keys(): - if opt not in self.known_opt: - del options[opt] - - before = self.options - self.options.update(options) - return self.options != before - - def getAccountRequest(self): - return self.core.requestFactory.getRequest(self.__name__, self.cj) - - def getDownloadSettings(self): - """ Can be overwritten to change download settings. Default is no chunkLimit, max dl limit, resumeDownload - - :return: (chunkLimit, limitDL, resumeDownload) / (int, int ,bool) - """ - return -1, 0, True - - @lock - def getAccountInfo(self, force=False): - """retrieve account info's for an user, do **not** overwrite this method!\\ - just use it to retrieve info's in hoster plugins. see `loadAccountInfo` - - :param name: username - :param force: reloads cached account information - :return: dictionary with information - """ - if force or self.timestamp + self.info_threshold * 60 < time(): - - # make sure to login - req = self.getAccountRequest() - self.checkLogin(req) - self.logInfo(_("Get Account Info for %s") % self.loginname) - try: - try: - infos = self.loadAccountInfo(req) - except TypeError: #TODO: temporary - self.logDebug("Deprecated .loadAccountInfo(...) signature, omit user argument.") - infos = self.loadAccountInfo(self.loginname, req) - except Exception, e: - infos = {"error": str(e)} - self.logError(_("Error: %s") % e) - finally: - req.close() - - self.logDebug("Account Info: %s" % str(infos)) - self.timestamp = time() - - self.restoreDefaults() # reset to initial state - if type(infos) == dict: # copy result from dict to class - for k, v in infos.iteritems(): - if hasattr(self, k): - setattr(self, k, v) - else: - self.logDebug("Unknown attribute %s=%s" % (k, v)) - - #TODO: remove user - def loadAccountInfo(self, req): - """ Overwrite this method and set account attributes within this method. - - :param user: Deprecated - :param req: Request instance - :return: - """ - pass - - def getAccountCookies(self, user): - self.logDebug("Deprecated method .getAccountCookies -> use account.cj") - return self.cj - - def getAccountData(self, user): - self.logDebug("Deprecated method .getAccountData -> use fields directly") - return {"password": self.password} - - def isPremium(self, user=None): - if user: self.logDebug("Deprecated Argument user for .isPremium()", user) - return self.premium - - def isUsable(self): - """Check several constraints to determine if account should be used""" - if not self.valid or not self.activated: return False - - if self.options["time"]: - time_data = "" - try: - time_data = self.options["time"] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - return False - except: - self.logWarning(_("Your Time %s has a wrong format, use: 1:22-3:44") % time_data) - - if 0 <= self.validuntil < time(): - return False - if self.trafficleft is 0: # test explicitly for 0 - return False - - return True - - def parseTraffic(self, string): #returns kbyte - return parseFileSize(string) / 1024 - - def formatTrafficleft(self): - if self.trafficleft is None: - self.getAccountInfo(force=True) - return format_size(self.trafficleft * 1024) - - def wrongPassword(self): - raise WrongPassword - - def empty(self, user=None): - if user: self.logDebug("Deprecated argument user for .empty()", user) - - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - - self.trafficleft = 0 - self.scheduleRefresh(30 * 60) - - def expired(self, user=None): - if user: self.logDebug("Deprecated argument user for .expired()", user) - - self.logWarning(_("Account %s is expired, checking again in 1h") % user) - - self.validuntil = time() - 1 - self.scheduleRefresh(60 * 60) - - def scheduleRefresh(self, time=0, force=True): - """ add a task for refreshing the account info to the scheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [force]) - - @lock - def checkLogin(self, req): - """ checks if the user is still logged in """ - if self.login_ts + self.login_timeout * 60 < time(): - if self.login_ts: # separate from fresh login to have better debug logs - self.logDebug("Reached login timeout for %s" % self.loginname) - else: - self.logInfo(_("Login with %s") % self.loginname) - - self._login(req) - return False - - return True diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py deleted file mode 100644 index ff9c57bef..000000000 --- a/module/plugins/Addon.py +++ /dev/null @@ -1,205 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from traceback import print_exc - -#from functools import wraps -from module.utils import has_method, to_list - -from Base import Base - -def class_name(p): - return p.rpartition(".")[2] - - -def AddEventListener(event): - """ Used to register method for events. Arguments needs to match parameter of event - - :param event: Name of event or list of them. - """ - class _klass(object): - def __new__(cls, f, *args, **kwargs): - for ev in to_list(event): - addonManager.addEventListener(class_name(f.__module__), f.func_name, ev) - return f - return _klass - -class ConfigHandler(object): - """ Register method as config handler. - - Your method signature has to be: - def foo(value=None): - - value will be passed to use your method to set the config. - When value is None your method needs to return an interaction task for configuration. - """ - - def __new__(cls, f, *args, **kwargs): - addonManager.addConfigHandler(class_name(f.__module__), f.func_name) - return f - -def AddonHandler(desc, media=None): - """ Register Handler for files, packages, or arbitrary callable methods. - To let the method work on packages/files, media must be set and the argument named pid or fid. - - :param desc: verbose description - :param media: if True or bits of media type - """ - pass - -def AddonInfo(desc): - """ Called to retrieve information about the current state. - Decorated method must return anything convertable into string. - - :param desc: verbose description - """ - pass - -def threaded(f): - """ Decorator to run method in a thread. """ - - #@wraps(f) - def run(*args,**kwargs): - addonManager.startThread(f, *args, **kwargs) - return run - -class Addon(Base): - """ - Base class for addon plugins. Use @threaded decorator for all longer running tasks. - - Decorate methods with @Expose, @AddEventListener, @ConfigHandler - - """ - - #: automatically register event listeners for functions, attribute will be deleted don't use it yourself - event_map = None - - # Alternative to event_map - #: List of events the plugin can handle, name the functions exactly like eventname. - event_list = None # dont make duplicate entries in event_map - - #: periodic call interval in seconds - interval = 60 - - def __init__(self, core, manager, user=None): - Base.__init__(self, core, user) - - #: Provide information in dict here, usable by API `getInfo` - self.info = None - - #: Callback of periodical job task, used by addonmanager - self.cb = None - - #: `AddonManager` - self.manager = manager - - #register events - if self.event_map: - for event, funcs in self.event_map.iteritems(): - if type(funcs) in (list, tuple): - for f in funcs: - self.evm.addEvent(event, getattr(self,f)) - else: - self.evm.addEvent(event, getattr(self,funcs)) - - #delete for various reasons - self.event_map = None - - if self.event_list: - for f in self.event_list: - self.evm.addEvent(f, getattr(self,f)) - - self.event_list = None - - self.initPeriodical() - self.init() - self.setup() - - def initPeriodical(self): - if self.interval >=1: - self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) - - def _periodical(self): - try: - if self.isActivated(): self.periodical() - except Exception, e: - self.core.log.error(_("Error executing addons: %s") % str(e)) - if self.core.debug: - print_exc() - - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) - - def __repr__(self): - return "<Addon %s>" % self.__name__ - - def isActivated(self): - """ checks if addon is activated""" - return True if self.__internal__ else self.getConfig("activated") - - def getCategory(self): - return self.core.pluginManager.getCategory(self.__name__) - - def init(self): - pass - - def setup(self): - """ more init stuff if needed """ - pass - - def activate(self): - """ Used to activate the addon """ - if has_method(self.__class__, "coreReady"): - self.logDebug("Deprecated method .coreReady() use activate() instead") - self.coreReady() - - def deactivate(self): - """ Used to deactivate the addon. """ - pass - - def periodical(self): - pass - - def newInteractionTask(self, task): - """ new interaction task for the plugin, it MUST set the handler and timeout or will be ignored """ - pass - - def taskCorrect(self, task): - pass - - def taskInvalid(self, task): - pass - - # public events starts from here - def downloadPreparing(self, pyfile): - pass - - def downloadFinished(self, pyfile): - pass - - def downloadFailed(self, pyfile): - pass - - def packageFinished(self, pypack): - pass - - def beforeReconnecting(self, ip): - pass - - def afterReconnecting(self, ip): - pass
\ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py deleted file mode 100644 index 7825e1c49..000000000 --- a/module/plugins/Base.py +++ /dev/null @@ -1,343 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import sys -from time import time, sleep -from random import randint - -from module.utils import decode -from module.utils.fs import exists, makedirs, join, remove - -# TODO -# more attributes if needed -# get rid of catpcha & container plugins ?! (move to crypter & internals) -# adapt old plugins as needed - -class Fail(Exception): - """ raised when failed """ - -class Retry(Exception): - """ raised when start again from beginning """ - -class Abort(Exception): - """ raised when aborted """ - -class Base(object): - """ - The Base plugin class with all shared methods and every possible attribute for plugin definition. - """ - __version__ = "0.1" - #: Regexp pattern which will be matched for download/crypter plugins - __pattern__ = r"" - #: Internal addon plugin which is always loaded - __internal__ = False - #: When True this addon can be enabled by every user - __user_context__ = False - #: Config definition: list of (name, type, label, default_value) or - #: (name, type, label, short_description, default_value) - __config__ = list() - #: Short description, one liner - __label__ = "" - #: More detailed text - __description__ = """""" - #: List of needed modules - __dependencies__ = tuple() - #: Used to assign a category for addon plugins - __category__ = "" - #: Tags to categorize the plugin, see documentation for further info - __tags__ = tuple() - #: Base64 encoded .png icon, should be 32x32, please don't use sizes above ~2KB, for bigger icons use url. - __icon__ = "" - #: Alternative, link to png icon - __icon_url__ = "" - #: Url with general information/support/discussion - __url__ = "" - #: Url to term of content, user is accepting these when using the plugin - __toc_url__ = "" - #: Url to service (to buy premium) for accounts - __ref_url__ = "" - - __author_name__ = tuple() - __author_mail__ = tuple() - - - def __init__(self, core, user=None): - self.__name__ = self.__class__.__name__ - - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - #: :class:`EventManager` - self.evm = core.eventManager - #: :class:`InteractionManager` - self.im = core.interactionManager - if user: - #: :class:`Api`, user api when user is set - self.api = self.core.api.withUserContext(user) - if self.api: - #: :class:`User`, user related to this plugin - self.user = self.api.user - else: - self.api = self.core.api - self.user = None - else: - self.api = self.core.api - self.user = None - - #: last interaction task - self.task = None - - def logInfo(self, *args, **kwargs): - """ Print args to log at specific level - - :param args: Arbitrary object which should be logged - :param kwargs: sep=(how to separate arguments), default = " | " - """ - self._log("info", *args, **kwargs) - - def logWarning(self, *args, **kwargs): - self._log("warning", *args, **kwargs) - - def logError(self, *args, **kwargs): - self._log("error", *args, **kwargs) - - def logDebug(self, *args, **kwargs): - self._log("debug", *args, **kwargs) - - def _log(self, level, *args, **kwargs): - if "sep" in kwargs: - sep = "%s" % kwargs["sep"] - else: - sep = " | " - - strings = [] - for obj in args: - if type(obj) == unicode: - strings.append(obj) - elif type(obj) == str: - strings.append(decode(obj)) - else: - strings.append(str(obj)) - - getattr(self.log, level)("%s: %s" % (self.__name__, sep.join(strings))) - - def getName(self): - """ Name of the plugin class """ - return self.__name__ - - def setConfig(self, option, value): - """ Set config value for current plugin """ - self.core.config.set(self.__name__, option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.getConfig(option) - - def getConfig(self, option): - """ Returns config value for current plugin """ - return self.core.config.get(self.__name__, option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - def shell(self): - """ open ipython shell """ - if self.core.debug: - from IPython import embed - #noinspection PyUnresolvedReferences - sys.stdout = sys._stdout - embed() - - def abort(self): - """ Check if plugin is in an abort state, is overwritten by subtypes""" - return False - - def checkAbort(self): - """ Will be overwritten to determine if control flow should be aborted """ - if self.abort(): raise Abort() - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it - - :param url: url as string - :param get: GET as dict - :param post: POST as dict, list or string - :param ref: Set HTTP_REFERER header - :param cookies: use saved cookies - :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Whether to decode the output according to http header, should be True in most cases - :return: Loaded content - """ - if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") - self.checkAbort() - - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) - - if self.core.debug: - from inspect import currentframe - - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - - try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() - - if just_header: - #parse header - header = {"code": self.req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - res = header - - return res - - def invalidTask(self): - if self.task: - self.task.invalid() - - def invalidCaptcha(self): - self.logDebug("Deprecated method .invalidCaptcha, use .invalidTask") - self.invalidTask() - - def correctTask(self): - if self.task: - self.task.correct() - - def correctCaptcha(self): - self.logDebug("Deprecated method .correctCaptcha, use .correctTask") - self.correctTask() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - name = "%sOCR" % self.__name__ - has_plugin = name in self.core.pluginManager.getPlugins("internal") - - if self.core.captcha: - OCR = self.core.pluginManager.loadClass("internal", name) - else: - OCR = None - - if OCR and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - self.checkAbort() - - ocr = OCR() - result = ocr.get_captcha(temp_file.name) - else: - task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type) - self.task = task - - while task.isWaiting(): - if self.abort(): - self.im.removeTask(task) - raise Abort() - sleep(1) - - #TODO task handling - self.im.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropriate time.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason)
\ No newline at end of file diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py deleted file mode 100644 index c665888ad..000000000 --- a/module/plugins/Crypter.py +++ /dev/null @@ -1,271 +0,0 @@ -# -*- coding: utf-8 -*- - -from traceback import print_exc - -from module.utils import to_list, has_method, uniqify -from module.utils.fs import exists, remove, fs_encode -from module.utils.packagetools import parseNames - -from Base import Base, Retry - -class Package: - """ Container that indicates that a new package should be created """ - def __init__(self, name, urls=None, folder=None): - self.name = name - self.urls = urls if urls else [] - self.folder = folder - - # nested packages - self.packs = [] - - def addURL(self, url): - self.urls.append(url) - - def addPackage(self, pack): - self.packs.append(pack) - - def getAllURLs(self): - urls = self.urls - for p in self.packs: - urls.extend(p.getAllURLs()) - return urls - - # same name and urls is enough to be equal for packages - def __eq__(self, other): - return self.name == other.name and self.urls == other.urls - - def __repr__(self): - return u"<CrypterPackage name=%s, links=%s, packs=%s" % (self.name, self.urls, self.packs) - - def __hash__(self): - return hash(self.name) ^ hash(frozenset(self.urls)) ^ hash(self.name) - -class PyFileMockup: - """ Legacy class needed by old crypter plugins """ - def __init__(self, url, pack): - self.url = url - self.name = url - self._package = pack - self.packageid = pack.id if pack else -1 - - def package(self): - return self._package - -class Crypter(Base): - """ - Base class for (de)crypter plugins. Overwrite decrypt* methods. - - How to use decrypt* methods: - - You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. - - After decrypting and generating urls/packages you have to return the result. - Valid return Data is: - - :class:`Package` instance Crypter.Package - A **new** package will be created with the name and the urls of the object. - - List of urls and `Package` instances - All urls in the list will be added to the **current** package. For each `Package`\ - instance a new package will be created. - - """ - - #: Prefix to annotate that the submited string for decrypting is indeed file content - CONTENT_PREFIX = "filecontent:" - - @classmethod - def decrypt(cls, core, url_or_urls): - """Static method to decrypt urls or content. Can be used by other plugins. - To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. - - :param core: pyLoad `Core`, needed in decrypt context - :param url_or_urls: List of urls or single url/ file content - :return: List of decrypted urls, all package info removed - """ - urls = to_list(url_or_urls) - p = cls(core) - try: - result = p.processDecrypt(urls) - finally: - p.clean() - - ret = [] - - for url_or_pack in result: - if isinstance(url_or_pack, Package): #package - ret.extend(url_or_pack.getAllURLs()) - else: # single url - ret.append(url_or_pack) - # eliminate duplicates - return uniqify(ret) - - def __init__(self, core, package=None, password=None): - Base.__init__(self, core) - self.req = core.requestFactory.getRequest(self.__name__) - - # Package the plugin was initialized for, don't use this, its not guaranteed to be set - self.package = package - #: Password supplied by user - self.password = password - #: Propose a renaming of the owner package - self.rename = None - - # For old style decrypter, do not use these! - self.packages = [] - self.urls = [] - self.pyfile = None - - self.init() - - def init(self): - """More init stuff if needed""" - - def setup(self): - """Called everytime before decrypting. A Crypter plugin will be most likely used for several jobs.""" - - def decryptURL(self, url): - """Decrypt a single url - - :param url: url to decrypt - :return: See :class:`Crypter` Documentation - """ - if url.startswith("http"): # basic method to redirect - return self.decryptFile(self.load(url)) - else: - self.fail(_("Not existing file or unsupported protocol")) - - def decryptURLs(self, urls): - """Decrypt a bunch of urls - - :param urls: list of urls - :return: See :class:`Crypter` Documentation - """ - raise NotImplementedError - - def decryptFile(self, content): - """Decrypt file content - - :param content: content to decrypt as string - :return: See :class:`Crypter` Documentation - """ - raise NotImplementedError - - def generatePackages(self, urls): - """Generates :class:`Package` instances and names from urls. Useful for many different links and no\ - given package name. - - :param urls: list of urls - :return: list of `Package` - """ - return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] - - def _decrypt(self, urls): - """ Internal method to select decrypting method - - :param urls: List of urls/content - :return: - """ - cls = self.__class__ - - # separate local and remote files - content, urls = self.getLocalContent(urls) - - if has_method(cls, "decryptURLs"): - self.setup() - result = to_list(self.decryptURLs(urls)) - elif has_method(cls, "decryptURL"): - result = [] - for url in urls: - self.setup() - result.extend(to_list(self.decryptURL(url))) - elif has_method(cls, "decrypt"): - self.logDebug("Deprecated .decrypt() method in Crypter plugin") - result = [] - for url in urls: - self.pyfile = PyFileMockup(url, self.package) - self.setup() - self.decrypt(self.pyfile) - result.extend(self.convertPackages()) - else: - if not has_method(cls, "decryptFile") or urls: - self.logDebug("No suited decrypting method was overwritten in plugin") - result = [] - - if has_method(cls, "decryptFile"): - for f, c in content: - self.setup() - result.extend(to_list(self.decryptFile(c))) - try: - if f.startswith("tmp_"): remove(f) - except : - pass - - return result - - def processDecrypt(self, urls): - """Catches all exceptions in decrypt methods and return results - - :return: Decrypting results - """ - try: - return self._decrypt(urls) - except Exception: - if self.core.debug: - print_exc() - return [] - - def getLocalContent(self, urls): - """Load files from disk and separate to file content and url list - - :param urls: - :return: list of (filename, content), remote urls - """ - content = [] - # do nothing if no decryptFile method - if hasattr(self.__class__, "decryptFile"): - remote = [] - for url in urls: - path = None - if url.startswith("http"): # skip urls directly - pass - elif url.startswith(self.CONTENT_PREFIX): - path = url - elif exists(url): - path = url - elif exists(self.core.path(url)): - path = self.core.path(url) - - if path: - try: - if path.startswith(self.CONTENT_PREFIX): - content.append(("", path[len(self.CONTENT_PREFIX)])) - else: - f = open(fs_encode(path), "rb") - content.append((f.name, f.read())) - f.close() - except IOError, e: - self.logError("IOError", e) - else: - remote.append(url) - - #swap filtered url list - urls = remote - - return content, urls - - def retry(self): - """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ - raise Retry() - - def convertPackages(self): - """ Deprecated """ - self.logDebug("Deprecated method .convertPackages()") - res = [Package(name, urls) for name, urls in self.packages] - res.extend(self.urls) - return res - - def clean(self): - if hasattr(self, "req"): - self.req.close() - del self.req
\ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py deleted file mode 100644 index 651471a93..000000000 --- a/module/plugins/Hoster.py +++ /dev/null @@ -1,400 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" - -import os -from time import time - -if os.name != "nt": - from module.utils.fs import chown - from pwd import getpwnam - from grp import getgrnam - -from Base import Base, Fail, Retry -from module.utils import chunks as _chunks -from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ - remove, makedirs, chmod, stat, exists, join - -# Import for Hoster Plugins -chunks = _chunks - -class Reconnect(Exception): - """ raised when reconnected """ - -class SkipDownload(Exception): - """ raised when download should be skipped """ - -class Hoster(Base): - """ - Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. - """ - - @staticmethod - def getInfo(urls): - """This method is used to retrieve the online status of files for hoster plugins. - It has to *yield* list of tuples with the result in this format (name, size, status, url), - where status is one of API pyfile statuses. - - :param urls: List of urls - :return: yield list of tuple with results (name, size, status, url) - """ - pass - - def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = False - - #: plugin is waiting - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = self.core.accountManager.getAccountForPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.isUsable(): self.account = None - if self.account: - self.user = self.account.loginname - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest() - # Default: -1, True, True - self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings() - self.premium = self.account.isPremium() - else: - self.req = self.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getMultiDL(self): - return self.limitDL <= 0 - - def setMultiDL(self, val): - self.limitDL = 0 if val else 1 - - #: virtual attribute using self.limitDL on behind - multiDL = property(getMultiDL, setMultiDL) - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) - - def getDownloadLimit(self): - if self.account: - limit = self.account.options.get("limitDL", 0) - if limit == "": limit = 0 - if self.limitDL > 0: # a limit is already set, we use the minimum - return min(int(limit), self.limitDL) - else: - return int(limit) - else: - return self.limitDL - - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for environment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - # will force a re-login or reload of account info if necessary - self.account.getAccountInfo() - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def abort(self): - return self.pyfile.abort - - def resetAccount(self): - """ don't use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check addon - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - self.checkAbort() - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - self.checkForSameFiles() - self.checkAbort() - - self.pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = save_filename(self.pyfile.name) - - filename = join(location, name) - - self.core.addonManager.dispatchEvent("download:start", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises SkipDownload: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cachedFiles(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was apparently started before - raise SkipDownload(pyfile.pluginname) - - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - raise SkipDownload("File exists.") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - raise SkipDownload(pyfile[0]) - - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py deleted file mode 100644 index 1936478b4..000000000 --- a/module/plugins/MultiHoster.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -from time import time - -from module.utils import remove_chars - -from Account import Account - -def normalize(domain): - """ Normalize domain/plugin name, so they are comparable """ - return remove_chars(domain.strip().lower(), "-.") - -#noinspection PyUnresolvedReferences -class MultiHoster(Account): - """ - Base class for MultiHoster services. - This is also an Account instance so you should see :class:`Account` and overwrite necessary methods. - Multihoster becomes only active when an Account was entered and the MultiHoster addon was activated. - You need to overwrite `loadHosterList` and a corresponding :class:`Hoster` plugin with the same name should - be available to make your service working. - """ - - #: List of hoster names that will be replaced so pyLoad will recognize them: (orig_name, pyload_name) - replacements = [("freakshare.net", "freakshare.com")] - - #: Load new hoster list every x seconds - hoster_timeout = 300 - - def __init__(self, *args, **kwargs): - - # Hoster list - self.hoster = [] - # Timestamp - self.ts = 0 - - Account.__init__(self, *args, **kwargs) - - def loadHosterList(self, req): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - - def isHosterUsuable(self, domain): - """ Determine before downloading if hoster should be used. - - :param domain: domain name - :return: True to let the MultiHoster download, False to fallback to default plugin - """ - return True - - def getHosterList(self, force=False): - if self.ts + self.hoster_timeout < time() or force: - req = self.getAccountRequest() - try: - self.hoster = self.loadHosterList(req) - except Exception, e: - self.logError(e) - return [] - finally: - req.close() - - for rep in self.replacements: - if rep[0] in self.hoster: - self.hoster.remove(rep[0]) - if rep[1] not in self.hoster: - self.hoster.append(rep[1]) - - self.ts = time() - - return self.hoster
\ No newline at end of file diff --git a/module/plugins/ReCaptcha.py b/module/plugins/ReCaptcha.py deleted file mode 100644 index e47522b4a..000000000 --- a/module/plugins/ReCaptcha.py +++ /dev/null @@ -1,22 +0,0 @@ -import re - -class ReCaptcha(): - def __init__(self, plugin): - self.plugin = plugin - self.plugin.logDebug("Deprecated usage of ReCaptcha: Use CaptchaService instead") - - def challenge(self, id): - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k":id}, cookies=True) - - try: - challenge = re.search("challenge : '(.*?)',", js).group(1) - server = re.search("server : '(.*?)',", js).group(1) - except: - self.plugin.fail("recaptcha error") - result = self.result(server,challenge) - - return challenge, result - - def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage"%server, get={"c":challenge}, cookies=True, imgtype="jpg") - diff --git a/module/plugins/__init__.py b/module/plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/__init__.py +++ /dev/null diff --git a/module/plugins/accounts/AlldebridCom.py b/module/plugins/accounts/AlldebridCom.py deleted file mode 100644 index beaddeac9..000000000 --- a/module/plugins/accounts/AlldebridCom.py +++ /dev/null @@ -1,49 +0,0 @@ -from module.plugins.Account import Account
-import xml.dom.minidom as dom
-from BeautifulSoup import BeautifulSoup
-from time import time
-import re
-import urllib -
-class AlldebridCom(Account):
- __name__ = "AlldebridCom"
- __version__ = "0.21"
- __type__ = "account"
- __description__ = """AllDebrid.com account plugin"""
- __author_name__ = ("Andy, Voigt")
- __author_mail__ = ("spamsales@online.de")
-
- def loadAccountInfo(self, user, req):
- data = self.getAccountData(user)
- page = req.load("http://www.alldebrid.com/account/")
- soup=BeautifulSoup(page)
- #Try to parse expiration date directly from the control panel page (better accuracy)
- try:
- time_text=soup.find('div',attrs={'class':'remaining_time_text'}).strong.string
- self.log.debug("Account expires in: %s" % time_text)
- p = re.compile('\d+')
- exp_data=p.findall(time_text)
- exp_time=time()+int(exp_data[0])*24*60*60+int(exp_data[1])*60*60+(int(exp_data[2])-1)*60
- #Get expiration date from API
- except:
- data = self.getAccountData(user)
- page = req.load("http://www.alldebrid.com/api.php?action=info_user&login=%s&pw=%s" % (user, data["password"]))
- self.log.debug(page)
- xml = dom.parseString(page)
- exp_time=time()+int(xml.getElementsByTagName("date")[0].childNodes[0].nodeValue)*86400
- account_info = {"validuntil": exp_time, "trafficleft": -1}
- return account_info
-
- def login(self, user, data, req):
- - urlparams = urllib.urlencode({'action':'login','login_login':user,'login_password':data["password"]}) - page = req.load("http://www.alldebrid.com/register/?%s" % (urlparams)) -
- if "This login doesn't exist" in page:
- self.wrongPassword()
- - if "The password is not valid" in page: - self.wrongPassword() - - if "Invalid captcha" in page: - self.wrongPassword() diff --git a/module/plugins/accounts/BayfilesCom.py b/module/plugins/accounts/BayfilesCom.py deleted file mode 100644 index 0d036488b..000000000 --- a/module/plugins/accounts/BayfilesCom.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from module.common.json_layer import json_loads -import re -from time import time, mktime, strptime - -class BayfilesCom(Account): - __name__ = "BayfilesCom" - __version__ = "0.02" - __type__ = "account" - __description__ = """bayfiles.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - for i in range(2): - response = json_loads(req.load("http://api.bayfiles.com/v1/account/info")) - self.logDebug(response) - if not response["error"]: - break - self.logWarning(response["error"]) - self.relogin() - - return {"premium": bool(response['premium']), \ - "trafficleft": -1, \ - "validuntil": response['expires'] if response['expires'] >= int(time()) else -1} - - def login(self, user, data, req): - response = json_loads(req.load("http://api.bayfiles.com/v1/account/login/%s/%s" % (user, data["password"]))) - self.logDebug(response) - if response["error"]: - self.logError(response["error"]) - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/BitshareCom.py b/module/plugins/accounts/BitshareCom.py deleted file mode 100644 index a4f56e31c..000000000 --- a/module/plugins/accounts/BitshareCom.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: pking -""" - -from module.plugins.Account import Account - -class BitshareCom(Account): - __name__ = "BitshareCom" - __version__ = "0.11" - __type__ = "account" - __description__ = """Bitshare account plugin""" - __author_name__ = ("Paul King") - - def loadAccountInfo(self, user, req): - page = req.load("http://bitshare.com/mysettings.html") - - if "\"http://bitshare.com/myupgrade.html\">Free" in page: - return {"validuntil": -1, "trafficleft":-1, "premium": False} - - if not '<input type="checkbox" name="directdownload" checked="checked" />' in page: - self.core.log.warning(_("Activate direct Download in your Bitshare Account")) - - return {"validuntil": -1, "trafficleft": -1, "premium": True} - - - def login(self, user, data, req): - page = req.load("http://bitshare.com/login.html", post={ "user" : user, "password" : data["password"], "submit" :"Login"}, cookies=True) - if "login" in req.lastEffectiveURL: - self.wrongPassword() diff --git a/module/plugins/accounts/BoltsharingCom.py b/module/plugins/accounts/BoltsharingCom.py deleted file mode 100644 index 678591d1d..000000000 --- a/module/plugins/accounts/BoltsharingCom.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class BoltsharingCom(XFSPAccount): - __name__ = "BoltsharingCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """Boltsharing.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://boltsharing.com/" diff --git a/module/plugins/accounts/CramitIn.py b/module/plugins/accounts/CramitIn.py deleted file mode 100644 index 182c9d647..000000000 --- a/module/plugins/accounts/CramitIn.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class CramitIn(XFSPAccount): - __name__ = "CramitIn" - __version__ = "0.01" - __type__ = "account" - __description__ = """cramit.in account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://cramit.in/"
\ No newline at end of file diff --git a/module/plugins/accounts/CyberlockerCh.py b/module/plugins/accounts/CyberlockerCh.py deleted file mode 100644 index 31e0c3e24..000000000 --- a/module/plugins/accounts/CyberlockerCh.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount -from module.plugins.internal.SimpleHoster import parseHtmlForm - -class CyberlockerCh(XFSPAccount): - __name__ = "CyberlockerCh" - __version__ = "0.01" - __type__ = "account" - __description__ = """CyberlockerCh account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - MAIN_PAGE = "http://cyberlocker.ch/" - - def login(self, user, data, req): - html = req.load(self.MAIN_PAGE + 'login.html', decode = True) - - action, inputs = parseHtmlForm('name="FL"', html) - if not inputs: - inputs = {"op": "login", - "redirect": self.MAIN_PAGE} - - inputs.update({"login": user, - "password": data['password']}) - - # Without this a 403 Forbidden is returned - req.http.lastURL = self.MAIN_PAGE + 'login.html' - html = req.load(self.MAIN_PAGE, post = inputs, decode = True) - - if 'Incorrect Login or Password' in html or '>Error<' in html: - self.wrongPassword() diff --git a/module/plugins/accounts/CzshareCom.py b/module/plugins/accounts/CzshareCom.py deleted file mode 100644 index 7b1a8edc5..000000000 --- a/module/plugins/accounts/CzshareCom.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from time import mktime, strptime -import re - -from module.plugins.Account import Account - - -class CzshareCom(Account): - __name__ = "CzshareCom" - __version__ = "0.13" - __type__ = "account" - __description__ = """czshare.com account plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - CREDIT_LEFT_PATTERN = r'<tr class="active">\s*<td>([0-9 ,]+) (KiB|MiB|GiB)</td>\s*<td>([^<]*)</td>\s*</tr>' - - def loadAccountInfo(self, user, req): - html = req.load("http://czshare.com/prehled_kreditu/") - - found = re.search(self.CREDIT_LEFT_PATTERN, html) - if not found: - return {"validuntil": 0, "trafficleft": 0} - else: - credits = float(found.group(1).replace(' ', '').replace(',', '.')) - credits = credits * 1024 ** {'KiB': 0, 'MiB': 1, 'GiB': 2}[found.group(2)] - validuntil = mktime(strptime(found.group(3), '%d.%m.%y %H:%M')) - return {"validuntil": validuntil, "trafficleft": credits} - - def login(self, user, data, req): - - html = req.load('https://czshare.com/index.php', post={ - "Prihlasit": "Prihlasit", - "login-password": data["password"], - "login-name": user - }) - - if '<div class="login' in html: - self.wrongPassword() diff --git a/module/plugins/accounts/DdlstorageCom.py b/module/plugins/accounts/DdlstorageCom.py deleted file mode 100644 index 01d165f23..000000000 --- a/module/plugins/accounts/DdlstorageCom.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class DdlstorageCom(XFSPAccount): - __name__ = "DdlstorageCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """DDLStorage.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://ddlstorage.com/"
\ No newline at end of file diff --git a/module/plugins/accounts/DebridItaliaCom.py b/module/plugins/accounts/DebridItaliaCom.py deleted file mode 100644 index 91dd3787f..000000000 --- a/module/plugins/accounts/DebridItaliaCom.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -import re -import _strptime -import time - -from module.plugins.Account import Account - - -class DebridItaliaCom(Account): - __name__ = "DebridItaliaCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """debriditalia.com account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - WALID_UNTIL_PATTERN = r"Premium valid till: (?P<D>[^|]+) \|" - - def loadAccountInfo(self, user, req): - if 'Account premium not activated' in self.html: - return {"premium": False, "validuntil": None, "trafficleft": None} - - m = re.search(self.WALID_UNTIL_PATTERN, self.html) - if m: - validuntil = int(time.mktime(time.strptime(m.group('D'), "%d/%m/%Y %H:%M"))) - return {"premium": True, "validuntil": validuntil, "trafficleft": -1} - else: - self.logError('Unable to retrieve account information - Plugin may be out of date') - - def login(self, user, data, req): - self.html = req.load("http://debriditalia.com/login.php", - get={"u": user, "p": data["password"]}) - if 'NO' in self.html: - self.wrongPassword() diff --git a/module/plugins/accounts/DepositfilesCom.py b/module/plugins/accounts/DepositfilesCom.py deleted file mode 100644 index b0730de8e..000000000 --- a/module/plugins/accounts/DepositfilesCom.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account -import re -from time import strptime, mktime - -class DepositfilesCom(Account): - __name__ = "DepositfilesCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """depositfiles.com account plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def loadAccountInfo(self, user, req): - - src = req.load("http://depositfiles.com/de/gold/") - validuntil = re.search("noch den Gold-Zugriff: <b>(.*?)</b></div>", src).group(1) - - validuntil = int(mktime(strptime(validuntil, "%Y-%m-%d %H:%M:%S"))) - - tmp = {"validuntil":validuntil, "trafficleft":-1} - return tmp - - def login(self, user, data, req): - - req.load("http://depositfiles.com/de/gold/payment.php") - src = req.load("http://depositfiles.com/de/login.php", get={"return": "/de/gold/payment.php"}, post={"login": user, "password": data["password"]}) - if r'<div class="error_message">Sie haben eine falsche Benutzername-Passwort-Kombination verwendet.</div>' in src: - self.wrongPassword() diff --git a/module/plugins/accounts/EasybytezCom.py b/module/plugins/accounts/EasybytezCom.py deleted file mode 100644 index ba7829b83..000000000 --- a/module/plugins/accounts/EasybytezCom.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm -import re -from module.utils import parseFileSize -from time import mktime, strptime - -class EasybytezCom(Account): - __name__ = "EasybytezCom" - __version__ = "0.02" - __type__ = "account" - __description__ = """EasyBytez.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - VALID_UNTIL_PATTERN = r'<TR><TD>Premium account expire:</TD><TD><b>([^<]+)</b>' - TRAFFIC_LEFT_PATTERN = r'<TR><TD>Traffic available today:</TD><TD><b>(?P<S>[^<]+)</b>' - - def loadAccountInfo(self, user, req): - html = req.load("http://www.easybytez.com/?op=my_account", decode = True) - - validuntil = trafficleft = None - premium = False - - found = re.search(self.VALID_UNTIL_PATTERN, html) - if found: - premium = True - trafficleft = -1 - try: - self.logDebug(found.group(1)) - validuntil = mktime(strptime(found.group(1), "%d %B %Y")) - except Exception, e: - self.logError(e) - else: - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if found: - trafficleft = found.group(1) - if "Unlimited" in trafficleft: - premium = True - else: - trafficleft = parseFileSize(trafficleft) / 1024 - - return ({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) - - def login(self, user, data, req): - html = req.load('http://www.easybytez.com/login.html', decode = True) - action, inputs = parseHtmlForm('name="FL"', html) - inputs.update({"login": user, - "password": data['password'], - "redirect": "http://www.easybytez.com/"}) - - html = req.load(action, post = inputs, decode = True) - - if 'Incorrect Login or Password' in html or '>Error<' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/EgoFilesCom.py b/module/plugins/accounts/EgoFilesCom.py deleted file mode 100644 index da1ed03ad..000000000 --- a/module/plugins/accounts/EgoFilesCom.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account -import re -import time -from module.utils import parseFileSize - -class EgoFilesCom(Account): - __name__ = "EgoFilesCom" - __version__ = "0.2" - __type__ = "account" - __description__ = """egofiles.com account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - PREMIUM_ACCOUNT_PATTERN = '<br/>\s*Premium: (?P<P>[^/]*) / Traffic left: (?P<T>[\d.]*) (?P<U>\w*)\s*\\n\s*<br/>' - - def loadAccountInfo(self, user, req): - html = req.load("http://egofiles.com") - if 'You are logged as a Free User' in html: - return {"premium": False, "validuntil": None, "trafficleft": None} - - m = re.search(self.PREMIUM_ACCOUNT_PATTERN, html) - if m: - validuntil = int(time.mktime(time.strptime(m.group('P'), "%Y-%m-%d %H:%M:%S"))) - trafficleft = parseFileSize(m.group('T'), m.group('U')) / 1024 - return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} - else: - self.logError('Unable to retrieve account information - Plugin may be out of date') - - def login(self, user, data, req): - # Set English language - req.load("https://egofiles.com/ajax/lang.php?lang=en", just_header=True) - - html = req.load("http://egofiles.com/ajax/register.php", - post={"log": 1, - "loginV": user, - "passV": data["password"]}) - if 'Login successful' not in html: - self.wrongPassword() diff --git a/module/plugins/accounts/EuroshareEu.py b/module/plugins/accounts/EuroshareEu.py deleted file mode 100644 index 42967d975..000000000 --- a/module/plugins/accounts/EuroshareEu.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from time import mktime, strptime -from string import replace -import re - -class EuroshareEu(Account): - __name__ = "EuroshareEu" - __version__ = "0.01" - __type__ = "account" - __description__ = """euroshare.eu account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - self.relogin(user) - html = req.load("http://euroshare.eu/customer-zone/settings/") - - found = re.search('id="input_expire_date" value="(\d+\.\d+\.\d+ \d+:\d+)"', html) - if found is None: - premium, validuntil = False, -1 - else: - premium = True - validuntil = mktime(strptime(found.group(1), "%d.%m.%Y %H:%M")) - - return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - - def login(self, user, data, req): - - html = req.load('http://euroshare.eu/customer-zone/login/', post={ - "trvale": "1", - "login": user, - "password": data["password"] - }, decode=True) - - if u">Nesprávne prihlasovacie meno alebo heslo" in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/FastshareCz.py b/module/plugins/accounts/FastshareCz.py deleted file mode 100644 index 69bbb0827..000000000 --- a/module/plugins/accounts/FastshareCz.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Account import Account -from module.utils import parseFileSize - - -class FastshareCz(Account): - __name__ = "FastshareCz" - __version__ = "0.02" - __type__ = "account" - __description__ = """fastshare.cz account plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - def loadAccountInfo(self, user, req): - html = req.load("http://www.fastshare.cz/user", decode=True) - - found = re.search(r'(?:Kredit|Credit)\s*: </td><td>(.+?) ', html) - if found: - trafficleft = parseFileSize(found.group(1)) / 1024 - premium = True if trafficleft else False - else: - trafficleft = None - premium = False - - return {"validuntil": -1, "trafficleft": trafficleft, "premium": premium} - - def login(self, user, data, req): - req.load('http://www.fastshare.cz/login') # Do not remove or it will not login - html = req.load('http://www.fastshare.cz/sql.php', post={ - "heslo": data['password'], - "login": user - }, decode=True) - - if u'>Špatné uşivatelské jméno nebo heslo.<' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/FilebeerInfo.py b/module/plugins/accounts/FilebeerInfo.py deleted file mode 100644 index 40ab70519..000000000 --- a/module/plugins/accounts/FilebeerInfo.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from time import mktime, strptime -from module.plugins.Account import Account -from module.utils import parseFileSize - -class FilebeerInfo(Account): - __name__ = "FilebeerInfo" - __version__ = "0.02" - __type__ = "account" - __description__ = """filebeer.info account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - VALID_UNTIL_PATTERN = r'Reverts To Free Account:\s</td>\s*<td>\s*(.*?)\s*</td>' - - def loadAccountInfo(self, user, req): - html = req.load("http://filebeer.info/upgrade.php", decode = True) - premium = not 'Free User </td>' in html - - validuntil = None - if premium: - try: - validuntil = mktime(strptime(re.search(self.VALID_UNTIL_PATTERN, html).group(1), "%d/%m/%Y %H:%M:%S")) - except Exception, e: - self.logError("Unable to parse account info", e) - - return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - - def login(self, user, data, req): - html = req.load('http://filebeer.info/login.php', post = { - "submit": 'Login', - "loginPassword": data['password'], - "loginUsername": user, - "submitme": '1' - }, decode = True) - - if "<ul class='pageErrors'>" in html or ">Your username and password are invalid<" in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/FilecloudIo.py b/module/plugins/accounts/FilecloudIo.py deleted file mode 100644 index cf9f92209..000000000 --- a/module/plugins/accounts/FilecloudIo.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account - -class FilecloudIo(Account): - __name__ = "FilecloudIo" - __version__ = "0.01" - __type__ = "account" - __description__ = """FilecloudIo account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - return ({"validuntil": -1, "trafficleft": -1, "premium": False}) - - def login(self, user, data, req): - req.cj.setCookie("secure.filecloud.io", "lang", "en") - html = req.load('https://secure.filecloud.io/user-login.html') - - if not hasattr(self, "form_data"): - self.form_data = {} - - self.form_data["username"] = user - self.form_data["password"] = data['password'] - - html = req.load('https://secure.filecloud.io/user-login_p.html', - post = self.form_data, - multipart = True) - - self.logged_in = True if "you have successfully logged in - filecloud.io" in html else False - self.form_data = {} -
\ No newline at end of file diff --git a/module/plugins/accounts/FilefactoryCom.py b/module/plugins/accounts/FilefactoryCom.py deleted file mode 100644 index 8e163e2f6..000000000 --- a/module/plugins/accounts/FilefactoryCom.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -from time import mktime, strptime - -class FilefactoryCom(Account): - __name__ = "FilefactoryCom" - __version__ = "0.13" - __type__ = "account" - __description__ = """filefactory.com account plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - ACCOUNT_INFO_PATTERN = r'<time datetime="([\d-]+)">' - - def loadAccountInfo(self, user, req): - html = req.load("http://www.filefactory.com/member/") - - found = re.search(self.ACCOUNT_INFO_PATTERN, html) - if found: - premium = True - validuntil = mktime(strptime(found.group(1),"%Y-%m-%d")) - else: - premium = False - validuntil = -1 - - return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} - - def login(self, user, data, req): - html = req.load("http://www.filefactory.com/member/login.php", post={ - "email": user, - "password": data["password"], - "redirect": "/"}) - - if '/member/login.php?err=1' in req.http.header: - self.wrongPassword() diff --git a/module/plugins/accounts/FilejungleCom.py b/module/plugins/accounts/FilejungleCom.py deleted file mode 100644 index 8ac25c201..000000000 --- a/module/plugins/accounts/FilejungleCom.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -from time import mktime, strptime - -class FilejungleCom(Account): - __name__ = "FilejungleCom" - __version__ = "0.11" - __type__ = "account" - __description__ = """filejungle.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - login_timeout = 60 - - URL = "http://filejungle.com/" - TRAFFIC_LEFT_PATTERN = r'"/extend_premium\.php">Until (\d+ [A-Za-z]+ \d+)<br' - LOGIN_FAILED_PATTERN = r'<span htmlfor="loginUser(Name|Password)" generated="true" class="fail_info">' - - def loadAccountInfo(self, user, req): - html = req.load(self.URL + "dashboard.php") - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if found: - premium = True - validuntil = mktime(strptime(found.group(1), "%d %b %Y")) - else: - premium = False - validuntil = -1 - - return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} - - def login(self, user, data, req): - html = req.load(self.URL + "login.php", post={ - "loginUserName": user, - "loginUserPassword": data["password"], - "loginFormSubmit": "Login", - "recaptcha_challenge_field": "", - "recaptcha_response_field": "", - "recaptcha_shortencode_field": ""}) - - if re.search(self.LOGIN_FAILED_PATTERN, html): - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/FilerioCom.py b/module/plugins/accounts/FilerioCom.py deleted file mode 100644 index feacacaf5..000000000 --- a/module/plugins/accounts/FilerioCom.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class FilerioCom(XFSPAccount): - __name__ = "FilerioCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """FileRio.in account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://filerio.in/"
\ No newline at end of file diff --git a/module/plugins/accounts/FilesMailRu.py b/module/plugins/accounts/FilesMailRu.py deleted file mode 100644 index 98fe13248..000000000 --- a/module/plugins/accounts/FilesMailRu.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from module.plugins.Account import Account -import re -from time import time - -class FilesMailRu(Account): - __name__ = "FilesMailRu" - __version__ = "0.1" - __type__ = "account" - __description__ = """filesmail.ru account plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def loadAccountInfo(self, user, req): - return {"validuntil": None, "trafficleft": None} - - def login(self, user, data,req): - user, domain = user.split("@") - - page = req.load("http://swa.mail.ru/cgi-bin/auth", None, { "Domain" : domain, "Login": user, "Password" : data['password'], "Page" : "http://files.mail.ru/"}, cookies=True) - - if "ÐевеÑМПе ÐžÐŒÑ Ð¿ÐŸÐ»ÑзПваÑÐµÐ»Ñ ÐžÐ»Ðž паÑПлÑ" in page: # @TODO seems not to work - self.wrongPassword() diff --git a/module/plugins/accounts/FileserveCom.py b/module/plugins/accounts/FileserveCom.py deleted file mode 100644 index 5e5068f22..000000000 --- a/module/plugins/accounts/FileserveCom.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from time import mktime, strptime - -from module.plugins.Account import Account -from module.common.json_layer import json_loads - -class FileserveCom(Account): - __name__ = "FileserveCom" - __version__ = "0.2" - __type__ = "account" - __description__ = """fileserve.com account plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def loadAccountInfo(self, user, req): - data = self.getAccountData(user) - - page = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data["password"], - "submit": "Submit+Query"}) - res = json_loads(page) - - if res["type"] == "premium": - validuntil = mktime(strptime(res["expireTime"], "%Y-%m-%d %H:%M:%S")) - return {"trafficleft": res["traffic"], "validuntil": validuntil} - else: - return {"premium": False, "trafficleft": None, "validuntil": None} - - - def login(self, user, data, req): - page = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data["password"], - "submit": "Submit+Query"}) - res = json_loads(page) - - if not res["type"]: - self.wrongPassword() - - #login at fileserv page - req.load("http://www.fileserve.com/login.php", - post={"loginUserName": user, "loginUserPassword": data["password"], "autoLogin": "checked", - "loginFormSubmit": "Login"}) diff --git a/module/plugins/accounts/FourSharedCom.py b/module/plugins/accounts/FourSharedCom.py deleted file mode 100644 index bd3820277..000000000 --- a/module/plugins/accounts/FourSharedCom.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from module.common.json_layer import json_loads - -class FourSharedCom(Account): - __name__ = "FourSharedCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """FourSharedCom account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - #fixme - return ({"validuntil": -1, "trafficleft": -1, "premium": False}) - - def login(self, user, data, req): - req.cj.setCookie("www.4shared.com", "4langcookie", "en") - response = req.load('http://www.4shared.com/login', - post = {"login": user, - "password": data['password'], - "remember": "false", - "doNotRedirect": "true"}) - self.logDebug(response) - response = json_loads(response) - - if not "ok" in response or response['ok'] != True: - if "rejectReason" in response and response['rejectReason'] != True: - self.logError(response['rejectReason']) - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/FreakshareCom.py b/module/plugins/accounts/FreakshareCom.py deleted file mode 100644 index 732f9e203..000000000 --- a/module/plugins/accounts/FreakshareCom.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -import re -from time import strptime, mktime - -from module.plugins.Account import Account - -class FreakshareCom(Account): - __name__ = "FreakshareCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """freakshare.com account plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def loadAccountInfo(self, user, req): - page = req.load("http://freakshare.com/") - - validuntil = r"ltig bis:</td>\s*<td><b>([0-9 \-:.]+)</b></td>" - validuntil = re.search(validuntil, page, re.MULTILINE) - validuntil = validuntil.group(1).strip() - validuntil = mktime(strptime(validuntil, "%d.%m.%Y - %H:%M")) - - traffic = r"Traffic verbleibend:</td>\s*<td>([^<]+)" - traffic = re.search(traffic, page, re.MULTILINE) - traffic = traffic.group(1).strip() - traffic = self.parseTraffic(traffic) - - return {"validuntil": validuntil, "trafficleft": traffic} - - def login(self, user, data, req): - page = req.load("http://freakshare.com/login.html", None, { "submit" : "Login", "user" : user, "pass" : data['password']}, cookies=True) - - if "Falsche Logindaten!" in page or "Wrong Username or Password!" in page: - self.wrongPassword() diff --git a/module/plugins/accounts/FshareVn.py b/module/plugins/accounts/FshareVn.py deleted file mode 100644 index 9b22cbafb..000000000 --- a/module/plugins/accounts/FshareVn.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from time import mktime, strptime -from pycurl import REFERER -import re - -class FshareVn(Account): - __name__ = "FshareVn" - __version__ = "0.04" - __type__ = "account" - __description__ = """fshare.vn account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - VALID_UNTIL_PATTERN = ur'<dt>Thá»i hạn dùng:</dt>\s*<dd>([^<]+)</dd>' - TRAFFIC_LEFT_PATTERN = ur'<dt>Tá»ng Dung Lượng Tà i Khoản</dt>\s*<dd[^>]*>([0-9.]+) ([kKMG])B</dd>' - DIRECT_DOWNLOAD_PATTERN = ur'<input type="checkbox"\s*([^=>]*)[^>]*/>KÃch hoạt download trá»±c tiếp</dt>' - - def loadAccountInfo(self, user, req): - html = req.load("http://www.fshare.vn/account_info.php", decode = True) - found = re.search(self.VALID_UNTIL_PATTERN, html) - if found: - premium = True - validuntil = mktime(strptime(found.group(1), '%I:%M:%S %p %d-%m-%Y')) - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - trafficleft = float(found.group(1)) * 1024 ** {'k': 0, 'K': 0, 'M': 1, 'G': 2}[found.group(2)] if found else 0 - else: - premium = False - validuntil = None - trafficleft = None - - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} - - def login(self, user, data, req): - req.http.c.setopt(REFERER, "https://www.fshare.vn/login.php") - - html = req.load('https://www.fshare.vn/login.php', post = { - "login_password" : data['password'], - "login_useremail" : user, - "url_refe" : "https://www.fshare.vn/login.php" - }, referer = True, decode = True) - - if not '<img alt="VIP"' in html: - self.wrongPassword() diff --git a/module/plugins/accounts/Ftp.py b/module/plugins/accounts/Ftp.py deleted file mode 100644 index 9c1081662..000000000 --- a/module/plugins/accounts/Ftp.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account - -class Ftp(Account): - __name__ = "Ftp" - __version__ = "0.01" - __type__ = "account" - __description__ = """Ftp dummy account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - login_timeout = info_threshold = 1000000
\ No newline at end of file diff --git a/module/plugins/accounts/HellshareCz.py b/module/plugins/accounts/HellshareCz.py deleted file mode 100644 index c7a918dec..000000000 --- a/module/plugins/accounts/HellshareCz.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -import time - -class HellshareCz(Account): - __name__ = "HellshareCz" - __version__ = "0.14" - __type__ = "account" - __description__ = """hellshare.cz account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - CREDIT_LEFT_PATTERN = r'<div class="credit-link">\s*<table>\s*<tr>\s*<th>(\d+|\d\d\.\d\d\.)</th>' - - def loadAccountInfo(self, user, req): - self.relogin(user) - html = req.load("http://www.hellshare.com/") - - found = re.search(self.CREDIT_LEFT_PATTERN, html) - if found is None: - trafficleft = None - validuntil = None - premium = False - else: - credit = found.group(1) - premium = True - try: - if "." in credit: - #Time-based account - vt = [int(x) for x in credit.split('.')[:2]] - lt = time.localtime() - year = lt.tm_year + int(vt[1] < lt.tm_mon or (vt[1] == lt.tm_mon and vt[0] < lt.tm_mday)) - validuntil = time.mktime(time.strptime("%s%d 23:59:59" % (credit,year), "%d.%m.%Y %H:%M:%S")) - trafficleft = -1 - else: - #Traffic-based account - trafficleft = int(credit) * 1024 - validuntil = -1 - except Exception, e: - self.logError('Unable to parse credit info', e) - validuntil = -1 - trafficleft = -1 - - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} - - def login(self, user, data, req): - html = req.load('http://www.hellshare.com/') - if req.lastEffectiveURL != 'http://www.hellshare.com/': - #Switch to English - self.logDebug('Switch lang - URL: %s' % req.lastEffectiveURL) - json = req.load("%s?do=locRouter-show" % req.lastEffectiveURL) - hash = re.search(r"(--[0-9a-f]+-)", json).group(1) - self.logDebug('Switch lang - HASH: %s' % hash) - html = req.load('http://www.hellshare.com/%s/' % hash) - - if re.search(self.CREDIT_LEFT_PATTERN, html): - self.logDebug('Already logged in') - return - - html = req.load('http://www.hellshare.com/login?do=loginForm-submit', post={ - "login": "Log in", - "password": data["password"], - "username": user, - "perm_login": "on" - }) - - if "<p>You input a wrong user name or wrong password</p>" in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/HellspyCz.py b/module/plugins/accounts/HellspyCz.py deleted file mode 100644 index 5f14a093e..000000000 --- a/module/plugins/accounts/HellspyCz.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -import string - -class HellspyCz(Account): - __name__ = "HellspyCz" - __version__ = "0.2" - __type__ = "account" - __description__ = """hellspy.cz account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - CREDIT_LEFT_PATTERN = r'<strong>Credits: </strong>\s*(\d+)' - WRONG_PASSWORD_PATTERN = r'<p class="block-error-3 marg-tb-050">\s*Wrong user or password was entered<br />' - - phpsessid = '' - - def loadAccountInfo(self, user, req): - cj = self.getAccountCookies(user) - cj.setCookie(".hellspy.com", "PHPSESSID", self.phpsessid) - - html = req.load("http://www.hellspy.com/") - - found = re.search(self.CREDIT_LEFT_PATTERN, html) - if found is None: - credits = 0 - else: - credits = int(found.group(1)) * 1024 - - return {"validuntil": -1, "trafficleft": credits} - - def login(self, user, data,req): - header = req.load('http://www.hellspy.com/', just_header = True) - self.phpsessid = re.search(r'PHPSESSID=(\w+)', header).group(1) - self.logDebug("PHPSESSID:" + self.phpsessid) - - html = req.load("http://www.hellspy.com/--%s-" % self.phpsessid) - - html = req.load("http://www.hell-share.com/user/login/?do=apiLoginForm-submit&api_hash=hellspy_iq&user_hash=%s" % self.phpsessid, post={ - "login": "1", - "password": data["password"], - "username": user, - "redir_url": 'http://www.hellspy.com/?do=loginBox-login', - "permanent_login": "1" - }) - - cj = self.getAccountCookies(user) - cj.setCookie(".hellspy.com", "PHPSESSID", self.phpsessid) - - if not re.search(self.CREDIT_LEFT_PATTERN, html): - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/HotfileCom.py b/module/plugins/accounts/HotfileCom.py deleted file mode 100644 index 23e42dacf..000000000 --- a/module/plugins/accounts/HotfileCom.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, JoKoT3 -""" - -from module.plugins.Account import Account -from time import strptime, mktime -import hashlib - -class HotfileCom(Account): - __name__ = "HotfileCom" - __version__ = "0.2" - __type__ = "account" - __description__ = """hotfile.com account plugin""" - __author_name__ = ("mkaay","JoKoT3") - __author_mail__ = ("mkaay@mkaay.de","jokot3@gmail.com") - - def loadAccountInfo(self, user, req): - resp = self.apiCall("getuserinfo", user=user) - if resp.startswith("."): - self.core.debug("HotfileCom API Error: %s" % resp) - raise Exception - info = {} - for p in resp.split("&"): - key, value = p.split("=") - info[key] = value - - if info['is_premium'] == '1': - info["premium_until"] = info["premium_until"].replace("T"," ") - zone = info["premium_until"][19:] - info["premium_until"] = info["premium_until"][:19] - zone = int(zone[:3]) - - validuntil = int(mktime(strptime(info["premium_until"], "%Y-%m-%d %H:%M:%S"))) + (zone*3600) - tmp = {"validuntil":validuntil, "trafficleft":-1, "premium":True} - - elif info['is_premium'] == '0': - tmp = {"premium":False} - - return tmp - - def apiCall(self, method, post={}, user=None): - if user: - data = self.getAccountData(user) - else: - user, data = self.selectAccount() - - req = self.getAccountRequest(user) - - digest = req.load("http://api.hotfile.com/", post={"action":"getdigest"}) - h = hashlib.md5() - h.update(data["password"]) - hp = h.hexdigest() - h = hashlib.md5() - h.update(hp) - h.update(digest) - pwhash = h.hexdigest() - - post.update({"action": method}) - post.update({"username":user, "passwordmd5dig":pwhash, "digest":digest}) - resp = req.load("http://api.hotfile.com/", post=post) - req.close() - return resp - - def login(self, user, data, req): - cj = self.getAccountCookies(user) - cj.setCookie("hotfile.com", "lang", "en") - req.load("http://hotfile.com/", cookies=True) - page = req.load("http://hotfile.com/login.php", post={"returnto": "/", "user": user, "pass": data["password"]}, cookies=True) - - if "Bad username/password" in page: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/Http.py b/module/plugins/accounts/Http.py deleted file mode 100644 index 805d19900..000000000 --- a/module/plugins/accounts/Http.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account - -class Http(Account): - __name__ = "Http" - __version__ = "0.01" - __type__ = "account" - __description__ = """Http dummy account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - login_timeout = info_threshold = 1000000
\ No newline at end of file diff --git a/module/plugins/accounts/MegasharesCom.py b/module/plugins/accounts/MegasharesCom.py deleted file mode 100644 index 91601fc95..000000000 --- a/module/plugins/accounts/MegasharesCom.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account -import re -from time import mktime, strptime - -class MegasharesCom(Account): - __name__ = "MegasharesCom" - __version__ = "0.02" - __type__ = "account" - __description__ = """megashares.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - VALID_UNTIL_PATTERN = r'<p class="premium_info_box">Period Ends: (\w{3} \d{1,2}, \d{4})</p>' - - def loadAccountInfo(self, user, req): - #self.relogin(user) - html = req.load("http://d01.megashares.com/myms.php", decode = True) - - premium = False if '>Premium Upgrade<' in html else True - - validuntil = trafficleft = -1 - try: - timestr = re.search(self.VALID_UNTIL_PATTERN, html).group(1) - self.logDebug(timestr) - validuntil = mktime(strptime(timestr, "%b %d, %Y")) - except Exception, e: - self.logError(e) - - return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - - def login(self, user, data, req): - html = req.load('http://d01.megashares.com/myms_login.php', post = { - "httpref": "", - "myms_login": "Login", - "mymslogin_name": user, - "mymspassword": data['password'] - }, decode = True) - - if not '<span class="b ml">%s</span>' % user in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/MultiDebridCom.py b/module/plugins/accounts/MultiDebridCom.py deleted file mode 100644 index 904be5ee7..000000000 --- a/module/plugins/accounts/MultiDebridCom.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -from time import time - -from module.plugins.Account import Account -from module.common.json_layer import json_loads - - -class MultiDebridCom(Account): - __name__ = "MultiDebridCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """Multi-debrid.com account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - def loadAccountInfo(self, user, req): - if 'days_left' in self.json_data: - validuntil = int(time() + self.json_data['days_left'] * 86400) - return {"premium": True, "validuntil": validuntil, "trafficleft": -1} - else: - self.logError('Unable to get account information') - - def login(self, user, data, req): - # Password to use is the API-Password written in http://multi-debrid.com/myaccount - self.html = req.load("http://multi-debrid.com/api.php", - get={"user": user, "pass": data["password"]}) - self.logDebug('JSON data: ' + self.html) - self.json_data = json_loads(self.html) - if self.json_data['status'] != 'ok': - self.logError('Invalid login. The password to use is the API-Password you find in your "My Account" page') - self.wrongPassword() diff --git a/module/plugins/accounts/MultishareCz.py b/module/plugins/accounts/MultishareCz.py deleted file mode 100644 index 39439cbbe..000000000 --- a/module/plugins/accounts/MultishareCz.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -#from time import mktime, strptime -#from pycurl import REFERER -import re -from module.utils import parseFileSize - -class MultishareCz(Account): - __name__ = "MultishareCz" - __version__ = "0.02" - __type__ = "account" - __description__ = """multishare.cz account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - TRAFFIC_LEFT_PATTERN = r'<span class="profil-zvyrazneni">Kredit:</span>\s*<strong>(?P<S>[0-9,]+) (?P<U>\w+)</strong>' - ACCOUNT_INFO_PATTERN = r'<input type="hidden" id="(u_ID|u_hash)" name="[^"]*" value="([^"]+)">' - - def loadAccountInfo(self, user, req): - #self.relogin(user) - html = req.load("http://www.multishare.cz/profil/", decode = True) - - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - trafficleft = parseFileSize(found.group('S'), found.group('U')) / 1024 if found else 0 - self.premium = True if trafficleft else False - - html = req.load("http://www.multishare.cz/", decode = True) - mms_info = dict(re.findall(self.ACCOUNT_INFO_PATTERN, html)) - - return dict(mms_info, **{"validuntil": -1, "trafficleft": trafficleft}) - - def login(self, user, data, req): - html = req.load('http://www.multishare.cz/html/prihlaseni_process.php', post = { - "akce": "PÅihlásit", - "heslo": data['password'], - "jmeno": user - }, decode = True) - - if '<div class="akce-chyba akce">' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/NetloadIn.py b/module/plugins/accounts/NetloadIn.py deleted file mode 100755 index cef3e298b..000000000 --- a/module/plugins/accounts/NetloadIn.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account -import re -from time import time - -class NetloadIn(Account): - __name__ = "NetloadIn" - __version__ = "0.22" - __type__ = "account" - __description__ = """netload.in account plugin""" - __author_name__ = ("RaNaN", "CryNickSystems") - __author_mail__ = ("RaNaN@pyload.org", "webmaster@pcProfil.de") - - def loadAccountInfo(self, user, req): - page = req.load("http://netload.in/index.php?id=2&lang=de") - left = r">(\d+) (Tag|Tage), (\d+) Stunden<" - left = re.search(left, page) - if left: - validuntil = time() + int(left.group(1)) * 24 * 60 * 60 + int(left.group(3)) * 60 * 60 - trafficleft = -1 - premium = True - else: - validuntil = None - premium = False - trafficleft = None - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium" : premium} - - def login(self, user, data,req): - page = req.load("http://netload.in/index.php", None, { "txtuser" : user, "txtpass" : data['password'], "txtcheck" : "login", "txtlogin" : "Login"}, cookies=True) - if "password or it might be invalid!" in page: - self.wrongPassword() diff --git a/module/plugins/accounts/Premium4Me.py b/module/plugins/accounts/Premium4Me.py deleted file mode 100644 index 6a52cb61a..000000000 --- a/module/plugins/accounts/Premium4Me.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*-
-from module.plugins.MultiHoster import MultiHoster
-
-class Premium4Me(MultiHoster):
- __name__ = "Premium4Me"
- __version__ = "0.10"
- __type__ = "account"
- __description__ = """Premium4.me account plugin"""
- __author_name__ = ("RaNaN", "zoidberg")
- __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz")
-
- def loadAccountInfo(self, req):
- traffic = req.load("http://premium4.me/api/traffic.php?authcode=%s" % self.authcode)
-
- account_info = {"trafficleft": int(traffic) / 1024, "validuntil": -1}
-
- return account_info
-
- def login(self, req):
- self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (self.loginname, self.password)).strip()
-
- if "wrong username" in self.authcode:
- self.wrongPassword()
-
- def loadHosterList(self, req):
- page = req.load("http://premium4.me/api/hosters.php?authcode=%s" % self.authcode)
- return [x.strip() for x in page.replace("\"", "").split(";")]
\ No newline at end of file diff --git a/module/plugins/accounts/PremiumizeMe.py b/module/plugins/accounts/PremiumizeMe.py deleted file mode 100644 index 1a446b842..000000000 --- a/module/plugins/accounts/PremiumizeMe.py +++ /dev/null @@ -1,44 +0,0 @@ -from module.plugins.Account import Account
-
-from module.common.json_layer import json_loads
-
-class PremiumizeMe(Account):
- __name__ = "PremiumizeMe"
- __version__ = "0.11"
- __type__ = "account"
- __description__ = """Premiumize.Me account plugin"""
-
- __author_name__ = ("Florian Franzen")
- __author_mail__ = ("FlorianFranzen@gmail.com")
-
- def loadAccountInfo(self, user, req):
-
- # Get user data from premiumize.me
- status = self.getAccountStatus(user, req)
- self.logDebug(status)
-
- # Parse account info
- account_info = {"validuntil": float(status['result']['expires']),
- "trafficleft": max(0, status['result']['trafficleft_bytes'] / 1024)}
-
- if status['result']['type'] == 'free':
- account_info['premium'] = False
-
- return account_info
-
- def login(self, user, data, req):
-
- # Get user data from premiumize.me
- status = self.getAccountStatus(user, req)
-
- # Check if user and password are valid
- if status['status'] != 200:
- self.wrongPassword()
-
-
- def getAccountStatus(self, user, req):
-
- # Use premiumize.me API v1 (see https://secure.premiumize.me/?show=api) to retrieve account info and return the parsed json answer
- answer = req.load("https://api.premiumize.me/pm-api/v1.php?method=accountstatus¶ms[login]=%s¶ms[pass]=%s" % (user, self.accounts[user]['password']))
- return json_loads(answer)
-
diff --git a/module/plugins/accounts/QuickshareCz.py b/module/plugins/accounts/QuickshareCz.py deleted file mode 100644 index 94649cc43..000000000 --- a/module/plugins/accounts/QuickshareCz.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Account import Account -from module.utils import parseFileSize - -class QuickshareCz(Account): - __name__ = "QuickshareCz" - __version__ = "0.01" - __type__ = "account" - __description__ = """quickshare.cz account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - html = req.load("http://www.quickshare.cz/premium", decode = True) - - found = re.search(r'Stav kreditu: <strong>(.+?)</strong>', html) - if found: - trafficleft = parseFileSize(found.group(1)) / 1024 - premium = True if trafficleft else False - else: - trafficleft = None - premium = False - - return {"validuntil": -1, "trafficleft": trafficleft, "premium": premium} - - def login(self, user, data, req): - html = req.load('http://www.quickshare.cz/html/prihlaseni_process.php', post = { - "akce": u'PÅihlásit', - "heslo": data['password'], - "jmeno": user - }, decode = True) - - if u'>TakovÃœ uÅŸivatel neexistuje.<' in html or u'>Å patné heslo.<' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/RapidgatorNet.py b/module/plugins/accounts/RapidgatorNet.py deleted file mode 100644 index 85adc71a3..000000000 --- a/module/plugins/accounts/RapidgatorNet.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Account import Account -from module.utils import parseFileSize -from module.common.json_layer import json_loads - -class RapidgatorNet(Account): - __name__ = "RapidgatorNet" - __version__ = "0.04" - __type__ = "account" - __description__ = """rapidgator.net account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - API_URL = 'http://rapidgator.net/api/user' - - def loadAccountInfo(self, user, req): - try: - sid = self.getAccountData(user).get('SID') - assert sid - - json = req.load("%s/info?sid=%s" % (self.API_URL, sid)) - self.logDebug("API:USERINFO", json) - json = json_loads(json) - - if json['response_status'] == 200: - if "reset_in" in json['response']: - self.scheduleRefresh(user, json['response']['reset_in']) - - return {"validuntil": json['response']['expire_date'], - "trafficleft": int(json['response']['traffic_left']) / 1024, - "premium": True} - else: - self.logError(json['response_details']) - except Exception, e: - self.logError(e) - - return {"validuntil": None, "trafficleft": None, "premium": False} - - def login(self, user, data, req): - try: - json = req.load('%s/login' % self.API_URL, - post = {"username": user, - "password": data['password']}) - self.logDebug("API:LOGIN", json) - json = json_loads(json) - - if json['response_status'] == 200: - data['SID'] = str(json['response']['session_id']) - return - else: - self.logError(json['response_details']) - except Exception, e: - self.logError(e) - - self.wrongPassword() diff --git a/module/plugins/accounts/RapidshareCom.py b/module/plugins/accounts/RapidshareCom.py deleted file mode 100644 index 15722e099..000000000 --- a/module/plugins/accounts/RapidshareCom.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account - -class RapidshareCom(Account): - __name__ = "RapidshareCom" - __version__ = "0.22" - __type__ = "account" - __description__ = """Rapidshare.com account plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def loadAccountInfo(self, user, req): - data = self.getAccountData(user) - api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi" - api_param_prem = {"sub": "getaccountdetails", "type": "prem", "login": user, "password": data["password"], "withcookie": 1} - src = req.load(api_url_base, cookies=False, get=api_param_prem) - if src.startswith("ERROR"): - raise Exception(src) - fields = src.split("\n") - info = {} - for t in fields: - if not t.strip(): - continue - k, v = t.split("=") - info[k] = v - - validuntil = int(info["billeduntil"]) - premium = True if validuntil else False - - tmp = {"premium": premium, "validuntil": validuntil, "trafficleft":-1, "maxtraffic":-1} - - return tmp - - def login(self, user, data, req): - api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi" - api_param_prem = {"sub": "getaccountdetails", "type": "prem", "login": user, "password": data["password"], "withcookie": 1} - src = req.load(api_url_base, cookies=False, get=api_param_prem) - if src.startswith("ERROR"): - raise Exception(src+"### Note you have to use your account number for login, instead of name.") - fields = src.split("\n") - info = {} - for t in fields: - if not t.strip(): - continue - k, v = t.split("=") - info[k] = v - cj = self.getAccountCookies(user) - cj.setCookie("rapidshare.com", "enc", info["cookie"]) - - diff --git a/module/plugins/accounts/RarefileNet.py b/module/plugins/accounts/RarefileNet.py deleted file mode 100644 index 90ad02d43..000000000 --- a/module/plugins/accounts/RarefileNet.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class RarefileNet(XFSPAccount): - __name__ = "RarefileNet" - __version__ = "0.02" - __type__ = "account" - __description__ = """RareFile.net account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://rarefile.net/" diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py deleted file mode 100644 index 9460fc815..000000000 --- a/module/plugins/accounts/RealdebridCom.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.MultiHoster import MultiHoster -import xml.dom.minidom as dom - -class RealdebridCom(MultiHoster): - __name__ = "RealdebridCom" - __version__ = "0.5" - __type__ = "account" - __description__ = """Real-Debrid.com account plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - def loadAccountInfo(self, req): - page = req.load("http://real-debrid.com/api/account.php") - xml = dom.parseString(page) - account_info = {"validuntil": int(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue), - "trafficleft": -1} - - return account_info - - def login(self, req): - page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (self.loginname, self.password)) - #page = req.load("https://real-debrid.com/login.html", post={"user": user, "pass": data["password"]}, cookies=True) - - if "Your login informations are incorrect" in page: - self.wrongPassword() - - - def loadHosterList(self, req): - https = "https" if self.getConfig("https") else "http" - page = req.load(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip() - - return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/accounts/RehostTo.py b/module/plugins/accounts/RehostTo.py deleted file mode 100644 index e1cb2668f..000000000 --- a/module/plugins/accounts/RehostTo.py +++ /dev/null @@ -1,37 +0,0 @@ -from module.plugins.Account import Account - - -class RehostTo(Account): - __name__ = "RehostTo" - __version__ = "0.1" - __type__ = "account" - __description__ = """Rehost.to account plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - - def loadAccountInfo(self, user, req): - - data = self.getAccountData(user) - page = req.load("http://rehost.to/api.php?cmd=login&user=%s&pass=%s" % (user, data["password"])) - data = [x.split("=") for x in page.split(",")] - ses = data[0][1] - long_ses = data[1][1] - - page = req.load("http://rehost.to/api.php?cmd=get_premium_credits&long_ses=%s" % long_ses) - traffic, valid = page.split(",") - - account_info = {"trafficleft": int(traffic) * 1024, - "validuntil": int(valid), - "long_ses": long_ses, - "ses": ses} - - return account_info - - def login(self, user, data, req): - page = req.load("http://rehost.to/api.php?cmd=login&user=%s&pass=%s" % (user, data["password"])) - - if "Login failed." in page: - self.wrongPassword() - - diff --git a/module/plugins/accounts/ReloadCc.py b/module/plugins/accounts/ReloadCc.py deleted file mode 100644 index e4cb32c42..000000000 --- a/module/plugins/accounts/ReloadCc.py +++ /dev/null @@ -1,73 +0,0 @@ -from module.plugins.Account import Account - -from module.common.json_layer import json_loads - -from module.network.HTTPRequest import BadHeader - -class ReloadCc(Account): - __name__ = "ReloadCc" - __version__ = "0.3" - __type__ = "account" - __description__ = """Reload.Cc account plugin""" - - __author_name__ = ("Reload Team") - __author_mail__ = ("hello@reload.cc") - - def loadAccountInfo(self, user, req): - - # Get user data from reload.cc - status = self.getAccountStatus(user, req) - - # Parse account info - account_info = {"validuntil": float(status['msg']['expires']), - "pwdhash": status['msg']['hash'], - "trafficleft": -1} - - return account_info - - def login(self, user, data, req): - - # Get user data from reload.cc - status = self.getAccountStatus(user, req) - - if not status: - raise Exception("There was an error upon logging in to Reload.cc!") - - # Check if user and password are valid - if status['status'] != "ok": - self.wrongPassword() - - - def getAccountStatus(self, user, req): - # Use reload.cc API v1 to retrieve account info and return the parsed json answer - query_params = dict( - via='pyload', - v=1, - get_traffic='true', - user=user - ) - - try: - query_params.update(dict(hash=self.infos[user]['pwdhash'])) - except Exception: - query_params.update(dict(pwd=self.accounts[user]['password'])) - - try: - answer = req.load("http://api.reload.cc/login", get=query_params) - except BadHeader, e: - if e.code == 400: - raise Exception("There was an unknown error within the Reload.cc plugin.") - elif e.code == 401: - self.wrongPassword() - elif e.code == 402: - self.expired(user) - elif e.code == 403: - raise Exception("Your account is disabled. Please contact the Reload.cc support!") - elif e.code == 409: - self.empty(user) - elif e.code == 503: - self.logInfo("Reload.cc is currently in maintenance mode! Please check again later.") - self.wrongPassword() - return None - - return json_loads(answer) diff --git a/module/plugins/accounts/RyushareCom.py b/module/plugins/accounts/RyushareCom.py deleted file mode 100644 index f734eb11b..000000000 --- a/module/plugins/accounts/RyushareCom.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class RyushareCom(XFSPAccount): - __name__ = "RyushareCom" - __version__ = "0.03" - __type__ = "account" - __description__ = """ryushare.com account plugin""" - __author_name__ = ("zoidberg", "trance4us") - __author_mail__ = ("zoidberg@mujmail.cz", "") - - MAIN_PAGE = "http://ryushare.com/" - - def login(self, user, data, req): - req.lastURL = "http://ryushare.com/login.python" - html = req.load("http://ryushare.com/login.python", post={"login": user, "password": data["password"], "op": "login"}) - if 'Incorrect Login or Password' in html or '>Error<' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/Share76Com.py b/module/plugins/accounts/Share76Com.py deleted file mode 100644 index 9c946ae50..000000000 --- a/module/plugins/accounts/Share76Com.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class Share76Com(XFSPAccount): - __name__ = "Share76Com" - __version__ = "0.02" - __type__ = "account" - __description__ = """Share76.com account plugin""" - __author_name__ = ("me") - - MAIN_PAGE = "http://Share76.com/" diff --git a/module/plugins/accounts/ShareFilesCo.py b/module/plugins/accounts/ShareFilesCo.py deleted file mode 100644 index 0d8ea6635..000000000 --- a/module/plugins/accounts/ShareFilesCo.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class ShareFilesCo(XFSPAccount): - __name__ = "ShareFilesCo" - __version__ = "0.01" - __type__ = "account" - __description__ = """ShareFilesCo account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - MAIN_PAGE = "http://sharefiles.co/" diff --git a/module/plugins/accounts/ShareRapidCom.py b/module/plugins/accounts/ShareRapidCom.py deleted file mode 100644 index f8043449c..000000000 --- a/module/plugins/accounts/ShareRapidCom.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from time import mktime, strptime -from module.plugins.Account import Account - -class ShareRapidCom(Account): - __name__ = "ShareRapidCom" - __version__ = "0.32" - __type__ = "account" - __description__ = """ShareRapid account plugin""" - __author_name__ = ("MikyWoW", "zoidberg") - - login_timeout = 60 - - def loadAccountInfo(self, user, req): - src = req.load("http://share-rapid.com/mujucet/", decode=True) - - found = re.search(ur'<td>Max. poÄet paralelnÃch stahovánÃ: </td><td>(\d+)', src) - if found: - data = self.getAccountData(user) - data["options"]["limitDL"] = [int(found.group(1))] - - found = re.search(ur'<td>Paušálnà stahovánà aktivnÃ. VyprÅ¡Ã </td><td><strong>(.*?)</strong>', src) - if found: - validuntil = mktime(strptime(found.group(1), "%d.%m.%Y - %H:%M")) - return {"premium": True, "trafficleft": -1, "validuntil": validuntil} - - found = re.search(r'<tr><td>GB:</td><td>(.*?) GB', src) - if found: - trafficleft = float(found.group(1)) * (1 << 20) - return {"premium": True, "trafficleft": trafficleft, "validuntil": -1} - - return {"premium": False, "trafficleft": None, "validuntil": None} - - def login(self, user, data, req): - htm = req.load("http://share-rapid.com/prihlaseni/", cookies=True) - if "Heslo:" in htm: - start = htm.index('id="inp_hash" name="hash" value="') - htm = htm[start+33:] - hashes = htm[0:32] - htm = req.load("http://share-rapid.com/prihlaseni/", - post={"hash": hashes, - "login": user, - "pass1": data["password"], - "remember": 0, - "sbmt": u"PÅihlásit"}, cookies=True)
\ No newline at end of file diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py deleted file mode 100644 index fe2b412db..000000000 --- a/module/plugins/accounts/ShareonlineBiz.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account - -class ShareonlineBiz(Account): - __name__ = "ShareonlineBiz" - __version__ = "0.24" - __type__ = "account" - __description__ = """share-online.biz account plugin""" - __author_name__ = ("mkaay", "zoidberg") - __author_mail__ = ("mkaay@mkaay.de", "zoidberg@mujmail.cz") - - def getUserAPI(self, user, req): - return req.load("http://api.share-online.biz/account.php", - {"username": user, "password": self.accounts[user]["password"], "act": "userDetails"}) - - def loadAccountInfo(self, user, req): - src = self.getUserAPI(user, req) - - info = {} - for line in src.splitlines(): - if "=" in line: - key, value = line.split("=") - info[key] = value - self.logDebug(info) - - if "dl" in info and info["dl"].lower() != "not_available": - req.cj.setCookie("share-online.biz", "dl", info["dl"]) - if "a" in info and info["a"].lower() != "not_available": - req.cj.setCookie("share-online.biz", "a", info["a"]) - - return {"validuntil": int(info["expire_date"]) if "expire_date" in info else -1, - "trafficleft": -1, - "premium": True if ("dl" in info or "a" in info) and (info["group"] != "Sammler") else False} - - def login(self, user, data, req): - src = self.getUserAPI(user, req) - if "EXCEPTION" in src: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/SpeedLoadOrg.py b/module/plugins/accounts/SpeedLoadOrg.py deleted file mode 100644 index 4eb2b52de..000000000 --- a/module/plugins/accounts/SpeedLoadOrg.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class SpeedLoadOrg(XFSPAccount): - __name__ = "SpeedLoadOrg" - __version__ = "0.01" - __type__ = "account" - __description__ = """SpeedLoadOrg account plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - MAIN_PAGE = "http://speedload.org/" diff --git a/module/plugins/accounts/StahnuTo.py b/module/plugins/accounts/StahnuTo.py deleted file mode 100644 index 8a4523bc5..000000000 --- a/module/plugins/accounts/StahnuTo.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -from module.utils import parseFileSize -import re - -class StahnuTo(Account): - __name__ = "StahnuTo" - __version__ = "0.02" - __type__ = "account" - __description__ = """StahnuTo account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - #login_timeout = 60 - - def loadAccountInfo(self, user, req): - html = req.load("http://www.stahnu.to/") - - found = re.search(r'>VIP: (\d+.*)<', html) - trafficleft = parseFileSize(found.group(1)) * 1024 if found else 0 - - return {"premium": trafficleft > (512 * 1024), "trafficleft": trafficleft, "validuntil": -1} - - def login(self, user, data, req): - html = req.load("http://www.stahnu.to/login.php", post={ - "username": user, - "password": data["password"], - "submit": "Login"}) - - if not '<a href="logout.php">' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/TurbobitNet.py b/module/plugins/accounts/TurbobitNet.py deleted file mode 100644 index c4b819131..000000000 --- a/module/plugins/accounts/TurbobitNet.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -from time import mktime, strptime - -class TurbobitNet(Account): - __name__ = "TurbobitNet" - __version__ = "0.01" - __type__ = "account" - __description__ = """TurbobitNet account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - #login_timeout = 60 - - def loadAccountInfo(self, user, req): - html = req.load("http://turbobit.net") - - found = re.search(r'<u>Turbo Access</u> to ([0-9.]+)', html) - if found: - premium = True - validuntil = mktime(strptime(found.group(1), "%d.%m.%Y")) - else: - premium = False - validuntil = -1 - - return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} - - def login(self, user, data, req): - req.cj.setCookie("turbobit.net", "user_lang", "en") - - html = req.load("http://turbobit.net/user/login", post={ - "user[login]": user, - "user[pass]": data["password"], - "user[submit]": "Login"}) - - if not '<div class="menu-item user-name">' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/UlozTo.py b/module/plugins/accounts/UlozTo.py deleted file mode 100644 index 6652c8b7c..000000000 --- a/module/plugins/accounts/UlozTo.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account -import re - -class UlozTo(Account): - __name__ = "UlozTo" - __version__ = "0.04" - __type__ = "account" - __description__ = """uloz.to account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - TRAFFIC_LEFT_PATTERN = r'<li class="menu-kredit"><a href="http://www.ulozto.net/kredit" title="[^"]*?GB = ([0-9.]+) MB"' - - def loadAccountInfo(self, user, req): - #this cookie gets lost somehow after each request - self.phpsessid = req.cj.getCookie("ULOSESSID") - html = req.load("http://www.ulozto.net/", decode = True) - req.cj.setCookie("www.ulozto.net", "ULOSESSID", self.phpsessid) - - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - trafficleft = int(float(found.group(1).replace(' ','').replace(',','.')) * 1000 / 1.024) if found else 0 - self.premium = True if trafficleft else False - - return {"validuntil": -1, "trafficleft": trafficleft} - - def login(self, user, data, req): - html = req.load('http://www.ulozto.net/login?do=loginForm-submit', post = { - "login": "Submit", - "password": data['password'], - "username": user - }, decode = True) - - if '<ul class="error">' in html: - self.wrongPassword() diff --git a/module/plugins/accounts/UploadedTo.py b/module/plugins/accounts/UploadedTo.py deleted file mode 100644 index e10b93e8d..000000000 --- a/module/plugins/accounts/UploadedTo.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account -import re -from time import time - -class UploadedTo(Account): - __name__ = "UploadedTo" - __version__ = "0.23" - __type__ = "account" - __description__ = """ul.net account plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def loadAccountInfo(self, user, req): - - req.load("http://uploaded.net/language/en") - html = req.load("http://uploaded.net/me") - - premium = '<a href="register"><em>Premium</em>' in html or '<em>Premium</em></th>' in html - - if premium: - raw_traffic = re.search(r'<th colspan="2"><b class="cB">([^<]+)', html).group(1) - raw_valid = re.search(r"<td>Duration:</td>\s*<th>([^<]+)", html, re.MULTILINE).group(1).strip() - - traffic = int(self.parseTraffic(raw_traffic)) - - if raw_valid == "unlimited": - validuntil = -1 - else: - raw_valid = re.findall(r"(\d+) (weeks|days|hours)", raw_valid) - validuntil = time() - for n, u in raw_valid: - validuntil += 3600 * int(n) * {"weeks": 168, "days": 24, "hours": 1}[u] - - return {"validuntil":validuntil, "trafficleft":traffic, "maxtraffic":50*1024*1024} - else: - return {"premium" : False, "validuntil" : -1} - - def login(self, user, data, req): - - req.load("http://uploaded.net/language/en") - req.cj.setCookie("uploaded.net", "lang", "en") - - page = req.load("http://uploaded.net/io/login", post={ "id" : user, "pw" : data["password"], "_" : ""}) - - if "User and password do not match!" in page: - self.wrongPassword() diff --git a/module/plugins/accounts/UploadheroCom.py b/module/plugins/accounts/UploadheroCom.py deleted file mode 100644 index f1e0649e6..000000000 --- a/module/plugins/accounts/UploadheroCom.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from module.plugins.Account import Account
-import re,datetime,time
-
-class UploadheroCom(Account):
- __name__ = "UploadheroCom"
- __version__ = "0.1"
- __type__ = "account"
- __description__ = """Uploadhero.com account plugin"""
- __author_name__ = ("mcmyst")
- __author_mail__ = ("mcmyst@hotmail.fr")
-
-
- def loadAccountInfo(self, user, req):
- premium_pattern = re.compile('Il vous reste <span class="bleu">([0-9]+)</span> jours premium.')
-
- data = self.getAccountData(user)
- page = req.load("http://uploadhero.com/my-account")
-
- if premium_pattern.search(page):
- end_date = datetime.date.today() + datetime.timedelta(days=int(premium_pattern.search(page).group(1)))
- end_date = time.mktime(future.timetuple())
- account_info = {"validuntil": end_date, "trafficleft": -1, "premium": True}
- else:
- account_info = {"validuntil": -1, "trafficleft": -1, "premium": False}
-
- return account_info
-
- def login(self, user, data, req):
- page = req.load("http://uploadhero.com/lib/connexion.php", post={"pseudo_login": user, "password_login": data["password"]})
-
- if "mot de passe invalide" in page:
- self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/UploadingCom.py b/module/plugins/accounts/UploadingCom.py deleted file mode 100644 index 507e4ab18..000000000 --- a/module/plugins/accounts/UploadingCom.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Account import Account -from time import time, strptime, mktime -import re - -class UploadingCom(Account): - __name__ = "UploadingCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """uploading.com account plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def loadAccountInfo(self, user, req): - src = req.load("http://uploading.com/") - premium = True - if "UPGRADE TO PREMIUM" in src: - return {"validuntil": -1, "trafficleft": -1, "premium": False} - - m = re.search("Valid Until:(.*?)<", src) - if m: - validuntil = int(mktime(strptime(m.group(1).strip(), "%b %d, %Y"))) - else: - validuntil = -1 - - return {"validuntil": validuntil, "trafficleft": -1, "premium": True} - - def login(self, user, data, req): - req.cj.setCookie("uploading.com", "lang", "1") - req.cj.setCookie("uploading.com", "language", "1") - req.cj.setCookie("uploading.com", "setlang", "en") - req.cj.setCookie("uploading.com", "_lang", "en") - req.load("http://uploading.com/") - req.load("http://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time()*1000), post={"email": user, "password": data["password"], "remember": "on"}) diff --git a/module/plugins/accounts/UploadstationCom.py b/module/plugins/accounts/UploadstationCom.py deleted file mode 100644 index e86cec7ce..000000000 --- a/module/plugins/accounts/UploadstationCom.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.accounts.FilejungleCom import FilejungleCom - -class UploadstationCom(FilejungleCom): - __name__ = "UploadstationCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """uploadstation.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - URL = "http://uploadstation.com/" diff --git a/module/plugins/accounts/UptoboxCom.py b/module/plugins/accounts/UptoboxCom.py deleted file mode 100644 index b07991817..000000000 --- a/module/plugins/accounts/UptoboxCom.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.XFSPAccount import XFSPAccount - -class UptoboxCom(XFSPAccount): - __name__ = "UptoboxCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """DDLStorage.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = "http://uptobox.com/"
\ No newline at end of file diff --git a/module/plugins/accounts/WarserverCz.py b/module/plugins/accounts/WarserverCz.py deleted file mode 100644 index 21961956b..000000000 --- a/module/plugins/accounts/WarserverCz.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re -from module.utils import parseFileSize -from time import mktime, strptime - -class WarserverCz(Account): - __name__ = "WarserverCz" - __version__ = "0.02" - __type__ = "account" - __description__ = """Warserver.cz account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - VALID_UNTIL_PATTERN = ur'<li>Neomezené stahovánà do: <strong>(.+?)<' - TRAFFIC_LEFT_PATTERN = ur'<li>Kredit: <strong>(.+?)<' - - DOMAIN = "http://www.warserver.cz" - - def loadAccountInfo(self, user, req): - html = req.load("%s/uzivatele/prehled" % self.DOMAIN, decode = True) - - validuntil = trafficleft = None - premium = False - - found = re.search(self.VALID_UNTIL_PATTERN, html) - if found: - self.logDebug("VALID_UNTIL", found.group(1)) - try: - #validuntil = mktime(strptime(found.group(1), "%d %B %Y")) - premium = True - trafficleft = -1 - except Exception, e: - self.logError(e) - - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if found: - self.logDebug("TRAFFIC_LEFT", found.group(1)) - trafficleft = parseFileSize((found.group(1).replace(" ",""))) // 1024 - premium = True if trafficleft > 1 << 18 else False - - return ({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) - - def login(self, user, data, req): - html = req.load('%s/uzivatele/prihlaseni?do=prihlaseni-submit' % self.DOMAIN, - post = {"username": user, - "password": data['password'], - "send": u"PÅihlásit"}, - decode = True) - - if '<p class="chyba">' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/WuploadCom.py b/module/plugins/accounts/WuploadCom.py deleted file mode 100644 index 3d9ddfffa..000000000 --- a/module/plugins/accounts/WuploadCom.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from types import MethodType - -from module.plugins.Account import Account -from module.common.json_layer import json_loads - -class WuploadCom(Account): - __name__ = "WuploadCom" - __version__ = "0.1" - __type__ = "account" - __description__ = """wupload.com account plugin""" - __author_name__ = ("RaNaN", "Paul King") - __author_mail__ = ("RaNaN@pyload.org", "") - - API_URL = "http://api.wupload.com" - - def init(self): - fs = self.core.pluginManager.loadClass("accounts", "FilesonicCom") - - methods = ["loadAccountInfo", "login"] - #methods to bind from fs - - for m in methods: - setattr(self, m, MethodType(fs.__dict__[m], self, WuploadCom)) - - def getDomain(self, req): - xml = req.load(self.API_URL + "/utility?method=getWuploadDomainForCurrentIp&format=json", - decode=True) - return json_loads(xml)["FSApi_Utility"]["getWuploadDomainForCurrentIp"]["response"]
\ No newline at end of file diff --git a/module/plugins/accounts/X7To.py b/module/plugins/accounts/X7To.py deleted file mode 100644 index 8c2bf245a..000000000 --- a/module/plugins/accounts/X7To.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: ernieb
-"""
-
-import re
-from time import strptime, mktime
-
-from module.plugins.Account import Account
-
-class X7To(Account):
- __name__ = "X7To"
- __version__ = "0.1"
- __type__ = "account"
- __description__ = """X7.To account plugin"""
- __author_name__ = ("ernieb")
- __author_mail__ = ("ernieb")
-
- def loadAccountInfo(self, user, req):
- page = req.load("http://www.x7.to/my")
-
- validCheck = re.search("Premium-Mitglied bis ([0-9]*-[0-9]*-[0-9]*)", page, re.IGNORECASE)
- if validCheck:
- valid = validCheck.group(1)
- valid = int(mktime(strptime(valid, "%Y-%m-%d")))
- else:
- validCheck = re.search("Premium member until ([0-9]*-[0-9]*-[0-9]*)", page, re.IGNORECASE)
- if validCheck:
- valid = validCheck.group(1)
- valid = int(mktime(strptime(valid, "%Y-%m-%d")))
- else:
- valid = 0
-
- trafficleft = re.search(r'<em style="white-space:nowrap">([\d]*[,]?[\d]?[\d]?) (KB|MB|GB)</em>', page, re.IGNORECASE)
- if trafficleft:
- units = float(trafficleft.group(1).replace(",", "."))
- pow = {'KB': 0, 'MB': 1, 'GB': 2}[trafficleft.group(2)]
- trafficleft = int(units * 1024 ** pow)
- else:
- trafficleft = -1
-
- return {"trafficleft": trafficleft, "validuntil": valid}
-
-
- def login(self, user, data, req):
- #req.cj.setCookie("share.cx", "lang", "english")
- page = req.load("http://x7.to/lang/en", None, {})
- page = req.load("http://x7.to/james/login", None,
- {"redirect": "http://www.x7.to/", "id": user, "pw": data['password'], "submit": "submit"})
-
- if "Username and password are not matching." in page:
- self.wrongPassword()
diff --git a/module/plugins/accounts/YibaishiwuCom.py b/module/plugins/accounts/YibaishiwuCom.py deleted file mode 100644 index e2aa6f11d..000000000 --- a/module/plugins/accounts/YibaishiwuCom.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.Account import Account -import re - -class YibaishiwuCom(Account): - __name__ = "YibaishiwuCom" - __version__ = "0.01" - __type__ = "account" - __description__ = """115.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - ACCOUNT_INFO_PATTERN = r'var USER_PERMISSION = {(.*?)}' - - def loadAccountInfo(self, user, req): - #self.relogin(user) - html = req.load("http://115.com/", decode = True) - - found = re.search(self.ACCOUNT_INFO_PATTERN, html, re.S) - premium = True if (found and 'is_vip: 1' in found.group(1)) else False - validuntil = trafficleft = (-1 if found else 0) - return dict({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) - - def login(self, user, data, req): - html = req.load('http://passport.115.com/?ac=login', post = { - "back": "http://www.115.com/", - "goto": "http://115.com/", - "login[account]": user, - "login[passwd]": data['password'] - }, decode = True) - - if not 'var USER_PERMISSION = {' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py deleted file mode 100644 index 61a66cd89..000000000 --- a/module/plugins/accounts/ZeveraCom.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.Account import Account - -import re -from time import mktime, strptime - -class ZeveraCom(Account): - __name__ = "ZeveraCom" - __version__ = "0.21" - __type__ = "account" - __description__ = """Zevera.com account plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def loadAccountInfo(self, user, req): - data = self.getAPIData(req) - if data == "No traffic": - account_info = {"trafficleft": 0, "validuntil": 0, "premium": False} - else: - account_info = { - "trafficleft": int(data['availabletodaytraffic']) * 1024, - "validuntil": mktime(strptime(data['endsubscriptiondate'],"%Y/%m/%d %H:%M:%S")), - "premium": True - } - return account_info - - def login(self, user, data, req): - self.loginname = user - self.password = data["password"] - if self.getAPIData(req) == "No traffic": - self.wrongPassword() - - def getAPIData(self, req, just_header = False, **kwargs): - get_data = { - 'cmd': 'accountinfo', - 'login': self.loginname, - 'pass': self.password - } - get_data.update(kwargs) - - response = req.load("http://www.zevera.com/jDownloader.ashx", get = get_data, decode = True, just_header = just_header) - self.logDebug(response) - - if ':' in response: - if not just_header: - response = response.replace(',','\n') - return dict((y.strip().lower(), z.strip()) for (y,z) in [x.split(':',1) for x in response.splitlines() if ':' in x]) - else: - return response
\ No newline at end of file diff --git a/module/plugins/accounts/__init__.py b/module/plugins/accounts/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/accounts/__init__.py +++ /dev/null diff --git a/module/plugins/addons/AlldebridCom.py b/module/plugins/addons/AlldebridCom.py deleted file mode 100644 index 6818b8c43..000000000 --- a/module/plugins/addons/AlldebridCom.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -# should be working - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class AlldebridCom(MultiHoster): - __name__ = "AlldebridCom" - __version__ = "0.13" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("https", "bool", "Enable HTTPS", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to stanard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - - __description__ = """Real-Debrid.com hook plugin""" - __author_name__ = ("Andy, Voigt") - __author_mail__ = ("spamsales@online.de") - - def getHoster(self): - https = "https" if self.getConfig("https") else "http" - page = getURL(https + "://www.alldebrid.com/api.php?action=get_host").replace("\"","").strip() - - return [x.strip() for x in page.split(",") if x.strip()] diff --git a/module/plugins/addons/BypassCaptcha.py b/module/plugins/addons/BypassCaptcha.py deleted file mode 100644 index 24ad17dd8..000000000 --- a/module/plugins/addons/BypassCaptcha.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, Godofdream, zoidberg -""" - -from thread import start_new_thread -from pycurl import FORM_FILE, LOW_SPEED_TIME - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader - -from module.plugins.Hook import Hook - -PYLOAD_KEY = "4f771155b640970d5607f919a615bdefc67e7d32" - -class BypassCaptchaException(Exception): - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def __str__(self): - return "<BypassCaptchaException %s>" % self.err - - def __repr__(self): - return "<BypassCaptchaException %s>" % self.err - -class BypassCaptcha(Hook): - __name__ = "BypassCaptcha" - __version__ = "0.03" - __description__ = """send captchas to BypassCaptcha.com""" - __config__ = [("activated", "bool", "Activated", True), - ("force", "bool", "Force BC even if client is connected", False), - ("passkey", "password", "Passkey", "")] - __author_name__ = ("RaNaN", "Godofdream", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "soilfcition@gmail.com", "zoidberg@mujmail.cz") - - SUBMIT_URL = "http://bypasscaptcha.com/upload.php" - RESPOND_URL = "http://bypasscaptcha.com/check_value.php" - GETCREDITS_URL = "http://bypasscaptcha.com/ex_left.php" - - def setup(self): - self.info = {} - - def getCredits(self): - response = getURL(self.GETCREDITS_URL, - post = {"key": self.getConfig("passkey")} - ) - - data = dict([x.split(' ',1) for x in response.splitlines()]) - return int(data['Left']) - - - def submit(self, captcha, captchaType="file", match=None): - req = getRequest() - - #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) - - try: - response = req.load(self.SUBMIT_URL, - post={"vendor_key": PYLOAD_KEY, - "key": self.getConfig("passkey"), - "gen_task_id": "1", - "file": (FORM_FILE, captcha)}, - multipart=True) - finally: - req.close() - - data = dict([x.split(' ',1) for x in response.splitlines()]) - if not data or "Value" not in data: - raise BypassCaptchaException(response) - - result = data['Value'] - ticket = data['TaskId'] - self.logDebug("result %s : %s" % (ticket,result)) - - return ticket, result - - def respond(self, ticket, success): - try: - response = getURL(self.RESPOND_URL, - post={"task_id": ticket, - "key": self.getConfig("passkey"), - "cv": 1 if success else 0} - ) - except BadHeader, e: - self.logError("Could not send response.", str(e)) - - def newCaptchaTask(self, task): - if "service" in task.data: - return False - - if not task.isTextual(): - return False - - if not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 0: - task.handler.append(self) - task.data['service'] = self.__name__ - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - - else: - self.logInfo("Your %s account has not enough credits" % self.__name__) - - def captchaCorrect(self, task): - if task.data['service'] == self.__name__ and "ticket" in task.data: - self.respond(task.data["ticket"], True) - - def captchaInvalid(self, task): - if task.data['service'] == self.__name__ and "ticket" in task.data: - self.respond(task.data["ticket"], False) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except BypassCaptchaException, e: - task.error = e.getCode() - return - - task.data["ticket"] = ticket - task.setResult(result)
\ No newline at end of file diff --git a/module/plugins/addons/CaptchaBrotherhood.py b/module/plugins/addons/CaptchaBrotherhood.py deleted file mode 100644 index bdf547827..000000000 --- a/module/plugins/addons/CaptchaBrotherhood.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN, zoidberg -""" -from __future__ import with_statement - -from thread import start_new_thread - -import pycurl -import StringIO -from urllib import urlencode -from time import sleep -import Image - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader -from module.plugins.Hook import Hook - -class CaptchaBrotherhoodException(Exception): - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def __str__(self): - return "<CaptchaBrotherhoodException %s>" % self.err - - def __repr__(self): - return "<CaptchaBrotherhoodException %s>" % self.err - -class CaptchaBrotherhood(Hook): - __name__ = "CaptchaBrotherhood" - __version__ = "0.04" - __description__ = """send captchas to CaptchaBrotherhood.com""" - __config__ = [("activated", "bool", "Activated", False), - ("username", "str", "Username", ""), - ("force", "bool", "Force CT even if client is connected", False), - ("passkey", "password", "Password", ""),] - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - API_URL = "http://www.captchabrotherhood.com/" - - def setup(self): - self.info = {} - - def getCredits(self): - response = getURL(self.API_URL + "askCredits.aspx", - get = {"username": self.getConfig("username"), - "password": self.getConfig("passkey")}) - if not response.startswith("OK"): - raise CaptchaBrotherhoodException(response) - else: - credits = int(response[3:]) - self.logInfo(_("%d credits left") % credits) - self.info["credits"] = credits - return credits - - def submit(self, captcha, captchaType="file", match=None): - try: - img = Image.open(captcha) - output = StringIO.StringIO() - self.logDebug("CAPTCHA IMAGE", img, img.format, img.mode) - if img.format in ("GIF", "JPEG"): - img.save(output, img.format) - else: - if img.mode != "RGB": - img = img.convert("RGB") - img.save(output, "JPEG") - data = output.getvalue() - output.close() - except Exception, e: - raise CaptchaBrotherhoodException("Reading or converting captcha image failed: %s" % e) - - req = getRequest() - - url = "%ssendNewCaptcha.aspx?%s" % (self.API_URL, - urlencode({"username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "captchaSource": "pyLoad", - "timeout": "80"}) - ) - - req.c.setopt(pycurl.URL, url) - req.c.setopt(pycurl.POST, 1) - req.c.setopt(pycurl.POSTFIELDS, data) - req.c.setopt(pycurl.HTTPHEADER, [ "Content-Type: text/html" ]) - - try: - req.c.perform() - response = req.getResponse() - except Exception, e: - raise CaptchaBrotherhoodException("Submit captcha image failed") - - req.close() - - if not response.startswith("OK"): - raise CaptchaBrotherhoodException(response[1]) - - ticket = response[3:] - - for i in range(15): - sleep(5) - response = self.get_api("askCaptchaResult", ticket) - if response.startswith("OK-answered"): - return ticket, response[12:] - - raise CaptchaBrotherhoodException("No solution received in time") - - def get_api(self, api, ticket): - response = getURL("%s%s.aspx" % (self.API_URL, api), - get={"username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "captchaID": ticket} - ) - if not response.startswith("OK"): - raise CaptchaBrotherhoodException("Unknown response: %s" % response) - - return response - - def newCaptchaTask(self, task): - if "service" in task.data: - return False - - if not task.isTextual(): - return False - - if not self.getConfig("username") or not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 10: - task.handler.append(self) - task.data['service'] = self.__name__ - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - else: - self.logInfo("Your CaptchaBrotherhood Account has not enough credits") - - def captchaInvalid(self, task): - if task.data['service'] == self.__name__ and "ticket" in task.data: - response = self.get_api("complainCaptcha", task.data['ticket']) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except CaptchaBrotherhoodException, e: - task.error = e.getCode() - return - - task.data["ticket"] = ticket - task.setResult(result)
\ No newline at end of file diff --git a/module/plugins/addons/CaptchaTrader.py b/module/plugins/addons/CaptchaTrader.py deleted file mode 100644 index 889fa38ef..000000000 --- a/module/plugins/addons/CaptchaTrader.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" - -try: - from json import loads -except ImportError: - from simplejson import loads - -from thread import start_new_thread -from pycurl import FORM_FILE, LOW_SPEED_TIME - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader - -from module.plugins.Addon import Addon - -PYLOAD_KEY = "9f65e7f381c3af2b076ea680ae96b0b7" - -class CaptchaTraderException(Exception): - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def __str__(self): - return "<CaptchaTraderException %s>" % self.err - - def __repr__(self): - return "<CaptchaTraderException %s>" % self.err - -class CaptchaTrader(Addon): - __name__ = "CaptchaTrader" - __version__ = "0.14" - __description__ = """send captchas to captchatrader.com""" - __config__ = [("activated", "bool", "Activated", True), - ("username", "str", "Username", ""), - ("force", "bool", "Force CT even if client is connected", False), - ("passkey", "password", "Password", ""),] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - SUBMIT_URL = "http://api.captchatrader.com/submit" - RESPOND_URL = "http://api.captchatrader.com/respond" - GETCREDITS_URL = "http://api.captchatrader.com/get_credits/username:%(user)s/password:%(password)s/" - - def setup(self): - self.info = {} - - def getCredits(self): - json = getURL(CaptchaTrader.GETCREDITS_URL % {"user": self.getConfig("username"), - "password": self.getConfig("passkey")}) - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - else: - self.logInfo(_("%s credits left") % response[1]) - self.info["credits"] = response[1] - return response[1] - - def submit(self, captcha, captchaType="file", match=None): - if not PYLOAD_KEY: - raise CaptchaTraderException("No API Key Specified!") - - #if type(captcha) == str and captchaType == "file": - # raise CaptchaTraderException("Invalid Type") - assert captchaType in ("file", "url-jpg", "url-jpeg", "url-png", "url-bmp") - - req = getRequest() - - #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) - - try: - json = req.load(CaptchaTrader.SUBMIT_URL, post={"api_key": PYLOAD_KEY, - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "value": (FORM_FILE, captcha), - "type": captchaType}, multipart=True) - finally: - req.close() - - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - - ticket = response[0] - result = response[1] - self.logDebug("result %s : %s" % (ticket,result)) - - return ticket, result - - def respond(self, ticket, success): - try: - json = getURL(CaptchaTrader.RESPOND_URL, post={"is_correct": 1 if success else 0, - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "ticket": ticket}) - - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - - except BadHeader, e: - self.logError(_("Could not send response."), str(e)) - - def newCaptchaTask(self, task): - if not task.isTextual(): - return False - - if not self.getConfig("username") or not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 10: - task.handler.append(self) - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - - else: - self.logInfo(_("Your CaptchaTrader Account has not enough credits")) - - def captchaCorrect(self, task): - if "ticket" in task.data: - ticket = task.data["ticket"] - self.respond(ticket, True) - - def captchaInvalid(self, task): - if "ticket" in task.data: - ticket = task.data["ticket"] - self.respond(ticket, False) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except CaptchaTraderException, e: - task.error = e.getCode() - return - - task.data["ticket"] = ticket - task.setResult(result) diff --git a/module/plugins/addons/Checksum.py b/module/plugins/addons/Checksum.py deleted file mode 100644 index b290838bb..000000000 --- a/module/plugins/addons/Checksum.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" -from __future__ import with_statement -import hashlib, zlib -from os import remove -from os.path import getsize, isfile, splitext -import re - -from module.utils import save_join, fs_encode -from module.plugins.Hook import Hook - -def computeChecksum(local_file, algorithm): - if algorithm in getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")): - h = getattr(hashlib, algorithm)() - chunk_size = 128 * h.block_size - - with open(local_file, 'rb') as f: - for chunk in iter(lambda: f.read(chunk_size), ''): - h.update(chunk) - - return h.hexdigest() - - elif algorithm in ("adler32", "crc32"): - hf = getattr(zlib, algorithm) - last = 0 - - with open(local_file, 'rb') as f: - for chunk in iter(lambda: f.read(8192), ''): - last = hf(chunk, last) - - return "%x" % last - - else: - return None - -class Checksum(Hook): - __name__ = "Checksum" - __version__ = "0.07" - __description__ = "Verify downloaded file size and checksum (enable in general preferences)" - __config__ = [("activated", "bool", "Activated", True), - ("action", "fail;retry;nothing", "What to do if check fails?", "retry"), - ("max_tries", "int", "Number of retries", 2)] - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - methods = { 'sfv':'crc32', 'crc': 'crc32', 'hash': 'md5'} - regexps = { 'sfv': r'^(?P<name>[^;].+)\s+(?P<hash>[0-9A-Fa-f]{8})$', - 'md5': r'^(?P<name>[0-9A-Fa-f]{32}) (?P<file>.+)$', - 'crc': r'filename=(?P<name>.+)\nsize=(?P<size>\d+)\ncrc32=(?P<hash>[0-9A-Fa-f]{8})$', - 'default': r'^(?P<hash>[0-9A-Fa-f]+)\s+\*?(?P<name>.+)$' } - - def setup(self): - if not self.config['general']['checksum']: - self.logInfo("Checksum validation is disabled in general configuration") - - self.algorithms = sorted(getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")), reverse = True) - self.algorithms.extend(["crc32", "adler32"]) - self.formats = self.algorithms + ['sfv', 'crc', 'hash'] - - def downloadFinished(self, pyfile): - """ - Compute checksum for the downloaded file and compare it with the hash provided by the hoster. - pyfile.plugin.check_data should be a dictionary which can contain: - a) if known, the exact filesize in bytes (e.g. "size": 123456789) - b) hexadecimal hash string with algorithm name as key (e.g. "md5": "d76505d0869f9f928a17d42d66326307") - """ - if hasattr(pyfile.plugin, "check_data") and (isinstance(pyfile.plugin.check_data, dict)): - data = pyfile.plugin.check_data.copy() - elif hasattr(pyfile.plugin, "api_data") and (isinstance(pyfile.plugin.api_data, dict)): - data = pyfile.plugin.api_data.copy() - else: - return - - self.logDebug(data) - - if not pyfile.plugin.lastDownload: - self.checkFailed(pyfile, None, "No file downloaded") - - local_file = fs_encode(pyfile.plugin.lastDownload) - #download_folder = self.config['general']['download_folder'] - #local_file = fs_encode(save_join(download_folder, pyfile.package().folder, pyfile.name)) - - if not isfile(local_file): - self.checkFailed(pyfile, None, "File does not exist") - - # validate file size - if "size" in data: - api_size = int(data['size']) - file_size = getsize(local_file) - if api_size != file_size: - self.logWarning("File %s has incorrect size: %d B (%d expected)" % (pyfile.name, file_size, api_size)) - self.checkFailed(pyfile, local_file, "Incorrect file size") - del data['size'] - - # validate checksum - if data and self.config['general']['checksum']: - if "checksum" in data: - data['md5'] = data['checksum'] - - for key in self.algorithms: - if key in data: - checksum = computeChecksum(local_file, key.replace("-","").lower()) - if checksum: - if checksum == data[key]: - self.logInfo('File integrity of "%s" verified by %s checksum (%s).' % (pyfile.name, key.upper(), checksum)) - return - else: - self.logWarning("%s checksum for file %s does not match (%s != %s)" % (key.upper(), pyfile.name, checksum, data[key])) - self.checkFailed(pyfile, local_file, "Checksums do not match") - else: - self.logWarning("Unsupported hashing algorithm: %s" % key.upper()) - else: - self.logWarning("Unable to validate checksum for file %s" % (pyfile.name)) - - def checkFailed(self, pyfile, local_file, msg): - action = self.getConfig("action") - if action == "fail": - pyfile.plugin.fail(reason = msg) - elif action == "retry": - if local_file: - remove(local_file) - pyfile.plugin.retry(reason = msg, max_tries = self.getConfig("max_tries")) - - - def packageFinished(self, pypack): - download_folder = save_join(self.config['general']['download_folder'], pypack.folder, "") - - for link in pypack.getChildren().itervalues(): - file_type = splitext(link["name"])[1][1:].lower() - #self.logDebug(link, file_type) - - if file_type not in self.formats: - continue - - hash_file = fs_encode(save_join(download_folder, link["name"])) - if not isfile(hash_file): - self.logWarning("File not found: %s" % link["name"]) - continue - - with open(hash_file) as f: - text = f.read() - - for m in re.finditer(self.regexps.get(file_type, self.regexps['default']), text): - data = m.groupdict() - self.logDebug(link["name"], data) - - local_file = fs_encode(save_join(download_folder, data["name"])) - algorithm = self.methods.get(file_type, file_type) - checksum = computeChecksum(local_file, algorithm) - if checksum == data["hash"]: - self.logInfo('File integrity of "%s" verified by %s checksum (%s).' % (data["name"], algorithm, checksum)) - else: - self.logWarning("%s checksum for file %s does not match (%s != %s)" % (algorithm, data["name"], checksum, data["hash"]))
\ No newline at end of file diff --git a/module/plugins/addons/ClickAndLoad.py b/module/plugins/addons/ClickAndLoad.py deleted file mode 100644 index 6d6928557..000000000 --- a/module/plugins/addons/ClickAndLoad.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - @interface-version: 0.2 -""" - -import socket -import thread - -from module.plugins.Addon import Addon - -class ClickAndLoad(Addon): - __name__ = "ClickAndLoad" - __version__ = "0.2" - __description__ = """Gives abillity to use jd's click and load. depends on webinterface""" - __config__ = [("activated", "bool", "Activated", "True"), - ("extern", "bool", "Allow external link adding", "False")] - __author_name__ = ("RaNaN", "mkaay") - __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") - - def activate(self): - self.port = int(self.core.config['webinterface']['port']) - if self.core.config['webinterface']['activated']: - try: - if self.getConfig("extern"): - ip = "0.0.0.0" - else: - ip = "127.0.0.1" - - thread.start_new_thread(proxy, (self, ip, self.port, 9666)) - except: - self.log.error("ClickAndLoad port already in use.") - - -def proxy(self, *settings): - thread.start_new_thread(server, (self,) + settings) - lock = thread.allocate_lock() - lock.acquire() - lock.acquire() - - -def server(self, *settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind((settings[0], settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect(("127.0.0.1", settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except socket.error, e: - if hasattr(e, "errno"): - errno = e.errno - else: - errno = e.args[0] - - if errno == 98: - self.core.log.warning(_("Click'N'Load: Port 9666 already in use")) - return - thread.start_new_thread(server, (self,) + settings) - except: - thread.start_new_thread(server, (self,) + settings) - - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) diff --git a/module/plugins/addons/DeathByCaptcha.py b/module/plugins/addons/DeathByCaptcha.py deleted file mode 100644 index 59ff40ded..000000000 --- a/module/plugins/addons/DeathByCaptcha.py +++ /dev/null @@ -1,210 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN, zoidberg -""" -from __future__ import with_statement - -from thread import start_new_thread -from pycurl import FORM_FILE, HTTPHEADER, RESPONSE_CODE -from time import sleep -from base64 import b64encode -import re - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader -from module.plugins.Hook import Hook -from module.common.json_layer import json_loads - -class DeathByCaptchaException(Exception): - DBC_ERRORS = {'not-logged-in': 'Access denied, check your credentials', - 'invalid-credentials': 'Access denied, check your credentials', - 'banned': 'Access denied, account is suspended', - 'insufficient-funds': 'Insufficient account balance to decrypt CAPTCHA', - 'invalid-captcha': 'CAPTCHA is not a valid image', - 'service-overload': 'CAPTCHA was rejected due to service overload, try again later', - 'invalid-request': 'Invalid request', - 'timed-out': 'No CAPTCHA solution received in time' } - - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def getDesc(self): - if self.err in self.DBC_ERRORS.keys(): - return self.DBC_ERRORS[self.err] - else: - return self.err - - def __str__(self): - return "<DeathByCaptchaException %s>" % self.err - - def __repr__(self): - return "<DeathByCaptchaException %s>" % self.err - -class DeathByCaptcha(Hook): - __name__ = "DeathByCaptcha" - __version__ = "0.03" - __description__ = """send captchas to DeathByCaptcha.com""" - __config__ = [("activated", "bool", "Activated", False), - ("username", "str", "Username", ""), - ("passkey", "password", "Password", ""), - ("force", "bool", "Force DBC even if client is connected", False)] - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - API_URL = "http://api.dbcapi.me/api/" - - def setup(self): - self.info = {} - - def call_api(self, api="captcha", post=False, multipart=False): - req = getRequest() - req.c.setopt(HTTPHEADER, ["Accept: application/json", - "User-Agent: pyLoad %s" % self.core.version]) - - if post: - if not isinstance(post, dict): - post = {} - post.update({"username": self.getConfig("username"), - "password": self.getConfig("passkey")}) - - response = None - try: - json = req.load("%s%s" % (self.API_URL, api), - post = post, - multipart=multipart) - self.logDebug(json) - response = json_loads(json) - - if "error" in response: - raise DeathByCaptchaException(response['error']) - elif "status" not in response: - raise DeathByCaptchaException(str(response)) - - except BadHeader, e: - if 403 == e.code: - raise DeathByCaptchaException('not-logged-in') - elif 413 == e.code: - raise DeathByCaptchaException('invalid-captcha') - elif 503 == e.code: - raise DeathByCaptchaException('service-overload') - elif e.code in (400, 405): - raise DeathByCaptchaException('invalid-request') - else: - raise - - finally: - req.close() - - return response - - def getCredits(self): - response = self.call_api("user", True) - - if 'is_banned' in response and response['is_banned']: - raise DeathByCaptchaException('banned') - elif 'balance' in response and 'rate' in response: - self.info.update(response) - else: - raise DeathByCaptchaException(response) - - def getStatus(self): - response = self.call_api("status", False) - - if 'is_service_overloaded' in response and response['is_service_overloaded']: - raise DeathByCaptchaException('service-overload') - - def submit(self, captcha, captchaType="file", match=None): - #workaround multipart-post bug in HTTPRequest.py - if re.match("^[A-Za-z0-9]*$", self.getConfig("passkey")): - multipart = True - data = (FORM_FILE, captcha) - else: - multipart = False - with open(captcha, 'rb') as f: - data = f.read() - data = "base64:" + b64encode(data) - - response = self.call_api("captcha", {"captchafile": data}, multipart) - - if "captcha" not in response: - raise DeathByCaptchaException(response) - ticket = response['captcha'] - - for i in range(24): - sleep(5) - response = self.call_api("captcha/%d" % ticket, False) - if response['text'] and response['is_correct']: - break - else: - raise DeathByCaptchaException('timed-out') - - result = response['text'] - self.logDebug("result %s : %s" % (ticket,result)) - - return ticket, result - - def newCaptchaTask(self, task): - if "service" in task.data: - return False - - if not task.isTextual(): - return False - - if not self.getConfig("username") or not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - try: - self.getStatus() - self.getCredits() - except DeathByCaptchaException, e: - self.logError(e.getDesc()) - return False - - balance, rate = self.info["balance"], self.info["rate"] - self.logInfo("Account balance: US$%.3f (%d captchas left at %.2f cents each)" % (balance / 100, balance // rate, rate)) - - if balance > rate: - task.handler.append(self) - task.data['service'] = self.__name__ - task.setWaiting(180) - start_new_thread(self.processCaptcha, (task,)) - - def captchaInvalid(self, task): - if task.data['service'] == self.__name__ and "ticket" in task.data: - try: - response = self.call_api("captcha/%d/report" % task.data["ticket"], True) - except DeathByCaptchaException, e: - self.logError(e.getDesc()) - except Exception, e: - self.logError(e) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except DeathByCaptchaException, e: - task.error = e.getCode() - self.logError(e.getDesc()) - return - - task.data["ticket"] = ticket - task.setResult(result)
\ No newline at end of file diff --git a/module/plugins/addons/DebridItaliaCom.py b/module/plugins/addons/DebridItaliaCom.py deleted file mode 100644 index 80cdc45f6..000000000 --- a/module/plugins/addons/DebridItaliaCom.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -from module.plugins.internal.MultiHoster import MultiHoster - - -class DebridItaliaCom(MultiHoster): - __name__ = "DebridItaliaCom" - __version__ = "0.05" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to standard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - - __description__ = """Debriditalia.com hook plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - def getHoster(self): - return ["netload.in", "hotfile.com", "rapidshare.com", "multiupload.com", - "uploading.com", "megashares.com", "crocko.com", "filepost.com", - "bitshare.com", "share-links.biz", "putlocker.com", "uploaded.to", - "speedload.org", "rapidgator.net", "likeupload.net", "cyberlocker.ch", - "depositfiles.com", "extabit.com", "filefactory.com", "sharefiles.co", - "ryushare.com", "tusfiles.net", "nowvideo.co", "cloudzer.net", "letitbit.net"] diff --git a/module/plugins/addons/DeleteFinished.py b/module/plugins/addons/DeleteFinished.py deleted file mode 100644 index e0df69eef..000000000 --- a/module/plugins/addons/DeleteFinished.py +++ /dev/null @@ -1,86 +0,0 @@ - # -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: Walter Purcaro -""" - -from module.plugins.Hook import Hook -from time import time - - -class DeleteFinished(Hook): - __name__ = "DeleteFinished" - __version__ = "0.5" - __description__ = "Automatically delete finished packages from queue" - __config__ = [ - ("activated", "bool", "Activated", "False"), - ("interval", "int", "Delete every (hours)", "72") - ] - __author_name__ = ("Walter Purcaro") - __author_mail__ = ("vuolter@gmail.com") - - def wakeup(self, pypack): - # self.logDebug("self.wakeup") - self.removeEvent("packageFinished", self.wakeup) - self.info["sleep"] = False - - def periodical(self): - # self.logDebug("self.periodical") - if not self.info["sleep"]: - self.core.api.deleteFinished() - self.logDebug("called self.core.api.deleteFinished") - self.info["sleep"] = True - self.addEvent("packageFinished", self.wakeup) - - def addEvent(self, event, handler): - if event in self.manager.events: - if handler not in self.manager.events[event]: - self.manager.events[event].append(handler) - # self.logDebug("self.addEvent: " + event + ": added handler") - else: - # self.logDebug("self.addEvent: " + event + ": NOT added handler") - return False - else: - self.manager.events[event] = [handler] - # self.logDebug("self.addEvent: " + event + ": added event and handler") - return True - - def removeEvent(self, event, handler): - if event in self.manager.events and handler in self.manager.events[event]: - self.manager.events[event].remove(handler) - # self.logDebug("self.removeEvent: " + event + ": removed handler") - return True - else: - # self.logDebug("self.removeEvent: " + event + ": NOT removed handler") - return False - - def configEvents(self, plugin=None, name=None, value=None): - # self.logDebug("self.configEvents") - interval = self.getConfig("interval") * 3600 - if interval != self.interval: - self.interval = interval - - def unload(self): - # self.logDebug("self.unload") - self.removeEvent("pluginConfigChanged", self.configEvents) - self.removeEvent("packageFinished", self.wakeup) - - def coreReady(self): - # self.logDebug("self.coreReady") - self.info = {"sleep": True} - self.addEvent("pluginConfigChanged", self.configEvents) - self.configEvents() - self.addEvent("packageFinished", self.wakeup) diff --git a/module/plugins/addons/DownloadScheduler.py b/module/plugins/addons/DownloadScheduler.py deleted file mode 100644 index 4049d71c5..000000000 --- a/module/plugins/addons/DownloadScheduler.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg - Original idea by new.cze -""" - -import re -from time import localtime - -from module.plugins.Hook import Hook - - -class DownloadScheduler(Hook): - __name__ = "DownloadScheduler" - __version__ = "0.21" - __description__ = """Download Scheduler""" - __config__ = [("activated", "bool", "Activated", "False"), - ("timetable", "str", "List time periods as hh:mm full or number(kB/s)", - "0:00 full, 7:00 250, 10:00 0, 17:00 150"), - ("abort", "bool", "Abort active downloads when start period with speed 0", "False")] - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - def setup(self): - self.cb = None # callback to scheduler job; will be by removed hookmanager when hook unloaded - - def coreReady(self): - self.updateSchedule() - - def updateSchedule(self, schedule=None): - if schedule is None: - schedule = self.getConfig("timetable") - - schedule = re.findall("(\d{1,2}):(\d{2})[\s]*(-?\d+)", - schedule.lower().replace("full", "-1").replace("none", "0")) - if not schedule: - self.logError("Invalid schedule") - return - - t0 = localtime() - now = (t0.tm_hour, t0.tm_min, t0.tm_sec, "X") - schedule = sorted([(int(x[0]), int(x[1]), 0, int(x[2])) for x in schedule] + [now]) - - self.logDebug("Schedule", schedule) - - for i, v in enumerate(schedule): - if v[3] == "X": - last, next = schedule[i - 1], schedule[(i + 1) % len(schedule)] - self.logDebug("Now/Last/Next", now, last, next) - - self.setDownloadSpeed(last[3]) - - next_time = (((24 + next[0] - now[0]) * 60 + next[1] - now[1]) * 60 + next[2] - now[2]) % 86400 - self.core.scheduler.removeJob(self.cb) - self.cb = self.core.scheduler.addJob(next_time, self.updateSchedule, threaded=False) - - def setDownloadSpeed(self, speed): - if speed == 0: - abort = self.getConfig("abort") - self.logInfo("Stopping download server. (Running downloads will %sbe aborted.)" % ('' if abort else 'not ')) - self.core.api.pauseServer() - if abort: - self.core.api.stopAllDownloads() - else: - self.core.api.unpauseServer() - - if speed > 0: - self.logInfo("Setting download speed to %d kB/s" % speed) - self.core.api.setConfigValue("download", "limit_speed", 1) - self.core.api.setConfigValue("download", "max_speed", speed) - else: - self.logInfo("Setting download speed to FULL") - self.core.api.setConfigValue("download", "limit_speed", 0) - self.core.api.setConfigValue("download", "max_speed", -1) diff --git a/module/plugins/addons/EasybytezCom.py b/module/plugins/addons/EasybytezCom.py deleted file mode 100644 index 6a4ded85b..000000000 --- a/module/plugins/addons/EasybytezCom.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster -import re - -class EasybytezCom(MultiHoster): - __name__ = "EasybytezCom" - __version__ = "0.03" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", "")] - __description__ = """EasyBytez.com hook plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def getHoster(self): - self.account = self.core.accountManager.getAccountPlugin(self.__name__) - user = self.account.selectAccount()[0] - - try: - req = self.account.getAccountRequest(user) - page = req.load("http://www.easybytez.com") - - found = re.search(r'</textarea>\s*Supported sites:(.*)', page) - return found.group(1).split(',') - except Exception, e: - self.logDebug(e) - self.logWarning("Unable to load supported hoster list, using last known") - return ['bitshare.com', 'crocko.com', 'ddlstorage.com', 'depositfiles.com', 'extabit.com', 'hotfile.com', 'mediafire.com', 'netload.in', 'rapidgator.net', 'rapidshare.com', 'uploading.com', 'uload.to', 'uploaded.to']
\ No newline at end of file diff --git a/module/plugins/addons/Ev0InFetcher.py b/module/plugins/addons/Ev0InFetcher.py deleted file mode 100644 index 608baf217..000000000 --- a/module/plugins/addons/Ev0InFetcher.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" -from module.lib import feedparser -from time import mktime, time - -from module.plugins.Addon import Addon - -class Ev0InFetcher(Addon): - __name__ = "Ev0InFetcher" - __version__ = "0.2" - __description__ = """checks rss feeds for ev0.in""" - __config__ = [("activated", "bool", "Activated", "False"), - ("interval", "int", "Check interval in minutes", "10"), - ("queue", "bool", "Move new shows directly to Queue", False), - ("shows", "str", "Shows to check for (comma separated)", ""), - ("quality", "xvid;x264;rmvb", "Video Format", "xvid"), - ("hoster", "str", "Hoster to use (comma separated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.interval = self.getConfig("interval") * 60 - - def filterLinks(self, links): - results = self.core.pluginManager.parseUrls(links) - sortedLinks = {} - - for url, hoster in results[0]: - if hoster not in sortedLinks: - sortedLinks[hoster] = [] - sortedLinks[hoster].append(url) - - for h in self.getConfig("hoster").split(","): - try: - return sortedLinks[h.strip()] - except: - continue - return [] - - def periodical(self): - def normalizefiletitle(filename): - filename = filename.replace('.', ' ') - filename = filename.replace('_', ' ') - filename = filename.lower() - return filename - - shows = [s.strip() for s in self.getConfig("shows").split(",")] - - feed = feedparser.parse("http://feeds.feedburner.com/ev0in/%s?format=xml" % self.getConfig("quality")) - - showStorage = {} - for show in shows: - showStorage[show] = int(self.getStorage("show_%s_lastfound" % show, 0)) - - found = False - for item in feed['items']: - for show, lastfound in showStorage.iteritems(): - if show.lower() in normalizefiletitle(item['title']) and lastfound < int(mktime(item.date_parsed)): - links = self.filterLinks(item['description'].split("<br />")) - packagename = item['title'].encode("utf-8") - self.core.log.info("Ev0InFetcher: new episode '%s' (matched '%s')" % (packagename, show)) - self.core.api.addPackage(packagename, links, 1 if self.getConfig("queue") else 0) - self.setStorage("show_%s_lastfound" % show, int(mktime(item.date_parsed))) - found = True - if not found: - #self.core.log.debug("Ev0InFetcher: no new episodes found") - pass - - for show, lastfound in self.getStorage().iteritems(): - if int(lastfound) > 0 and int(lastfound) + (3600*24*30) < int(time()): - self.delStorage("show_%s_lastfound" % show) - self.core.log.debug("Ev0InFetcher: cleaned '%s' record" % show) diff --git a/module/plugins/addons/ExpertDecoders.py b/module/plugins/addons/ExpertDecoders.py deleted file mode 100644 index 2e66e49ca..000000000 --- a/module/plugins/addons/ExpertDecoders.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN, zoidberg -""" -from __future__ import with_statement - -from thread import start_new_thread -from pycurl import FORM_FILE, LOW_SPEED_TIME -from uuid import uuid4 -from base64 import b64encode - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader - -from module.plugins.Hook import Hook - -class ExpertDecoders(Hook): - __name__ = "ExpertDecoders" - __version__ = "0.01" - __description__ = """send captchas to expertdecoders.com""" - __config__ = [("activated", "bool", "Activated", False), - ("force", "bool", "Force CT even if client is connected", False), - ("passkey", "password", "Access key", ""),] - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - API_URL = "http://www.fasttypers.org/imagepost.ashx" - - def setup(self): - self.info = {} - - def getCredits(self): - response = getURL(self.API_URL, post = { "key": self.getConfig("passkey"), "action": "balance" }) - - if response.isdigit(): - self.logInfo(_("%s credits left") % response) - self.info["credits"] = credits = int(response) - return credits - else: - self.logError(response) - return 0 - - def processCaptcha(self, task): - task.data["ticket"] = ticket = uuid4() - result = None - - with open(task.captchaFile, 'rb') as f: - data = f.read() - data = b64encode(data) - #self.logDebug("%s: %s : %s" % (ticket, task.captchaFile, data)) - - req = getRequest() - #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) - - try: - result = req.load(self.API_URL, - post={ "action": "upload", - "key": self.getConfig("passkey"), - "file": data, - "gen_task_id": ticket } - ) - finally: - req.close() - - self.logDebug("result %s : %s" % (ticket, result)) - task.setResult(result) - - def newCaptchaTask(self, task): - if not task.isTextual(): - return False - - if not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 0: - task.handler.append(self) - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - - else: - self.logInfo(_("Your ExpertDecoders Account has not enough credits")) - - def captchaInvalid(self, task): - if "ticket" in task.data: - - try: - response = getURL(self.API_URL, - post={ "action": "refund", - "key": self.getConfig("passkey"), - "gen_task_id": task.data["ticket"] } - ) - self.logInfo("Request refund: %s" % response) - - except BadHeader, e: - self.logError("Could not send refund request.", str(e))
\ No newline at end of file diff --git a/module/plugins/addons/ExternalScripts.py b/module/plugins/addons/ExternalScripts.py deleted file mode 100644 index 8f5a5841e..000000000 --- a/module/plugins/addons/ExternalScripts.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import subprocess -from os import access, X_OK, makedirs -from os.path import basename - -from module.plugins.Addon import Addon -from module.utils.fs import save_join, exists, join, listdir - -class ExternalScripts(Addon): - __name__ = "ExternalScripts" - __version__ = "0.22" - __description__ = """Run external scripts""" - __config__ = [("activated", "bool", "Activated", "True")] - __author_name__ = ("mkaay", "RaNaN", "spoob") - __author_mail__ = ("mkaay@mkaay.de", "ranan@pyload.org", "spoob@pyload.org") - - event_list = ["unrarFinished", "allDownloadsFinished", "allDownloadsProcessed"] - - def setup(self): - self.scripts = {} - - folders = ['download_preparing', 'download_finished', 'package_finished', - 'before_reconnect', 'after_reconnect', 'unrar_finished', - 'all_dls_finished', 'all_dls_processed'] - - for folder in folders: - - self.scripts[folder] = [] - - self.initPluginType(folder, join(pypath, 'scripts', folder)) - self.initPluginType(folder, join('scripts', folder)) - - for script_type, names in self.scripts.iteritems(): - if names: - self.logInfo((_("Installed scripts for %s: ") % script_type ) + ", ".join([basename(x) for x in names])) - - - def initPluginType(self, folder, path): - if not exists(path): - try: - makedirs(path) - except : - self.logDebug("Script folder %s not created" % folder) - return - - for f in listdir(path): - if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): - continue - - if not access(join(path,f), X_OK): - self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) - - self.scripts[folder].append(join(path, f)) - - def callScript(self, script, *args): - try: - cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args] - #output goes to pyload - subprocess.Popen(cmd, bufsize=-1) - except Exception, e: - self.logError(_("Error in %(script)s: %(error)s") % { "script" :basename(script), "error": str(e)}) - - def downloadPreparing(self, pyfile): - for script in self.scripts['download_preparing']: - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id) - - def downloadFinished(self, pyfile): - for script in self.scripts['download_finished']: - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, - save_join(self.core.config['general']['download_folder'], pyfile.package().folder, pyfile.name), - pyfile.id) - - - def packageFinished(self, pypack): - for script in self.scripts['package_finished']: - folder = self.core.config['general']['download_folder'] - folder = save_join(folder, pypack.folder) - - self.callScript(script, pypack.name, folder, pypack.password, pypack.id) - - def beforeReconnecting(self, ip): - for script in self.scripts['before_reconnect']: - self.callScript(script, ip) - - def afterReconnecting(self, ip): - for script in self.scripts['after_reconnect']: - self.callScript(script, ip) - - def unrarFinished(self, folder, fname): - for script in self.scripts["unrar_finished"]: - self.callScript(script, folder, fname) - - def allDownloadsFinished(self): - for script in self.scripts["all_dls_finished"]: - self.callScript(script) - - def allDownloadsProcessed(self): - for script in self.scripts["all_dls_processed"]: - self.callScript(script) - diff --git a/module/plugins/addons/ExtractArchive.py b/module/plugins/addons/ExtractArchive.py deleted file mode 100644 index 369b20ba9..000000000 --- a/module/plugins/addons/ExtractArchive.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import os -from os.path import basename, isfile, isdir, join -from traceback import print_exc -from copy import copy - -# monkey patch bug in python 2.6 and lower -# see http://bugs.python.org/issue6122 -# http://bugs.python.org/issue1236 -# http://bugs.python.org/issue1731717 -if sys.version_info < (2, 7) and os.name != "nt": - from subprocess import Popen - import errno - - def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except OSError, e: - if e.errno == errno.EINTR: - continue - raise - - # unsued timeout option for older python version - def wait(self, timeout=0): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - Popen.wait = wait - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - -from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs -from module.plugins.Addon import Addon, threaded, Expose -from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword - -class ExtractArchive(Addon): - """ - Provides: unrarFinished (folder, filename) - """ - __name__ = "ExtractArchive" - __version__ = "0.12" - __description__ = "Extract different kind of archives" - __config__ = [("activated", "bool", "Activated", True), - ("fullpath", "bool", "Extract full path", True), - ("overwrite", "bool", "Overwrite files", True), - ("passwordfile", "file", "password file", "unrar_passwords.txt"), - ("deletearchive", "bool", "Delete archives when done", False), - ("subfolder", "bool", "Create subfolder for each package", False), - ("destination", "folder", "Extract files to", ""), - ("recursive", "bool", "Extract archives in archvies", True), - ("queue", "bool", "Wait for all downloads to be finished", True), - ("renice", "int", "CPU Priority", 0), ] - __author_name__ = ("pyload Team") - __author_mail__ = ("admin<at>pyload.org") - - event_list = ["allDownloadsProcessed"] - - def setup(self): - self.plugins = [] - self.passwords = [] - names = [] - - for p in ("UnRar", "UnZip"): - try: - module = self.core.pluginManager.loadModule("internal", p) - klass = getattr(module, p) - if klass.checkDeps(): - names.append(p) - self.plugins.append(klass) - - except OSError, e: - if e.errno == 2: - self.logInfo(_("No %s installed") % p) - else: - self.logWarning(_("Could not activate %s") % p, str(e)) - if self.core.debug: - print_exc() - - except Exception, e: - self.logWarning(_("Could not activate %s") % p, str(e)) - if self.core.debug: - print_exc() - - if names: - self.logInfo(_("Activated") + " " + " ".join(names)) - else: - self.logInfo(_("No Extract plugins activated")) - - # queue with package ids - self.queue = [] - - @Expose - def extractPackage(self, id): - """ Extract package with given id""" - self.manager.startThread(self.extract, [id]) - - def packageFinished(self, pypack): - if self.getConfig("queue"): - self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pypack.id) - else: - self.manager.startThread(self.extract, [pypack.id]) - - - @threaded - def allDownloadsProcessed(self, thread): - local = copy(self.queue) - del self.queue[:] - self.extract(local, thread) - - - def extract(self, ids, thread=None): - # reload from txt file - self.reloadPasswords() - - # dl folder - dl = self.config['general']['download_folder'] - - extracted = [] - - #iterate packages -> plugins -> targets - for pid in ids: - p = self.core.files.getPackage(pid) - self.logInfo(_("Check package %s") % p.name) - if not p: continue - - # determine output folder - out = save_join(dl, p.folder, "") - # force trailing slash - - if self.getConfig("destination") and self.getConfig("destination").lower() != "none": - - out = save_join(dl, p.folder, self.getConfig("destination"), "") - #relative to package folder if destination is relative, otherwise absolute path overwrites them - - if self.getConf("subfolder"): - out = join(out, fs_encode(p.folder)) - - if not exists(out): - makedirs(out) - - files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] - matched = False - - # check as long there are unseen files - while files_ids: - new_files_ids = [] - - for plugin in self.plugins: - targets = plugin.getTargets(files_ids) - if targets: - self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) - matched = True - for target, fid in targets: - if target in extracted: - self.logDebug(basename(target), "skipped") - continue - extracted.append(target) #prevent extracting same file twice - - klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), - self.getConfig("renice")) - klass.init() - - self.logInfo(basename(target), _("Extract to %s") % out) - new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread) - self.logDebug("Extracted: %s" % new_files) - self.setPermissions(new_files) - - for file in new_files: - if not exists(file): - self.logDebug("new file %s does not exists" % file) - continue - if self.getConfig("recursive") and isfile(file): - new_files_ids.append((file, fid)) #append as new target - - files_ids = new_files_ids # also check extracted files - - if not matched: self.logInfo(_("No files found to extract")) - - def startExtracting(self, plugin, fid, passwords, thread): - pyfile = self.core.files.getFile(fid) - if not pyfile: return [] - - pyfile.setCustomStatus(_("extracting")) - thread.addActive(pyfile) #keep this file until everything is done - - try: - progress = lambda x: pyfile.setProgress(x) - success = False - - if not plugin.checkArchive(): - plugin.extract(progress) - success = True - else: - self.logInfo(basename(plugin.file), _("Password protected")) - self.logDebug("Passwords: %s" % str(passwords)) - - pwlist = copy(self.getPasswords()) - #remove already supplied pws from list (only local) - for pw in passwords: - if pw in pwlist: pwlist.remove(pw) - - for pw in passwords + pwlist: - try: - self.logDebug("Try password: %s" % pw) - if plugin.checkPassword(pw): - plugin.extract(progress, pw) - self.addPassword(pw) - success = True - break - except WrongPassword: - self.logDebug("Password was wrong") - - if not success: - self.logError(basename(plugin.file), _("Wrong password")) - return [] - - if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) - - if self.getConfig("deletearchive"): - files = plugin.getDeleteFiles() - self.logInfo(_("Deleting %s files") % len(files)) - for f in files: - if exists(f): remove(f) - else: self.logDebug("%s does not exists" % f) - - self.logInfo(basename(plugin.file), _("Extracting finished")) - self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file) - - return plugin.getExtractedFiles() - - - except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), str(e)) - except CRCError: - self.logError(basename(plugin.file), _("CRC Mismatch")) - except Exception, e: - if self.core.debug: - print_exc() - self.logError(basename(plugin.file), _("Unknown Error"), str(e)) - - return [] - - @Expose - def getPasswords(self): - """ List of saved passwords """ - return self.passwords - - - def reloadPasswords(self): - pwfile = self.getConfig("passwordfile") - if not exists(pwfile): - open(pwfile, "wb").close() - - passwords = [] - f = open(pwfile, "rb") - for pw in f.read().splitlines(): - passwords.append(pw) - f.close() - - self.passwords = passwords - - - @Expose - def addPassword(self, pw): - """ Adds a password to saved list""" - pwfile = self.getConfig("passwordfile") - - if pw in self.passwords: self.passwords.remove(pw) - self.passwords.insert(0, pw) - - f = open(pwfile, "wb") - for pw in self.passwords: - f.write(pw + "\n") - f.close() - - def setPermissions(self, files): - for f in files: - if not exists(f): continue - try: - if self.core.config["permission"]["change_file"]: - if isfile(f): - chmod(f, int(self.core.config["permission"]["file"], 8)) - elif isdir(f): - chmod(f, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - chown(f, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed"), e) diff --git a/module/plugins/addons/HotFolder.py b/module/plugins/addons/HotFolder.py deleted file mode 100644 index d05026448..000000000 --- a/module/plugins/addons/HotFolder.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - @interface-version: 0.2 -""" - -from os import makedirs -from os import listdir -from os.path import exists -from os.path import join -from os.path import isfile -from shutil import move -import time - -from module.plugins.Addon import Addon - -class HotFolder(Addon): - __name__ = "HotFolder" - __version__ = "0.1" - __description__ = """observe folder and file for changes and add container and links""" - __config__ = [ ("activated", "bool", "Activated" , "False"), - ("folder", "str", "Folder to observe", "container"), - ("watch_file", "bool", "Observe link file", "False"), - ("keep", "bool", "Keep added containers", "True"), - ("file", "str", "Link file", "links.txt")] - __threaded__ = [] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.de") - - def setup(self): - self.interval = 10 - - def periodical(self): - - if not exists(join(self.getConfig("folder"), "finished")): - makedirs(join(self.getConfig("folder"), "finished")) - - if self.getConfig("watch_file"): - - if not exists(self.getConfig("file")): - f = open(self.getConfig("file"), "wb") - f.close() - - - f = open(self.getConfig("file"), "rb") - content = f.read().strip() - f.close() - f = open(self.getConfig("file"), "wb") - f.close() - if content: - name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y") ) - - f = open(join(self.getConfig("folder"), "finished", name), "wb") - f.write(content) - f.close() - - self.core.api.addPackage(f.name, [f.name], 1) - - for f in listdir(self.getConfig("folder")): - path = join(self.getConfig("folder"), f) - - if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): - continue - - newpath = join(self.getConfig("folder"), "finished", f if self.getConfig("keep") else "tmp_"+f) - move(path, newpath) - - self.log.info(_("Added %s from HotFolder") % f) - self.core.api.addPackage(f, [newpath], 1) - -
\ No newline at end of file diff --git a/module/plugins/addons/IRCInterface.py b/module/plugins/addons/IRCInterface.py deleted file mode 100644 index c261fc6f3..000000000 --- a/module/plugins/addons/IRCInterface.py +++ /dev/null @@ -1,431 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - @author: jeix - @interface-version: 0.2 -""" - -from select import select -import socket -from threading import Thread -import time -from time import sleep -from traceback import print_exc -import re - -from module.plugins.Addon import Addon -from module.network.RequestFactory import getURL -from module.utils import formatSize -from module.Api import PackageDoesNotExists, FileDoesNotExists - -from pycurl import FORM_FILE - -class IRCInterface(Thread, Addon): - __name__ = "IRCInterface" - __version__ = "0.1" - __description__ = """connect to irc and let owner perform different tasks""" - __config__ = [("activated", "bool", "Activated", "False"), - ("host", "str", "IRC-Server Address", "Enter your server here!"), - ("port", "int", "IRC-Server Port", "6667"), - ("ident", "str", "Clients ident", "pyload-irc"), - ("realname", "str", "Realname", "pyload-irc"), - ("nick", "str", "Nickname the Client will take", "pyLoad-IRC"), - ("owner", "str", "Nickname the Client will accept commands from", "Enter your nick here!"), - ("info_file", "bool", "Inform about every file finished", "False"), - ("info_pack", "bool", "Inform about every package finished", "True"), - ("captcha", "bool", "Send captcha requests", "True")] - __author_name__ = ("Jeix") - __author_mail__ = ("Jeix@hasnomail.com") - - def __init__(self, core, manager): - Thread.__init__(self) - Addon.__init__(self, core, manager) - self.setDaemon(True) - # self.sm = core.server_methods - self.api = core.api #todo, only use api - - def coreReady(self): - self.new_package = {} - - self.abort = False - - self.links_added = 0 - self.more = [] - - self.start() - - - def packageFinished(self, pypack): - try: - if self.getConfig("info_pack"): - self.response(_("Package finished: %s") % pypack.name) - except: - pass - - def downloadFinished(self, pyfile): - try: - if self.getConfig("info_file"): - self.response(_("Download finished: %(name)s @ %(plugin)s ") % { "name" : pyfile.name, "plugin": pyfile.pluginname} ) - except: - pass - - def newCaptchaTask(self, task): - if self.getConfig("captcha") and task.isTextual(): - task.handler.append(self) - task.setWaiting(60) - - page = getURL("http://www.freeimagehosting.net/upload.php", post={"attached" : (FORM_FILE, task.captchaFile)}, multipart=True) - - url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", page).group(1) - self.response(_("New Captcha Request: %s") % url) - self.response(_("Answer with 'c %s text on the captcha'") % task.id) - - def run(self): - # connect to IRC etc. - self.sock = socket.socket() - host = self.getConfig("host") - self.sock.connect((host, self.getConfig("port"))) - nick = self.getConfig("nick") - self.sock.send("NICK %s\r\n" % nick) - self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick)) - for t in self.getConfig("owner").split(): - if t.strip().startswith("#"): - self.sock.send("JOIN %s\r\n" % t.strip()) - self.log.info("pyLoad IRC: Connected to %s!" % host) - self.log.info("pyLoad IRC: Switching to listening mode!") - try: - self.main_loop() - - except IRCError, ex: - self.sock.send("QUIT :byebye\r\n") - print_exc() - self.sock.close() - - - def main_loop(self): - readbuffer = "" - while True: - sleep(1) - fdset = select([self.sock], [], [], 0) - if self.sock not in fdset[0]: - continue - - if self.abort: - raise IRCError("quit") - - readbuffer += self.sock.recv(1024) - temp = readbuffer.split("\n") - readbuffer = temp.pop() - - for line in temp: - line = line.rstrip() - first = line.split() - - if first[0] == "PING": - self.sock.send("PONG %s\r\n" % first[1]) - - if first[0] == "ERROR": - raise IRCError(line) - - msg = line.split(None, 3) - if len(msg) < 4: - continue - - msg = { - "origin":msg[0][1:], - "action":msg[1], - "target":msg[2], - "text":msg[3][1:] - } - - self.handle_events(msg) - - - def handle_events(self, msg): - if not msg["origin"].split("!", 1)[0] in self.getConfig("owner").split(): - return - - if msg["target"].split("!", 1)[0] != self.getConfig("nick"): - return - - if msg["action"] != "PRIVMSG": - return - - # HANDLE CTCP ANTI FLOOD/BOT PROTECTION - if msg["text"] == "\x01VERSION\x01": - self.log.debug("Sending CTCP VERSION.") - self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) - return - elif msg["text"] == "\x01TIME\x01": - self.log.debug("Sending CTCP TIME.") - self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) - return - elif msg["text"] == "\x01LAG\x01": - self.log.debug("Received CTCP LAG.") # don't know how to answer - return - - trigger = "pass" - args = None - - try: - temp = msg["text"].split() - trigger = temp[0] - if len(temp) > 1: - args = temp[1:] - except: - pass - - handler = getattr(self, "event_%s" % trigger, self.event_pass) - try: - res = handler(args) - for line in res: - self.response(line, msg["origin"]) - except Exception, e: - self.log.error("pyLoad IRC: "+ repr(e)) - - - def response(self, msg, origin=""): - if origin == "": - for t in self.getConfig("owner").split(): - self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg)) - else: - self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg)) - - -#### Events - def event_pass(self, args): - return [] - - def event_status(self, args): - downloads = self.api.statusDownloads() - if not downloads: - return ["INFO: There are no active downloads currently."] - - temp_progress = "" - lines = ["ID - Name - Status - Speed - ETA - Progress"] - for data in downloads: - - if data.status == 5: - temp_progress = data.format_wait - else: - temp_progress = "%d%% (%s)" % (data.percent, data.format_size) - - lines.append("#%d - %s - %s - %s - %s - %s" % - ( - data.fid, - data.name, - data.statusmsg, - "%s/s" % formatSize(data.speed), - "%s" % data.format_eta, - temp_progress - ) - ) - return lines - - def event_queue(self, args): - ps = self.api.getQueueData() - - if not ps: - return ["INFO: There are no packages in queue."] - - lines = [] - for pack in ps: - lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links) )) - - return lines - - def event_collector(self, args): - ps = self.api.getCollectorData() - if not ps: - return ["INFO: No packages in collector!"] - - lines = [] - for pack in ps: - lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links) )) - - return lines - - def event_info(self, args): - if not args: - return ['ERROR: Use info like this: info <id>'] - - info = None - try: - info = self.api.getFileData(int(args[0])) - - except FileDoesNotExists: - return ["ERROR: Link doesn't exists."] - - return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.statusmsg, - info.plugin)] - - def event_packinfo(self, args): - if not args: - return ['ERROR: Use packinfo like this: packinfo <id>'] - - lines = [] - pack = None - try: - pack = self.api.getPackageData(int(args[0])) - - except PackageDoesNotExists: - return ["ERROR: Package doesn't exists."] - - id = args[0] - - self.more = [] - - lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links)) ) - for pyfile in pack.links: - self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size, - pyfile.statusmsg, pyfile.plugin)) - - if len(self.more) < 6: - lines.extend(self.more) - self.more = [] - else: - lines.extend(self.more[:6]) - self.more = self.more[6:] - lines.append("%d more links do display." % len(self.more)) - - - return lines - - def event_more(self, args): - if not self.more: - return ["No more information to display."] - - lines = self.more[:6] - self.more = self.more[6:] - lines.append("%d more links do display." % len(self.more)) - - return lines - - def event_start(self, args): - - self.api.unpauseServer() - return ["INFO: Starting downloads."] - - def event_stop(self, args): - - self.api.pauseServer() - return ["INFO: No new downloads will be started."] - - - def event_add(self, args): - if len(args) < 2: - return ['ERROR: Add links like this: "add <packagename|id> links". ', - 'This will add the link <link> to to the package <package> / the package with id <id>!'] - - - - pack = args[0].strip() - links = [x.strip() for x in args[1:]] - - count_added = 0 - count_failed = 0 - try: - id = int(pack) - pack = self.api.getPackageData(id) - if not pack: - return ["ERROR: Package doesn't exists."] - - #TODO add links - - return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack["name"], id)] - - except: - # create new package - id = self.api.addPackage(pack, links, 1) - return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))] - - - def event_del(self, args): - if len(args) < 2: - return ["ERROR: Use del command like this: del -p|-l <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] - - if args[0] == "-p": - ret = self.api.deletePackages(map(int, args[1:])) - return ["INFO: Deleted %d packages!" % len(args[1:])] - - elif args[0] == "-l": - ret = self.api.delLinks(map(int, args[1:])) - return ["INFO: Deleted %d links!" % len(args[1:])] - - else: - return ["ERROR: Use del command like this: del <-p|-l> <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] - - def event_push(self, args): - if not args: - return ["ERROR: Push package to queue like this: push <package id>"] - - id = int(args[0]) - try: - info = self.api.getPackageInfo(id) - except PackageDoesNotExists: - return ["ERROR: Package #%d does not exist." % id] - - self.api.pushToQueue(id) - return ["INFO: Pushed package #%d to queue." % id] - - def event_pull(self, args): - if not args: - return ["ERROR: Pull package from queue like this: pull <package id>."] - - id = int(args[0]) - if not self.api.getPackageData(id): - return ["ERROR: Package #%d does not exist." % id] - - self.api.pullFromQueue(id) - return ["INFO: Pulled package #%d from queue to collector." % id] - - def event_c(self, args): - """ captcha answer """ - if not args: - return ["ERROR: Captcha ID missing."] - - task = self.core.captchaManager.getTaskByID(args[0]) - if not task: - return ["ERROR: Captcha Task with ID %s does not exists." % args[0]] - - task.setResult(" ".join(args[1:])) - return ["INFO: Result %s saved." % " ".join(args[1:])] - - - def event_help(self, args): - lines = ["The following commands are available:", - "add <package|packid> <links> [...] Adds link to package. (creates new package if it does not exist)", - "queue Shows all packages in the queue", - "collector Shows all packages in collector", - "del -p|-l <id> [...] Deletes all packages|links with the ids specified", - "info <id> Shows info of the link with id <id>", - "packinfo <id> Shows info of the package with id <id>", - "more Shows more info when the result was truncated", - "start Starts all downloads", - "stop Stops the download (but not abort active downloads)", - "push <id> Push package to queue", - "pull <id> Pull package from queue", - "status Show general download status", - "help Shows this help message"] - return lines - - -class IRCError(Exception): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) diff --git a/module/plugins/addons/ImageTyperz.py b/module/plugins/addons/ImageTyperz.py deleted file mode 100644 index 59b6334a7..000000000 --- a/module/plugins/addons/ImageTyperz.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN, zoidberg -""" -from __future__ import with_statement - -from thread import start_new_thread -from pycurl import FORM_FILE, LOW_SPEED_TIME - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader - -from module.plugins.Hook import Hook -import re -from base64 import b64encode - -class ImageTyperzException(Exception): - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def __str__(self): - return "<ImageTyperzException %s>" % self.err - - def __repr__(self): - return "<ImageTyperzException %s>" % self.err - -class ImageTyperz(Hook): - __name__ = "ImageTyperz" - __version__ = "0.03" - __description__ = """send captchas to ImageTyperz.com""" - __config__ = [("activated", "bool", "Activated", True), - ("username", "str", "Username", ""), - ("passkey", "password", "Password", ""), - ("force", "bool", "Force IT even if client is connected", False)] - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - SUBMIT_URL = "http://captchatypers.com/Forms/UploadFileAndGetTextNEW.ashx" - RESPOND_URL = "http://captchatypers.com/Forms/SetBadImage.ashx" - GETCREDITS_URL = "http://captchatypers.com/Forms/RequestBalance.ashx" - - def setup(self): - self.info = {} - - def getCredits(self): - response = getURL(self.GETCREDITS_URL, - post = {"action": "REQUESTBALANCE", - "username": self.getConfig("username"), - "password": self.getConfig("passkey")} - ) - - if response.startswith('ERROR'): - raise ImageTyperzException(response) - - try: - balance = float(response) - except: - raise ImageTyperzException("invalid response") - - self.logInfo("Account balance: $%s left" % response) - return balance - - def submit(self, captcha, captchaType="file", match=None): - req = getRequest() - #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) - - try: - #workaround multipart-post bug in HTTPRequest.py - if re.match("^[A-Za-z0-9]*$", self.getConfig("passkey")): - multipart = True - data = (FORM_FILE, captcha) - else: - multipart = False - with open(captcha, 'rb') as f: - data = f.read() - data = b64encode(data) - - response = req.load(self.SUBMIT_URL, - post={ "action": "UPLOADCAPTCHA", - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "file": data}, - multipart = multipart) - finally: - req.close() - - if response.startswith("ERROR"): - raise ImageTyperzException(response) - else: - data = response.split('|') - if len(data) == 2: - ticket, result = data - else: - raise ImageTyperzException("Unknown response %s" % response) - - return ticket, result - - def newCaptchaTask(self, task): - if "service" in task.data: - return False - - if not task.isTextual(): - return False - - if not self.getConfig("username") or not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 0: - task.handler.append(self) - task.data['service'] = self.__name__ - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - - else: - self.logInfo("Your %s account has not enough credits" % self.__name__) - - def captchaInvalid(self, task): - if task.data['service'] == self.__name__ and "ticket" in task.data: - response = getURL(self.RESPOND_URL, - post={"action": "SETBADIMAGE", - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "imageid": task.data["ticket"]} - ) - - if response == "SUCCESS": - self.logInfo("Bad captcha solution received, requested refund") - else: - self.logError("Bad captcha solution received, refund request failed", response) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except ImageTyperzException, e: - task.error = e.getCode() - return - - task.data["ticket"] = ticket - task.setResult(result)
\ No newline at end of file diff --git a/module/plugins/addons/LinkdecrypterCom.py b/module/plugins/addons/LinkdecrypterCom.py deleted file mode 100644 index d3d6bce68..000000000 --- a/module/plugins/addons/LinkdecrypterCom.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re - -from module.plugins.Hook import Hook -from module.network.RequestFactory import getURL -from module.utils import remove_chars - -class LinkdecrypterCom(Hook): - __name__ = "LinkdecrypterCom" - __version__ = "0.17" - __description__ = """linkdecrypter.com - regexp loader""" - __config__ = [ ("activated", "bool", "Activated" , "True") ] - __author_name__ = ("zoidberg") - - def coreReady(self): - page = getURL("http://linkdecrypter.com/") - m = re.search(r'<b>Supported\(\d+\)</b>: <i>([^+<]*)', page) - if not m: - self.logError(_("Crypter list not found")) - return - - builtin = [ name.lower() for name in self.core.pluginManager.crypterPlugins.keys() ] - builtin.extend([ "downloadserienjunkiesorg" ]) - - crypter_pattern = re.compile("(\w[\w.-]+)") - online = [] - for crypter in m.group(1).split(', '): - m = re.match(crypter_pattern, crypter) - if m and remove_chars(m.group(1), "-.") not in builtin: - online.append(m.group(1).replace(".", "\\.")) - - if not online: - self.logError(_("Crypter list is empty")) - return - - regexp = r"https?://([^.]+\.)*?(%s)/.*" % "|".join(online) - - dict = self.core.pluginManager.crypterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - - self.logDebug("REGEXP: " + regexp) diff --git a/module/plugins/addons/MergeFiles.py b/module/plugins/addons/MergeFiles.py deleted file mode 100644 index 48f997681..000000000 --- a/module/plugins/addons/MergeFiles.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: and9000 -""" - -import os -import re -import sys -import traceback - -from os.path import join -from module.utils import save_join, fs_encode -from module.plugins.Addon import Addon - -BUFFER_SIZE = 4096 - -class MergeFiles(Addon): - __name__ = "MergeFiles" - __version__ = "0.1" - __description__ = "Merges parts splitted with hjsplit" - __config__ = [ - ("activated" , "bool" , "Activated" , "False"), - ] - __threaded__ = ["packageFinished"] - __author_name__ = ("and9000") - __author_mail__ = ("me@has-no-mail.com") - - def setup(self): - # nothing to do - pass - - def packageFinished(self, pack): - files = {} - fid_dict = {} - for fid, data in pack.getChildren().iteritems(): - if re.search("\.[0-9]{3}$", data["name"]): - if data["name"][:-4] not in files: - files[data["name"][:-4]] = [] - files[data["name"][:-4]].append(data["name"]) - files[data["name"][:-4]].sort() - fid_dict[data["name"]] = fid - - download_folder = self.core.config['general']['download_folder'] - - if self.core.config['general']['folder_per_package']: - download_folder = save_join(download_folder, pack.folder) - - for name, file_list in files.iteritems(): - self.core.log.info("Starting merging of %s" % name) - final_file = open(join(download_folder, fs_encode(name)), "wb") - - for splitted_file in file_list: - self.core.log.debug("Merging part %s" % splitted_file) - pyfile = self.core.files.getFile(fid_dict[splitted_file]) - pyfile.setStatus("processing") - try: - s_file = open(os.path.join(download_folder, splitted_file), "rb") - size_written = 0 - s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file))) - while True: - f_buffer = s_file.read(BUFFER_SIZE) - if f_buffer: - final_file.write(f_buffer) - size_written += BUFFER_SIZE - pyfile.setProgress((size_written*100)/s_file_size) - else: - break - s_file.close() - self.core.log.debug("Finished merging part %s" % splitted_file) - except Exception, e: - print traceback.print_exc() - finally: - pyfile.setProgress(100) - pyfile.setStatus("finished") - pyfile.release() - - final_file.close() - self.core.log.info("Finished merging of %s" % name) - - diff --git a/module/plugins/addons/MultiDebridCom.py b/module/plugins/addons/MultiDebridCom.py deleted file mode 100644 index c95138648..000000000 --- a/module/plugins/addons/MultiDebridCom.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -from module.plugins.internal.MultiHoster import MultiHoster -from module.network.RequestFactory import getURL -from module.common.json_layer import json_loads - - -class MultiDebridCom(MultiHoster): - __name__ = "MultiDebridCom" - __version__ = "0.01" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to standard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - - __description__ = """Multi-debrid.com hook plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - def getHoster(self): - json_data = getURL('http://multi-debrid.com/api.php?hosts', decode=True) - self.logDebug('JSON data: ' + json_data) - json_data = json_loads(json_data) - - return json_data['hosts'] diff --git a/module/plugins/addons/MultiHome.py b/module/plugins/addons/MultiHome.py deleted file mode 100644 index af3f55416..000000000 --- a/module/plugins/addons/MultiHome.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Addon import Addon -from time import time - -class MultiHome(Addon): - __name__ = "MultiHome" - __version__ = "0.1" - __description__ = """ip address changer""" - __config__ = [ ("activated", "bool", "Activated" , "False"), - ("interfaces", "str", "Interfaces" , "None") ] - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.register = {} - self.interfaces = [] - self.parseInterfaces(self.getConfig("interfaces").split(";")) - if not self.interfaces: - self.parseInterfaces([self.config["download"]["interface"]]) - self.setConfig("interfaces", self.toConfig()) - - def toConfig(self): - return ";".join([i.adress for i in self.interfaces]) - - def parseInterfaces(self, interfaces): - for interface in interfaces: - if not interface or str(interface).lower() == "none": - continue - self.interfaces.append(Interface(interface)) - - def coreReady(self): - requestFactory = self.core.requestFactory - oldGetRequest = requestFactory.getRequest - def getRequest(pluginName, account=None): - iface = self.bestInterface(pluginName, account) - if iface: - iface.useFor(pluginName, account) - requestFactory.iface = lambda: iface.adress - self.log.debug("Multihome: using address: "+iface.adress) - return oldGetRequest(pluginName, account) - requestFactory.getRequest = getRequest - - def bestInterface(self, pluginName, account): - best = None - for interface in self.interfaces: - if not best or interface.lastPluginAccess(pluginName, account) < best.lastPluginAccess(pluginName, account): - best = interface - return best - -class Interface(object): - def __init__(self, adress): - self.adress = adress - self.history = {} - - def lastPluginAccess(self, pluginName, account): - if (pluginName, account) in self.history: - return self.history[(pluginName, account)] - return 0 - - def useFor(self, pluginName, account): - self.history[(pluginName, account)] = time() - - def __repr__(self): - return "<Interface - %s>" % self.adress diff --git a/module/plugins/addons/MultiHoster.py b/module/plugins/addons/MultiHoster.py deleted file mode 100644 index 825085df8..000000000 --- a/module/plugins/addons/MultiHoster.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from types import MethodType - -from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize -from module.plugins.Addon import Addon, AddEventListener -from module.plugins.PluginManager import PluginTuple - -class MultiHoster(Addon): - __version__ = "0.1" - __internal__ = True - __description__ = "Gives ability to use MultiHoster services. You need to add your account first." - __config__ = [] - __author_mail__ = ("pyLoad Team",) - __author_mail__ = ("support@pyload.org",) - - #TODO: multiple accounts - multihoster / config options - - def init(self): - - # overwritten plugins - self.plugins = {} - - def addHoster(self, account): - - self.logDebug("New MultiHoster %s" % account.__name__) - - pluginMap = {} - for name in self.core.pluginManager.getPlugins("hoster").keys(): - pluginMap[name.lower()] = name - - supported = [] - new_supported = [] - - for hoster in account.getHosterList(): - name = normalize(hoster) - - if name in pluginMap: - supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not supported and not new_supported: - account.logError(_("No Hoster loaded")) - return - - klass = self.core.pluginManager.getPluginClass(account.__name__) - - # inject plugin plugin - account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) - for hoster in supported: - self.plugins[hoster] = klass - - account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - # recreate plugin tuple for new regexp - hoster = self.core.pluginManager.getPlugins("hoster") - p = hoster[account.__name__] - new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) - hoster[account.__name__] = new - - - - @AddEventListener("account:deleted") - def refreshAccounts(self, plugin=None, user=None): - - self.plugins = {} - - for name, account in self.core.accountManager.iterAccounts(): - if isinstance(account, MultiHosterAccount) and account.isUsable(): - self.addHoster(account) - - @AddEventListener("account:updated") - def refreshAccount(self, plugin, user): - - account = self.core.accountManager.getAccount(plugin, user) - if isinstance(account, MultiHosterAccount) and account.isUsable(): - self.addHoster(account) - - def activate(self): - self.refreshAccounts() - - # new method for plugin manager - def getPlugin(self2, name): - if name in self.plugins: - return self.plugins[name] - return self2.getPluginClass(name) - - pm = self.core.pluginManager - pm.getPlugin = MethodType(getPlugin, pm, object) - - - def deactivate(self): - #restore state - pm = self.core.pluginManager - pm.getPlugin = pm.getPluginClass - diff --git a/module/plugins/addons/MultishareCz.py b/module/plugins/addons/MultishareCz.py deleted file mode 100644 index 7e5a3e007..000000000 --- a/module/plugins/addons/MultishareCz.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster -import re - -class MultishareCz(MultiHoster): - __name__ = "MultishareCz" - __version__ = "0.04" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", "uloz.to")] - __description__ = """MultiShare.cz hook plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - HOSTER_PATTERN = r'<img class="logo-shareserveru"[^>]*?alt="([^"]+)"></td>\s*<td class="stav">[^>]*?alt="OK"' - - def getHoster(self): - - page = getURL("http://www.multishare.cz/monitoring/") - return re.findall(self.HOSTER_PATTERN, page)
\ No newline at end of file diff --git a/module/plugins/addons/Premium4Me.py b/module/plugins/addons/Premium4Me.py deleted file mode 100644 index b49eb41a9..000000000 --- a/module/plugins/addons/Premium4Me.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*-
-
-from module.network.RequestFactory import getURL
-from module.plugins.internal.MultiHoster import MultiHoster
-
-class Premium4Me(MultiHoster):
- __name__ = "Premium4Me"
- __version__ = "0.02"
- __type__ = "hook"
-
- __config__ = [("activated", "bool", "Activated", "False"),
- ("hosterListMode", "all;listed;unlisted", "Use for downloads from supported hosters:", "all"),
- ("hosterList", "str", "Hoster list (comma separated)", "")]
- __description__ = """premium4.me hook plugin"""
- __author_name__ = ("RaNaN", "zoidberg")
- __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz")
-
- def getHoster(self):
-
- page = getURL("http://premium4.me/api/hosters.php?authcode=%s" % self.account.authcode)
- return [x.strip() for x in page.replace("\"", "").split(";")]
-
- def coreReady(self):
-
- self.account = self.core.accountManager.getAccountPlugin("Premium4Me")
-
- user = self.account.selectAccount()[0]
-
- if not user:
- self.logError(_("Please add your premium4.me account first and restart pyLoad"))
- return
-
- return MultiHoster.coreReady(self)
\ No newline at end of file diff --git a/module/plugins/addons/PremiumizeMe.py b/module/plugins/addons/PremiumizeMe.py deleted file mode 100644 index a10c24f85..000000000 --- a/module/plugins/addons/PremiumizeMe.py +++ /dev/null @@ -1,50 +0,0 @@ -from module.plugins.internal.MultiHoster import MultiHoster - -from module.common.json_layer import json_loads -from module.network.RequestFactory import getURL - -class PremiumizeMe(MultiHoster): - __name__ = "PremiumizeMe" - __version__ = "0.12" - __type__ = "hook" - __description__ = """Premiumize.Me hook plugin""" - - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported):", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to stanard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - - __author_name__ = ("Florian Franzen") - __author_mail__ = ("FlorianFranzen@gmail.com") - - def getHoster(self): - # If no accounts are available there will be no hosters available - if not self.account or not self.account.canUse(): - return [] - - # Get account data - (user, data) = self.account.selectAccount() - - # Get supported hosters list from premiumize.me using the json API v1 (see https://secure.premiumize.me/?show=api) - answer = getURL("https://api.premiumize.me/pm-api/v1.php?method=hosterlist¶ms[login]=%s¶ms[pass]=%s" % (user, data['password'])) - data = json_loads(answer) - - - # If account is not valid thera are no hosters available - if data['status'] != 200: - return [] - - # Extract hosters from json file - return data['result']['hosterlist'] - - def coreReady(self): - # Get account plugin and check if there is a valid account available - self.account = self.core.accountManager.getAccountPlugin("PremiumizeMe") - if not self.account.canUse(): - self.account = None - self.logError(_("Please add a valid premiumize.me account first and restart pyLoad.")) - return - - # Run the overwriten core ready which actually enables the multihoster hook - return MultiHoster.coreReady(self)
\ No newline at end of file diff --git a/module/plugins/addons/RealdebridCom.py b/module/plugins/addons/RealdebridCom.py deleted file mode 100644 index be74b47c3..000000000 --- a/module/plugins/addons/RealdebridCom.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class RealdebridCom(MultiHoster): - __name__ = "RealdebridCom" - __version__ = "0.43" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("https", "bool", "Enable HTTPS", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported):", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to stanard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - __description__ = """Real-Debrid.com hook plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - def getHoster(self): - https = "https" if self.getConfig("https") else "http" - page = getURL(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip() - - return [x.strip() for x in page.split(",") if x.strip()] diff --git a/module/plugins/addons/RehostTo.py b/module/plugins/addons/RehostTo.py deleted file mode 100644 index 7ca5e5cde..000000000 --- a/module/plugins/addons/RehostTo.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class RehostTo(MultiHoster): - __name__ = "RehostTo" - __version__ = "0.42" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", ""), - ("unloadFailing", "bool", "Revert to stanard download if download fails", "False"), - ("interval", "int", "Reload interval in hours (0 to disable)", "24")] - - __description__ = """rehost.to hook plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def getHoster(self): - - page = getURL("http://rehost.to/api.php?cmd=get_supported_och_dl&long_ses=%s" % self.long_ses) - return [x.strip() for x in page.replace("\"", "").split(",")] - - - def coreReady(self): - - self.account = self.core.accountManager.getAccountPlugin("RehostTo") - - user = self.account.selectAccount()[0] - - if not user: - self.log.error("Rehost.to: "+ _("Please add your rehost.to account first and restart pyLoad")) - return - - data = self.account.getAccountInfo(user) - self.ses = data["ses"] - self.long_ses = data["long_ses"] - - return MultiHoster.coreReady(self)
\ No newline at end of file diff --git a/module/plugins/addons/RestartFailed.py b/module/plugins/addons/RestartFailed.py deleted file mode 100644 index 7ee53deb9..000000000 --- a/module/plugins/addons/RestartFailed.py +++ /dev/null @@ -1,124 +0,0 @@ - # -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: Walter Purcaro -""" - -from module.plugins.Hook import Hook -from time import time - - -class RestartFailed(Hook): - __name__ = "RestartFailed" - __version__ = "1.5" - __description__ = "Automatically restart failed/aborted downloads" - __config__ = [ - ("activated", "bool", "Activated", "True"), - ("dlFail", "bool", "Restart when download fail", "True"), - ("dlFail_n", "int", "Only when failed downloads are at least", "5"), - ("dlFail_i", "int", "Only when elapsed time since last restart is (min)", "10"), - ("dlPrcs", "bool", "Restart after all downloads are processed", "True"), - ("recnt", "bool", "Restart after reconnecting", "True"), - ("rsLoad", "bool", "Restart on plugin activation", "False") - ] - __author_name__ = ("Walter Purcaro") - __author_mail__ = ("vuolter@gmail.com") - - def restart(self, arg=None): - # self.logDebug("self.restart") - self.info["timerflag"] = False - self.info["dlfailed"] = 0 - self.core.api.restartFailed() - self.logDebug("self.restart: self.core.api.restartFailed") - self.info["lastrstime"] = time() - - def periodical(self): - # self.logDebug("self.periodical") - if self.info["timerflag"]: - self.restart() - - def checkInterval(self, arg=None): - # self.logDebug("self.checkInterval") - now = time() - lastrstime = self.info["lastrstime"] - interval = self.getConfig("dlFail_i") * 60 - if now < lastrstime + interval: - self.info["timerflag"] = True - else: - self.restart() - - def checkFailed(self, pyfile): - # self.logDebug("self.checkFailed") - self.info["dlfailed"] += 1 - curr = self.info["dlfailed"] - max = self.getConfig("dlFail_n") - if curr >= max: - self.checkInterval() - - def addEvent(self, event, handler): - if event in self.manager.events: - if handler not in self.manager.events[event]: - self.manager.events[event].append(handler) - # self.logDebug("self.addEvent: " + event + ": added handler") - else: - # self.logDebug("self.addEvent: " + event + ": NOT added handler") - return False - else: - self.manager.events[event] = [handler] - # self.logDebug("self.addEvent: " + event + ": added event and handler") - return True - - def removeEvent(self, event, handler): - if event in self.manager.events and handler in self.manager.events[event]: - self.manager.events[event].remove(handler) - # self.logDebug("self.removeEvent: " + event + ": removed handler") - return True - else: - # self.logDebug("self.removeEvent: " + event + ": NOT removed handler") - return False - - def configEvents(self, plugin=None, name=None, value=None): - # self.logDebug("self.configEvents") - self.interval = self.getConfig("dlFail_i") * 60 - dlFail = self.getConfig("dlFail") - dlPrcs = self.getConfig("dlPrcs") - recnt = self.getConfig("recnt") - if dlPrcs: - self.addEvent("allDownloadsProcessed", self.checkInterval) - else: - self.removeEvent("allDownloadsProcessed", self.checkInterval) - if not dlFail: - self.info["timerflag"] = False - if recnt: - self.addEvent("afterReconnecting", self.restart) - else: - self.removeEvent("afterReconnecting", self.restart) - - def unload(self): - # self.logDebug("self.unload") - self.removeEvent("pluginConfigChanged", self.configEvents) - self.removeEvent("downloadFailed", self.checkFailed) - self.removeEvent("allDownloadsProcessed", self.checkInterval) - self.removeEvent("afterReconnecting", self.restart) - - def coreReady(self): - # self.logDebug("self.coreReady") - self.info = {"dlfailed": 0, "lastrstime": 0, "timerflag": False} - if self.getConfig("rsLoad"): - self.restart() - self.addEvent("downloadFailed", self.checkFailed) - self.addEvent("pluginConfigChanged", self.configEvents) - self.configEvents() diff --git a/module/plugins/addons/UnSkipOnFail.py b/module/plugins/addons/UnSkipOnFail.py deleted file mode 100644 index 4b7a58be8..000000000 --- a/module/plugins/addons/UnSkipOnFail.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: hgg -""" -from os.path import basename - -from module.utils import fs_encode -from module.plugins.Hook import Hook -from module.PyFile import PyFile - -class UnSkipOnFail(Hook): - __name__ = 'UnSkipOnFail' - __version__ = '0.01' - __description__ = 'When a download fails, restart "skipped" duplicates.' - __config__ = [('activated', 'bool', 'Activated', True),] - __author_name__ = ('hagg',) - __author_mail__ = ('') - - def downloadFailed(self, pyfile): - pyfile_name = basename(pyfile.name) - pid = pyfile.package().id - msg = 'look for skipped duplicates for %s (pid:%s)...' - self.logInfo(msg % (pyfile_name, pid)) - dups = self.findDuplicates(pyfile) - for link in dups: - # check if link is "skipped"(=4) - if link.status == 4: - lpid = link.packageID - self.logInfo('restart "%s" (pid:%s)...' % (pyfile_name, lpid)) - self.setLinkStatus(link, "queued") - - def findDuplicates(self, pyfile): - """ Search all packages for duplicate links to "pyfile". - Duplicates are links that would overwrite "pyfile". - To test on duplicity the package-folder and link-name - of twolinks are compared (basename(link.name)). - So this method returns a list of all links with equal - package-folders and filenames as "pyfile", but except - the data for "pyfile" iotselöf. - It does MOT check the link's status. - """ - dups = [] - pyfile_name = fs_encode(basename(pyfile.name)) - # get packages (w/o files, as most file data is useless here) - queue = self.core.api.getQueue() - for package in queue: - # check if package-folder equals pyfile's package folder - if fs_encode(package.folder) == fs_encode(pyfile.package().folder): - # now get packaged data w/ files/links - pdata = self.core.api.getPackageData(package.pid) - if pdata.links: - for link in pdata.links: - link_name = fs_encode(basename(link.name)) - # check if link name collides with pdata's name - if link_name == pyfile_name: - # at last check if it is not pyfile itself - if link.fid != pyfile.id: - dups.append(link) - return dups - - def setLinkStatus(self, link, new_status): - """ Change status of "link" to "new_status". - "link" has to be a valid FileData object, - "new_status" has to be a valid status name - (i.e. "queued" for this Plugin) - It creates a temporary PyFile object using - "link" data, changes its status, and tells - the core.files-manager to save its data. - """ - pyfile = PyFile(self.core.files, - link.fid, - link.url, - link.name, - link.size, - link.status, - link.error, - link.plugin, - link.packageID, - link.order) - pyfile.setStatus(new_status) - self.core.files.save() - pyfile.release() - diff --git a/module/plugins/addons/UpdateManager.py b/module/plugins/addons/UpdateManager.py deleted file mode 100644 index c800b44bf..000000000 --- a/module/plugins/addons/UpdateManager.py +++ /dev/null @@ -1,201 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import sys -import re -from os import stat -from os.path import join, exists -from time import time - -from module.ConfigParser import IGNORE -from module.network.RequestFactory import getURL -from module.plugins.Hook import threaded, Expose, Hook - -class UpdateManager(Hook): - __name__ = "UpdateManager" - __version__ = "0.13" - __description__ = """checks for updates""" - __config__ = [("activated", "bool", "Activated", "True"), - ("interval", "int", "Check interval in minutes", "480"), - ("debug", "bool", "Check for plugin changes when in debug mode", False)] - __author_name__ = ("RaNaN") - __author_mail__ = ("ranan@pyload.org") - - URL = "http://get.pyload.org/check2/%s/" - MIN_TIME = 3 * 60 * 60 # 3h minimum check interval - - @property - def debug(self): - return self.core.debug and self.getConfig("debug") - - def setup(self): - if self.debug: - self.logDebug("Monitoring file changes") - self.interval = 4 - self.last_check = 0 #timestamp of updatecheck - self.old_periodical = self.periodical - self.periodical = self.checkChanges - self.mtimes = {} #recordes times - else: - self.interval = max(self.getConfig("interval") * 60, self.MIN_TIME) - - self.updated = False - self.reloaded = True - self.version = "None" - - self.info = {"pyload": False, "plugins": False} - - @threaded - def periodical(self): - - updates = self.checkForUpdate() - if updates: - self.checkPlugins(updates) - - if self.updated and not self.reloaded: - self.info["plugins"] = True - self.log.info(_("*** Plugins have been updated, please restart pyLoad ***")) - elif self.updated and self.reloaded: - self.log.info(_("Plugins updated and reloaded")) - self.updated = False - elif self.version == "None": - self.log.info(_("No plugin updates available")) - - @Expose - def recheckForUpdates(self): - """recheck if updates are available""" - self.periodical() - - def checkForUpdate(self): - """checks if an update is available, return result""" - - try: - if self.version == "None": # No updated known - version_check = getURL(self.URL % self.core.api.getServerVersion()).splitlines() - self.version = version_check[0] - - # Still no updates, plugins will be checked - if self.version == "None": - self.log.info(_("No Updates for pyLoad")) - return version_check[1:] - - - self.info["pyload"] = True - self.log.info(_("*** New pyLoad Version %s available ***") % self.version) - self.log.info(_("*** Get it here: http://pyload.org/download ***")) - - except: - self.log.warning(_("Not able to connect server for updates")) - - return None # Nothing will be done - - - def checkPlugins(self, updates): - """ checks for plugins updates""" - - # plugins were already updated - if self.info["plugins"]: return - - reloads = [] - - vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - url = updates[0] - schema = updates[1].split("|") - updates = updates[2:] - - for plugin in updates: - info = dict(zip(schema, plugin.split("|"))) - filename = info["name"] - prefix = info["type"] - version = info["version"] - - if filename.endswith(".pyc"): - name = filename[:filename.find("_")] - else: - name = filename.replace(".py", "") - - #TODO: obsolete in 0.5.0 - if prefix.endswith("s"): - type = prefix[:-1] - else: - type = prefix - - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) - - if name in plugins: - if float(plugins[name]["v"]) >= float(version): - continue - - if name in IGNORE or (type, name) in IGNORE: - continue - - self.log.info(_("New version of %(type)s|%(name)s : %(version).2f") % { - "type": type, - "name": name, - "version": float(version) - }) - - try: - content = getURL(url % info) - except Exception, e: - self.logWarning(_("Error when updating %s") % filename, str(e)) - continue - - m = vre.search(content) - if not m or m.group(2) != version: - self.logWarning(_("Error when updating %s") % name, _("Version mismatch")) - continue - - f = open(join("userplugins", prefix, filename), "wb") - f.write(content) - f.close() - self.updated = True - - reloads.append((prefix, name)) - - self.reloaded = self.core.pluginManager.reloadPlugins(reloads) - - def checkChanges(self): - - if self.last_check + max(self.getConfig("interval") * 60, self.MIN_TIME) < time(): - self.old_periodical() - self.last_check = time() - - modules = filter( - lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, - sys.modules.itervalues()) - - reloads = [] - - for m in modules: - root, type, name = m.__name__.rsplit(".", 2) - id = (type, name) - if type in self.core.pluginManager.plugins: - f = m.__file__.replace(".pyc", ".py") - if not exists(f): continue - - mtime = stat(f).st_mtime - - if id not in self.mtimes: - self.mtimes[id] = mtime - elif self.mtimes[id] < mtime: - reloads.append(id) - self.mtimes[id] = mtime - - self.core.pluginManager.reloadPlugins(reloads) diff --git a/module/plugins/addons/XFileSharingPro.py b/module/plugins/addons/XFileSharingPro.py deleted file mode 100644 index 105c70113..000000000 --- a/module/plugins/addons/XFileSharingPro.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Hook import Hook -import re - -class XFileSharingPro(Hook): - __name__ = "XFileSharingPro" - __version__ = "0.04" - __type__ = "hook" - __config__ = [ ("activated" , "bool" , "Activated" , "True"), - ("loadDefault", "bool", "Include default (built-in) hoster list" , "True"), - ("includeList", "str", "Include hosters (comma separated)", ""), - ("excludeList", "str", "Exclude hosters (comma separated)", "") ] - __description__ = """Hoster URL pattern loader for the generic XFileSharingPro plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def coreReady(self): - self.loadPattern() - - def loadPattern(self): - hosterList = self.getConfigSet('includeList') - excludeList = self.getConfigSet('excludeList') - - if self.getConfig('loadDefault'): - hosterList |= set(( - #WORKING HOSTERS: - "aieshare.com", "asixfiles.com", "banashare.com", "cyberlocker.ch", "eyesfile.co", "eyesfile.com", - "fileband.com", "filedwon.com", "filedownloads.org", "hipfile.com", "kingsupload.com", "mlfat4arab.com", - "netuploaded.com", "odsiebie.pl", "q4share.com", "ravishare.com", "uptobox.com", "verzend.be", - #NOT TESTED: - "bebasupload.com", "boosterking.com", "divxme.com", "filevelocity.com", "glumbouploads.com", "grupload.com", "heftyfile.com", - "host4desi.com", "laoupload.com", "linkzhost.com", "movreel.com", "rockdizfile.com", "limfile.com" - "share76.com", "sharebeast.com", "sharehut.com", "sharerun.com", "shareswift.com", "sharingonline.com", "6ybh-upload.com", - "skipfile.com", "spaadyshare.com", "space4file.com", "uploadbaz.com", "uploadc.com", - "uploaddot.com", "uploadfloor.com", "uploadic.com", "uploadville.com", "vidbull.com", "zalaa.com", - "zomgupload.com", "kupload.org", "movbay.org", "multishare.org", "omegave.org", "toucansharing.org", "uflinq.org", - "banicrazy.info", "flowhot.info", "upbrasil.info", "shareyourfilez.biz", "bzlink.us", "cloudcache.cc", "fileserver.cc" - "farshare.to", "filemaze.ws", "filehost.ws", "filestock.ru", "moidisk.ru", "4up.im", "100shared.com", - #WRONG FILE NAME: - "sendmyway.com", "upchi.co.il", "180upload.com", - #NOT WORKING: - "amonshare.com", "imageporter.com", "file4safe.com", - #DOWN OR BROKEN: - "ddlanime.com", "fileforth.com", "loombo.com", "goldfile.eu", "putshare.com" - )) - - hosterList -= (excludeList) - hosterList -= set(('', u'')) - - if not hosterList: - self.unload() - return - - regexp = r"http://(?:[^/]*\.)?(%s)/\w{12}" % ("|".join(sorted(hosterList)).replace('.','\.')) - #self.logDebug(regexp) - - dict = self.core.pluginManager.hosterPlugins['XFileSharingPro'] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - self.logDebug("Pattern loaded - handling %d hosters" % len(hosterList)) - - def getConfigSet(self, option): - s = self.getConfig(option).lower().replace('|',',').replace(';',',') - return set([x.strip() for x in s.split(',')]) - - def unload(self): - dict = self.core.pluginManager.hosterPlugins['XFileSharingPro'] - dict["pattern"] = r"^unmatchable$" - dict["re"] = re.compile(r"^unmatchable$")
\ No newline at end of file diff --git a/module/plugins/addons/XMPPInterface.py b/module/plugins/addons/XMPPInterface.py deleted file mode 100644 index e8ef1d2ca..000000000 --- a/module/plugins/addons/XMPPInterface.py +++ /dev/null @@ -1,276 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - @interface-version: 0.2 -""" - -from pyxmpp import streamtls -from pyxmpp.all import JID, Message, Presence -from pyxmpp.jabber.client import JabberClient -from pyxmpp.interface import implements -from pyxmpp.interfaces import * - -from module.plugins.addons.IRCInterface import IRCInterface - -class XMPPInterface(IRCInterface, JabberClient): - __name__ = "XMPPInterface" - __version__ = "0.1" - __description__ = """connect to jabber and let owner perform different tasks""" - __config__ = [("activated", "bool", "Activated", "False"), - ("jid", "str", "Jabber ID", "user@exmaple-jabber-server.org"), - ("pw", "str", "Password", ""), - ("tls", "bool", "Use TLS", False), - ("owners", "str", "List of JIDs accepting commands from", "me@icq-gateway.org;some@msn-gateway.org"), - ("info_file", "bool", "Inform about every file finished", "False"), - ("info_pack", "bool", "Inform about every package finished", "True"), - ("captcha", "bool", "Send captcha requests", "True")] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - implements(IMessageHandlersProvider) - - def __init__(self, core, manager): - IRCInterface.__init__(self, core, manager) - - self.jid = JID(self.getConfig("jid")) - password = self.getConfig("pw") - - # if bare JID is provided add a resource -- it is required - if not self.jid.resource: - self.jid = JID(self.jid.node, self.jid.domain, "pyLoad") - - if self.getConfig("tls"): - tls_settings = streamtls.TLSSettings(require=True, verify_peer=False) - auth = ("sasl:PLAIN", "sasl:DIGEST-MD5") - else: - tls_settings = None - auth = ("sasl:DIGEST-MD5", "digest") - - # setup client with provided connection information - # and identity data - JabberClient.__init__(self, self.jid, password, - disco_name="pyLoad XMPP Client", disco_type="bot", - tls_settings=tls_settings, auth_methods=auth) - - self.interface_providers = [ - VersionHandler(self), - self, - ] - - def coreReady(self): - self.new_package = {} - - self.start() - - def packageFinished(self, pypack): - try: - if self.getConfig("info_pack"): - self.announce(_("Package finished: %s") % pypack.name) - except: - pass - - def downloadFinished(self, pyfile): - try: - if self.getConfig("info_file"): - self.announce( - _("Download finished: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.pluginname}) - except: - pass - - def run(self): - # connect to IRC etc. - self.connect() - try: - self.loop() - except Exception, ex: - self.core.log.error("pyLoad XMPP: %s" % str(ex)) - - def stream_state_changed(self, state, arg): - """This one is called when the state of stream connecting the component - to a server changes. This will usually be used to let the user - know what is going on.""" - self.log.debug("pyLoad XMPP: *** State changed: %s %r ***" % (state, arg)) - - def disconnected(self): - self.log.debug("pyLoad XMPP: Client was disconnected") - - def stream_closed(self, stream): - self.log.debug("pyLoad XMPP: Stream was closed | %s" % stream) - - def stream_error(self, err): - self.log.debug("pyLoad XMPP: Stream Error: %s" % err) - - def get_message_handlers(self): - """Return list of (message_type, message_handler) tuples. - - The handlers returned will be called when matching message is received - in a client session.""" - return [ - ("normal", self.message), - ] - - def presence_control(self, stanza): - from_jid = unicode(stanza.get_from_jid()) - stanza_type = stanza.get_type() - self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, - from_jid)) - - if from_jid in self.getConfig("owners"): - return stanza.make_accept_response() - - return stanza.make_deny_response() - - def session_started(self): - self.stream.send(Presence()) - - self.stream.set_presence_handler("subscribe", self.presence_control) - self.stream.set_presence_handler("subscribed", self.presence_control) - self.stream.set_presence_handler("unsubscribe", self.presence_control) - self.stream.set_presence_handler("unsubscribed", self.presence_control) - - def message(self, stanza): - """Message handler for the component.""" - subject = stanza.get_subject() - body = stanza.get_body() - t = stanza.get_type() - self.log.debug(u'pyLoad XMPP: Message from %s received.' % (unicode(stanza.get_from(), ))) - self.log.debug(u'pyLoad XMPP: Body: %s Subject: %s Type: %s' % (body, subject, t)) - - if t == "headline": - # 'headline' messages should never be replied to - return True - if subject: - subject = u"Re: " + subject - - to_jid = stanza.get_from() - from_jid = stanza.get_to() - - #j = JID() - to_name = to_jid.as_utf8() - from_name = from_jid.as_utf8() - - names = self.getConfig("owners").split(";") - - if to_name in names or to_jid.node + "@" + to_jid.domain in names: - messages = [] - - trigger = "pass" - args = None - - try: - temp = body.split() - trigger = temp[0] - if len(temp) > 1: - args = temp[1:] - except: - pass - - handler = getattr(self, "event_%s" % trigger, self.event_pass) - try: - res = handler(args) - for line in res: - m = Message( - to_jid=to_jid, - from_jid=from_jid, - stanza_type=stanza.get_type(), - subject=subject, - body=line) - - messages.append(m) - except Exception, e: - self.log.error("pyLoad XMPP: " + repr(e)) - - return messages - - else: - return True - - def response(self, msg, origin=""): - return self.announce(msg) - - def announce(self, message): - """ send message to all owners""" - for user in self.getConfig("owners").split(";"): - self.log.debug("pyLoad XMPP: Send message to %s" % user) - - to_jid = JID(user) - - m = Message(from_jid=self.jid, - to_jid=to_jid, - stanza_type="chat", - body=message) - - stream = self.get_stream() - if not stream: - self.connect() - stream = self.get_stream() - - stream.send(m) - - def beforeReconnecting(self, ip): - self.disconnect() - - def afterReconnecting(self, ip): - self.connect() - - -class VersionHandler(object): - """Provides handler for a version query. - - This class will answer version query and announce 'jabber:iq:version' namespace - in the client's disco#info results.""" - - implements(IIqHandlersProvider, IFeaturesProvider) - - def __init__(self, client): - """Just remember who created this.""" - self.client = client - - def get_features(self): - """Return namespace which should the client include in its reply to a - disco#info query.""" - return ["jabber:iq:version"] - - def get_iq_get_handlers(self): - """Return list of tuples (element_name, namespace, handler) describing - handlers of <iq type='get'/> stanzas""" - return [ - ("query", "jabber:iq:version", self.get_version), - ] - - def get_iq_set_handlers(self): - """Return empty list, as this class provides no <iq type='set'/> stanza handler.""" - return [] - - def get_version(self, iq): - """Handler for jabber:iq:version queries. - - jabber:iq:version queries are not supported directly by PyXMPP, so the - XML node is accessed directly through the libxml2 API. This should be - used very carefully!""" - iq = iq.make_result_response() - q = iq.new_query("jabber:iq:version") - q.newTextChild(q.ns(), "name", "Echo component") - q.newTextChild(q.ns(), "version", "1.0") - return iq - - def unload(self): - self.log.debug("pyLoad XMPP: unloading") - self.disconnect() - - def deactivate(self): - self.unload() diff --git a/module/plugins/addons/ZeveraCom.py b/module/plugins/addons/ZeveraCom.py deleted file mode 100644 index cadf60069..000000000 --- a/module/plugins/addons/ZeveraCom.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class ZeveraCom(MultiHoster): - __name__ = "ZeveraCom" - __version__ = "0.02" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported)", "all"), - ("hosterList", "str", "Hoster list (comma separated)", "")] - __description__ = """Real-Debrid.com hook plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def getHoster(self): - page = getURL("http://www.zevera.com/jDownloader.ashx?cmd=gethosters") - return [x.strip() for x in page.replace("\"", "").split(",")]
\ No newline at end of file diff --git a/module/plugins/addons/__init__.py b/module/plugins/addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/addons/__init__.py +++ /dev/null diff --git a/module/plugins/crypter/C1neonCom.py b/module/plugins/crypter/C1neonCom.py deleted file mode 100644 index 36b84764e..000000000 --- a/module/plugins/crypter/C1neonCom.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: godofdream -""" - -import re -import random -from module.plugins.Crypter import Crypter -from module.common.json_layer import json_loads -class C1neonCom(Crypter): - __name__ = "C1neonCom" - __type__ = "container" - __pattern__ = r"http://(www\.)?c1neon.com/.*?" - __version__ = "0.05" - __config__ = [ - ("changeNameS", "Packagename;Show;Season;Episode", "Rename Show by", "Show"), - ("changeName", "Packagename;Movie", "Rename Movie by", "Movie"), - ("useStreams", "bool", "Use Streams too", False), - ("hosterListMode", "all;onlypreferred", "Use for hosters (if supported)", "all"), - ("randomPreferred", "bool", "Randomize Preferred-List", False), - ("hosterList", "str", "Preferred Hoster list (comma separated, no ending)", "2shared,Bayfiles,Netload,Rapidshare,Share-online"), - ("ignoreList", "str", "Ignored Hoster list (comma separated, no ending)", "Megaupload") - ] - __description__ = """C1neon.Com Container Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - VALUES_PATTERN = r"var subcats = (.*?)(;</script>|;var)" - SHOW_PATTERN = r"title='(.*?)'" - SERIE_PATTERN = r"<title>.*Serie.*</title>" - - def decrypt(self, pyfile): - src = self.req.load(str(pyfile.url)) - - pattern = re.compile(self.VALUES_PATTERN, re.DOTALL) - data = json_loads(re.search(pattern, src).group(1)) - - # Get package info - links = [] - Showname = re.search(self.SHOW_PATTERN, src) - if Showname: - Showname = Showname.group(1).decode("utf-8") - else: - Showname = self.pyfile.package().name - - if re.search(self.SERIE_PATTERN, src): - for Season in data: - self.logDebug("Season " + Season) - for Episode in data[Season]: - self.logDebug("Episode " + Episode) - links.extend(self.getpreferred(data[Season][Episode])) - if self.getConfig("changeNameS") == "Episode": - self.packages.append((data[Season][Episode]['info']['name'].split("»")[0], links, data[Season][Episode]['info']['name'].split("»")[0])) - links = [] - - if self.getConfig("changeNameS") == "Season": - self.packages.append((Showname + " Season " + Season, links, Showname + " Season " + Season)) - links = [] - - if self.getConfig("changeNameS") == "Show": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.packages.append((Showname, links, Showname)) - - elif self.getConfig("changeNameS") == "Packagename": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.core.files.addLinks(links, self.pyfile.package().id) - else: - for Movie in data: - links.extend(self.getpreferred(data[Movie])) - if self.getConfig("changeName") == "Movie": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.packages.append((Showname, links, Showname)) - - elif self.getConfig("changeName") == "Packagename": - if links == []: - self.fail('Could not extract any links (Out of Date?)') - else: - self.core.files.addLinks(links, self.pyfile.package().id) - - #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) - #selects only one Hoster - def getpreferred(self, hosterslist): - hosterlist = {} - if 'u' in hosterslist: - hosterlist.update(hosterslist['u']) - if ('d' in hosterslist): - hosterlist.update(hosterslist['d']) - if self.getConfig("useStreams") and 's' in hosterslist: - hosterlist.update(hosterslist['s']) - - result = [] - preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig("randomPreferred") == True: - random.shuffle(preferredList) - for preferred in preferredList: - for Hoster in hosterlist: - if preferred == Hoster.split('<')[0].strip().lower().replace('.',''): - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part[3]) - result.append(str(Part[3])) - return result - - ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig('hosterListMode') == "all": - for Hoster in hosterlist: - if Hoster.split('<')[0].strip().lower().replace('.','') not in ignorelist: - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part[3]) - result.append(str(Part[3])) - return result - return result - - - diff --git a/module/plugins/crypter/CCF.py b/module/plugins/crypter/CCF.py deleted file mode 100644 index ab7ff1099..000000000 --- a/module/plugins/crypter/CCF.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from urllib2 import build_opener - -from module.plugins.Crypter import Crypter -from module.lib.MultipartPostHandler import MultipartPostHandler - -from os import makedirs -from os.path import exists, join - -class CCF(Crypter): - __name__ = "CCF" - __version__ = "0.2" - __pattern__ = r"(?!http://).*\.ccf$" - __description__ = """CCF Container Convert Plugin""" - __author_name__ = ("Willnix") - __author_mail__ = ("Willnix@pyload.org") - - def decrypt(self, pyfile): - - infile = pyfile.url.replace("\n", "") - - opener = build_opener(MultipartPostHandler) - params = {"src": "ccf", - "filename": "test.ccf", - "upload": open(infile, "rb")} - tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', params).read() - - download_folder = self.config['general']['download_folder'] - location = download_folder #join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding())) - if not exists(location): - makedirs(location) - - tempdlc_name = join(location, "tmp_%s.dlc" % pyfile.name) - tempdlc = open(tempdlc_name, "w") - tempdlc.write(re.search(r'<dlc>(.*)</dlc>', tempdlc_content, re.DOTALL).group(1)) - tempdlc.close() - - self.packages.append((tempdlc_name, [tempdlc_name], tempdlc_name)) - diff --git a/module/plugins/crypter/CrockoComFolder.py b/module/plugins/crypter/CrockoComFolder.py deleted file mode 100644 index d727ec7ab..000000000 --- a/module/plugins/crypter/CrockoComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class CrockoComFolder(SimpleCrypter): - __name__ = "CrockoComFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?crocko.com/f/.*" - __version__ = "0.01" - __description__ = """Crocko.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<td class="last"><a href="([^"]+)">download</a>'
\ No newline at end of file diff --git a/module/plugins/crypter/CryptItCom.py b/module/plugins/crypter/CryptItCom.py deleted file mode 100644 index 4935758c7..000000000 --- a/module/plugins/crypter/CryptItCom.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- - - -import re - -from random import randint - -from module.plugins.Crypter import Crypter - - -class CryptItCom(Crypter): - __name__ = "CryptItCom" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?crypt-it\.com/(s|e|d|c)/[\w]+" - __version__ = "0.1" - __description__ = """Crypt.It.com Container Plugin""" - __author_name__ = ("jeix") - __author_mail__ = ("jeix@hasnomail.de") - - def file_exists(self): - html = self.load(self.pyfile.url) - if r'<div class="folder">Was ist Crypt-It</div>' in html: - return False - return True - - def decrypt(self, pyfile): - if not self.file_exists(): - self.offline() - - # @TODO parse name and password - repl_pattern = r"/(s|e|d|c)/" - url = re.sub(repl_pattern, r"/d/", self.pyfile.url) - - pyfile.name = "tmp_cryptit_%s.ccf" % randint(0,1000) - location = self.download(url) - - self.packages.append(["Crypt-it Package", [location], "Crypt-it Package"]) -
\ No newline at end of file diff --git a/module/plugins/crypter/CzshareComFolder.py b/module/plugins/crypter/CzshareComFolder.py deleted file mode 100644 index c240c6a70..000000000 --- a/module/plugins/crypter/CzshareComFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class CzshareComFolder(Crypter): - __name__ = "CzshareComFolder" - __type__ = "crypter" - __pattern__ = r"http://(\w*\.)*czshare\.(com|cz)/folders/.*" - __version__ = "0.1" - __description__ = """Czshare.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'<tr class="subdirectory">\s*<td>\s*<table>(.*?)</table>' - LINK_PATTERN = r'<td class="col2"><a href="([^"]+)">info</a></td>' - #NEXT_PAGE_PATTERN = r'<a class="next " href="/([^"]+)"> </a>' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/DDLMusicOrg.py b/module/plugins/crypter/DDLMusicOrg.py deleted file mode 100644 index f7cc996d0..000000000 --- a/module/plugins/crypter/DDLMusicOrg.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import sleep - -from module.plugins.Crypter import Crypter - -class DDLMusicOrg(Crypter): - __name__ = "DDLMusicOrg" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?ddl-music\.org/captcha/ddlm_cr\d\.php\?\d+\?\d+" - __version__ = "0.3" - __description__ = """ddl-music.org Container Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.multiDL = False - - def decrypt(self, pyfile): - html = self.req.load(self.pyfile.url, cookies=True) - - if re.search(r"Wer dies nicht rechnen kann", html) is not None: - self.offline() - - math = re.search(r"(\d+) ([\+-]) (\d+) =\s+<inp", self.html) - id = re.search(r"name=\"id\" value=\"(\d+)\"", self.html).group(1) - linknr = re.search(r"name=\"linknr\" value=\"(\d+)\"", self.html).group(1) - - solve = "" - if math.group(2) == "+": - solve = int(math.group(1)) + int(math.group(3)) - else: - solve = int(math.group(1)) - int(math.group(3)) - sleep(3) - htmlwithlink = self.req.load(self.pyfile.url, cookies=True, post={"calc%s" % linknr:solve, "send%s" % linknr:"Send", "id":id, "linknr":linknr}) - m = re.search(r"<form id=\"ff\" action=\"(.*?)\" method=\"post\">", htmlwithlink) - if m: - self.packages.append((self.pyfile.package().name, [m.group(1)], self.pyfile.package().folder)) - else: - self.retry() diff --git a/module/plugins/crypter/DataHuFolder.py b/module/plugins/crypter/DataHuFolder.py deleted file mode 100644 index f710f60d7..000000000 --- a/module/plugins/crypter/DataHuFolder.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class DataHuFolder(SimpleCrypter): - __name__ = "DataHuFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?data.hu/dir/\w+" - __version__ = "0.03" - __description__ = """Data.hu Folder Plugin""" - __author_name__ = ("crash", "stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r"<a href='(http://data\.hu/get/.+)' target='_blank'>\1</a>" - TITLE_PATTERN = ur'<title>(?P<title>.+) Let\xf6lt\xe9se</title>' - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - if u'K\xe9rlek add meg a jelsz\xf3t' in self.html: # Password protected - password = self.getPassword() - if password is '': - self.fail("No password specified, please set right password on Add package form and retry") - self.logDebug('The folder is password protected', 'Using password: ' + password) - self.html = self.load(pyfile.url, post={'mappa_pass': password}, decode=True) - if u'Hib\xe1s jelsz\xf3' in self.html: # Wrong password - self.fail("Incorrect password, please set right password on Add package form and retry") - - package_name, folder_name = self.getPackageNameAndFolder() - - package_links = re.findall(self.LINK_PATTERN, self.html) - self.logDebug('Package has %d links' % len(package_links)) - - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/DdlstorageComFolder.py b/module/plugins/crypter/DdlstorageComFolder.py deleted file mode 100644 index d536032c6..000000000 --- a/module/plugins/crypter/DdlstorageComFolder.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.plugins.hoster.MediafireCom import checkHTMLHeader -from module.common.json_layer import json_loads - -class DdlstorageComFolder(Crypter): - __name__ = "DdlstorageComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:\w*\.)*?ddlstorage.com/folder/\w{10}" - __version__ = "0.01" - __description__ = """DDLStorage.com Folder Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - FILE_URL_PATTERN = '<a style="text-decoration:none;" href="http://www.ddlstorage.com/(.*)">' - - def decrypt(self, pyfile): - new_links = [] - # load and parse html - html = self.load(pyfile.url) - found = re.findall(self.FILE_URL_PATTERN, html) - self.logDebug(found) - for link in found: - # file page - new_links.append("http://www.ddlstorage.com/%s" % link) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/DepositfilesComFolder.py b/module/plugins/crypter/DepositfilesComFolder.py deleted file mode 100644 index 9023b238f..000000000 --- a/module/plugins/crypter/DepositfilesComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class DepositfilesComFolder(SimpleCrypter): - __name__ = "DepositfilesComFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?depositfiles.com/folders/\w+" - __version__ = "0.01" - __description__ = """Depositfiles.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<div class="progressName"[^>]*>\s*<a href="([^"]+)" title="[^"]*" target="_blank">'
\ No newline at end of file diff --git a/module/plugins/crypter/Dereferer.py b/module/plugins/crypter/Dereferer.py deleted file mode 100644 index 584835e18..000000000 --- a/module/plugins/crypter/Dereferer.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" - -import re -import urllib - -from module.plugins.Crypter import Crypter - -class Dereferer(Crypter): - __name__ = "Dereferer" - __type__ = "crypter" - __pattern__ = r'https?://([^/]+)/.*?(?P<url>(ht|f)tps?(://|%3A%2F%2F).*)' - __version__ = "0.1" - __description__ = """Crypter for dereferers""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def decrypt(self, pyfile): - link = re.match(self.__pattern__, self.pyfile.url).group('url') - self.core.files.addLinks([ urllib.unquote(link).rstrip('+') ], self.pyfile.package().id) diff --git a/module/plugins/crypter/DontKnowMe.py b/module/plugins/crypter/DontKnowMe.py deleted file mode 100644 index dfa72df47..000000000 --- a/module/plugins/crypter/DontKnowMe.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import urllib - -from module.plugins.Crypter import Crypter - -class DontKnowMe(Crypter): - __name__ = "DontKnowMe" - __type__ = "crypter" - __pattern__ = r"http://dontknow.me/at/\?.+$" - __version__ = "0.1" - __description__ = """DontKnowMe""" - __author_name__ = ("selaux") - __author_mail__ = ("") - - LINK_PATTERN = r"http://dontknow.me/at/\?(.+)$" - - def decrypt(self, pyfile): - link = re.findall(self.LINK_PATTERN, self.pyfile.url)[0] - self.core.files.addLinks([ urllib.unquote(link) ], self.pyfile.package().id) diff --git a/module/plugins/crypter/DownloadVimeoCom.py b/module/plugins/crypter/DownloadVimeoCom.py deleted file mode 100644 index 88310915b..000000000 --- a/module/plugins/crypter/DownloadVimeoCom.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import HTMLParser -from module.plugins.Crypter import Crypter - -class DownloadVimeoCom(Crypter): - __name__ = 'DownloadVimeoCom' - __type__ = 'crypter' - __pattern__ = r'(?:http://vimeo\.com/\d*|http://smotri\.com/video/view/\?id=.*)' - ## The download from dailymotion failed with a 403 - __version__ = '0.1' - __description__ = """Video Download Plugin based on downloadvimeo.com""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - BASE_URL = 'http://downloadvimeo.com' - - def decrypt(self, pyfile): - self.package = pyfile.package() - html = self.load('%s/generate?url=%s' % (self.BASE_URL, pyfile.url)) - h = HTMLParser.HTMLParser() - try: - f = re.search(r'cmd quality="(?P<quality>[^"]+?)">\s*?(?P<URL>[^<]*?)</cmd>', html) - except: - self.logDebug('Failed to find the URL') - else: - url = h.unescape(f.group('URL')) - self.logDebug('Quality: %s, URL: %s' % (f.group('quality'), url)) - self.packages.append((self.package.name, [url], self.package.folder)) diff --git a/module/plugins/crypter/DuckCryptInfo.py b/module/plugins/crypter/DuckCryptInfo.py deleted file mode 100644 index 4886d24db..000000000 --- a/module/plugins/crypter/DuckCryptInfo.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.lib.BeautifulSoup import BeautifulSoup -from module.plugins.Crypter import Crypter - -class DuckCryptInfo(Crypter): - __name__ = "DuckCryptInfo" - __type__ = "container" - __pattern__ = r"http://(?:www\.)?duckcrypt.info/(folder|wait|link)/(\w+)/?(\w*)" - __version__ = "0.02" - __description__ = """DuckCrypt.Info Container Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - TIMER_PATTERN = r'<span id="timer">(.*)</span>' - - def decrypt(self, pyfile): - url = pyfile.url - # seems we don't need to wait - #src = self.req.load(str(url)) - #found = re.search(self.TIMER_PATTERN, src) - #if found: - # self.logDebug("Sleeping for" % found.group(1)) - # self.setWait(int(found.group(1)) ,False) - found = re.search(self.__pattern__, url) - if not found: - self.fail('Weird error in link') - if str(found.group(1)) == "link": - self.handleLink(url) - else: - self.handleFolder(found) - - - - def handleFolder(self, found): - src = self.load("http://duckcrypt.info/ajax/auth.php?hash=" + str(found.group(2))) - found = re.search(self.__pattern__, src) - self.logDebug("Redirectet to " + str(found.group(0))) - src = self.load(str(found.group(0))) - soup = BeautifulSoup(src) - cryptlinks = soup.findAll("div", attrs={"class": "folderbox"}) - self.logDebug("Redirectet to " + str(cryptlinks)) - if not cryptlinks: - self.fail('no links found - (Plugin out of date?)') - for clink in cryptlinks: - if clink.find("a"): - self.handleLink(clink.find("a")['href']) - - def handleLink(self, url): - src = self.load(url) - soup = BeautifulSoup(src) - link = soup.find("iframe")["src"] - if not link: - self.logDebug('no links found - (Plugin out of date?)') - else: - self.core.files.addLinks([link], self.pyfile.package().id) - diff --git a/module/plugins/crypter/EasybytezComFolder.py b/module/plugins/crypter/EasybytezComFolder.py deleted file mode 100644 index 1b887e421..000000000 --- a/module/plugins/crypter/EasybytezComFolder.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class EasybytezComFolder(SimpleCrypter): - __name__ = "EasybytezComFolder" - __type__ = "crypter" - __pattern__ = r"https?://(www\.)?easybytez\.com/users/\w+/\w+" - __version__ = "0.01" - __description__ = """Easybytez Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r'<div class="link"><a href="(http://www\.easybytez\.com/\w+)" target="_blank">.+</a></div>' - TITLE_PATTERN = r'<Title>Files of (?P<title>.+) folder</Title>' - PAGES_PATTERN = r"<a href='[^']+'>(\d+)</a><a href='[^']+'>Next »</a><br><small>\(\d+ total\)</small></div>" - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - package_name, folder_name = self.getPackageNameAndFolder() - - package_links = re.findall(self.LINK_PATTERN, self.html) - - pages = re.search(self.PAGES_PATTERN, self.html) - if pages: - pages = int(pages.group(1)) - else: - pages = 1 - - p = 2 - while p <= pages: - self.html = self.load(pyfile.url, get={'page': p}, decode=True) - package_links += re.findall(self.LINK_PATTERN, self.html) - p += 1 - - self.logDebug('Package has %d links' % len(package_links)) - - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/EmbeduploadCom.py b/module/plugins/crypter/EmbeduploadCom.py deleted file mode 100644 index 8fd70882f..000000000 --- a/module/plugins/crypter/EmbeduploadCom.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.network.HTTPRequest import BadHeader - -class EmbeduploadCom(Crypter): - __name__ = "EmbeduploadCom" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?embedupload.com/\?d=.*" - __version__ = "0.02" - __description__ = """EmbedUpload.com crypter""" - __config__ = [("preferedHoster", "str", "Prefered hoster list (bar-separated) ", "embedupload"), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated) ", "")] - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<div id="([^"]+)"[^>]*>\s*<a href="([^"]+)" target="_blank" (?:class="DownloadNow"|style="color:red")>' - - def decrypt(self, pyfile): - self.html = self.load(self.pyfile.url, decode=True) - tmp_links = [] - new_links = [] - - found = re.findall(self.LINK_PATTERN, self.html) - if found: - prefered_set = set(self.getConfig("preferedHoster").split('|')) - prefered_set = map(lambda s: s.lower().split('.')[0], prefered_set) - print "PF", prefered_set - tmp_links.extend([x[1] for x in found if x[0] in prefered_set]) - self.getLocation(tmp_links, new_links) - - if not new_links: - ignored_set = set(self.getConfig("ignoredHoster").split('|')) - ignored_set = map(lambda s: s.lower().split('.')[0], ignored_set) - print "IG", ignored_set - tmp_links.extend([x[1] for x in found if x[0] not in ignored_set]) - self.getLocation(tmp_links, new_links) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') - - def getLocation(self, tmp_links, new_links): - for link in tmp_links: - try: - header = self.load(link, just_header = True) - if "location" in header: - new_links.append(header['location']) - except BadHeader: - pass - -
\ No newline at end of file diff --git a/module/plugins/crypter/FilebeerInfoFolder.py b/module/plugins/crypter/FilebeerInfoFolder.py deleted file mode 100644 index f45144f14..000000000 --- a/module/plugins/crypter/FilebeerInfoFolder.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class FilebeerInfoFolder(Crypter): - __name__ = "FilebeerInfoFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?filebeer\.info/(\d+~f).*" - __version__ = "0.01" - __description__ = """Filebeer.info Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<td title="[^"]*"><a href="([^"]+)" target="_blank">' - PAGE_COUNT_PATTERN = r'<p class="introText">\s*Total Pages (\d+)' - - def decrypt(self, pyfile): - pyfile.url = re.sub(self.__pattern__, r'http://filebeer.info/\1?page=1', pyfile.url) - html = self.load(pyfile.url) - - page_count = int(re.search(self.PAGE_COUNT_PATTERN, html).group(1)) - new_links = [] - - for i in range(1, page_count + 1): - self.logInfo("Fetching links from page %i" % i) - new_links.extend(re.findall(self.LINK_PATTERN, html)) - - if i < page_count: - html = self.load("%s?page=%d" % (pyfile.url, i+1)) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/FilefactoryComFolder.py b/module/plugins/crypter/FilefactoryComFolder.py deleted file mode 100644 index 32793b491..000000000 --- a/module/plugins/crypter/FilefactoryComFolder.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class FilefactoryComFolder(Crypter): - __name__ = "FilefactoryComFolder" - __type__ = "crypter" - __pattern__ = r"(http://(www\.)?filefactory\.com/f/\w+).*" - __version__ = "0.1" - __description__ = """Filefactory.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'<table class="items" cellspacing="0" cellpadding="0">(.*?)</table>' - LINK_PATTERN = r'<td class="name"><a href="([^"]+)">' - PAGINATOR_PATTERN = r'<div class="list">\s*<label>Pages</label>\s*<ul>(.*?)</ul>\s*</div>' - NEXT_PAGE_PATTERN = r'<li class="current">.*?</li>\s*<li class=""><a href="([^"]+)">' - - def decrypt(self, pyfile): - url_base = re.search(self.__pattern__, self.pyfile.url).group(1) - html = self.load(url_base) - - new_links = [] - for i in range(1,100): - self.logInfo("Fetching links from page %i" % i) - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - try: - paginator = re.search(self.PAGINATOR_PATTERN, html, re.DOTALL).group(1) - next_page = re.search(self.NEXT_PAGE_PATTERN, paginator).group(1) - html = self.load("%s/%s" % (url_base, next_page)) - except Exception, e: - break - else: - self.logInfo("Limit of 99 pages reached, aborting") - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/FileserveComFolder.py b/module/plugins/crypter/FileserveComFolder.py deleted file mode 100644 index 9fe806971..000000000 --- a/module/plugins/crypter/FileserveComFolder.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class FileserveComFolder(Crypter): - __name__ = "FileserveComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?fileserve.com/list/\w+" - __version__ = "0.11" - __description__ = """FileServeCom.com Folder Plugin""" - __author_name__ = ("fionnc") - __author_mail__ = ("fionnc@gmail.com") - - FOLDER_PATTERN = r'<table class="file_list">(.*?)</table>' - LINK_PATTERN = r'<a href="([^"]+)" class="sheet_icon wbold">' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) - - if new_links: - self.core.files.addLinks(map(lambda s:"http://fileserve.com%s" % s, new_links), self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/FourChanOrg.py b/module/plugins/crypter/FourChanOrg.py deleted file mode 100644 index 5c96e723d..000000000 --- a/module/plugins/crypter/FourChanOrg.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class FourChanOrg(Crypter): - # Based on 4chandl by Roland Beermann - # https://gist.github.com/enkore/3492599 - __name__ = "FourChanOrg" - __type__ = "container" - __version__ = "0.3" - __pattern__ = r"http://boards\.4chan.org/\w+/res/(\d+)" - __description__ = "Downloader for entire 4chan threads" - - def decrypt(self, pyfile): - pagehtml = self.load(pyfile.url) - - images = set(re.findall(r'(images\.4chan\.org/[^/]*/src/[^"<]*)', pagehtml)) - urls = [] - for image in images: - urls.append("http://" + image) - - self.core.files.addLinks(urls, self.pyfile.package().id) diff --git a/module/plugins/crypter/FshareVnFolder.py b/module/plugins/crypter/FshareVnFolder.py deleted file mode 100644 index 2515e7edd..000000000 --- a/module/plugins/crypter/FshareVnFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class FshareVnFolder(SimpleCrypter): - __name__ = "FshareVnFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?fshare.vn/folder/.*" - __version__ = "0.01" - __description__ = """Fshare.vn Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<li class="w_80pc"><a href="([^"]+)" target="_blank">'
\ No newline at end of file diff --git a/module/plugins/crypter/GooGl.py b/module/plugins/crypter/GooGl.py deleted file mode 100644 index bcb1d7494..000000000 --- a/module/plugins/crypter/GooGl.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -from module.plugins.Crypter import Crypter -from module.common.json_layer import json_loads - - -class GooGl(Crypter): - __name__ = "GooGl" - __type__ = "crypter" - __pattern__ = r"https?://(www\.)?goo\.gl/\w+" - __version__ = "0.01" - __description__ = """Goo.gl Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - API_URL = 'https://www.googleapis.com/urlshortener/v1/url' - - def decrypt(self, pyfile): - rep = self.load(self.API_URL, get={'shortUrl': pyfile.url}) - self.logDebug('JSON data: ' + rep) - rep = json_loads(rep) - - if 'longUrl' in rep: - self.core.files.addLinks([rep['longUrl']], self.pyfile.package().id) - else: - self.fail('Unable to expand shortened link') diff --git a/module/plugins/crypter/HoerbuchIn.py b/module/plugins/crypter/HoerbuchIn.py deleted file mode 100644 index 6f23b2eb9..000000000 --- a/module/plugins/crypter/HoerbuchIn.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup - -class HoerbuchIn(Crypter): - __name__ = "HoerbuchIn" - __type__ = "container" - __pattern__ = r"http://(www\.)?hoerbuch\.in/(wp/horbucher/\d+/.+/|tp/out.php\?.+|protection/folder_\d+\.html)" - __version__ = "0.7" - __description__ = """Hoerbuch.in Container Plugin""" - __author_name__ = ("spoob", "mkaay") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de") - - article = re.compile("http://(www\.)?hoerbuch\.in/wp/horbucher/\d+/.+/") - protection = re.compile("http://(www\.)?hoerbuch\.in/protection/folder_\d+.html") - - def decrypt(self, pyfile): - self.pyfile = pyfile - - if self.article.match(self.pyfile.url): - src = self.load(self.pyfile.url) - soup = BeautifulSoup(src, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) - - abookname = soup.find("a", attrs={"rel": "bookmark"}).text - for a in soup.findAll("a", attrs={"href": self.protection}): - package = "%s (%s)" % (abookname, a.previousSibling.previousSibling.text[:-1]) - links = self.decryptFolder(a["href"]) - - self.packages.append((package, links, self.pyfile.package().folder)) - else: - links = self.decryptFolder(self.pyfile.url) - - self.packages.append((self.pyfile.package().name, links, self.pyfile.package().folder)) - - def decryptFolder(self, url): - m = self.protection.search(url) - if not m: - self.fail("Bad URL") - url = m.group(0) - - self.pyfile.url = url - src = self.req.load(url, post={"viewed": "adpg"}) - - links = [] - pattern = re.compile(r'<div class="container"><a href="(.*?)"') - for hoster_url in pattern.findall(src): - self.req.lastURL = url - self.load(hoster_url) - links.append(self.req.lastEffectiveURL) - - return links diff --git a/module/plugins/crypter/HotfileFolderCom.py b/module/plugins/crypter/HotfileFolderCom.py deleted file mode 100644 index ea7311e3c..000000000 --- a/module/plugins/crypter/HotfileFolderCom.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class HotfileFolderCom(Crypter): - __name__ = "HotfileFolderCom" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?hotfile.com/list/\w+/\w+" - __version__ = "0.2" - __description__ = """HotfileFolder Download Plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def decryptURL(self, url): - html = self.load(url) - - new_links = [] - for link in re.findall(r'href="(http://(www.)?hotfile\.com/dl/\d+/[0-9a-zA-Z]+[^"]+)', html): - new_links.append(link[0]) - - if new_links: - self.logDebug("Found %d new links" % len(new_links)) - return new_links - else: - self.fail('Could not extract any links') - diff --git a/module/plugins/crypter/ILoadTo.py b/module/plugins/crypter/ILoadTo.py deleted file mode 100644 index 9815ae266..000000000 --- a/module/plugins/crypter/ILoadTo.py +++ /dev/null @@ -1,62 +0,0 @@ -
-import re
-import urllib
-
-from module.plugins.Crypter import Crypter
-from module.lib.BeautifulSoup import BeautifulSoup
-
-class ILoadTo(Crypter):
- __name__ = "ILoadTo"
- __type__ = "crypter"
- __pattern__ = r"http://iload\.to/go/\d+-[\w\.-]+/"
- __config__ = []
- __version__ = "0.1"
- __description__ = """iload.to Crypter Plugin"""
- __author_name__ = ("hzpz")
- __author_mail__ = ("none")
-
-
- def decrypt(self, pyfile):
- url = pyfile.url
- src = self.req.load(str(url))
- soup = BeautifulSoup(src)
-
- # find captcha URL and decrypt
- captchaTag = soup.find("img", attrs={"id": "Captcha"})
- if not captchaTag:
- self.fail("Cannot find Captcha")
-
- captchaUrl = "http://iload.to" + captchaTag["src"]
- self.logDebug("Captcha URL: %s" % captchaUrl)
- result = self.decryptCaptcha(str(captchaUrl))
-
- # find captcha form URL
- formTag = soup.find("form", attrs={"id": "CaptchaForm"})
- formUrl = "http://iload.to" + formTag["action"]
- self.logDebug("Form URL: %s" % formUrl)
-
- # submit decrypted captcha
- self.req.lastURL = url
- src = self.req.load(str(formUrl), post={'captcha': result})
-
- # find decrypted links
- links = re.findall(r"<a href=\"(.+)\" style=\"text-align:center;font-weight:bold;\" class=\"button\" target=\"_blank\" onclick=\"this.className\+=' success';\">", src)
-
- if not len(links) > 0:
- self.retry()
-
- self.correctCaptcha()
-
- cleanedLinks = []
- for link in links:
- if link.startswith("http://dontknow.me/at/?"):
- cleanedLink = urllib.unquote(link[23:])
- else:
- cleanedLink = link
- self.logDebug("Link: %s" % cleanedLink)
- cleanedLinks.append(cleanedLink)
-
- self.logDebug("Decrypted %d links" % len(links))
-
- self.pyfile.package().password = "iload.to"
- self.packages.append((self.pyfile.package().name, cleanedLinks, self.pyfile.package().folder))
\ No newline at end of file diff --git a/module/plugins/crypter/LetitbitNetFolder.py b/module/plugins/crypter/LetitbitNetFolder.py deleted file mode 100644 index 68aad9dd7..000000000 --- a/module/plugins/crypter/LetitbitNetFolder.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - - -class LetitbitNetFolder(Crypter): - __name__ = "LetitbitNetFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?letitbit.net/folder/\w+" - __version__ = "0.1" - __description__ = """Letitbit.net Folder Plugin""" - __author_name__ = ("DHMH", "z00nx") - __author_mail__ = ("webmaster@pcProfil.de", "z00nx0@gmail.com") - - FOLDER_PATTERN = r'<table>(.*)</table>' - LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: - self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') diff --git a/module/plugins/crypter/LinkList.py b/module/plugins/crypter/LinkList.py deleted file mode 100644 index ebfa373eb..000000000 --- a/module/plugins/crypter/LinkList.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.Crypter import Crypter, Package - -class LinkList(Crypter): - __name__ = "LinkList" - __version__ = "0.11" - __pattern__ = r".+\.txt$" - __description__ = """Read Link Lists in txt format""" - __author_name__ = ("spoob", "jeix") - __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") - - # method declaration is needed here - def decryptURL(self, url): - return Crypter.decryptURL(self, url) - - def decryptFile(self, content): - links = content.splitlines() - - curPack = "default" - packages = {curPack:[]} - - for link in links: - link = link.strip() - if not link: continue - - if link.startswith(";"): - continue - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - packages[curPack].append(link) - - # empty packages fix - delete = [] - - for key,value in packages.iteritems(): - if not value: - delete.append(key) - - for key in delete: - del packages[key] - - urls = [] - - for name, links in packages.iteritems(): - if name == "default": - urls.extend(links) - else: - urls.append(Package(name, links)) - - return urls
\ No newline at end of file diff --git a/module/plugins/crypter/LinkSaveIn.py b/module/plugins/crypter/LinkSaveIn.py deleted file mode 100644 index 30cc61055..000000000 --- a/module/plugins/crypter/LinkSaveIn.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*-
-
-#
-# v2.01 - hagg
-# * cnl2 and web links are skipped if JS is not available (instead of failing the package)
-# * only best available link source is used (priority: cnl2>rsdf>ccf>dlc>web
-#
-
-from Crypto.Cipher import AES
-from module.plugins.Crypter import Crypter
-from module.unescape import unescape
-import base64
-import binascii
-import re
-
-class LinkSaveIn(Crypter):
- __name__ = "LinkSaveIn"
- __type__ = "crypter"
- __pattern__ = r"http://(www\.)?linksave.in/(?P<id>\w+)$"
- __version__ = "2.01"
- __description__ = """LinkSave.in Crypter Plugin"""
- __author_name__ = ("fragonib")
- __author_mail__ = ("fragonib[AT]yahoo[DOT]es")
-
- # Constants
- _JK_KEY_ = "jk"
- _CRYPTED_KEY_ = "crypted"
- HOSTER_DOMAIN = "linksave.in"
-
- def setup(self):
- self.html = None
- self.fileid = None
- self.captcha = False
- self.package = None
- self.preferred_sources = ['cnl2', 'rsdf', 'ccf', 'dlc', 'web']
-
- def decrypt(self, pyfile):
-
- # Init
- self.package = pyfile.package()
- self.fileid = re.match(self.__pattern__, pyfile.url).group('id')
- self.req.cj.setCookie(self.HOSTER_DOMAIN, "Linksave_Language", "english")
-
- # Request package
- self.html = self.load(self.pyfile.url)
- if not self.isOnline():
- self.offline()
-
- # Check for protection
- if self.isPasswordProtected():
- self.unlockPasswordProtection()
- self.handleErrors()
-
- if self.isCaptchaProtected():
- self.captcha = True
- self.unlockCaptchaProtection()
- self.handleErrors()
-
- # Get package name and folder
- (package_name, folder_name) = self.getPackageInfo()
-
- # Extract package links
- package_links = []
- for type_ in self.preferred_sources:
- package_links.extend(self.handleLinkSource(type_))
- if package_links: # use only first source which provides links
- break
- package_links = set(package_links)
-
- # Pack
- if package_links:
- self.packages = [(package_name, package_links, folder_name)]
- else:
- self.fail('Could not extract any links')
-
- def isOnline(self):
- if "<big>Error 404 - Folder not found!</big>" in self.html:
- self.logDebug("File not found")
- return False
- return True
-
- def isPasswordProtected(self):
- if re.search(r'''<input.*?type="password"''', self.html):
- self.logDebug("Links are password protected")
- return True
-
- def isCaptchaProtected(self):
- if "<b>Captcha:</b>" in self.html:
- self.logDebug("Links are captcha protected")
- return True
- return False
-
- def unlockPasswordProtection(self):
- password = self.getPassword()
- self.logDebug("Submitting password [%s] for protected links" % password)
- post = {"id": self.fileid, "besucherpasswort": password, 'login': 'submit'}
- self.html = self.load(self.pyfile.url, post=post)
-
- def unlockCaptchaProtection(self):
- captcha_hash = re.search(r'name="hash" value="([^"]+)', self.html).group(1)
- captcha_url = re.search(r'src=".(/captcha/cap.php\?hsh=[^"]+)', self.html).group(1)
- captcha_code = self.decryptCaptcha("http://linksave.in" + captcha_url, forceUser=True)
- self.html = self.load(self.pyfile.url, post={"id": self.fileid, "hash": captcha_hash, "code": captcha_code})
-
- def getPackageInfo(self):
- name = self.pyfile.package().name
- folder = self.pyfile.package().folder
- self.logDebug("Defaulting to pyfile name [%s] and folder [%s] for package" % (name, folder))
- return name, folder
-
- def handleErrors(self):
- if "The visitorpassword you have entered is wrong" in self.html:
- self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")
- self.fail("Incorrect password, please set right password on 'Edit package' form and retry")
-
- if self.captcha:
- if "Wrong code. Please retry" in self.html:
- self.logDebug("Invalid captcha, retrying")
- self.invalidCaptcha()
- self.retry()
- else:
- self.correctCaptcha()
-
- def handleLinkSource(self, type_):
- if type_ == 'cnl2':
- return self.handleCNL2()
- elif type_ in ('rsdf', 'ccf', 'dlc'):
- return self.handleContainer(type_)
- elif type_ == 'web':
- return self.handleWebLinks()
- else:
- self.fail('unknown source type "%s" (this is probably a bug)' % type_)
-
- def handleWebLinks(self):
- package_links = []
- self.logDebug("Search for Web links")
- if not self.js:
- self.logDebug("no JS -> skip Web links")
- else:
- #@TODO: Gather paginated web links
- pattern = r'<a href="http://linksave\.in/(\w{43})"'
- ids = re.findall(pattern, self.html)
- self.logDebug("Decrypting %d Web links" % len(ids))
- for i, weblink_id in enumerate(ids):
- try:
- webLink = "http://linksave.in/%s" % weblink_id
- self.logDebug("Decrypting Web link %d, %s" % (i+1, webLink))
- fwLink = "http://linksave.in/fw-%s" % weblink_id
- response = self.load(fwLink)
- jscode = re.findall(r'<script type="text/javascript">(.*)</script>', response)[-1]
- jseval = self.js.eval("document = { write: function(e) { return e; } }; %s" % jscode)
- dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0)
- self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink)
- response = self.load(dlLink)
- link = unescape(re.search(r'<iframe src="(.+?)"', response).group(1))
- package_links.append(link)
- except Exception, detail:
- self.logDebug("Error decrypting Web link %s, %s" % (webLink, detail))
- return package_links
-
- def handleContainer(self, type_):
- package_links = []
- type_ = type_.lower()
- self.logDebug('Seach for %s Container links' % type_.upper())
- if not type_.isalnum(): # check to prevent broken re-pattern (cnl2,rsdf,ccf,dlc,web are all alpha-numeric)
- self.fail('unknown container type "%s" (this is probably a bug)' % type_)
- pattern = r"\('%s_link'\).href=unescape\('(.*?\.%s)'\)" % (type_, type_)
- containersLinks = re.findall(pattern, self.html)
- self.logDebug("Found %d %s Container links" % (len(containersLinks), type_.upper()))
- for containerLink in containersLinks:
- link = "http://linksave.in/%s" % unescape(containerLink)
- package_links.append(link)
- return package_links
-
- def handleCNL2(self):
- package_links = []
- self.logDebug("Search for CNL2 links")
- if not self.js:
- self.logDebug("no JS -> skip CNL2 links")
- elif 'cnl2_load' in self.html:
- try:
- (vcrypted, vjk) = self._getCipherParams()
- for (crypted, jk) in zip(vcrypted, vjk):
- package_links.extend(self._getLinks(crypted, jk))
- except:
- self.fail("Unable to decrypt CNL2 links")
- return package_links
-
- def _getCipherParams(self):
-
- # Get jk
- jk_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._JK_KEY_
- vjk = re.findall(jk_re, self.html)
-
- # Get crypted
- crypted_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._CRYPTED_KEY_
- vcrypted = re.findall(crypted_re, self.html)
-
- # Log and return
- self.logDebug("Detected %d crypted blocks" % len(vcrypted))
- return vcrypted, vjk
-
- def _getLinks(self, crypted, jk):
-
- # Get key
- jreturn = self.js.eval("%s f()" % jk)
- self.logDebug("JsEngine returns value [%s]" % jreturn)
- key = binascii.unhexlify(jreturn)
-
- # Decode crypted
- crypted = base64.standard_b64decode(crypted)
-
- # Decrypt
- Key = key
- IV = key
- obj = AES.new(Key, AES.MODE_CBC, IV)
- text = obj.decrypt(crypted)
-
- # Extract links
- text = text.replace("\x00", "").replace("\r", "")
- links = text.split("\n")
- links = filter(lambda x: x != "", links)
-
- # Log and return
- self.logDebug("Package has %d links" % len(links))
- return links
-
diff --git a/module/plugins/crypter/LinkdecrypterCom.py b/module/plugins/crypter/LinkdecrypterCom.py deleted file mode 100644 index 69d2f8192..000000000 --- a/module/plugins/crypter/LinkdecrypterCom.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Crypter import Crypter - -class LinkdecrypterCom(Crypter): - __name__ = "LinkdecrypterCom" - __type__ = "crypter" - __version__ = "0.27" - __description__ = """linkdecrypter.com""" - __author_name__ = ("zoidberg", "flowlee") - - TEXTAREA_PATTERN = r'<textarea name="links" wrap="off" readonly="1" class="caja_des">(.+)</textarea>' - PASSWORD_PATTERN = r'<input type="text" name="password"' - CAPTCHA_PATTERN = r'<img class="captcha" src="(.+?)"(.*?)>' - REDIR_PATTERN = r'<i>(Click <a href="./">here</a> if your browser does not redirect you).</i>' - - def decrypt(self, pyfile): - - self.passwords = self.getPassword().splitlines() - - # API not working anymore - new_links = self.decryptHTML() - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') - - def decryptAPI(self): - - get_dict = { "t": "link", "url": self.pyfile.url, "lcache": "1" } - self.html = self.load('http://linkdecrypter.com/api', get = get_dict) - if self.html.startswith('http://'): return self.html.splitlines() - - if self.html == 'INTERRUPTION(PASSWORD)': - for get_dict['pass'] in self.passwords: - self.html = self.load('http://linkdecrypter.com/api', get= get_dict) - if self.html.startswith('http://'): return self.html.splitlines() - - self.logError('API', self.html) - if self.html == 'INTERRUPTION(PASSWORD)': - self.fail("No or incorrect password") - - return None - - def decryptHTML(self): - - retries = 5 - - post_dict = { "link_cache": "on", "pro_links": self.pyfile.url, "modo_links": "text" } - self.html = self.load('http://linkdecrypter.com/', post=post_dict, cookies=True, decode=True) - - while self.passwords or retries: - found = re.search(self.TEXTAREA_PATTERN, self.html, flags=re.DOTALL) - if found: return [ x for x in found.group(1).splitlines() if '[LINK-ERROR]' not in x ] - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if found: - captcha_url = 'http://linkdecrypter.com/' + found.group(1) - result_type = "positional" if "getPos" in found.group(2) else "textual" - - found = re.search(r"<p><i><b>([^<]+)</b></i></p>", self.html) - msg = found.group(1) if found else "" - self.logInfo("Captcha protected link", result_type, msg) - - captcha = self.decryptCaptcha(captcha_url, result_type = result_type) - if result_type == "positional": - captcha = "%d|%d" % captcha - self.html = self.load('http://linkdecrypter.com/', post={ "captcha": captcha }, decode=True) - retries -= 1 - - elif self.PASSWORD_PATTERN in self.html: - if self.passwords: - password = self.passwords.pop(0) - self.logInfo("Password protected link, trying " + password) - self.html = self.load('http://linkdecrypter.com/', post={'password': password}, decode=True) - else: - self.fail("No or incorrect password") - - else: - retries -= 1 - self.html = self.load('http://linkdecrypter.com/', cookies=True, decode=True) - - return None diff --git a/module/plugins/crypter/LixIn.py b/module/plugins/crypter/LixIn.py deleted file mode 100644 index e2ee30731..000000000 --- a/module/plugins/crypter/LixIn.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class LixIn(Crypter): - __name__ = "LixIn" - __type__ = "container" - __pattern__ = r"http://(www.)?lix.in/(?P<id>.*)" - __version__ = "0.22" - __description__ = """Lix.in Container Plugin""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - CAPTCHA_PATTERN='<img src="(?P<image>captcha_img.php\?.*?)"' - SUBMIT_PATTERN=r"value='continue.*?'" - LINK_PATTERN=r'name="ifram" src="(?P<link>.*?)"' - - - def decrypt(self, pyfile): - url = pyfile.url - - matches = re.search(self.__pattern__,url) - if not matches: - self.fail("couldn't identify file id") - - id = matches.group("id") - self.logDebug("File id is %s" % id) - - self.html = self.req.load(url, decode=True) - - matches = re.search(self.SUBMIT_PATTERN,self.html) - if not matches: - self.fail("link doesn't seem valid") - - matches = re.search(self.CAPTCHA_PATTERN, self.html) - if matches: - for i in range(5): - matches = re.search(self.CAPTCHA_PATTERN, self.html) - if matches: - self.logDebug("trying captcha") - captcharesult = self.decryptCaptcha("http://lix.in/"+matches.group("image")) - self.html = self.req.load(url, decode=True, post={"capt" : captcharesult, "submit":"submit","tiny":id}) - else: - self.logDebug("no captcha/captcha solved") - break - else: - self.html = self.req.load(url, decode=True, post={"submit" : "submit", - "tiny" : id}) - - matches = re.search(self.LINK_PATTERN, self.html) - if not matches: - self.fail("can't find destination url") - - new_link = matches.group("link") - self.logDebug("Found link %s, adding to package" % new_link) - - self.packages.append((self.pyfile.package().name, [new_link], self.pyfile.package().name)) diff --git a/module/plugins/crypter/LofCc.py b/module/plugins/crypter/LofCc.py deleted file mode 100644 index 9c98c48a0..000000000 --- a/module/plugins/crypter/LofCc.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import sleep -from os.path import join - -from module.plugins.Crypter import Crypter -from module.plugins.ReCaptcha import ReCaptcha - -class LofCc(Crypter): - __name__ = "LofCc" - __type__ = "container" - __pattern__ = r"http://lof.cc/(.*)" - __version__ = "0.1" - __description__ = """lof.cc Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.multiDL = False - - def decrypt(self, pyfile): - html = self.req.load(self.pyfile.url, cookies=True) - - m = re.search(r"src=\"http://www.google.com/recaptcha/api/challenge\?k=(.*?)\"></script>", html) - if not m: - self.offline() - - recaptcha = ReCaptcha(self) - challenge, code = recaptcha.challenge(m.group(1)) - - resultHTML = self.req.load(self.pyfile.url, post={"recaptcha_challenge_field":challenge, "recaptcha_response_field":code}, cookies=True) - - if re.search("class=\"error\"", resultHTML): - self.retry() - - self.correctCaptcha() - - dlc = self.req.load(self.pyfile.url+"/dlc", cookies=True) - - name = re.search(self.__pattern__, self.pyfile.url).group(1)+".dlc" - - dlcFile = join(self.config["general"]["download_folder"], name) - f = open(dlcFile, "wb") - f.write(dlc) - f.close() - - self.packages.append((self.pyfile.package().name, [dlcFile], self.pyfile.package().folder)) diff --git a/module/plugins/crypter/MBLinkInfo.py b/module/plugins/crypter/MBLinkInfo.py deleted file mode 100644 index e266c7722..000000000 --- a/module/plugins/crypter/MBLinkInfo.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - - -class MBLinkInfo(Crypter): - __name__ = "MBLinkInfo" - __type__ = "container" - __pattern__ = r"http://(?:www\.)?mblink\.info/?\?id=(\d+)" - __version__ = "0.02" - __description__ = """MBLink.Info Container Plugin""" - __author_name__ = ("Gummibaer", "stickell") - __author_mail__ = ("Gummibaer@wiki-bierkiste.de", "l.stickell@yahoo.it") - - URL_PATTERN = r'<meta[^;]+; URL=(.*)["\']>' - - def decrypt(self, pyfile): - src = self.load(pyfile.url) - found = re.search(self.URL_PATTERN, src) - if found: - link = found.group(1) - self.logDebug("Redirected to " + link) - self.core.files.addLinks([link], self.pyfile.package().id) - else: - self.fail('Unable to detect valid link') diff --git a/module/plugins/crypter/MediafireComFolder.py b/module/plugins/crypter/MediafireComFolder.py deleted file mode 100644 index ddd61379c..000000000 --- a/module/plugins/crypter/MediafireComFolder.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.plugins.hoster.MediafireCom import checkHTMLHeader -from module.common.json_layer import json_loads - -class MediafireComFolder(Crypter): - __name__ = "MediafireComFolder" - __type__ = "crypter" - __pattern__ = r"http://(\w*\.)*mediafire\.com/(folder/|\?sharekey=|\?\w{13}($|[/#]))" - __version__ = "0.14" - __description__ = """Mediafire.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_KEY_PATTERN = r"var afI= '(\w+)';" - FILE_URL_PATTERN = '<meta property="og:url" content="http://www.mediafire.com/\?(\w+)"/>' - - def decrypt(self, pyfile): - new_links = [] - - url, result = checkHTMLHeader(pyfile.url) - self.logDebug('Location (%d): %s' % (result, url)) - - if result == 0: - # load and parse html - html = self.load(pyfile.url) - found = re.search(self.FILE_URL_PATTERN, html) - if found: - # file page - new_links.append("http://www.mediafire.com/file/%s" % found.group(1)) - else: - # folder page - found = re.search(self.FOLDER_KEY_PATTERN, html) - if found: - folder_key = found.group(1) - self.logDebug("FOLDER KEY: %s" % folder_key) - - json_resp = json_loads(self.load("http://www.mediafire.com/api/folder/get_info.php?folder_key=%s&response_format=json&version=1" % folder_key)) - #self.logInfo(json_resp) - if json_resp['response']['result'] == "Success": - for link in json_resp['response']['folder_info']['files']: - new_links.append("http://www.mediafire.com/file/%s" % link['quickkey']) - else: - self.fail(json_resp['response']['message']) - elif result == 1: - self.offline() - else: - new_links.append(url) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/Movie2kTo.py b/module/plugins/crypter/Movie2kTo.py deleted file mode 100644 index 097cb702e..000000000 --- a/module/plugins/crypter/Movie2kTo.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from collections import defaultdict - -class Movie2kTo(Crypter): - __name__ = 'Movie2kTo' - __type__ = 'container' - __pattern__ = r'http://(?:www\.)?movie2k\.to/(.*)\.html' - __version__ = '0.5' - __config__ = [('accepted_hosters', 'str', 'List of accepted hosters', 'Xvidstage, Putlocker, '), - ('dir_quality', 'bool', 'Show the quality of the footage in the folder name', 'True'), - ('whole_season', 'bool', 'Download whole season', 'False'), - ('everything', 'bool', 'Download everything', 'False'), - ('firstN', 'int', 'Download the first N files for each episode (the first file is probably all you will need)', '1')] - __description__ = """Movie2k.to Container Plugin""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - BASE_URL_PATTERN = r'http://(?:www\.)?movie2k\.to/' - TVSHOW_URL_PATH_PATTERN = r'tvshows-(?P<id>\d+?)-(?P<name>.+)' - FILM_URL_PATH_PATTERN = r'(?P<name>.+?)-(?:online-film|watch-movie)-(?P<id>\d+)' - SEASON_PATTERN = r'<div id="episodediv(\d+?)" style="display:(inline|none)">(.*?)</div>' - EP_PATTERN = r'<option value="(.+?)"( selected)?>Episode\s*?(\d+?)</option>' - BASE_URL = 'http://www.movie2k.to' - - def decrypt(self, pyfile): - self.package = pyfile.package() - self.folder = self.package.folder - self.qStatReset() - whole_season = self.getConfig('whole_season') - everything = self.getConfig('everything') - self.getInfo(pyfile.url) - - if (whole_season or everything) and self.format == 'tvshow': - self.logDebug('Downloading the whole season') - for season, season_sel, html in re.findall(self.SEASON_PATTERN, self.html, re.DOTALL | re.I): - if (season_sel == 'inline') or everything: - season_links = [] - for url_path, ep_sel, ep in re.findall(self.EP_PATTERN, html, re.I): - season_name = self.name_tvshow(season, ep) - self.logDebug('%s: %s' % (season_name, url_path)) - if ep_sel and (season_sel == 'inline'): - self.logDebug('%s selected (in the start URL: %s)' % (season_name, pyfile.url)) - season_links += self.getInfoAndLinks('%s/%s' % (self.BASE_URL, url_path)) - elif (whole_season and (season_sel == 'inline')) or everything: - season_links += self.getInfoAndLinks('%s/%s' % (self.BASE_URL, url_path)) - - self.logDebug(season_links) - folder = '%s: Season %s' % (self.name, season) - name = '%s%s' % (folder, self.qStat()) - self.packages.append((name, season_links, folder)) - self.qStatReset() - else: - links = self.getLinks() - name = '%s%s' % (self.package.name, self.qStat()) - self.packages.append((name, links , self.package.folder)) - - def qStat(self): - if len(self.q) == 0: return '' - if not self.getConfig('dir_quality'): return '' - if len(self.q) == 1: return (' (Quality: %d, max (all hosters): %d)' % (self.q[0], self.max_q)) - return (' (Average quality: %d, min: %d, max: %d, %s, max (all hosters): %d)' - % (sum(self.q) / float(len(self.q)), min(self.q), max(self.q), self.q, self.max_q)) - - def qStatReset(self): - self.q = [] ## to calculate the average, min and max of the quality - self.max_q = None ## maximum quality of all hosters - - def tvshow_number(self, number): - if int(number) < 10: - return '0%s' % number - else: - return number - - def name_tvshow(self, season, ep): - return '%s S%sE%s' % (self.name, self.tvshow_number(season), self.tvshow_number(ep)) - - def getInfo(self, url): - self.html = self.load(url) - self.url_path = re.match(self.__pattern__, url).group(1) - self.format = pattern_re = None - if re.match(r'tvshows', self.url_path): - self.format = 'tvshow' - pattern_re = re.search(self.TVSHOW_URL_PATH_PATTERN, self.url_path) - elif re.search(self.FILM_URL_PATH_PATTERN, self.url_path): - self.format = 'film' - pattern_re = re.search(self.FILM_URL_PATH_PATTERN, self.url_path) - self.name = pattern_re.group('name') - self.id = pattern_re.group('id') - self.logDebug('URL Path: %s (ID: %s, Name: %s, Format: %s)' - % (self.url_path, self.id, self.name, self.format)) - - def getInfoAndLinks(self, url): - self.getInfo(url) - return self.getLinks() - - ## This function returns the links for one episode as list - def getLinks(self): - accepted_hosters = re.findall(r'\b(\w+?)\b', self.getConfig('accepted_hosters')) - firstN = self.getConfig('firstN') - links = [] - re_quality = re.compile(r'.+?Quality:.+?smileys/(\d)\.gif') - ## The quality is one digit. 0 is the worst and 5 is the best. - ## Is not always there ⊠- re_hoster_id_js = re.compile(r'links\[(\d+?)\].+ (.+?)</a>(.+?)</tr>') - re_hoster_id_html = re.compile(r'(?:<td height|<tr id).+?<a href=".*?(\d{7}).*?".+? ([^<>]+?)</a>(.+?)</tr>') - ## I assume that the ID is 7 digits longs - count = defaultdict(int) - matches = re_hoster_id_html.findall(self.html) - matches += re_hoster_id_js.findall(self.html) - # self.logDebug(matches) - ## h_id: hoster_id of a possible hoster - for h_id, hoster, q_html in matches: - match_q = re_quality.search(q_html) - if match_q: - quality = int(match_q.group(1)) - if self.max_q == None: - self.max_q = quality - else: - if self.max_q < quality: self.max_q = quality - q_s = ', Quality: %d' % quality - else: - q_s = ', unknown quality' - if hoster in accepted_hosters: - self.logDebug('Accepted: %s, ID: %s%s' % (hoster, h_id, q_s)) - count[hoster] += 1 - if count[hoster] <= firstN: - if match_q: self.q.append(quality) - if h_id != self.id: - self.html = self.load('%s/tvshows-%s-%s.html' % (self.BASE_URL, h_id, self.name)) - else: - self.logDebug('This is already the right ID') - # The iframe tag must continue with a width. There where - # two iframes in the site and I try to make sure that it - # matches the right one. This is not (yet) nessesary - # because the right iframe happens to be the first iframe. - for pattern in (r'<a target="_blank" href="(http://[^"]*?)"', r'<iframe src="(http://[^"]*?)" width'): - try: - url = re.search(pattern, self.html).group(1) - except: - self.logDebug('Failed to find the URL (pattern %s)' % pattern) - else: - self.logDebug('id: %s, %s: %s' % (h_id, hoster, url)) - links.append(url) - break - else: - self.logDebug('Not accepted: %s, ID: %s%s' % (hoster, h_id, q_s)) - # self.logDebug(links) - return links diff --git a/module/plugins/crypter/MultiloadCz.py b/module/plugins/crypter/MultiloadCz.py deleted file mode 100644 index 2c71b8fea..000000000 --- a/module/plugins/crypter/MultiloadCz.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class MultiloadCz(Crypter): - __name__ = "MultiloadCz" - __type__ = "crypter" - __pattern__ = r"http://.*multiload.cz/(stahnout|slozka)/.*" - __version__ = "0.4" - __description__ = """multiload.cz""" - __config__ = [("usedHoster", "str", "Prefered hoster list (bar-separated) ", ""), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated) ", "")] - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'<form action="" method="get"><textarea[^>]*>([^>]*)</textarea></form>' - LINK_PATTERN = r'<p class="manager-server"><strong>([^<]+)</strong></p><p class="manager-linky"><a href="([^"]+)">' - - def decrypt(self, pyfile): - self.html = self.load(self.pyfile.url, decode=True) - new_links = [] - - if re.search(self.__pattern__, self.pyfile.url).group(1) == "slozka": - found = re.search(self.FOLDER_PATTERN, self.html) - if found is not None: - new_links.extend(found.group(1).split()) - else: - found = re.findall(self.LINK_PATTERN, self.html) - if found: - prefered_set = set(self.getConfig("usedHoster").split('|')) - new_links.extend([x[1] for x in found if x[0] in prefered_set]) - - if not new_links: - ignored_set = set(self.getConfig("ignoredHoster").split('|')) - new_links.extend([x[1] for x in found if x[0] not in ignored_set]) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/MultiuploadCom.py b/module/plugins/crypter/MultiuploadCom.py deleted file mode 100644 index bf5540982..000000000 --- a/module/plugins/crypter/MultiuploadCom.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter -from module.common.json_layer import json_loads -from time import time - -class MultiuploadCom(Crypter): - __name__ = "MultiuploadCom" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?multiupload.com/(\w+)" - __version__ = "0.01" - __description__ = """MultiUpload.com crypter""" - __config__ = [("preferedHoster", "str", "Prefered hoster list (bar-separated) ", "multiupload"), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated) ", "")] - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - ML_LINK_PATTERN = r'<div id="downloadbutton_" style=""><a href="([^"]+)"' - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url) - found = re.search(self.ML_LINK_PATTERN, self.html) - ml_url = found.group(1) if found else None - - json_list = json_loads(self.load("http://multiupload.com/progress/", get = { - "d": re.search(self.__pattern__, pyfile.url).group(1), - "r": str(int(time()*1000)) - })) - new_links = [] - - prefered_set = map(lambda s: s.lower().split('.')[0], set(self.getConfig("preferedHoster").split('|'))) - - if ml_url and 'multiupload' in prefered_set: new_links.append(ml_url) - - for link in json_list: - if link['service'].lower() in prefered_set and int(link['status']) and not int(link['deleted']): - url = self.getLocation(link['url']) - if url: new_links.append(url) - - if not new_links: - ignored_set = map(lambda s: s.lower().split('.')[0], set(self.getConfig("ignoredHoster").split('|'))) - - if 'multiupload' not in ignored_set: new_links.append(ml_url) - - for link in json_list: - if link['service'].lower() not in ignored_set and int(link['status']) and not int(link['deleted']): - url = self.getLocation(link['url']) - if url: new_links.append(url) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links') - - def getLocation(self, url): - header = self.load(url, just_header = True) - return header['location'] if "location" in header else None
\ No newline at end of file diff --git a/module/plugins/crypter/NCryptIn.py b/module/plugins/crypter/NCryptIn.py deleted file mode 100644 index 5e1ea347c..000000000 --- a/module/plugins/crypter/NCryptIn.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*-
-
-from Crypto.Cipher import AES
-from module.plugins.Crypter import Crypter
-from module.plugins.ReCaptcha import ReCaptcha
-import base64
-import binascii
-import re
-
-class NCryptIn(Crypter):
- __name__ = "NCryptIn"
- __type__ = "crypter"
- __pattern__ = r"http://(?:www\.)?ncrypt.in/folder-([^/\?]+)"
- __version__ = "1.21"
- __description__ = """NCrypt.in Crypter Plugin"""
- __author_name__ = ("fragonib")
- __author_mail__ = ("fragonib[AT]yahoo[DOT]es")
-
- # Constants
- _JK_KEY_ = "jk"
- _CRYPTED_KEY_ = "crypted"
-
- def setup(self):
- self.html = None
- self.cleanedHtml = None
- self.captcha = False
- self.package = None
-
- def decrypt(self, pyfile):
-
- # Init
- self.package = pyfile.package()
-
- # Request package
- self.html = self.load(self.pyfile.url)
- self.cleanedHtml = self.removeCrap(self.html)
- if not self.isOnline():
- self.offline()
-
- # Check for protection
- if self.isProtected():
- self.html = self.unlockProtection()
- self.cleanedHtml = self.removeCrap(self.html)
- self.handleErrors()
-
- # Get package name and folder
- (package_name, folder_name) = self.getPackageInfo()
-
- # Extract package links
- package_links = []
- package_links.extend(self.handleWebLinks())
- package_links.extend(self.handleContainers())
- package_links.extend(self.handleCNL2())
- package_links = self.removeContainers(package_links)
- package_links = set(package_links)
-
- # Pack
- self.packages = [(package_name, package_links, folder_name)]
-
- def removeCrap(self, content):
- patterns = (r'(type="hidden".*?(name=".*?")?.*?value=".*?")',
- r'display:none;">(.*?)</(div|span)>',
- r'<div\s+class="jdownloader"(.*?)</div>',
- r'<iframe\s+style="display:none(.*?)</iframe>')
- for pattern in patterns:
- rexpr = re.compile(pattern, re.DOTALL)
- content = re.sub(rexpr, "", content)
- return content
-
- def removeContainers(self,package_links):
- tmp_package_links = package_links[:]
- for link in tmp_package_links:
- self.logDebug(link)
- if ".dlc" in link or ".ccf" in link or ".rsdf" in link:
- self.logDebug("Removing [%s] from package_links" % link)
- package_links.remove(link)
-
- if len(package_links) > 0:
- return package_links
- else:
- return tmp_package_links
-
- def isOnline(self):
- if "Your folder does not exist" in self.cleanedHtml:
- self.logDebug("File not found")
- return False
- return True
-
- def isProtected(self):
- if re.search(r'''<form.*?name.*?protected.*?>''', self.cleanedHtml):
- self.logDebug("Links are protected")
- return True
- return False
-
- def getPackageInfo(self):
- title_re = r'<h2><span.*?class="arrow".*?>(?P<title>[^<]+).*?</span>.*?</h2>'
- m = re.findall(title_re, self.html, re.DOTALL)
- if m is not None:
- title = m[-1].strip()
- name = folder = title
- self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder))
- else:
- name = self.package.name
- folder = self.package.folder
- self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder))
- return name, folder
-
- def unlockProtection(self):
-
- postData = {}
-
- form = re.search(r'''<form\ name="protected"(.*?)</form>''', self.cleanedHtml, re.DOTALL).group(1)
-
- # Submit package password
- if "password" in form:
- password = self.getPassword()
- self.logDebug("Submitting password [%s] for protected links" % password)
- postData['password'] = password
-
- # Resolve anicaptcha
- if "anicaptcha" in form:
- self.captcha = True
- self.logDebug("Captcha protected, resolving captcha")
- captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1)
- captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri)
- self.logDebug("Captcha resolved [%s]" % captcha)
- postData['captcha'] = captcha
-
- # Resolve recaptcha
- if "recaptcha" in form:
- self.captcha = True
- id = re.search(r'\?k=(.*?)"', form).group(1)
- self.logDebug("Resolving ReCaptcha with key [%s]" % id)
- recaptcha = ReCaptcha(self)
- challenge, code = recaptcha.challenge(id)
- postData['recaptcha_challenge_field'] = challenge
- postData['recaptcha_response_field'] = code
-
- # Resolve circlecaptcha
- if "circlecaptcha" in form:
- self.captcha = True
- self.logDebug("Captcha protected")
- captcha_img_url = "http://ncrypt.in/classes/captcha/circlecaptcha.php"
- coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional')
- self.logDebug("Captcha resolved, coords [%s]" % str(coords))
- self.captcha_post_url = self.pyfile.url
-
- postData['circle.x'] = coords[0]
- postData['circle.y'] = coords[1]
-
-
- # Unlock protection
- postData['submit_protected'] = 'Continue to folder '
- return self.load(self.pyfile.url, post=postData)
-
- def handleErrors(self):
-
- if "This password is invalid!" in self.cleanedHtml:
- self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")
- self.fail("Incorrect password, please set right password on 'Edit package' form and retry")
-
- if self.captcha:
- if "The securitycheck was wrong!" in self.cleanedHtml:
- self.logDebug("Invalid captcha, retrying")
- self.invalidCaptcha()
- self.retry()
- else:
- self.correctCaptcha()
-
- def handleWebLinks(self):
- package_links = []
- self.logDebug("Handling Web links")
-
- pattern = r"(http://ncrypt\.in/link-.*?=)"
- links = re.findall(pattern, self.html)
- self.logDebug("Decrypting %d Web links" % len(links))
- for i, link in enumerate(links):
- self.logDebug("Decrypting Web link %d, %s" % (i+1, link))
- try:
- url = link.replace("link-", "frame-")
- link = self.load(url, just_header=True)['location']
- package_links.append(link)
- except Exception, detail:
- self.logDebug("Error decrypting Web link %s, %s" % (link, detail))
- return package_links
-
- def handleContainers(self):
- package_links = []
- self.logDebug("Handling Container links")
-
- pattern = r"/container/(rsdf|dlc|ccf)/([a-z0-9]+)"
- containersLinks = re.findall(pattern, self.html)
- self.logDebug("Decrypting %d Container links" % len(containersLinks))
- for containerLink in containersLinks:
- link = "http://ncrypt.in/container/%s/%s.%s" % (containerLink[0], containerLink[1], containerLink[0])
- package_links.append(link)
- return package_links
-
- def handleCNL2(self):
- package_links = []
- self.logDebug("Handling CNL2 links")
-
- if 'cnl2_output' in self.cleanedHtml:
- try:
- (vcrypted, vjk) = self._getCipherParams()
- for (crypted, jk) in zip(vcrypted, vjk):
- package_links.extend(self._getLinks(crypted, jk))
- except:
- self.fail("Unable to decrypt CNL2 links")
- return package_links
-
- def _getCipherParams(self):
-
- pattern = r'<input.*?name="%s".*?value="(.*?)"'
-
- # Get jk
- jk_re = pattern % NCryptIn._JK_KEY_
- vjk = re.findall(jk_re, self.html)
-
- # Get crypted
- crypted_re = pattern % NCryptIn._CRYPTED_KEY_
- vcrypted = re.findall(crypted_re, self.html)
-
- # Log and return
- self.logDebug("Detected %d crypted blocks" % len(vcrypted))
- return vcrypted, vjk
-
- def _getLinks(self, crypted, jk):
-
- # Get key
- jreturn = self.js.eval("%s f()" % jk)
- self.logDebug("JsEngine returns value [%s]" % jreturn)
- key = binascii.unhexlify(jreturn)
-
- # Decode crypted
- crypted = base64.standard_b64decode(crypted)
-
- # Decrypt
- Key = key
- IV = key
- obj = AES.new(Key, AES.MODE_CBC, IV)
- text = obj.decrypt(crypted)
-
- # Extract links
- text = text.replace("\x00", "").replace("\r", "")
- links = text.split("\n")
- links = filter(lambda x: x != "", links)
-
- # Log and return
- self.logDebug("Block has %d links" % len(links))
- return links
diff --git a/module/plugins/crypter/NetfolderIn.py b/module/plugins/crypter/NetfolderIn.py deleted file mode 100644 index c5c602c27..000000000 --- a/module/plugins/crypter/NetfolderIn.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class NetfolderIn(SimpleCrypter): - __name__ = "NetfolderIn" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?netfolder.in/((?P<id1>\w+)/\w+|folder.php\?folder_id=(?P<id2>\w+))" - __version__ = "0.6" - __description__ = """NetFolder Crypter Plugin""" - __author_name__ = ("RaNaN", "fragonib") - __author_mail__ = ("RaNaN@pyload.org", "fragonib[AT]yahoo[DOT]es") - - TITLE_PATTERN = r'<div class="Text">Inhalt des Ordners <span(.*)>(?P<title>.+)</span></div>' - - def decrypt(self, pyfile): - # Request package - self.html = self.load(pyfile.url) - - # Check for password protection - if self.isPasswordProtected(): - self.html = self.submitPassword() - if self.html is None: - self.fail("Incorrect password, please set right password on Add package form and retry") - - # Get package name and folder - (package_name, folder_name) = self.getPackageNameAndFolder() - - # Get package links - package_links = self.getLinks() - - # Set package - self.packages = [(package_name, package_links, folder_name)] - - def isPasswordProtected(self): - - if '<input type="password" name="password"' in self.html: - self.logDebug("Links are password protected") - return True - return False - - def submitPassword(self): - # Gather data - try: - m = re.match(self.__pattern__, self.pyfile.url) - id = max(m.group('id1'), m.group('id2')) - except AttributeError: - self.logDebug("Unable to get package id from url [%s]" % self.pyfile.url) - return - url = "http://netfolder.in/folder.php?folder_id=" + id - password = self.getPassword() - - # Submit package password - post = {'password': password, 'save': 'Absenden'} - self.logDebug("Submitting password [%s] for protected links with id [%s]" % (password, id)) - html = self.load(url, {}, post) - - # Check for invalid password - if '<div class="InPage_Error">' in html: - self.logDebug("Incorrect password, please set right password on Edit package form and retry") - return None - - return html - - def getLinks(self): - links = re.search(r'name="list" value="(.*?)"', self.html).group(1).split(",") - self.logDebug("Package has %d links" % len(links)) - return links diff --git a/module/plugins/crypter/OneKhDe.py b/module/plugins/crypter/OneKhDe.py deleted file mode 100644 index c77203187..000000000 --- a/module/plugins/crypter/OneKhDe.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.unescape import unescape -from module.plugins.Crypter import Crypter - -class OneKhDe(Crypter): - __name__ = "OneKhDe" - __type__ = "container" - __pattern__ = r"http://(www\.)?1kh.de/f/" - __version__ = "0.1" - __description__ = """1kh.de Container Plugin""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - - def file_exists(self): - """ returns True or False - """ - return True - - def proceed(self, url, location): - url = self.parent.url - self.html = self.req.load(url) - temp_links = [] - link_ids = re.findall(r"<a id=\"DownloadLink_(\d*)\" href=\"http://1kh.de/", self.html) - for id in link_ids: - new_link = unescape(re.search("width=\"100%\" src=\"(.*)\"></iframe>", self.req.load("http://1kh.de/l/" + id)).group(1)) - temp_links.append(new_link) - self.links = temp_links diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py deleted file mode 100755 index 726371966..000000000 --- a/module/plugins/crypter/OronComFolder.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class OronComFolder(Crypter): - __name__ = "OronComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?oron.com/folder/\w+" - __version__ = "0.2" - __description__ = """Oron.com Folder Plugin""" - __author_name__ = ("DHMH") - __author_mail__ = ("webmaster@pcProfil.de") - - FOLDER_PATTERN = r'<table(?:.*)class="tbl"(?:.*)>(?:.*)<table(?:.*)class="tbl2"(?:.*)>(?P<body>.*)</table>(?:.*)</table>' - LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' - - def decryptURL(self, url): - html = self.load(url) - - new_links = [] - - if 'No such folder exist' in html: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning("Folder does not exist") - return new_links - - folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning("Parse error (FOLDER)") - return new_links - - new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) - - if new_links: - self.logDebug("Found %d new links" % len(new_links)) - return new_links - else: - # Don't fail because if there's more than a folder for this package - # and only one of them fails, no urls at all will be added. - self.logWarning('Could not extract any links') - return new_links diff --git a/module/plugins/crypter/QuickshareCzFolder.py b/module/plugins/crypter/QuickshareCzFolder.py deleted file mode 100644 index 6cb049935..000000000 --- a/module/plugins/crypter/QuickshareCzFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class QuickshareCzFolder(Crypter): - __name__ = "QuickshareCzFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?quickshare.cz/slozka-\d+.*" - __version__ = "0.1" - __description__ = """Quickshare.cz Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - - FOLDER_PATTERN = r'<textarea[^>]*>(.*?)</textarea>' - LINK_PATTERN = r'(http://www.quickshare.cz/\S+)' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - - if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/RSDF.py b/module/plugins/crypter/RSDF.py deleted file mode 100644 index cbc9864b1..000000000 --- a/module/plugins/crypter/RSDF.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import base64 -import binascii -import re - -from module.plugins.Crypter import Crypter - -class RSDF(Crypter): - __name__ = "RSDF" - __version__ = "0.21" - __pattern__ = r".*\.rsdf" - __description__ = """RSDF Container Decode Plugin""" - __author_name__ = ("RaNaN", "spoob") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org") - - - def decrypt(self, pyfile): - - from Crypto.Cipher import AES - - infile = pyfile.url.replace("\n", "") - Key = binascii.unhexlify('8C35192D964DC3182C6F84F3252239EB4A320D2500000000') - - IV = binascii.unhexlify('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') - IV_Cipher = AES.new(Key, AES.MODE_ECB) - IV = IV_Cipher.encrypt(IV) - - obj = AES.new(Key, AES.MODE_CFB, IV) - - rsdf = open(infile, 'r') - - data = rsdf.read() - rsdf.close() - - if re.search(r"<title>404 - Not Found</title>", data) is None: - data = binascii.unhexlify(''.join(data.split())) - data = data.splitlines() - - links = [] - for link in data: - link = base64.b64decode(link) - link = obj.decrypt(link) - decryptedUrl = link.replace('CCF: ', '') - links.append(decryptedUrl) - - self.log.debug("%s: adding package %s with %d links" % (self.__name__,pyfile.package().name,len(links))) - self.packages.append((pyfile.package().name, links)) diff --git a/module/plugins/crypter/RSLayerCom.py b/module/plugins/crypter/RSLayerCom.py deleted file mode 100644 index 6e4266f2e..000000000 --- a/module/plugins/crypter/RSLayerCom.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup -from module.unescape import unescape - -class RSLayerCom(Crypter): - __name__ = "RSLayerCom" - __type__ = "container" - __pattern__ = r"http://(www\.)?rs-layer.com/directory-" - __config__ = [] - __version__ = "0.2" - __description__ = """RS-Layer.com Container Plugin""" - __author_name__ = ("hzpz") - __author_mail__ = ("none") - - def decrypt(self, pyfile): - url = pyfile.url - src = self.req.load(str(url)) - - soup = BeautifulSoup(src) - captchaTag = soup.find("img", attrs={"id": "captcha_image"}) - if captchaTag: - captchaUrl = "http://rs-layer.com/" + captchaTag["src"] - self.logDebug("Captcha URL: %s" % captchaUrl) - result = self.decryptCaptcha(str(captchaUrl), imgtype="png") - captchaInput = soup.find("input", attrs={"id": "captcha"}) - self.req.lastUrl = url - src = self.req.load(str(url), post={'captcha_input': result, 'image_name': captchaTag["src"]}) - - link_ids = re.findall(r"onclick=\"getFile\(\'([0-9]{7}-.{8})\'\);changeBackgroundColor", src) - - if not len(link_ids) > 0: - self.retry() - - self.correctCaptcha() - - links = [] - for id in link_ids: - self.logDebug("ID: %s" % id) - new_link = unescape(re.search(r"<iframe style=\"width: 100%; height: 100%;\" src=\"(.*)\"></frame>", - self.req.load("http://rs-layer.com/link-" + id + ".html")).group(1)) - self.logDebug("Link: %s" % new_link) - links.append(new_link) - - self.packages.append((self.pyfile.package().name, links, self.pyfile.package().folder)) diff --git a/module/plugins/crypter/RelinkUs.py b/module/plugins/crypter/RelinkUs.py deleted file mode 100644 index 8f29a9158..000000000 --- a/module/plugins/crypter/RelinkUs.py +++ /dev/null @@ -1,264 +0,0 @@ -# -*- coding: utf-8 -*- - -from Crypto.Cipher import AES -from module.plugins.Crypter import Crypter -import base64 -import binascii -import re -import os - - -class RelinkUs(Crypter): - __name__ = "RelinkUs" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?relink.us/(f/|((view|go).php\?id=))(?P<id>.+)" - __version__ = "3.0" - __description__ = """Relink.us Crypter Plugin""" - __author_name__ = ("fragonib") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es") - - # Constants - PREFERRED_LINK_SOURCES = ['cnl2', 'dlc', 'web'] - - OFFLINE_TOKEN = "<title>Tattooside" - PASSWORD_TOKEN = "container_password.php" - PASSWORD_ERROR_ROKEN = "You have entered an incorrect password" - PASSWORD_SUBMIT_URL = "http://www.relink.us/container_password.php" - CAPTCHA_TOKEN = "container_captcha.php" - CAPTCHA_ERROR_ROKEN = "You have solved the captcha wrong" - CAPTCHA_IMG_URL = "http://www.relink.us/core/captcha/circlecaptcha.php" - CAPTCHA_SUBMIT_URL = "http://www.relink.us/container_captcha.php" - FILE_TITLE_REGEX = r"<th>Title</th><td><i>(.*)</i></td></tr>" - FILE_NOTITLE = 'No title' - - CNL2_FORM_REGEX = r'<form id="cnl_form-(.*?)</form>' - CNL2_FORMINPUT_REGEX = r'<input.*?name="%s".*?value="(.*?)"' - CNL2_JK_KEY = "jk" - CNL2_CRYPTED_KEY = "crypted" - DLC_LINK_REGEX = r'<a href=".*?" class="dlc_button" target="_blank">' - DLC_DOWNLOAD_URL = "http://www.relink.us/download.php" - WEB_FORWARD_REGEX = r"getFile\('(?P<link>.+)'\)"; - WEB_FORWARD_URL = "http://www.relink.us/frame.php" - WEB_LINK_REGEX = r'<iframe name="Container" height="100%" frameborder="no" width="100%" src="(?P<link>.+)"></iframe>' - - - def setup(self): - self.fileid = None - self.package = None - self.password = None - self.html = None - self.captcha = False - - def decrypt(self, pyfile): - - # Init - self.initPackage(pyfile) - - # Request package - self.requestPackage() - - # Check for online - if not self.isOnline(): - self.offline() - - # Check for protection - if self.isPasswordProtected(): - self.unlockPasswordProtection() - self.handleErrors() - - if self.isCaptchaProtected(): - self.captcha = True - self.unlockCaptchaProtection() - self.handleErrors() - - # Get package name and folder - (package_name, folder_name) = self.getPackageInfo() - - # Extract package links - package_links = [] - for sources in self.PREFERRED_LINK_SOURCES: - package_links.extend(self.handleLinkSource(sources)) - if package_links: # use only first source which provides links - break - package_links = set(package_links) - - # Pack - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') - - def initPackage(self, pyfile): - self.fileid = re.match(self.__pattern__, pyfile.url).group('id') - self.package = pyfile.package() - self.password = self.getPassword() - self.url = pyfile.url - - def requestPackage(self): - self.html = self.load(self.url, decode = True) - - def isOnline(self): - if self.OFFLINE_TOKEN in self.html: - self.logDebug("File not found") - return False - return True - - def isPasswordProtected(self): - if self.PASSWORD_TOKEN in self.html: - self.logDebug("Links are password protected") - return True - - def isCaptchaProtected(self): - if self.CAPTCHA_TOKEN in self.html: - self.logDebug("Links are captcha protected") - return True - return False - - def unlockPasswordProtection(self): - self.logDebug("Submitting password [%s] for protected links" % self.password) - passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.fileid - passwd_data = { 'id': self.fileid, 'password': self.password, 'pw': 'submit' } - self.html = self.load(passwd_url, post=passwd_data, decode=True) - - def unlockCaptchaProtection(self): - self.logDebug("Request user positional captcha resolving") - captcha_img_url = self.CAPTCHA_IMG_URL + "?id=%s" % self.fileid - coords = self.decryptCaptcha(captcha_img_url, forceUser=True, imgtype="png", result_type='positional') - self.logDebug("Captcha resolved, coords [%s]" % str(coords)) - captcha_post_url = self.CAPTCHA_SUBMIT_URL + "?id=%s" % self.fileid - captcha_post_data = { 'button.x': coords[0], 'button.y': coords[1], 'captcha': 'submit' } - self.html = self.load(captcha_post_url, post=captcha_post_data, decode=True) - - def getPackageInfo(self): - name = folder = None - - # Try to get info from web - m = re.search(self.FILE_TITLE_REGEX, self.html) - if m is not None: - title = m.group(1).strip() - if not self.FILE_NOTITLE in title: - name = folder = title - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - - # Fallback to defaults - if not name or not folder: - name = self.package.name - folder = self.package.folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - - # Return package info - return name, folder - - def handleErrors(self): - if self.PASSWORD_ERROR_ROKEN in self.html: - msg = "Incorrect password, please set right password on 'Edit package' form and retry" - self.logDebug(msg) - self.fail(msg) - - if self.captcha: - if self.CAPTCHA_ERROR_ROKEN in self.html: - self.logDebug("Invalid captcha, retrying") - self.invalidCaptcha() - self.retry() - else: - self.correctCaptcha() - - def handleLinkSource(self, source): - if source == 'cnl2': - return self.handleCNL2Links() - elif source == 'dlc': - return self.handleDLCLinks() - elif source == 'web': - return self.handleWEBLinks() - else: - self.fail('Unknown source [%s] (this is probably a bug)' % source) - - def handleCNL2Links(self): - self.logDebug("Search for CNL2 links") - package_links = [] - m = re.search(self.CNL2_FORM_REGEX, self.html, re.DOTALL) - if m is not None: - cnl2_form = m.group(1) - try: - (vcrypted, vjk) = self._getCipherParams(cnl2_form) - for (crypted, jk) in zip(vcrypted, vjk): - package_links.extend(self._getLinks(crypted, jk)) - except: - self.logDebug("Unable to decrypt CNL2 links") - return package_links - - def handleDLCLinks(self): - self.logDebug('Search for DLC links') - package_links = [] - m = re.search(self.DLC_LINK_REGEX, self.html) - if m is not None: - container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.fileid - self.logDebug("Downloading DLC container link [%s]" % container_url) - try: - dlc = self.load(container_url) - dlc_filename = self.fileid + ".dlc" - dlc_filepath = os.path.join(self.config["general"]["download_folder"], dlc_filename) - f = open(dlc_filepath, "wb") - f.write(dlc) - f.close() - package_links.append(dlc_filepath) - except: - self.logDebug("Unable to download DLC container") - return package_links - - def handleWEBLinks(self): - self.logDebug("Search for WEB links") - package_links = [] - fw_params = re.findall(self.WEB_FORWARD_REGEX, self.html) - self.logDebug("Decrypting %d Web links" % len(fw_params)) - for index, fw_param in enumerate(fw_params): - try: - fw_url = self.WEB_FORWARD_URL + "?%s" % fw_param - self.logDebug("Decrypting Web link %d, %s" % (index+1, fw_url)) - fw_response = self.load(fw_url, decode=True) - dl_link = re.search(self.WEB_LINK_REGEX, fw_response).group('link') - package_links.append(dl_link) - except Exception, detail: - self.logDebug("Error decrypting Web link %s, %s" % (index, detail)) - self.setWait(4) - self.wait() - return package_links - - def _getCipherParams(self, cnl2_form): - - # Get jk - jk_re = self.CNL2_FORMINPUT_REGEX % self.CNL2_JK_KEY - vjk = re.findall(jk_re, cnl2_form, re.IGNORECASE) - - # Get crypted - crypted_re = self.CNL2_FORMINPUT_REGEX % RelinkUs.CNL2_CRYPTED_KEY - vcrypted = re.findall(crypted_re, cnl2_form, re.IGNORECASE) - - # Log and return - self.logDebug("Detected %d crypted blocks" % len(vcrypted)) - return vcrypted, vjk - - def _getLinks(self, crypted, jk): - - # Get key - jreturn = self.js.eval("%s f()" % jk) - self.logDebug("JsEngine returns value [%s]" % jreturn) - key = binascii.unhexlify(jreturn) - - # Decode crypted - crypted = base64.standard_b64decode(crypted) - - # Decrypt - Key = key - IV = key - obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) - - # Extract links - text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) - - # Log and return - self.logDebug("Package has %d links" % len(links)) - return links diff --git a/module/plugins/crypter/SecuredIn.py b/module/plugins/crypter/SecuredIn.py deleted file mode 100644 index e41896c5f..000000000 --- a/module/plugins/crypter/SecuredIn.py +++ /dev/null @@ -1,334 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup - -from math import ceil - -class SecuredIn(Crypter): - __name__ = "SecuredIn" - __type__ = "container" - __pattern__ = r"http://[\w\.]*?secured\.in/download-[\d]+-[\w]{8}\.html" - __version__ = "0.1" - __description__ = """secured.in Container Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - self.multi_dl = False - - def file_exists(self): - return True - - def proceed(self, url, location): - links = [] - ajaxUrl = "http://secured.in/ajax-handler.php" - src = self.req.load(url, cookies=True) - soup = BeautifulSoup(src) - img = soup.find("img", attrs={"id":"captcha_img"}) - for i in range(3): - form = soup.find("form", attrs={"id":"frm_captcha"}) - captchaHash = form.find("input", attrs={"id":"captcha_hash"})["value"] - captchaUrl = "http://secured.in/%s" % img["src"] - captchaData = self.req.load(str(captchaUrl), cookies=True) - result = self.waitForCaptcha(captchaData, "jpg") - src = self.req.load(url, cookies=True, post={"captcha_key":result, "captcha_hash":captchaHash}) - soup = BeautifulSoup(src) - img = soup.find("img", attrs={"id":"captcha_img"}) - if not img: - files = soup.findAll("tr", attrs={"id":re.compile("file-\d+")}) - dlIDPattern = re.compile("accessDownload\(\d, \d+, '(.*?)', \d\)") - cypher = self.Cypher() - for cfile in files: - m = dlIDPattern.search(cfile["onclick"]) - if m: - crypted = self.req.load(ajaxUrl, cookies=True, post={"cmd":"download", "download_id":m.group(1)}) - cypher.reset() - link = cypher.cypher(crypted) - links.append(link) - break - self.links = links - - class Cypher(): - def __init__(self): - self.reset() - - def reset(self): - self.iatwbfrd = [ - 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, - 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, - 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, - 0xbd314b27, 0x78af2fda, 0x55605c60, 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, - 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, 0x7a325381, - 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, - 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, - 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, - 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f, - 0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, 0x075372c9, 0x80991b7b, - 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, - 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, - 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, - 0xdb3222f8, 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, - 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, - 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, - 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, - 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, - 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, - 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, - 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, - 0x02e5b9c5, 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, - 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, 0x53b02d5d, - 0xa99f8fa1, 0x08ba4799, 0x6e85076a - ] - - self.olkemfjq = [ - 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, - 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b - ] - - self.oqlaoymh = 0 - self.oqmykrna = 0 - self.pqmyzkid = 0 - self.pldmjnde = 0 - self.ldiwkqly = 0 - - self.plkodnyq = [ - 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, - 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, - 0xd5730a1d, 0x4cd04dc6, 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, - 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, - 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, - 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, - 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, - 0x88f46dba, 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, - 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, 0x4de81751, 0x3830dc8e, 0x379d5862, - 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, - 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, 0xdda26a7e, - 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, - 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, - 0xce6ea048, 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, - 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, - 0xc6913667, 0x8df9317c, 0xe0b12b4f, 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, - 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, - 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, - 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, - 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, - 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c, - 0xcd769c2b, 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, 0x85cbfe4e, 0x8ae88dd8, - 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, - 0xce77e25b, 0x578fdfe3, 0x3ac372e6 - ] - - self.pnjzokye = None - - self.thdlpsmy = [ - 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, - 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, - 0x7fac6dd0, 0x31cb8504, 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, - 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, - 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, - 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, - 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, - 0xe4c66d22, 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, - 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, 0x6b2395e0, 0x333e92e1, 0x3b240b62, - 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f, 0x877d48fa, - 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, 0xc67b5510, - 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, - 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, - 0x782ef11c, 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, - 0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, 0xd1fd8346, - 0xf6381fb0, 0x7745ae04, 0xd736fccc, 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, - 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, - 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, - 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, - 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, - 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, - 0xce591d76, 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, 0xed545578, 0x08fca5b5, - 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234, - 0x92638212, 0x670efa8e, 0x406000e0 - ] - - self.ybghjtik = [ - 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, - 0xc2b19ee1, 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, - 0x4d2d38e6, 0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, 0x687f3584, - 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, - 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, - 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, - 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, - 0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, - 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, - 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, - 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, - 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, - 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, - 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, - 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, - 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, - 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, 0x8fd948e4, - 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, - 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, - 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, - 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460, 0x4085f2a7, - 0xce77326e, 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, 0x9e447a2e, 0xc3453484, - 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, - 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7 - ] - - def cypher(self, code): - return self.lskdqpyr(code, "") - - def lskdqpyr(self, alokfmth, yoaksjdh): - if self.pnjzokye is None or self.pnjzokye.lower() == yoaksjdh: - self.yoliukev(yoaksjdh) - self.pnjzokye = yoaksjdh - alokfmth = self.plaomtje(alokfmth) - ykaiumgp = "" - alokijuh = len(alokfmth) - lokimyas = self.ylomiktb(alokfmth[0:8]) - palsiuzt = lokimyas[0] - tzghbndf = lokimyas[1] - awsedrft = [None, None] - for kiujzhqa in range(8, alokijuh, 8): - lokimyas = self.ylomiktb(alokfmth[kiujzhqa:kiujzhqa+8]) - awsedrft[0] = lokimyas[0] - awsedrft[1] = lokimyas[1] - lokimyas = self.okaqnhlp(lokimyas[0], lokimyas[1]) - lokimyas[0] ^= palsiuzt - lokimyas[1] ^= tzghbndf - palsiuzt = awsedrft[0] - tzghbndf = awsedrft[1] - ykaiumgp += self.ykijmtkd(lokimyas) - return ykaiumgp - - def okaqnhlp(self, lahgrnvp, trenlpys): - ujhaqylw = 0 - for yalmhopr in range(17, 1, -1): - lahgrnvp ^= self.ldiwkqly[yalmhopr] - trenlpys ^= (self.oqlaoymh[lahgrnvp >> 24 & 0xff] + self.oqmykrna[lahgrnvp >> 16 & 0xff] ^ self.pqmyzkid[lahgrnvp >> 8 & 0xff]) + self.pldmjnde[lahgrnvp & 0xff] - ujhaqylw = lahgrnvp - lahgrnvp = trenlpys - trenlpys = ujhaqylw - ujhaqylw = lahgrnvp - lahgrnvp = trenlpys - trenlpys = ujhaqylw - trenlpys ^= self.ldiwkqly[1] - lahgrnvp ^= self.ldiwkqly[0] - return [lahgrnvp, trenlpys] - - def plaomtje(self, yoiumqpy): - qkailkzt = "" - xoliuzem = 0 - lyomiujt = 0 - yploemju = -1 - for i in range(0, len(yoiumqpy)): - yploamzu = ord(yoiumqpy[i]) - if ord('A') <= yploamzu <= ord('Z'): - xoliuzem = ord(yoiumqpy[i]) - 65 - elif ord('a') <= yploamzu <= ord('z'): - xoliuzem = ord(yoiumqpy[i]) - 97 + 26 - elif ord('0') <= yploamzu <= ord('9'): - xoliuzem = ord(yoiumqpy[i]) - 48 + 52 - elif yploamzu == ord('+'): - xoliuzem = 62 - elif yploamzu == ord('/'): - xoliuzem = 63 - else: - continue - yploemju += 1 - - lxkdmizj = 0 - switch = yploemju % 4 - if switch == 0: - lyomiujt = xoliuzem - continue - elif switch == 1: - lxkdmizj = lyomiujt << 2 | xoliuzem >> 4 - lyomiujt = xoliuzem & 0x0F - elif switch == 2: - lxkdmizj = lyomiujt << 4 | xoliuzem >> 2 - lyomiujt = xoliuzem & 0x03 - elif switch == 3: - lxkdmizj = lyomiujt << 6 | xoliuzem >> 0 - lyomiujt = xoliuzem & 0x00 - qkailkzt += unichr(lxkdmizj) - return qkailkzt - - def qmyjuila(self, oqlamykt, yalkionj): - dolizmvw = 0 - for iumswkya in range(0, 16): - oqlamykt ^= self.ldiwkqly[iumswkya] - yalkionj ^= (self.oqlaoymh[oqlamykt >> 24 & 0xff] + self.oqmykrna[oqlamykt >> 16 & 0xff] ^ self.pqmyzkid[oqlamykt >> 8 & 0xff]) + self.pldmjnde[oqlamykt & 0xff] - dolizmvw = oqlamykt - oqlamykt = yalkionj - yalkionj = dolizmvw - dolizmvw = oqlamykt - oqlamykt = yalkionj - yalkionj = dolizmvw - yalkionj ^= self.ldiwkqly[16] - oqlamykt ^= self.ldiwkqly[17] - return [oqlamykt, yalkionj] - - def ykijmtkd(self, yoirlkqw): - loipamyu = len(yoirlkqw) - yoirlkqwchar = [] - for ymujtnbq in range(0, loipamyu): - yoir = [yoirlkqw[ymujtnbq] >> 24 & 0xff, yoirlkqw[ymujtnbq] >> 16 & 0xff, yoirlkqw[ymujtnbq] >> 8 & 0xff, yoirlkqw[ymujtnbq] & 0xff] - for c in yoir: - yoirlkqwchar.append(chr(c)) - return "".join(yoirlkqwchar) - - def ylomiktb(self, lofiuzmq): - plokimqw = int(ceil(len(lofiuzmq) / 4.0)) - lopkisdq = [] - for ypoqlktz in range(0, plokimqw): - lopkisdq.append(ord(lofiuzmq[(ypoqlktz << 2) + 3]) + (ord(lofiuzmq[(ypoqlktz << 2) + 2]) << 8) + (ord(lofiuzmq[(ypoqlktz << 2) + 1]) << 16) + (ord(lofiuzmq[(ypoqlktz << 2)]) << 24)) - return lopkisdq - - def yoliukev(self, kaiumylq): - self.oqlaoymh = self.iatwbfrd - self.oqmykrna = self.ybghjtik - self.pqmyzkid = self.thdlpsmy - self.pldmjnde = self.plkodnyq - - yaqpolft = [0 for i in range(len(kaiumylq))] - - yaqwsedr = 0 - btzqwsay = 0 - while yaqwsedr < len(kaiumylq): - wlqoakmy = 0 - for lopiuztr in range(0, 4): - wlqoakmy = wlqoakmy << 8 | ord(kaiumylq[yaqwsedr % len(kaiumylq)]) - yaqwsedr += 1 - yaqpolft[btzqwsay] = wlqoakmy - btzqwsay += 1 - self.ldiwkqly = [] - for btzqwsay in range(0, 18): - self.ldiwkqly.append(self.olkemfjq[btzqwsay]) - yalopiuq = [0, 0] - for btzqwsay in range(0, 18, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.ldiwkqly[btzqwsay] = yalopiuq[0] - self.ldiwkqly[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.oqlaoymh[btzqwsay] = yalopiuq[0] - self.oqlaoymh[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.oqmykrna[btzqwsay] = yalopiuq[0] - self.oqmykrna[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.pqmyzkid[btzqwsay] = yalopiuq[0] - self.pqmyzkid[btzqwsay + 1] = yalopiuq[1] - for btzqwsay in range(0, 256, 2): - yalopiuq = self.qmyjuila(yalopiuq[0], yalopiuq[1]) - self.pldmjnde[btzqwsay] = yalopiuq[0] - self.pldmjnde[btzqwsay + 1] = yalopiuq[1] - diff --git a/module/plugins/crypter/SerienjunkiesOrg.py b/module/plugins/crypter/SerienjunkiesOrg.py deleted file mode 100644 index 3fcc12e36..000000000 --- a/module/plugins/crypter/SerienjunkiesOrg.py +++ /dev/null @@ -1,321 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from time import sleep -import random -from module.plugins.Crypter import Crypter -from module.lib.BeautifulSoup import BeautifulSoup -from module.unescape import unescape - -class SerienjunkiesOrg(Crypter): - __name__ = "SerienjunkiesOrg" - __type__ = "container" - __pattern__ = r"http://.*?(serienjunkies.org|dokujunkies.org)/.*?" - __version__ = "0.38" - __config__ = [ - ("changeNameSJ", "Packagename;Show;Season;Format;Episode", "Take SJ.org name", "Show"), - ("changeNameDJ", "Packagename;Show;Format;Episode", "Take DJ.org name", "Show"), - ("randomPreferred", "bool", "Randomize Preferred-List", False), - ("hosterListMode", "OnlyOne;OnlyPreferred(One);OnlyPreferred(All);All", "Use for hosters (if supported)", "All"), - ("hosterList", "str", "Preferred Hoster list (comma separated)", "RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom"), - ("ignoreList", "str", "Ignored Hoster list (comma separated)", "MegauploadCom") - ] - __description__ = """serienjunkies.org Container Plugin""" - __author_name__ = ("mkaay", "godofdream") - __author_mail__ = ("mkaay@mkaay.de", "soilfiction@gmail.com") - - - def setup(self): - self.multiDL = False - - def getSJSrc(self, url): - src = self.req.load(str(url)) - if "This website is not available in your country" in src: - self.fail("Not available in your country") - if not src.find("Enter Serienjunkies") == -1: - sleep(1) - src = self.req.load(str(url)) - return src - - def handleShow(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - packageName = self.pyfile.package().name - if self.getConfig("changeNameSJ") == "Show": - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) - if found: - packageName = found - - nav = soup.find("div", attrs={"id": "scb"}) - - package_links = [] - for a in nav.findAll("a"): - if self.getConfig("changeNameSJ") == "Show": - package_links.append(a["href"]) - else: - package_links.append(a["href"] + "#hasName") - if self.getConfig("changeNameSJ") == "Show": - self.packages.append((packageName, package_links, packageName)) - else: - self.core.files.addLinks(package_links, self.pyfile.package().id) - - - def handleSeason(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - post = soup.find("div", attrs={"class": "post-content"}) - ps = post.findAll("p") - - seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") - groups = {} - gid = -1 - for p in ps: - if re.search("<strong>Sprache|<strong>Format", str(p)): - var = p.findAll("strong") - opts = {"Sprache": "", "Format": ""} - for v in var: - n = unescape(v.string).strip() - n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) - if n.strip() not in opts: - continue - val = v.nextSibling - if not val: - continue - val = val.replace("|", "").strip() - val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) - opts[n.strip()] = val.strip() - gid += 1 - groups[gid] = {} - groups[gid]["ep"] = {} - groups[gid]["opts"] = opts - elif re.search("<strong>Download:", str(p)): - parts = str(p).split("<br />") - if re.search("<strong>", parts[0]): - ename = re.search('<strong>(.*?)</strong>',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") - groups[gid]["ep"][ename] = {} - parts.remove(parts[0]) - for part in parts: - hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) - if hostername: - hostername = hostername.group(1) - groups[gid]["ep"][ename][hostername] = [] - links = re.findall('href="(.*?)"',part) - for link in links: - groups[gid]["ep"][ename][hostername].append(link + "#hasName") - - links = [] - for g in groups.values(): - for ename in g["ep"]: - links.extend(self.getpreferred(g["ep"][ename])) - if self.getConfig("changeNameSJ") == "Episode": - self.packages.append((ename, links, ename)) - links = [] - package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) - if self.getConfig("changeNameSJ") == "Format": - self.packages.append((package, links, package)) - links = [] - if (self.getConfig("changeNameSJ") == "Packagename") or re.search("#hasName", url): - self.core.files.addLinks(links, self.pyfile.package().id) - elif (self.getConfig("changeNameSJ") == "Season") or not re.search("#hasName", url): - self.packages.append((seasonName, links, seasonName)) - - def handleEpisode(self, url): - src = self.getSJSrc(url) - if not src.find( - "Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: - self.fail(_("Downloadlimit reached")) - else: - soup = BeautifulSoup(src) - form = soup.find("form") - h1 = soup.find("h1") - - if h1.get("class") == "wrap": - captchaTag = soup.find(attrs={"src": re.compile("^/secure/")}) - if not captchaTag: - sleep(5) - self.retry() - - captchaUrl = "http://download.serienjunkies.org" + captchaTag["src"] - result = self.decryptCaptcha(str(captchaUrl), imgtype="png") - sinp = form.find(attrs={"name": "s"}) - - self.req.lastURL = str(url) - sj = self.load(str(url), post={'s': sinp["value"], 'c': result, 'action': "Download"}) - - soup = BeautifulSoup(sj) - rawLinks = soup.findAll(attrs={"action": re.compile("^http://download.serienjunkies.org/")}) - - if not len(rawLinks) > 0: - sleep(1) - self.retry() - return - - self.correctCaptcha() - - links = [] - for link in rawLinks: - frameUrl = link["action"].replace("/go-", "/frame/go-") - links.append(self.handleFrame(frameUrl)) - if re.search("#hasName", url) or ((self.getConfig("changeNameSJ") == "Packagename") and (self.getConfig("changeNameDJ") == "Packagename")): - self.core.files.addLinks(links, self.pyfile.package().id) - else: - if h1.text[2] == "_": - eName = h1.text[3:] - else: - eName = h1.text - self.packages.append((eName, links, eName)) - - - def handleOldStyleLink(self, url): - sj = self.req.load(str(url)) - soup = BeautifulSoup(sj) - form = soup.find("form", attrs={"action": re.compile("^http://serienjunkies.org")}) - captchaTag = form.find(attrs={"src": re.compile("^/safe/secure/")}) - captchaUrl = "http://serienjunkies.org" + captchaTag["src"] - result = self.decryptCaptcha(str(captchaUrl)) - url = form["action"] - sinp = form.find(attrs={"name": "s"}) - - self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, - just_header=True) - decrypted = self.req.lastEffectiveURL - if decrypted == str(url): - self.retry() - self.core.files.addLinks([decrypted], self.pyfile.package().id) - - def handleFrame(self, url): - self.req.load(str(url)) - return self.req.lastEffectiveURL - - def handleShowDJ(self, url): - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - post = soup.find("div", attrs={"id": "page_post"}) - ps = post.findAll("p") - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) - if found: - seasonName = found - - groups = {} - gid = -1 - for p in ps: - if re.search("<strong>Sprache|<strong>Format", str(p)): - var = p.findAll("strong") - opts = {"Sprache": "", "Format": ""} - for v in var: - n = unescape(v.string).strip() - n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) - if n.strip() not in opts: - continue - val = v.nextSibling - if not val: - continue - val = val.replace("|", "").strip() - val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) - opts[n.strip()] = val.strip() - gid += 1 - groups[gid] = {} - groups[gid]["ep"] = {} - groups[gid]["opts"] = opts - elif re.search("<strong>Download:", str(p)): - parts = str(p).split("<br />") - if re.search("<strong>", parts[0]): - ename = re.search('<strong>(.*?)</strong>',parts[0]).group(1).strip().decode("utf-8").replace("–", "-") - groups[gid]["ep"][ename] = {} - parts.remove(parts[0]) - for part in parts: - hostername = re.search(" \| ([-a-zA-Z0-9]+\.\w+)",part) - if hostername: - hostername = hostername.group(1) - groups[gid]["ep"][ename][hostername] = [] - links = re.findall('href="(.*?)"',part) - for link in links: - groups[gid]["ep"][ename][hostername].append(link + "#hasName") - - links = [] - for g in groups.values(): - for ename in g["ep"]: - links.extend(self.getpreferred(g["ep"][ename])) - if self.getConfig("changeNameDJ") == "Episode": - self.packages.append((ename, links, ename)) - links = [] - package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) - if self.getConfig("changeNameDJ") == "Format": - self.packages.append((package, links, package)) - links = [] - if (self.getConfig("changeNameDJ") == "Packagename") or re.search("#hasName", url): - self.core.files.addLinks(links, self.pyfile.package().id) - elif (self.getConfig("changeNameDJ") == "Show") or not re.search("#hasName", url): - self.packages.append((seasonName, links, seasonName)) - - - - - - - - def handleCategoryDJ(self, url): - package_links = [] - src = self.getSJSrc(url) - soup = BeautifulSoup(src) - content = soup.find("div", attrs={"id": "content"}) - for a in content.findAll("a", attrs={"rel": "bookmark"}): - package_links.append(a["href"]) - self.core.files.addLinks(package_links, self.pyfile.package().id) - - def decrypt(self, pyfile): - showPattern = re.compile("^http://serienjunkies.org/serie/(.*)/$") - seasonPattern = re.compile("^http://serienjunkies.org/.*?/(.*)/$") - episodePattern = re.compile("^http://download.serienjunkies.org/f-.*?.html(#hasName)?$") - oldStyleLink = re.compile("^http://serienjunkies.org/safe/(.*)$") - categoryPatternDJ = re.compile("^http://dokujunkies.org/.*?(.*)$") - showPatternDJ = re.compile("^http://dokujunkies.org/.*?/(.*)\.html(#hasName)?$") - framePattern = re.compile("^http://download.(serienjunkies.org|dokujunkies.org)/frame/go-.*?/$") - url = pyfile.url - if framePattern.match(url): - self.packages.append((self.pyfile.package().name, [self.handleFrame(url)], self.pyfile.package().name)) - elif episodePattern.match(url): - self.handleEpisode(url) - elif oldStyleLink.match(url): - self.handleOldStyleLink(url) - elif showPattern.match(url): - self.handleShow(url) - elif showPatternDJ.match(url): - self.handleShowDJ(url) - elif seasonPattern.match(url): - self.handleSeason(url) - elif categoryPatternDJ.match(url): - self.handleCategoryDJ(url) - - #selects the preferred hoster, after that selects any hoster (ignoring the one to ignore) - def getpreferred(self, hosterlist): - - result = [] - preferredList = self.getConfig("hosterList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if (self.getConfig("randomPreferred") == True) and (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]) : - random.shuffle(preferredList) - # we don't want hosters be read two times - hosterlist2 = hosterlist.copy() - - for preferred in preferredList: - for Hoster in hosterlist: - if preferred == Hoster.lower().replace('.',''): - for Part in hosterlist[Hoster]: - self.logDebug("selected " + Part) - result.append(str(Part)) - del(hosterlist2[Hoster]) - if (self.getConfig("hosterListMode") in ["OnlyOne","OnlyPreferred(One)"]): - return result - - - ignorelist = self.getConfig("ignoreList").strip().lower().replace('|',',').replace('.','').replace(';',',').split(',') - if self.getConfig('hosterListMode') in ["OnlyOne","All"]: - for Hoster in hosterlist2: - if Hoster.strip().lower().replace('.','') not in ignorelist: - for Part in hosterlist2[Hoster]: - self.logDebug("selected2 " + Part) - result.append(str(Part)) - - if self.getConfig('hosterListMode') == "OnlyOne": - return result - return result diff --git a/module/plugins/crypter/ShareLinksBiz.py b/module/plugins/crypter/ShareLinksBiz.py deleted file mode 100644 index 1ffa5d41a..000000000 --- a/module/plugins/crypter/ShareLinksBiz.py +++ /dev/null @@ -1,269 +0,0 @@ -# -*- coding: utf-8 -*-
-
-from Crypto.Cipher import AES
-from module.plugins.Crypter import Crypter
-from module.plugins.ReCaptcha import ReCaptcha
-import base64
-import binascii
-import re
-
-
-class ShareLinksBiz(Crypter):
- __name__ = "ShareLinksBiz"
- __type__ = "crypter"
- __pattern__ = r"(?P<base>http://[\w\.]*?(share-links|s2l)\.biz)/(?P<id>_?[0-9a-z]+)(/.*)?"
- __version__ = "1.12"
- __description__ = """Share-Links.biz Crypter"""
- __author_name__ = ("fragonib")
- __author_mail__ = ("fragonib[AT]yahoo[DOT]es")
-
-
- def setup(self):
- self.baseUrl = None
- self.fileId = None
- self.package = None
- self.html = None
- self.captcha = False
-
- def decrypt(self, pyfile):
-
- # Init
- self.initFile(pyfile)
-
- # Request package
- url = self.baseUrl + '/' + self.fileId
- self.html = self.load(url, decode=True)
-
- # Unblock server (load all images)
- self.unblockServer()
-
- # Check for protection
- if self.isPasswordProtected():
- self.unlockPasswordProtection()
- self.handleErrors()
-
- if self.isCaptchaProtected():
- self.captcha = True
- self.unlockCaptchaProtection()
- self.handleErrors()
-
- # Extract package links
- package_links = []
- package_links.extend(self.handleWebLinks())
- package_links.extend(self.handleContainers())
- package_links.extend(self.handleCNL2())
- package_links = set(package_links)
-
- # Get package info
- package_name, package_folder = self.getPackageInfo()
-
- # Pack
- self.packages = [(package_name, package_links, package_folder)]
-
- def initFile(self, pyfile):
- url = pyfile.url
- if 's2l.biz' in url:
- url = self.load(url, just_header=True)['location']
- self.baseUrl = re.search(self.__pattern__, url).group(1)
- self.fileId = re.match(self.__pattern__, url).group('id')
- self.package = pyfile.package()
-
- def isOnline(self):
- if "No usable content was found" in self.html:
- self.logDebug("File not found")
- return False
- return True
-
- def isPasswordProtected(self):
- if re.search(r'''<form.*?id="passwordForm".*?>''', self.html):
- self.logDebug("Links are protected")
- return True
- return False
-
- def isCaptchaProtected(self):
- if '<map id="captchamap"' in self.html:
- self.logDebug("Links are captcha protected")
- return True
- return False
-
- def unblockServer(self):
- imgs = re.findall("(/template/images/.*?\.gif)", self.html)
- for img in imgs:
- self.load(self.baseUrl + img)
-
- def unlockPasswordProtection(self):
- password = self.getPassword()
- self.logDebug("Submitting password [%s] for protected links" % password)
- post = {"password": password, 'login': 'Submit form'}
- url = self.baseUrl + '/' + self.fileId
- self.html = self.load(url, post=post, decode=True)
-
- def unlockCaptchaProtection(self):
- # Get captcha map
- captchaMap = self._getCaptchaMap()
- self.logDebug("Captcha map with [%d] positions" % len(captchaMap.keys()))
-
- # Request user for captcha coords
- m = re.search(r'<img src="/captcha.gif\?d=(.*?)&PHPSESSID=(.*?)&legend=1"', self.html)
- captchaUrl = self.baseUrl + '/captcha.gif?d=%s&PHPSESSID=%s' % (m.group(1), m.group(2))
- self.logDebug("Waiting user for correct position")
- coords = self.decryptCaptcha(captchaUrl, forceUser=True, imgtype="gif", result_type='positional')
- self.logDebug("Captcha resolved, coords [%s]" % str(coords))
-
- # Resolve captcha
- href = self._resolveCoords(coords, captchaMap)
- if href is None:
- self.logDebug("Invalid captcha resolving, retrying")
- self.invalidCaptcha()
- self.setWait(5, False)
- self.wait()
- self.retry()
- url = self.baseUrl + href
- self.html = self.load(url, decode=True)
-
- def _getCaptchaMap(self):
- map = {}
- for m in re.finditer(r'<area shape="rect" coords="(.*?)" href="(.*?)"', self.html):
- rect = eval('(' + m.group(1) + ')')
- href = m.group(2)
- map[rect] = href
- return map
-
- def _resolveCoords(self, coords, captchaMap):
- x, y = coords
- for rect, href in captchaMap.items():
- x1, y1, x2, y2 = rect
- if (x>=x1 and x<=x2) and (y>=y1 and y<=y2):
- return href
-
- def handleErrors(self):
- if "The inserted password was wrong" in self.html:
- self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")
- self.fail("Incorrect password, please set right password on 'Edit package' form and retry")
-
- if self.captcha:
- if "Your choice was wrong" in self.html:
- self.logDebug("Invalid captcha, retrying")
- self.invalidCaptcha()
- self.setWait(5)
- self.wait()
- self.retry()
- else:
- self.correctCaptcha()
-
- def getPackageInfo(self):
- name = folder = None
-
- # Extract from web package header
- title_re = r'<h2><img.*?/>(.*)</h2>'
- m = re.search(title_re, self.html, re.DOTALL)
- if m is not None:
- title = m.group(1).strip()
- if 'unnamed' not in title:
- name = folder = title
- self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder))
-
- # Fallback to defaults
- if not name or not folder:
- name = self.package.name
- folder = self.package.folder
- self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder))
-
- # Return package info
- return name, folder
-
- def handleWebLinks(self):
- package_links = []
- self.logDebug("Handling Web links")
-
- #@TODO: Gather paginated web links
- pattern = r"javascript:_get\('(.*?)', \d+, ''\)"
- ids = re.findall(pattern, self.html)
- self.logDebug("Decrypting %d Web links" % len(ids))
- for i, id in enumerate(ids):
- try:
- self.logDebug("Decrypting Web link %d, [%s]" % (i+1, id))
- dwLink = self.baseUrl + "/get/lnk/" + id
- response = self.load(dwLink)
- code = re.search(r'frm/(\d+)', response).group(1)
- fwLink = self.baseUrl + "/get/frm/" + code
- response = self.load(fwLink)
- jscode = re.search(r'<script language="javascript">\s*eval\((.*)\)\s*</script>', response, re.DOTALL).group(1)
- jscode = self.js.eval("f = %s" % jscode)
- jslauncher = "window=''; parent={frames:{Main:{location:{href:''}}},location:''}; %s; parent.frames.Main.location.href"
- dlLink = self.js.eval(jslauncher % jscode)
- self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink)
- package_links.append(dlLink)
- except Exception, detail:
- self.logDebug("Error decrypting Web link [%s], %s" % (id, detail))
- return package_links
-
- def handleContainers(self):
- package_links = []
- self.logDebug("Handling Container links")
-
- pattern = r"javascript:_get\('(.*?)', 0, '(rsdf|ccf|dlc)'\)"
- containersLinks = re.findall(pattern, self.html)
- self.logDebug("Decrypting %d Container links" % len(containersLinks))
- for containerLink in containersLinks:
- link = "%s/get/%s/%s" % (self.baseUrl, containerLink[1], containerLink[0])
- package_links.append(link)
- return package_links
-
- def handleCNL2(self):
- package_links = []
- self.logDebug("Handling CNL2 links")
-
- if '/lib/cnl2/ClicknLoad.swf' in self.html:
- try:
- (crypted, jk) = self._getCipherParams()
- package_links.extend(self._getLinks(crypted, jk))
- except:
- self.fail("Unable to decrypt CNL2 links")
- return package_links
-
- def _getCipherParams(self):
-
- # Request CNL2
- code = re.search(r'ClicknLoad.swf\?code=(.*?)"', self.html).group(1)
- url = "%s/get/cnl2/%s" % (self.baseUrl, code)
- response = self.load(url)
- params = response.split(";;")
-
- # Get jk
- strlist = list(base64.standard_b64decode(params[1]))
- strlist.reverse()
- jk = ''.join(strlist)
-
- # Get crypted
- strlist = list(base64.standard_b64decode(params[2]))
- strlist.reverse()
- crypted = ''.join(strlist)
-
- # Log and return
- return crypted, jk
-
- def _getLinks(self, crypted, jk):
-
- # Get key
- jreturn = self.js.eval("%s f()" % jk)
- self.logDebug("JsEngine returns value [%s]" % jreturn)
- key = binascii.unhexlify(jreturn)
-
- # Decode crypted
- crypted = base64.standard_b64decode(crypted)
-
- # Decrypt
- Key = key
- IV = key
- obj = AES.new(Key, AES.MODE_CBC, IV)
- text = obj.decrypt(crypted)
-
- # Extract links
- text = text.replace("\x00", "").replace("\r", "")
- links = text.split("\n")
- links = filter(lambda x: x != "", links)
-
- # Log and return
- self.logDebug("Block has %d links" % len(links))
- return links
\ No newline at end of file diff --git a/module/plugins/crypter/ShareRapidComFolder.py b/module/plugins/crypter/ShareRapidComFolder.py deleted file mode 100644 index cb7f37525..000000000 --- a/module/plugins/crypter/ShareRapidComFolder.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class ShareRapidComFolder(SimpleCrypter): - __name__ = "ShareRapidComFolder" - __type__ = "crypter" - __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/(slozka/.+)" - __version__ = "0.01" - __description__ = """Share-Rapid.com Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - LINK_PATTERN = r'<td class="soubor"[^>]*><a href="([^"]+)">'
\ No newline at end of file diff --git a/module/plugins/crypter/SpeedLoadOrgFolder.py b/module/plugins/crypter/SpeedLoadOrgFolder.py deleted file mode 100644 index f85ede6f3..000000000 --- a/module/plugins/crypter/SpeedLoadOrgFolder.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - -class SpeedLoadOrgFolder(SimpleCrypter): - __name__ = "SpeedLoadOrgFolder" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?speedload\.org/(\d+~f$|folder/\d+/)" - __version__ = "0.2" - __description__ = """Speedload Crypter Plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - LINK_PATTERN = r'<div class="link"><a href="(http://speedload.org/\w+)"' - TITLE_PATTERN = r'<title>Files of: (?P<title>[^<]+) folder</title>' diff --git a/module/plugins/crypter/StealthTo.py b/module/plugins/crypter/StealthTo.py deleted file mode 100644 index cf7a79e9b..000000000 --- a/module/plugins/crypter/StealthTo.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter - -class StealthTo(Crypter): - __name__ = "StealthTo" - __type__ = "container" - __pattern__ = r"http://(www\.)?stealth.to/folder/" - __version__ = "0.1" - __description__ = """Stealth.to Container Plugin""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - def __init__(self, parent): - Crypter.__init__(self, parent) - self.parent = parent - self.html = None - - def file_exists(self): - """ returns True or False - """ - return True - - def proceed(self, url, location): - url = self.parent.url - self.html = self.req.load(url, cookies=True) - temp_links = [] - ids = [] - ats = [] # authenticity_token - inputs = re.findall(r"(<(input|form)[^>]+)", self.html) - for input in inputs: - if re.search(r"name=\"authenticity_token\"",input[0]): - ats.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) - if re.search(r"name=\"id\"",input[0]): - ids.append(re.search(r"value=\"([^\"]+)", input[0]).group(1)) - - for i in range(0, len(ids)): - self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": ("download_" + str(ids[i]))}, cookies=True) - new_html = self.req.load(url + "/web", post={"authenticity_token": ats[i], "id": str(ids[i]), "link": "1"}, cookies=True) - temp_links.append(re.search(r"iframe src=\"(.*)\" frameborder", new_html).group(1)) - - self.links = temp_links diff --git a/module/plugins/crypter/TrailerzoneInfo.py b/module/plugins/crypter/TrailerzoneInfo.py deleted file mode 100644 index 43a4fcce5..000000000 --- a/module/plugins/crypter/TrailerzoneInfo.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class TrailerzoneInfo(Crypter): - __name__ = "TrailerzoneInfo" - __type__ = "crypter" - __pattern__ = r"http://(www\.)?trailerzone.info/.*?" - __version__ = "0.02" - __description__ = """TrailerZone.info Crypter Plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - JS_KEY_PATTERN = r"<script>(.*)var t = window" - - def decrypt(self, pyfile): - protectPattern = re.compile("http://(www\.)?trailerzone.info/protect.html.*?") - goPattern = re.compile("http://(www\.)?trailerzone.info/go.html.*?") - url = pyfile.url - if protectPattern.match(url): - self.handleProtect(url) - elif goPattern.match(url): - self.handleGo(url) - - def handleProtect(self, url): - self.handleGo("http://trailerzone.info/go.html#:::" + url.split("#:::",1)[1]) - - def handleGo(self, url): - - src = self.req.load(str(url)) - pattern = re.compile(self.JS_KEY_PATTERN, re.DOTALL) - found = re.search(pattern, src) - - # Get package info - package_links = [] - try: - result = self.js.eval(found.group(1) + " decodeLink('" + url.split("#:::",1)[1] + "');") - result = str(result) - self.logDebug("RESULT: %s" % result) - package_links.append(result) - self.core.files.addLinks(package_links, self.pyfile.package().id) - except Exception, e: - self.logDebug(e) - self.fail('Could not extract any links by javascript') diff --git a/module/plugins/crypter/UlozToFolder.py b/module/plugins/crypter/UlozToFolder.py deleted file mode 100644 index 814d5240d..000000000 --- a/module/plugins/crypter/UlozToFolder.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class UlozToFolder(Crypter): - __name__ = "UlozToFolder" - __type__ = "crypter" - __pattern__ = r"http://.*(uloz\.to|ulozto\.(cz|sk|net)|bagruj.cz|zachowajto.pl)/(m|soubory)/.*" - __version__ = "0.2" - __description__ = """Uloz.to Folder Plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FOLDER_PATTERN = r'<ul class="profile_files">(.*?)</ul>' - LINK_PATTERN = r'<br /><a href="/([^"]+)">[^<]+</a>' - NEXT_PAGE_PATTERN = r'<a class="next " href="/([^"]+)"> </a>' - - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - - new_links = [] - for i in range(1,100): - self.logInfo("Fetching links from page %i" % i) - found = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if found is None: self.fail("Parse error (FOLDER)") - - new_links.extend(re.findall(self.LINK_PATTERN, found.group(1))) - found = re.search(self.NEXT_PAGE_PATTERN, html) - if found: - html = self.load("http://ulozto.net/" + found.group(1)) - else: - break - else: - self.logInfo("Limit of 99 pages reached, aborting") - - if new_links: - self.core.files.addLinks(map(lambda s:"http://ulozto.net/%s" % s, new_links), self.pyfile.package().id) - else: - self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/UploadedToFolder.py b/module/plugins/crypter/UploadedToFolder.py deleted file mode 100644 index c514f23d0..000000000 --- a/module/plugins/crypter/UploadedToFolder.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*-
-
-############################################################################
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU Affero General Public License as #
-# published by the Free Software Foundation, either version 3 of the #
-# License, or (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU Affero General Public License for more details. #
-# #
-# You should have received a copy of the GNU Affero General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-############################################################################
-
-import re
-
-from module.plugins.internal.SimpleCrypter import SimpleCrypter
-
-
-class UploadedToFolder(SimpleCrypter):
- __name__ = "UploadedToFolder"
- __type__ = "crypter"
- __pattern__ = r"http://(?:www\.)?(uploaded|ul)\.(to|net)/(f|folder|list)/(?P<id>\w+)"
- __version__ = "0.3"
- __description__ = """UploadedTo Crypter Plugin"""
- __author_name__ = ("stickell")
- __author_mail__ = ("l.stickell@yahoo.it")
-
- PLAIN_PATTERN = r'<small class="date"><a href="(?P<plain>[\w/]+)" onclick='
- TITLE_PATTERN = r'<title>(?P<title>[^<]+)</title>'
-
- def decrypt(self, pyfile):
- self.html = self.load(pyfile.url)
-
- package_name, folder_name = self.getPackageNameAndFolder()
-
- m = re.search(self.PLAIN_PATTERN, self.html)
- if m:
- plain_link = 'http://uploaded.net/' + m.group('plain')
- else:
- self.fail('Parse error - Unable to find plain url list')
-
- self.html = self.load(plain_link)
- package_links = self.html.split('\n')[:-1]
- self.logDebug('Package has %d links' % len(package_links))
-
- self.packages = [(package_name, package_links, folder_name)]
diff --git a/module/plugins/crypter/WiiReloadedOrg.py b/module/plugins/crypter/WiiReloadedOrg.py deleted file mode 100644 index 574a147c4..000000000 --- a/module/plugins/crypter/WiiReloadedOrg.py +++ /dev/null @@ -1,52 +0,0 @@ -
-import re
-
-from module.plugins.Crypter import Crypter
-
-class WiiReloadedOrg(Crypter):
- __name__ = "WiiReloadedOrg"
- __type__ = "crypter"
- __pattern__ = r"http://www\.wii-reloaded\.org/protect/get\.php\?i=.+"
- __config__ = [("changeName", "bool", "Use Wii-Reloaded.org folder name", "True")]
- __version__ = "0.1"
- __description__ = """Wii-Reloaded.org Crypter Plugin"""
- __author_name__ = ("hzpz")
- __author_mail__ = ("none")
-
-
- def decrypt(self, pyfile):
- url = pyfile.url
- src = self.req.load(str(url))
-
- ids = re.findall(r"onClick=\"popup_dl\((.+)\)\"", src)
- if len(ids) == 0:
- self.fail("Unable to decrypt links, this plugin probably needs to be updated")
-
- packageName = self.pyfile.package().name
- if self.getConfig("changeName"):
- packageNameMatch = re.search(r"<div id=\"foldername\">(.+)</div>", src)
- if not packageNameMatch:
- self.logWarning("Unable to get folder name, this plugin probably needs to be updated")
- else:
- packageName = packageNameMatch.group(1)
-
- self.pyfile.package().password = "wii-reloaded.info"
-
- self.logDebug("Processing %d links" % len(ids))
- links = []
- for id in ids:
- self.req.lastURL = str(url)
- header = self.req.load("http://www.wii-reloaded.org/protect/hastesosiehtsaus.php?i=" + id, just_header=True)
- self.logDebug("Header:\n" + header)
- redirectLocationMatch = re.search(r"^Location: (.+)$", header, flags=re.MULTILINE)
- if not redirectLocationMatch:
- self.offline()
- redirectLocation = redirectLocationMatch.group(1)
- self.logDebug(len(redirectLocation))
- if not redirectLocation.startswith("http"):
- self.offline()
- self.logDebug("Decrypted link: %s" % redirectLocation)
- links.append(redirectLocation)
-
- self.logDebug("Decrypted %d links" % len(links))
- self.packages.append((packageName, links, packageName))
\ No newline at end of file diff --git a/module/plugins/crypter/XfilesharingProFolder.py b/module/plugins/crypter/XfilesharingProFolder.py deleted file mode 100644 index 8e58c207d..000000000 --- a/module/plugins/crypter/XfilesharingProFolder.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*-
-
-from module.plugins.Crypter import Crypter, Package
-import re
-
-class XfilesharingProFolder(Crypter):
- __name__ = "XfilesharingProFolder"
- __type__ = "crypter"
- __pattern__ = r"http://(?:www\.)?((easybytez|turboupload|uploadville|file4safe|fileband|filebeep|grupload|247upload)\.com|(muchshare|annonhost).net|bzlink.us)/users/.*"
- __version__ = "0.01"
- __description__ = """Generic XfilesharingPro Folder Plugin"""
- __author_name__ = ("zoidberg")
- __author_mail__ = ("zoidberg@mujmail.cz")
-
- LINK_PATTERN = r'<div class="link"><a href="([^"]+)" target="_blank">[^<]*</a></div>'
- SUBFOLDER_PATTERN = r'<TD width="1%"><img src="[^"]*/images/folder2.gif"></TD><TD><a href="([^"]+)"><b>(?!\. \.<)([^<]+)</b></a></TD>'
-
- def decryptURL(self, url):
- return self.decryptFile(self.load(url, decode = True))
-
- def decryptFile(self, html):
- new_links = []
-
- new_links.extend(re.findall(self.LINK_PATTERN, html))
-
- subfolders = re.findall(self.SUBFOLDER_PATTERN, html)
- #self.logDebug(subfolders)
- for (url, name) in subfolders:
- if self.package: name = "%s/%s" % (self.package.name, name)
- new_links.append(Package(name, [url]))
-
- if not new_links: self.fail('Could not extract any links')
-
- return new_links
\ No newline at end of file diff --git a/module/plugins/crypter/YoutubeBatch.py b/module/plugins/crypter/YoutubeBatch.py deleted file mode 100644 index 72b72aab7..000000000 --- a/module/plugins/crypter/YoutubeBatch.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import json - -from module.plugins.Crypter import Crypter - -API_KEY = "AIzaSyCKnWLNlkX-L4oD1aEzqqhRw1zczeD6_k0" - -class YoutubeBatch(Crypter): - __name__ = "YoutubeBatch" - __type__ = "container" - __pattern__ = r"https?://(?:[^/]*?)youtube\.com/(?:(?:view_play_list|playlist|.*?feature=PlayList).*?[?&](?:list|p)=)([a-zA-Z0-9-_]+)" - __version__ = "0.93" - __description__ = """Youtube.com Channel Download Plugin""" - __author_name__ = ("RaNaN", "Spoob", "zoidberg", "roland") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "zoidberg@mujmail.cz", "roland@enkore.de") - - def get_videos(self, playlist_id, token=None): - url = "https://www.googleapis.com/youtube/v3/playlistItems?playlistId=%s&part=snippet&key=%s&maxResults=50" % (playlist_id, API_KEY) - if token: - url += "&pageToken=" + token - - response = json.loads(self.load(url)) - - for item in response["items"]: - if item["kind"] == "youtube#playlistItem" and item["snippet"]["resourceId"]["kind"] == "youtube#video": - yield "http://youtube.com/watch?v=" + item["snippet"]["resourceId"]["videoId"] - - if "nextPageToken" in response: - for item in self.get_videos(playlist_id, response["nextPageToken"]): - yield item - - def decrypt(self, pyfile): - match_id = re.match(self.__pattern__, self.pyfile.url) - new_links = [] - playlist_id = match_id.group(1) - - new_links.extend(self.get_videos(playlist_id)) - - self.packages.append((self.pyfile.package().name, new_links, self.pyfile.package().name)) diff --git a/module/plugins/crypter/__init__.py b/module/plugins/crypter/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/crypter/__init__.py +++ /dev/null diff --git a/module/plugins/hooks/Captcha9kw.py b/module/plugins/hooks/Captcha9kw.py deleted file mode 100755 index bb2b8c862..000000000 --- a/module/plugins/hooks/Captcha9kw.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: mkaay, RaNaN, zoidberg
-"""
-from __future__ import with_statement
-
-from thread import start_new_thread
-from base64 import b64encode
-import cStringIO
-import pycurl
-import time
-
-from module.network.RequestFactory import getURL, getRequest
-from module.network.HTTPRequest import BadHeader
-
-from module.plugins.Hook import Hook
-
-class Captcha9kw(Hook):
- __name__ = "Captcha9kw"
- __version__ = "0.04"
- __description__ = """send captchas to 9kw.eu"""
- __config__ = [("activated", "bool", "Activated", True),
- ("force", "bool", "Force CT even if client is connected", True),
- ("https", "bool", "Enable HTTPS", "False"),
- ("confirm", "bool", "Confirm Captcha", "False"),
- ("captchaperhour", "int", "Captcha per hour", "9999"),
- ("prio", "int", "Prio (1-10)", "0"),
- ("passkey", "password", "API key", ""),]
- __author_name__ = ("RaNaN")
- __author_mail__ = ("RaNaN@pyload.org")
-
- API_URL = "://www.9kw.eu/index.cgi"
-
- def setup(self):
- self.API_URL = "https"+self.API_URL if self.getConfig("https") else "http"+self.API_URL
- self.info = {}
-
- def getCredits(self):
- response = getURL(self.API_URL, get = { "apikey": self.getConfig("passkey"), "pyload": "1", "source": "pyload", "action": "usercaptchaguthaben" })
-
- if response.isdigit():
- self.logInfo(_("%s credits left") % response)
- self.info["credits"] = credits = int(response)
- return credits
- else:
- self.logError(response)
- return 0
-
- def processCaptcha(self, task):
- result = None
-
- with open(task.captchaFile, 'rb') as f:
- data = f.read()
- data = b64encode(data)
- self.logDebug("%s : %s" % (task.captchaFile, data))
- if task.isPositional():
- mouse = 1
- else:
- mouse = 0
-
- response = getURL(self.API_URL, post = {
- "apikey": self.getConfig("passkey"),
- "prio": self.getConfig("prio"),
- "confirm": self.getConfig("confirm"),
- "captchaperhour": self.getConfig("captchaperhour"),
- "maxtimeout": "220",
- "pyload": "1",
- "source": "pyload",
- "base64": "1",
- "mouse": mouse,
- "file-upload-01": data,
- "action": "usercaptchaupload" })
-
- if response.isdigit():
- self.logInfo(_("NewCaptchaID from upload: %s : %s" % (response,task.captchaFile)))
-
- for i in range(1, 220, 1):
- response2 = getURL(self.API_URL, get = { "apikey": self.getConfig("passkey"), "id": response,"pyload": "1","source": "pyload", "action": "usercaptchacorrectdata" })
-
- if(response2 != ""):
- break;
-
- time.sleep(1)
-
- result = response2
- task.data["ticket"] = response
- self.logInfo("result %s : %s" % (response, result))
- task.setResult(result)
- else:
- self.logError("Bad upload: %s" % response)
- return False
-
- def newCaptchaTask(self, task):
- if not task.isTextual() and not task.isPositional():
- return False
-
- if not self.getConfig("passkey"):
- return False
-
- if self.core.isClientConnected() and not self.getConfig("force"):
- return False
-
- if self.getCredits() > 0:
- task.handler.append(self)
- task.setWaiting(220)
- start_new_thread(self.processCaptcha, (task,))
-
- else:
- self.logError(_("Your Captcha 9kw.eu Account has not enough credits"))
-
- def captchaCorrect(self, task):
- if "ticket" in task.data:
-
- try:
- response = getURL(self.API_URL,
- post={ "action": "usercaptchacorrectback",
- "apikey": self.getConfig("passkey"),
- "api_key": self.getConfig("passkey"),
- "correct": "1",
- "pyload": "1",
- "source": "pyload",
- "id": task.data["ticket"] }
- )
- self.logInfo("Request correct: %s" % response)
-
- except BadHeader, e:
- self.logError("Could not send correct request.", str(e))
- else:
- self.logError("No CaptchaID for correct request (task %s) found." % task)
-
- def captchaInvalid(self, task):
- if "ticket" in task.data:
-
- try:
- response = getURL(self.API_URL,
- post={ "action": "usercaptchacorrectback",
- "apikey": self.getConfig("passkey"),
- "api_key": self.getConfig("passkey"),
- "correct": "2",
- "pyload": "1",
- "source": "pyload",
- "id": task.data["ticket"] }
- )
- self.logInfo("Request refund: %s" % response)
-
- except BadHeader, e:
- self.logError("Could not send refund request.", str(e))
- else:
- self.logError("No CaptchaID for not correct request (task %s) found." % task)
diff --git a/module/plugins/hooks/ReloadCc.py b/module/plugins/hooks/ReloadCc.py deleted file mode 100644 index dbd9d659b..000000000 --- a/module/plugins/hooks/ReloadCc.py +++ /dev/null @@ -1,65 +0,0 @@ -from module.plugins.internal.MultiHoster import MultiHoster - -from module.common.json_layer import json_loads -from module.network.RequestFactory import getURL - -class ReloadCc(MultiHoster): - __name__ = "ReloadCc" - __version__ = "0.3" - __type__ = "hook" - __description__ = """Reload.cc hook plugin""" - - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for hosters (if supported):", "all"), - ("hosterList", "str", "Hoster list (comma separated)", "")] - - __author_name__ = ("Reload Team") - __author_mail__ = ("hello@reload.cc") - - interval = 0 # Disable periodic calls - - def getHoster(self): - # If no accounts are available there will be no hosters available - if not self.account or not self.account.canUse(): - print "ReloadCc: No accounts available" - return [] - - # Get account data - (user, data) = self.account.selectAccount() - - # Get supported hosters list from reload.cc using the json API v1 - query_params = dict( - via='pyload', - v=1, - get_supported='true', - get_traffic='true', - user=user - ) - - try: - query_params.update(dict(hash=self.account.infos[user]['pwdhash'])) - except Exception: - query_params.update(dict(pwd=data['password'])) - - answer = getURL("http://api.reload.cc/login", get=query_params) - data = json_loads(answer) - - - # If account is not valid thera are no hosters available - if data['status'] != "ok": - print "ReloadCc: Status is not ok: %s" % data['status'] - return [] - - # Extract hosters from json file - return data['msg']['supportedHosters'] - - def coreReady(self): - # Get account plugin and check if there is a valid account available - self.account = self.core.accountManager.getAccountPlugin("ReloadCc") - if not self.account.canUse(): - self.account = None - self.logError("Please add a valid reload.cc account first and restart pyLoad.") - return - - # Run the overwriten core ready which actually enables the multihoster hook - return MultiHoster.coreReady(self) diff --git a/module/plugins/hoster/ARD.py b/module/plugins/hoster/ARD.py deleted file mode 100644 index 5ab65cd4b..000000000 --- a/module/plugins/hoster/ARD.py +++ /dev/null @@ -1,80 +0,0 @@ - -import subprocess -import re -import os.path -import os - -from module.utils import save_join, save_path -from module.plugins.Hoster import Hoster - -# Requires rtmpdump -# by Roland Beermann - -class RTMP: - # TODO: Port to some RTMP-library like rtmpy or similar - # TODO?: Integrate properly into the API of pyLoad - - command = "rtmpdump" - - @classmethod - def download_rtmp_stream(cls, url, output_file, playpath=None): - opts = [ - "-r", url, - "-o", output_file, - ] - if playpath: - opts.append("--playpath") - opts.append(playpath) - - cls._invoke_rtmpdump(opts) - - @classmethod - def _invoke_rtmpdump(cls, opts): - args = [ - cls.command - ] - args.extend(opts) - - return subprocess.check_call(args) - -class ARD(Hoster): - __name__ = "ARD Mediathek" - __version__ = "0.1" - __pattern__ = r"http://www\.ardmediathek\.de/.*" - __config__ = [] - - def process(self, pyfile): - site = self.load(pyfile.url) - - avail_videos = re.findall(r"""mediaCollection.addMediaStream\(0, ([0-9]*), "([^\"]*)", "([^\"]*)", "[^\"]*"\);""", site) - avail_videos.sort(key=lambda videodesc: int(videodesc[0]), reverse=True) # The higher the number, the better the quality - - quality, url, playpath = avail_videos[0] - - pyfile.name = re.search(r"<h1>([^<]*)</h1>", site).group(1) - - if url.startswith("http"): - # Best quality is available over HTTP. Very rare. - self.download(url) - else: - pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, pyfile.package().folder) - - if not os.path.exists(location): - os.makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - output_file = save_join(location, save_path(pyfile.name)) + os.path.splitext(playpath)[1] - - RTMP.download_rtmp_stream(url, playpath=playpath, output_file=output_file) diff --git a/module/plugins/hoster/AlldebridCom.py b/module/plugins/hoster/AlldebridCom.py deleted file mode 100644 index efc96ff28..000000000 --- a/module/plugins/hoster/AlldebridCom.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*-
-
-import re
-from urllib import unquote
-from random import randrange
-from module.plugins.Hoster import Hoster
-from module.common.json_layer import json_loads
-from module.utils import parseFileSize
-
-
-class AlldebridCom(Hoster):
- __name__ = "AlldebridCom"
- __version__ = "0.31"
- __type__ = "hoster"
-
- __pattern__ = r"https?://.*alldebrid\..*"
- __description__ = """Alldebrid.com hoster plugin"""
- __author_name__ = ("Andy, Voigt")
- __author_mail__ = ("spamsales@online.de")
-
- def getFilename(self, url):
- try:
- name = unquote(url.rsplit("/", 1)[1])
- except IndexError:
- name = "Unknown_Filename..."
- if name.endswith("..."): #incomplete filename, append random stuff
- name += "%s.tmp" % randrange(100, 999)
- return name
-
- def init(self):
- self.tries = 0
- self.chunkLimit = 3
- self.resumeDownload = True
-
- def process(self, pyfile):
- if not self.account:
- self.logError("Please enter your AllDebrid account or deactivate this plugin")
- self.fail("No AllDebrid account provided")
-
- self.log.debug("AllDebrid: Old URL: %s" % pyfile.url)
- if re.match(self.__pattern__, pyfile.url):
- new_url = pyfile.url
- else:
- password = self.getPassword().splitlines()
- password = "" if not password else password[0]
-
- url = "http://www.alldebrid.com/service.php?link=%s&json=true&pw=%s" % (pyfile.url, password)
- page = self.load(url)
- data = json_loads(page)
-
- self.logDebug("Json data: %s" % str(data))
-
- if data["error"]:
- if data["error"] == "This link isn't available on the hoster website.":
- self.offline()
- else:
- self.logWarning(data["error"])
- self.tempOffline()
- else:
- if self.pyfile.name and not self.pyfile.name.endswith('.tmp'):
- self.pyfile.name = data["filename"]
- self.pyfile.size = parseFileSize(data["filesize"])
- new_url = data["link"]
-
- if self.getConfig("https"):
- new_url = new_url.replace("http://", "https://")
- else:
- new_url = new_url.replace("https://", "http://")
-
- self.logDebug("AllDebrid: New URL: %s" % new_url)
-
- if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown"):
- #only use when name wasnt already set
- pyfile.name = self.getFilename(new_url)
-
- self.download(new_url, disposition=True)
-
- check = self.checkDownload({"error": "<title>An error occured while processing your request</title>",
- "empty": re.compile(r"^$")})
-
- if check == "error":
- self.retry(reason="An error occured while generating link.", wait_time=60)
- elif check == "empty":
- self.retry(reason="Downloaded File was empty.", wait_time=60)
diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py deleted file mode 100644 index 7070fafde..000000000 --- a/module/plugins/hoster/BasePlugin.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from urlparse import urlparse -from re import search -from urllib import unquote - -from module.network.HTTPRequest import BadHeader -from module.plugins.Hoster import Hoster -from module.utils import html_unescape, remove_chars - -class BasePlugin(Hoster): - __name__ = "BasePlugin" - __type__ = "hoster" - __pattern__ = r"^unmatchable$" - __version__ = "0.17" - __description__ = """Base Plugin when any other didnt fit""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def setup(self): - self.chunkLimit = -1 - self.resumeDownload = True - - def process(self, pyfile): - """main function""" - - #debug part, for api exerciser - if pyfile.url.startswith("DEBUG_API"): - self.multiDL = False - return - - #TODO: remove debug - if pyfile.url.lower().startswith("debug"): - self.decryptCaptcha("http://download.pyload.org/pie.png", imgtype="png") - self.download("http://download.pyload.org/random100.bin") - return -# -# if pyfile.url == "79": -# self.core.api.addPackage("test", [str(i) for i in range(80)], 1) -# -# return - if pyfile.url.startswith("http"): - - try: - self.downloadFile(pyfile) - except BadHeader, e: - if e.code in (401, 403): - self.logDebug("Auth required") - - account = self.core.accountManager.getAccountPlugin('Http') - servers = [ x['login'] for x in account.getAllAccounts() ] - server = urlparse(pyfile.url).netloc - - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]["password"]) - else: - for pwd in pyfile.package().password.splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break - else: - self.fail(_("Authorization required (username:password)")) - - self.downloadFile(pyfile) - else: - raise - - else: - self.fail("No Plugin matched and not a downloadable url.") - - - def downloadFile(self, pyfile): - url = pyfile.url - - for i in range(5): - header = self.load(url, just_header = True) - - # self.load does not raise a BadHeader on 404 responses, do it here - if header.has_key('code') and header['code'] == 404: - raise BadHeader(404) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - break - - name = html_unescape(unquote(urlparse(url).path.split("/")[-1])) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = remove_chars(disp['name'], "\"';/").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True) diff --git a/module/plugins/hoster/BayfilesCom.py b/module/plugins/hoster/BayfilesCom.py deleted file mode 100644 index 190d9a952..000000000 --- a/module/plugins/hoster/BayfilesCom.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads -from time import time - -class BayfilesCom(SimpleHoster): - __name__ = "BayfilesCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?bayfiles\.com/file/\w+/\w+/.*" - __version__ = "0.04" - __description__ = """Bayfiles.com plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<p title="(?P<N>[^"]+)">[^<]*<strong>(?P<S>[0-9., ]+)(?P<U>[kKMG])i?B</strong></p>' - FILE_OFFLINE_PATTERN = r'(<p>The requested file could not be found.</p>|<title>404 Not Found</title>)' - - WAIT_PATTERN = r'>Your IP [0-9.]* has recently downloaded a file\. Upgrade to premium or wait (\d+) minutes\.<' - VARS_PATTERN = r'var vfid = (\d+);\s*var delay = (\d+);' - LINK_PATTERN = r"javascript:window.location.href = '([^']+)';" - PREMIUM_LINK_PATTERN = r'(?:<a class="highlighted-btn" href="|(?=http://s\d+\.baycdn\.com/dl/))(.*?)"' - - def handleFree(self): - found = re.search(self.WAIT_PATTERN, self.html) - if found: - self.setWait(int(found.group(1)) * 60) - self.wait() - self.retry() - - # Get download token - found = re.search(self.VARS_PATTERN, self.html) - if not found: self.parseError('VARS') - vfid, delay = found.groups() - - response = json_loads(self.load('http://bayfiles.com/ajax_download', get = { - "_": time() * 1000, - "action": "startTimer", - "vfid": vfid}, decode = True)) - - if not "token" in response or not response['token']: - self.fail('No token') - - self.setWait(int(delay)) - self.wait() - - self.html = self.load('http://bayfiles.com/ajax_download', get = { - "token": response['token'], - "action": "getLink", - "vfid": vfid}) - - # Get final link and download - found = re.search(self.LINK_PATTERN, self.html) - if not found: self.parseError("Free link") - self.startDownload(found.group(1)) - - def handlePremium(self): - found = re.search(self.PREMIUM_LINK_PATTERN, self.html) - if not found: self.parseError("Premium link") - self.startDownload(found.group(1)) - - def startDownload(self, url): - self.logDebug("%s URL: %s" % ("Premium" if self.premium else "Free", url)) - self.download(url) - # check download - check = self.checkDownload({ - "waitforfreeslots": re.compile(r"<title>BayFiles</title>"), - "notfound": re.compile(r"<title>404 Not Found</title>") - }) - if check == "waitforfreeslots": - self.retry(60, 300, "Wait for free slot") - elif check == "notfound": - self.retry(60, 300, "404 Not found") - -getInfo = create_getInfo(BayfilesCom) diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py deleted file mode 100644 index 49299d463..000000000 --- a/module/plugins/hoster/BezvadataCz.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class BezvadataCz(SimpleHoster): - __name__ = "BezvadataCz" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)*bezvadata.cz/stahnout/.*" - __version__ = "0.24" - __description__ = """BezvaData.cz""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<p><b>Soubor: (?P<N>[^<]+)</b></p>' - FILE_SIZE_PATTERN = r'<li><strong>Velikost:</strong> (?P<S>[^<]+)</li>' - FILE_OFFLINE_PATTERN = r'<title>BezvaData \| Soubor nenalezen</title>' - - def setup(self): - self.multiDL = self.resumeDownload = True - - def handleFree(self): - #download button - found = re.search(r'<a class="stahnoutSoubor".*?href="(.*?)"', self.html) - if not found: self.parseError("page1 URL") - url = "http://bezvadata.cz%s" % found.group(1) - - #captcha form - self.html = self.load(url) - self.checkErrors() - for i in range(5): - action, inputs = self.parseHtmlForm('frm-stahnoutFreeForm') - if not inputs: self.parseError("FreeForm") - - found = re.search(r'<img src="data:image/png;base64,(.*?)"', self.html) - if not found: self.parseError("captcha img") - - #captcha image is contained in html page as base64encoded data but decryptCaptcha() expects image url - self.load, proper_load = self.loadcaptcha, self.load - try: - inputs['captcha'] = self.decryptCaptcha(found.group(1), imgtype='png') - finally: - self.load = proper_load - - if '<img src="data:image/png;base64' in self.html: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: - self.fail("No valid captcha code entered") - - #download url - self.html = self.load("http://bezvadata.cz%s" % action, post=inputs) - self.checkErrors() - found = re.search(r'<a class="stahnoutSoubor2" href="(.*?)">', self.html) - if not found: self.parseError("page2 URL") - url = "http://bezvadata.cz%s" % found.group(1) - self.logDebug("DL URL %s" % url) - - #countdown - found = re.search(r'id="countdown">(\d\d):(\d\d)<', self.html) - wait_time = (int(found.group(1)) * 60 + int(found.group(2)) + 1) if found else 120 - self.setWait(wait_time, False) - self.wait() - - self.download(url) - - def checkErrors(self): - if 'images/button-download-disable.png' in self.html: - self.longWait(300, 24) #parallel dl limit - elif '<div class="infobox' in self.html: - self.tempOffline() - - def loadcaptcha(self, data, *args, **kwargs): - return data.decode("base64") - -getInfo = create_getInfo(BezvadataCz) diff --git a/module/plugins/hoster/BillionuploadsCom.py b/module/plugins/hoster/BillionuploadsCom.py deleted file mode 100644 index 5b053d547..000000000 --- a/module/plugins/hoster/BillionuploadsCom.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class BillionuploadsCom(XFileSharingPro): - __name__ = "BillionuploadsCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?billionuploads.com/\w{12}" - __version__ = "0.01" - __description__ = """billionuploads.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<b>Filename:</b>(?P<N>.*?)<br>' - FILE_SIZE_PATTERN = r'<b>Size:</b>(?P<S>.*?)<br>' - HOSTER_NAME = "billionuploads.com" - -getInfo = create_getInfo(BillionuploadsCom) diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py deleted file mode 100644 index 644345387..000000000 --- a/module/plugins/hoster/BitshareCom.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import with_statement - -import re -from pycurl import FOLLOWLOCATION - -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getRequest - - -def getInfo(urls): - result = [] - - for url in urls: - - # Get file info html - req = getRequest() - req.cj.setCookie(BitshareCom.HOSTER_DOMAIN, "language_selection", "EN") - html = req.load(url) - req.close() - - # Check online - if re.search(BitshareCom.FILE_OFFLINE_PATTERN, html): - result.append((url, 0, 1, url)) - continue - - # Name - name1 = re.search(BitshareCom.__pattern__, url).group('name') - m = re.search(BitshareCom.FILE_INFO_PATTERN, html) - name2 = m.group('name') - name = max(name1, name2) - - # Size - value = float(m.group('size')) - units = m.group('units') - pow = {'KB' : 1, 'MB' : 2, 'GB' : 3}[units] - size = int(value*1024**pow) - - # Return info - result.append((name, size, 2, url)) - - yield result - -class BitshareCom(Hoster): - __name__ = "BitshareCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?bitshare\.com/(files/(?P<id1>[a-zA-Z0-9]+)(/(?P<name>.*?)\.html)?|\?f=(?P<id2>[a-zA-Z0-9]+))" - __version__ = "0.47" - __description__ = """Bitshare.Com File Download Hoster""" - __author_name__ = ("paulking", "fragonib") - __author_mail__ = (None, "fragonib[AT]yahoo[DOT]es") - - HOSTER_DOMAIN = "bitshare.com" - FILE_OFFLINE_PATTERN = r'''(>We are sorry, but the requested file was not found in our database|>Error - File not available<|The file was deleted either by the uploader, inactivity or due to copyright claim)''' - FILE_INFO_PATTERN = r'<h1>(Downloading|Streaming)\s(?P<name>.+?)\s-\s(?P<size>[\d.]+)\s(?P<units>..)yte</h1>' - FILE_AJAXID_PATTERN = r'var ajaxdl = "(.*?)";' - CAPTCHA_KEY_PATTERN = r"http://api\.recaptcha\.net/challenge\?k=(.*?) " - TRAFFIC_USED_UP = r"Your Traffic is used up for today. Upgrade to premium to continue!" - - def setup(self): - self.multiDL = self.premium - self.chunkLimit = 1 - - def process(self, pyfile): - if self.premium: - self.account.relogin(self.user) - - self.pyfile = pyfile - - # File id - m = re.match(self.__pattern__, self.pyfile.url) - self.file_id = max(m.group('id1'), m.group('id2')) - self.logDebug("File id is [%s]" % self.file_id) - - # Load main page - self.req.cj.setCookie(self.HOSTER_DOMAIN, "language_selection", "EN") - self.html = self.load(self.pyfile.url, ref=False, decode=True) - - # Check offline - if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: - self.offline() - - # Check Traffic used up - if re.search(BitshareCom.TRAFFIC_USED_UP, self.html) is not None: - self.logInfo("Your Traffic is used up for today. Wait 1800 seconds or reconnect!") - self.logDebug("Waiting %d seconds." % 1800) - self.setWait(1800, True) - self.wantReconnect = True - self.wait() - self.retry() - - # File name - m = re.search(BitshareCom.__pattern__, self.pyfile.url) - name1 = m.group('name') if m is not None else None - m = re.search(BitshareCom.FILE_INFO_PATTERN, self.html) - name2 = m.group('name') if m is not None else None - self.pyfile.name = max(name1, name2) - - # Ajax file id - self.ajaxid = re.search(BitshareCom.FILE_AJAXID_PATTERN, self.html).group(1) - self.logDebug("File ajax id is [%s]" % self.ajaxid) - - # This may either download our file or forward us to an error page - url = self.getDownloadUrl() - self.logDebug("Downloading file with url [%s]" % url) - self.download(url) - - - def getDownloadUrl(self): - # Return location if direct download is active - if self.premium: - header = self.load(self.pyfile.url, cookies = True, just_header = True) - if 'location' in header: - return header['location'] - - # Get download info - self.logDebug("Getting download info") - response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", - post={"request" : "generateID", "ajaxid" : self.ajaxid}) - self.handleErrors(response, ':') - parts = response.split(":") - filetype = parts[0] - wait = int(parts[1]) - captcha = int(parts[2]) - self.logDebug("Download info [type: '%s', waiting: %d, captcha: %d]" % (filetype, wait, captcha)) - - # Waiting - if wait > 0: - self.logDebug("Waiting %d seconds." % wait) - if wait < 120: - self.setWait(wait, False) - self.wait() - else: - self.setWait(wait - 55, True) - self.wait() - self.retry() - - # Resolve captcha - if captcha == 1: - self.logDebug("File is captcha protected") - id = re.search(BitshareCom.CAPTCHA_KEY_PATTERN, self.html).group(1) - # Try up to 3 times - for i in range(3): - self.logDebug("Resolving ReCaptcha with key [%s], round %d" % (id, i+1)) - recaptcha = ReCaptcha(self) - challenge, code = recaptcha.challenge(id) - response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", - post={"request" : "validateCaptcha", "ajaxid" : self.ajaxid, "recaptcha_challenge_field" : challenge, "recaptcha_response_field" : code}) - if self.handleCaptchaErrors(response): - break - - - # Get download URL - self.logDebug("Getting download url") - response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", - post={"request" : "getDownloadURL", "ajaxid" : self.ajaxid}) - self.handleErrors(response, '#') - url = response.split("#")[-1] - - return url - - def handleErrors(self, response, separator): - self.logDebug("Checking response [%s]" % response) - if "ERROR:Session timed out" in response: - self.retry() - elif "ERROR" in response: - msg = response.split(separator)[-1] - self.fail(msg) - - def handleCaptchaErrors(self, response): - self.logDebug("Result of captcha resolving [%s]" % response) - if "SUCCESS" in response: - self.correctCaptcha() - return True - elif "ERROR:SESSION ERROR" in response: - self.retry() - self.logDebug("Wrong captcha") - self.invalidCaptcha() diff --git a/module/plugins/hoster/BoltsharingCom.py b/module/plugins/hoster/BoltsharingCom.py deleted file mode 100644 index 2f42c8b23..000000000 --- a/module/plugins/hoster/BoltsharingCom.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class BoltsharingCom(XFileSharingPro): - __name__ = "BoltsharingCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?boltsharing.com/\w{12}" - __version__ = "0.01" - __description__ = """Boltsharing.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - HOSTER_NAME = "boltsharing.com" - -getInfo = create_getInfo(BoltsharingCom) diff --git a/module/plugins/hoster/CatShareNet.py b/module/plugins/hoster/CatShareNet.py deleted file mode 100644 index 47063096e..000000000 --- a/module/plugins/hoster/CatShareNet.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha - - -class CatShareNet(SimpleHoster): - __name__ = "CatShareNet" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?catshare.net/\w{16}.*" - __version__ = "0.01" - __description__ = """CatShare.net Download Hoster""" - __author_name__ = ("z00nx") - __author_mail__ = ("z00nx0@gmail.com") - - FILE_INFO_PATTERN = r'<h3 class="pull-left"[^>]+>(?P<N>.*)</h3>\s+<h3 class="pull-right"[^>]+>(?P<S>.*)</h3>' - FILE_OFFLINE_PATTERN = r'Podany plik zosta' - SECONDS_PATTERN = 'var\s+count\s+=\s+(\d+);' - RECAPTCHA_KEY = "6Lfln9kSAAAAANZ9JtHSOgxUPB9qfDFeLUI_QMEy" - - def handleFree(self): - found = re.search(self.SECONDS_PATTERN, self.html) - seconds = int(found.group(1)) - self.logDebug("Seconds found", seconds) - self.setWait(seconds + 1) - self.wait() - recaptcha = ReCaptcha(self) - challenge, code = recaptcha.challenge(self.RECAPTCHA_KEY) - post_data = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": code} - self.download(self.pyfile.url, post=post_data) - check = self.checkDownload({"html": re.compile("\A<!DOCTYPE html PUBLIC")}) - if check == "html": - self.logDebug("Wrong captcha entered") - self.invalidCaptcha() - self.retry() - -getInfo = create_getInfo(CatShareNet) diff --git a/module/plugins/hoster/ChipDe.py b/module/plugins/hoster/ChipDe.py deleted file mode 100644 index fcb84a300..000000000 --- a/module/plugins/hoster/ChipDe.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Crypter import Crypter - -class ChipDe(Crypter): - __name__ = "ChipDe" - __type__ = "container" - __pattern__ = r"http://(?:www\.)?chip.de/video/.*\.html" - __version__ = "0.1" - __description__ = """Chip.de Container Plugin""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url) - try: - url = re.search(r'"(http://video.chip.de/\d+?/.*)"', self.html).group(1) - self.logDebug('The file URL is %s' % url) - except: - self.fail('Failed to find the URL') - - self.packages.append((self.pyfile.package().name, [ url ], self.pyfile.package().folder)) diff --git a/module/plugins/hoster/CloudzerNet.py b/module/plugins/hoster/CloudzerNet.py deleted file mode 100644 index 7608b193d..000000000 --- a/module/plugins/hoster/CloudzerNet.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.common.json_layer import json_loads -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getURL - - -def getInfo(urls): - for url in urls: - header = getURL(url, just_header=True) - if 'Location: http://cloudzer.net/404' in header: - file_info = (url, 0, 1, url) - else: - file_info = parseFileInfo(CloudzerNet, url, getURL(url, decode=True)) - yield file_info - - -class CloudzerNet(SimpleHoster): - __name__ = "CloudzerNet" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?(cloudzer\.net/file/|clz\.to/(file/)?)(?P<ID>\w+).*" - __version__ = "0.02" - __description__ = """Cloudzer.net hoster plugin""" - __author_name__ = ("gs", "z00nx") - __author_mail__ = ("I-_-I-_-I@web.de", "z00nx0@gmail.com") - - FILE_SIZE_PATTERN = '<span class="size">(?P<S>[^<]+)</span>' - WAIT_PATTERN = '<meta name="wait" content="(\d+)">' - FILE_OFFLINE_PATTERN = r'Please check the URL for typing errors, respectively' - CAPTCHA_KEY = '6Lcqz78SAAAAAPgsTYF3UlGf2QFQCNuPMenuyHF3' - - def handleFree(self): - found = re.search(self.WAIT_PATTERN, self.html) - seconds = int(found.group(1)) - self.logDebug("Found wait", seconds) - self.setWait(seconds + 1) - self.wait() - response = self.load('http://cloudzer.net/io/ticket/slot/%s' % self.file_info['ID'], post=' ', cookies=True) - self.logDebug("Download slot request response", response) - response = json_loads(response) - if response["succ"] is not True: - self.fail("Unable to get a download slot") - - recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge(self.CAPTCHA_KEY) - post_data = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": response} - response = json_loads(self.load('http://cloudzer.net/io/ticket/captcha/%s' % self.file_info['ID'], post=post_data, cookies=True)) - self.logDebug("Captcha check response", response) - self.logDebug("First check") - - if "err" in response: - if response["err"] == "captcha": - self.logDebug("Wrong captcha") - self.invalidCaptcha() - self.retry() - elif "Sie haben die max" in response["err"] or "You have reached the max" in response["err"]: - self.logDebug("Download limit reached, waiting an hour") - self.setWait(3600, True) - self.wait() - if "type" in response: - if response["type"] == "download": - url = response["url"] - self.logDebug("Download link", url) - self.download(url, disposition=True) diff --git a/module/plugins/hoster/CramitIn.py b/module/plugins/hoster/CramitIn.py deleted file mode 100644 index 171fba0ff..000000000 --- a/module/plugins/hoster/CramitIn.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class CramitIn(XFileSharingPro): - __name__ = "CramitIn" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*cramit.in/\w{12}" - __version__ = "0.04" - __description__ = """Cramit.in hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<span class=t2>\s*(?P<N>.*?)</span>.*?<small>\s*\((?P<S>.*?)\)' - DIRECT_LINK_PATTERN = r'href="(http://cramit.in/file_download/.*?)"' - HOSTER_NAME = "cramit.in" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - -getInfo = create_getInfo(CramitIn)
\ No newline at end of file diff --git a/module/plugins/hoster/CrockoCom.py b/module/plugins/hoster/CrockoCom.py deleted file mode 100644 index 27ab52436..000000000 --- a/module/plugins/hoster/CrockoCom.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha -import re - -class CrockoCom(SimpleHoster): - __name__ = "CrockoCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?(crocko|easy-share).com/.*" - __version__ = "0.13" - __description__ = """Crocko Download Hoster""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<span class="fz24">Download:\s*<strong>(?P<N>.*)' - FILE_SIZE_PATTERN = r'<span class="tip1"><span class="inner">(?P<S>[^<]+)</span></span>' - FILE_OFFLINE_PATTERN = r"<h1>Sorry,<br />the page you're looking for <br />isn't here.</h1>" - DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" - CAPTCHA_URL_PATTERN = re.compile(r"u='(/file_contents/captcha/\w+)';\s*w='(\d+)';") - CAPTCHA_KEY_PATTERN = re.compile(r'Recaptcha.create\("([^"]+)"') - - FORM_PATTERN = r'<form method="post" action="([^"]+)">(.*?)</form>' - FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' - - FILE_NAME_REPLACEMENTS = [(r'<[^>]*>', '')] - - def handleFree(self): - if "You need Premium membership to download this file." in self.html: - self.fail("You need Premium membership to download this file.") - - url = False - for i in range(5): - found = re.search(self.CAPTCHA_URL_PATTERN, self.html) - if found: - url, wait_time = 'http://crocko.com' + found.group(1), found.group(2) - self.setWait(wait_time) - self.wait() - self.html = self.load(url) - else: - break - - found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) - if not found: self.parseError('Captcha KEY') - captcha_key = found.group(1) - - found = re.search(self.FORM_PATTERN, self.html, re.DOTALL) - if not found: self.parseError('ACTION') - action, form = found.groups() - inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) - - recaptcha = ReCaptcha(self) - - for i in range(5): - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - self.download(action, post = inputs) - - check = self.checkDownload({ - "captcha_err": self.CAPTCHA_KEY_PATTERN - }) - - if check == "captcha_err": - self.invalidCaptcha() - else: - break - else: - self.fail('No valid captcha solution received') - -getInfo = create_getInfo(CrockoCom) -
\ No newline at end of file diff --git a/module/plugins/hoster/CyberlockerCh.py b/module/plugins/hoster/CyberlockerCh.py deleted file mode 100644 index 57dd26787..000000000 --- a/module/plugins/hoster/CyberlockerCh.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class CyberlockerCh(XFileSharingPro): - __name__ = "CyberlockerCh" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?cyberlocker\.ch/\w{12}" - __version__ = "0.01" - __description__ = """Cyberlocker.ch hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - HOSTER_NAME = "cyberlocker.ch" - -getInfo = create_getInfo(CyberlockerCh) diff --git a/module/plugins/hoster/CzshareCom.py b/module/plugins/hoster/CzshareCom.py deleted file mode 100644 index 347427586..000000000 --- a/module/plugins/hoster/CzshareCom.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -# Test links (random.bin): -# http://czshare.com/5278880/random.bin - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError -from module.utils import parseFileSize - -class CzshareCom(SimpleHoster): - __name__ = "CzshareCom" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)*czshare\.(com|cz)/(\d+/|download.php\?).*" - __version__ = "0.93" - __description__ = """CZshare.com""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=[^>]*>(?P<N>[^<]+)</a>' - FILE_SIZE_PATTERN = r'<div class="tab" id="category">(?:\s*<p>[^\n]*</p>)*\s*Velikost:\s*(?P<S>[0-9., ]+)(?P<U>[kKMG])i?B\s*</div>' - FILE_OFFLINE_PATTERN = r'<div class="header clearfix">\s*<h2 class="red">' - - FILE_SIZE_REPLACEMENTS = [(' ', '')] - FILE_URL_REPLACEMENTS = [(r'http://[^/]*/download.php\?.*?id=(\w+).*', r'http://czshare.com/\1/x/')] - SH_CHECK_TRAFFIC = True - - FREE_URL_PATTERN = r'<a href="([^"]+)" class="page-download">[^>]*alt="([^"]+)" /></a>' - FREE_FORM_PATTERN = r'<form action="download.php" method="post">\s*<img src="captcha.php" id="captcha" />(.*?)</form>' - PREMIUM_FORM_PATTERN = r'<form action="/profi_down.php" method="post">(.*?)</form>' - FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)"[^>]*/>' - MULTIDL_PATTERN = r"<p><font color='red'>Z[^<]*PROFI.</font></p>" - USER_CREDIT_PATTERN = r'<div class="credit">\s*kredit: <strong>([0-9., ]+)([kKMG]i?B)</strong>\s*</div><!-- .credit -->' - - def setup(self): - self.multiDL = self.resumeDownload = True if self.premium else False - self.chunkLimit = 1 - - def checkTrafficLeft(self): - # check if user logged in - found = re.search(self.USER_CREDIT_PATTERN, self.html) - if not found: - self.account.relogin(self.user) - self.html = self.load(self.pyfile.url, cookies=True, decode=True) - found = re.search(self.USER_CREDIT_PATTERN, self.html) - if not found: return False - - # check user credit - try: - credit = parseFileSize(found.group(1).replace(' ',''), found.group(2)) - self.logInfo("Premium download for %i KiB of Credit" % (self.pyfile.size / 1024)) - self.logInfo("User %s has %i KiB left" % (self.user, credit / 1024)) - if credit < self.pyfile.size: - self.logInfo("Not enough credit to download file %s" % self.pyfile.name) - return False - except Exception, e: - # let's continue and see what happens... - self.logError('Parse error (CREDIT): %s' % e) - - return True - - def handlePremium(self): - # parse download link - try: - form = re.search(self.PREMIUM_FORM_PATTERN, self.html, re.DOTALL).group(1) - inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) - except Exception, e: - self.logError("Parse error (FORM): %s" % e) - self.resetAccount() - - # download the file, destination is determined by pyLoad - self.download("http://czshare.com/profi_down.php", post=inputs, disposition=True) - self.checkDownloadedFile() - - def handleFree(self): - # get free url - found = re.search(self.FREE_URL_PATTERN, self.html) - if found is None: - raise PluginParseError('Free URL') - parsed_url = "http://czshare.com" + found.group(1) - self.logDebug("PARSED_URL:" + parsed_url) - - # get download ticket and parse html - self.html = self.load(parsed_url, cookies=True, decode=True) - if re.search(self.MULTIDL_PATTERN, self.html): - self.longWait(300, 12) - - try: - form = re.search(self.FREE_FORM_PATTERN, self.html, re.DOTALL).group(1) - inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) - self.pyfile.size = int(inputs['size']) - except Exception, e: - self.logError(e) - raise PluginParseError('Form') - - # get and decrypt captcha - captcha_url = 'http://czshare.com/captcha.php' - for i in range(5): - inputs['captchastring2'] = self.decryptCaptcha(captcha_url) - self.html = self.load(parsed_url, cookies=True, post=inputs, decode=True) - if u"<li>ZadanÃœ ovÄÅovacà kód nesouhlasÃ!</li>" in self.html: - self.invalidCaptcha() - elif re.search(self.MULTIDL_PATTERN, self.html): - self.longWait(300, 12) - else: - self.correctCaptcha() - break - else: - self.fail("No valid captcha code entered") - - found = re.search("countdown_number = (\d+);", self.html) - self.setWait(int(found.group(1)) if found else 50) - - # download the file, destination is determined by pyLoad - self.logDebug("WAIT URL", self.req.lastEffectiveURL) - found = re.search("free_wait.php\?server=(.*?)&(.*)", self.req.lastEffectiveURL) - if not found: - raise PluginParseError('Download URL') - - url = "http://%s/download.php?%s" % (found.group(1), found.group(2)) - - self.wait() - self.multiDL = True - self.download(url) - self.checkDownloadedFile() - - def checkDownloadedFile(self): - # check download - check = self.checkDownload({ - "tempoffline": re.compile(r"^Soubor je do.*asn.* nedostupn.*$"), - "credit": re.compile(r"^Nem.*te dostate.*n.* kredit.$"), - "multi_dl": re.compile(self.MULTIDL_PATTERN), - "captcha_err": "<li>ZadanÃœ ovÄÅovacà kód nesouhlasÃ!</li>" - }) - - if check == "tempoffline": - self.fail("File not available - try later") - if check == "credit": - self.resetAccount() - elif check == "multi_dl": - self.longWait(300, 12) - elif check == "captcha_err": - self.invalidCaptcha() - self.retry() - -getInfo = create_getInfo(CzshareCom) diff --git a/module/plugins/hoster/DailymotionCom.py b/module/plugins/hoster/DailymotionCom.py deleted file mode 100644 index 1b411393d..000000000 --- a/module/plugins/hoster/DailymotionCom.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from urllib import unquote -from module.plugins.Hoster import Hoster - -class DailymotionCom(Hoster): - __name__ = 'DailymotionCom' - __type__ = 'hoster' - __pattern__ = r'http://www.dailymotion.com/.*' - __version__ = '0.1' - __description__ = """Dailymotion Video Download Hoster""" - __author_name__ = ("Peekayy") - __author_mail__ = ("peekayy.dev@gmail.com") - - def process(self, pyfile): - html = self.load(pyfile.url, decode=True) - - for pattern in (r'name="title" content="Dailymotion \\-(.*?)\\- ein Film', - r'class="title" title="(.*?)"', - r'<span class="title foreground" title="(.*?)">', - r'"(?:vs_videotitle|videoTitle|dm_title|ss_mediaTitle)": "(.*?)"'): - filename = re.search(pattern, html) - if filename is not None: break - else: - self.fail("Unable to find file name") - - pyfile.name = filename.group(1)+'.mp4' - self.logDebug('Filename='+pyfile.name) - allLinksInfo = re.search(r'"sequence":"(.*?)"', html) - self.logDebug(allLinksInfo.groups()) - allLinksInfo = unquote(allLinksInfo.group(1)) - - for quality in ('hd720URL', 'hqURL', 'sdURL', 'ldURL', ''): - dlLink = self.getQuality(quality, allLinksInfo) - if dlLink is not None: break - else: - self.fail(r'Unable to find video URL') - - self.logDebug(dlLink) - self.download(dlLink) - - def getQuality(self, quality, data): - link = re.search('"' + quality + '":"(http:[^<>"\']+)"', data) - if link is not None: - return link.group(1).replace('\\','')
\ No newline at end of file diff --git a/module/plugins/hoster/DataHu.py b/module/plugins/hoster/DataHu.py deleted file mode 100644 index 7abd93d1f..000000000 --- a/module/plugins/hoster/DataHu.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -# Test links (random.bin): -# http://data.hu/get/6381232/random.bin - -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - - -class DataHu(SimpleHoster): - __name__ = "DataHu" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?data.hu/get/\w+" - __version__ = "0.01" - __description__ = """Data.hu Download Hoster""" - __author_name__ = ("crash", "stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_INFO_PATTERN = ur'<title>(?P<N>.*) \((?P<S>[^)]+)\) let\xf6lt\xe9se</title>' - FILE_OFFLINE_PATTERN = ur'Az adott f\xe1jl nem l\xe9tezik' - DIRECT_LINK_PATTERN = r'<div class="download_box_button"><a href="([^"]+)">' - - def handleFree(self): - self.resumeDownload = True - self.html = self.load(self.pyfile.url, decode=True) - - m = re.search(self.DIRECT_LINK_PATTERN, self.html) - if m: - url = m.group(1) - self.logDebug('Direct link: ' + url) - else: - self.parseError('Unable to get direct link') - - self.download(url, disposition=True) - - -getInfo = create_getInfo(DataHu) diff --git a/module/plugins/hoster/DataportCz.py b/module/plugins/hoster/DataportCz.py deleted file mode 100644 index 3dc581bf1..000000000 --- a/module/plugins/hoster/DataportCz.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError -from pycurl import FOLLOWLOCATION - -class DataportCz(SimpleHoster): - __name__ = "DataportCz" - __type__ = "hoster" - __pattern__ = r"http://(?:.*?\.)?dataport.cz/file/(.*)" - __version__ = "0.37" - __description__ = """Dataport.cz plugin - free only""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<span itemprop="name">(?P<N>[^<]+)</span>' - FILE_SIZE_PATTERN = r'<td class="fil">Velikost</td>\s*<td>(?P<S>[^<]+)</td>' - FILE_OFFLINE_PATTERN = r'<h2>Soubor nebyl nalezen</h2>' - FILE_URL_REPLACEMENTS = [(__pattern__, r'http://www.dataport.cz/file/\1')] - - CAPTCHA_URL_PATTERN = r'<section id="captcha_bg">\s*<img src="(.*?)"' - FREE_SLOTS_PATTERN = ur'PoÄet volnÃœch slotů: <span class="darkblue">(\d+)</span><br />' - - def handleFree(self): - captchas = {"1": "jkeG", "2": "hMJQ", "3": "vmEK", "4": "ePQM", "5": "blBd"} - - for i in range(60): - action, inputs = self.parseHtmlForm('free_download_form') - self.logDebug(action, inputs) - if not action or not inputs: - raise PluginParseError('free_download_form') - - if "captchaId" in inputs and inputs["captchaId"] in captchas: - inputs['captchaCode'] = captchas[inputs["captchaId"]] - else: - raise PluginParseError('captcha') - - self.html = self.download("http://www.dataport.cz%s" % action, post = inputs) - - check = self.checkDownload({"captcha": 'alert("\u0160patn\u011b opsan\u00fd k\u00f3d z obr\u00e1zu");', - "slot": 'alert("Je n\u00e1m l\u00edto, ale moment\u00e1ln\u011b nejsou'}) - if check == "captcha": - raise PluginParseError('invalid captcha') - elif check == "slot": - self.logDebug("No free slots - wait 60s and retry") - self.setWait(60, False) - self.wait() - self.html = self.load(self.pyfile.url, decode = True) - continue - else: - break - -create_getInfo(DataportCz)
\ No newline at end of file diff --git a/module/plugins/hoster/DateiTo.py b/module/plugins/hoster/DateiTo.py deleted file mode 100644 index 8d994c179..000000000 --- a/module/plugins/hoster/DateiTo.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha - -class DateiTo(SimpleHoster): - __name__ = "DateiTo" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?datei\.to/datei/(?P<ID>\w+)\.html" - __version__ = "0.02" - __description__ = """Datei.to plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'Dateiname:</td>\s*<td colspan="2"><strong>(?P<N>.*?)</' - FILE_SIZE_PATTERN = r'Dateigröße:</td>\s*<td colspan="2">(?P<S>.*?)</' - FILE_OFFLINE_PATTERN = r'>Datei wurde nicht gefunden<|>Bitte wÀhle deine Datei aus... <' - PARALELL_PATTERN = r'>Du lÀdst bereits eine Datei herunter<' - - WAIT_PATTERN = r'countdown\({seconds: (\d+)' - DATA_PATTERN = r'url: "(.*?)", data: "(.*?)",' - RECAPTCHA_KEY_PATTERN = r'Recaptcha.create\("(.*?)"' - - def handleFree(self): - url = 'http://datei.to/ajax/download.php' - data = {'P': 'I', 'ID': self.file_info['ID']} - - recaptcha = ReCaptcha(self) - - for i in range(10): - self.logDebug("URL", url, "POST", data) - self.html = self.load(url, post = data) - self.checkErrors() - - if url.endswith('download.php') and 'P' in data: - if data['P'] == 'I': - self.doWait() - - elif data['P'] == 'IV': - break - - found = re.search(self.DATA_PATTERN, self.html) - if not found: self.parseError('data') - url = 'http://datei.to/' + found.group(1) - data = dict(x.split('=') for x in found.group(2).split('&')) - - if url.endswith('recaptcha.php'): - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - recaptcha_key = found.group(1) if found else "6LdBbL8SAAAAAI0vKUo58XRwDd5Tu_Ze1DA7qTao" - - data['recaptcha_challenge_field'], data['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key) - - else: - self.fail('Too bad...') - - download_url = self.html - self.logDebug('Download URL', download_url) - self.download(download_url) - - def checkErrors(self): - found = re.search(self.PARALELL_PATTERN, self.html) - if found: - found = re.search(self.WAIT_PATTERN, self.html) - wait_time = int(found.group(1)) if found else 30 - self.setWait(wait_time + 1, False) - self.wait(300) - self.retry() - - def doWait(self): - found = re.search(self.WAIT_PATTERN, self.html) - wait_time = int(found.group(1)) if found else 30 - self.setWait(wait_time + 1, False) - - self.load('http://datei.to/ajax/download.php', post = {'P': 'Ads'}) - self.wait() - -getInfo = create_getInfo(DateiTo) diff --git a/module/plugins/hoster/DdlstorageCom.py b/module/plugins/hoster/DdlstorageCom.py deleted file mode 100644 index 1ad5fa6d8..000000000 --- a/module/plugins/hoster/DdlstorageCom.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class DdlstorageCom(XFileSharingPro): - __name__ = "DdlstorageCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?ddlstorage.com/\w{12}" - __version__ = "0.06" - __description__ = """DDLStorage.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<h2>\s*Download File\s*<span[^>]*>(?P<N>[^>]+)</span></h2>\s*[^\(]*\((?P<S>[^\)]+)\)</h2>' - HOSTER_NAME = "ddlstorage.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - self.chunkLimit = 1 - -getInfo = create_getInfo(DdlstorageCom)
\ No newline at end of file diff --git a/module/plugins/hoster/DebridItaliaCom.py b/module/plugins/hoster/DebridItaliaCom.py deleted file mode 100644 index 470c4ae5d..000000000 --- a/module/plugins/hoster/DebridItaliaCom.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -import re - -from module.plugins.Hoster import Hoster - - -class DebridItaliaCom(Hoster): - __name__ = "DebridItaliaCom" - __version__ = "0.03" - __type__ = "hoster" - __pattern__ = r"https?://.*debriditalia\.com" - __description__ = """Debriditalia.com hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - def init(self): - self.chunkLimit = -1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: - self.logError("Please enter your DebridItalia account or deactivate this plugin") - self.fail("No DebridItalia account provided") - - self.logDebug("Old URL: %s" % pyfile.url) - if re.match(self.__pattern__, pyfile.url): - new_url = pyfile.url - else: - url = "http://debriditalia.com/linkgen2.php?xjxfun=convertiLink&xjxargs[]=S<![CDATA[%s]]>" % pyfile.url - page = self.load(url) - self.logDebug("XML data: %s" % page) - - if 'File not available' in page: - self.fail('File not available') - else: - new_url = re.search(r'<a href="(?:[^"]+)">(?P<direct>[^<]+)</a>', page).group('direct') - - self.logDebug("New URL: %s" % new_url) - - self.download(new_url, disposition=True) - - check = self.checkDownload({"empty": re.compile(r"^$")}) - - if check == "empty": - self.retry(5, 120, 'Empty file downloaded') diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py deleted file mode 100644 index eb64ae4de..000000000 --- a/module/plugins/hoster/DepositfilesCom.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from urllib import unquote -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.RequestFactory import getURL -from module.plugins.ReCaptcha import ReCaptcha - -class DepositfilesCom(SimpleHoster): - __name__ = "DepositfilesCom" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?(depositfiles\.com|dfiles\.eu)(/\w{1,3})?/files/[\w]+" - __version__ = "0.43" - __description__ = """Depositfiles.com Download Hoster""" - __author_name__ = ("spoob", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'File name: <b title="(?P<N>[^"]+)' - FILE_SIZE_PATTERN = r'File size: <b>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</b>' - FILE_INFO_PATTERN = r'<script type="text/javascript">eval\( unescape\(\'(?P<N>.*?)\'' - FILE_OFFLINE_PATTERN = r'<span class="html_download_api-not_exists"></span>' - FILE_URL_REPLACEMENTS = [(r"\.com(/.*?)?/files", ".com/en/files"), (r"\.html$", "")] - FILE_NAME_REPLACEMENTS = [(r'\%u([0-9A-Fa-f]{4})', lambda m: unichr(int(m.group(1), 16))), (r'.*<b title="(?P<N>[^"]+).*', "\g<N>" )] - - RECAPTCHA_PATTERN = r"Recaptcha.create\('([^']+)'" - DOWNLOAD_LINK_PATTERN = r'<form id="downloader_file_form" action="(http://.+?\.(dfiles\.eu|depositfiles\.com)/.+?)" method="post"' - - def setup(self): - self.multiDL = False - self.resumeDownload = self.premium - - def handleFree(self): - self.html = self.load(self.pyfile.url, post={"gateway_result":"1"}, cookies = True) - if re.search(self.FILE_OFFLINE_PATTERN, self.html): self.offline() - - if re.search(r'File is checked, please try again in a minute.', self.html) is not None: - self.log.info("DepositFiles.com: The file is being checked. Waiting 1 minute.") - self.setWait(61) - self.wait() - self.retry() - - wait = re.search(r'html_download_api-limit_interval\">(\d+)</span>', self.html) - if wait: - wait_time = int(wait.group(1)) - self.log.info( "%s: Traffic used up. Waiting %d seconds." % (self.__name__, wait_time) ) - self.setWait(wait_time) - self.wantReconnect = True - self.wait() - self.retry() - - wait = re.search(r'>Try in (\d+) minutes or use GOLD account', self.html) - if wait: - wait_time = int(wait.group(1)) - self.log.info( "%s: All free slots occupied. Waiting %d minutes." % (self.__name__, wait_time) ) - self.setWait(wait_time * 60, False) - - wait = re.search(r'Please wait (\d+) sec', self.html) - if wait: - self.setWait(int(wait.group(1))) - - found = re.search(r"var fid = '(\w+)';", self.html) - if not found: self.retry(wait_time=5) - params = {'fid' : found.group(1)} - self.logDebug ("FID: %s" % params['fid']) - - captcha_key = '6LdRTL8SAAAAAE9UOdWZ4d0Ky-aeA7XfSqyWDM2m' - found = re.search(self.RECAPTCHA_PATTERN, self.html) - if found: captcha_key = found.group(1) - self.logDebug ("CAPTCHA_KEY: %s" % captcha_key) - - self.wait() - recaptcha = ReCaptcha(self) - - for i in range(5): - self.html = self.load("http://depositfiles.com/get_file.php", get = params) - - if '<input type=button value="Continue" onclick="check_recaptcha' in self.html: - if not captcha_key: self.parseError('Captcha key') - if 'response' in params: self.invalidCaptcha() - params['challenge'], params['response'] = recaptcha.challenge(captcha_key) - self.logDebug(params) - continue - - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if found: - if 'response' in params: self.correctCaptcha() - link = unquote(found.group(1)) - self.logDebug ("LINK: %s" % link) - break - else: - self.parseError('Download link') - else: - self.fail('No valid captcha response received') - - try: - self.download(link, disposition = True) - except: - self.retry(wait_time = 60) - - def handlePremium(self): - if '<span class="html_download_api-gold_traffic_limit">' in self.html: - self.logWarning("Download limit reached") - self.retry(25, 3600, "Download limit reached") - elif 'onClick="show_gold_offer' in self.html: - self.account.relogin(self.user) - self.retry() - link = unquote(re.search('<div id="download_url">\s*<a href="(http://.+?\.depositfiles.com/.+?)"', self.html).group(1)) - self.multiDL = True - self.download(link, disposition = True) - -getInfo = create_getInfo(DepositfilesCom) diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py deleted file mode 100644 index 67c2d6c17..000000000 --- a/module/plugins/hoster/DlFreeFr.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns -from module.common.json_layer import json_loads - -import pycurl -from module.network.Browser import Browser -from module.network.CookieJar import CookieJar - -class CustomBrowser(Browser): - def __init__(self, bucket=None, options={}): - Browser.__init__(self, bucket, options) - - def load(self, *args, **kwargs): - post = kwargs.get("post") - if post is None: - if len(args) > 2: - post = args[2] - if post: - self.http.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.http.c.setopt(pycurl.POST, 1) - self.http.c.setopt(pycurl.CUSTOMREQUEST, "POST") - else: - self.http.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.http.c.setopt(pycurl.POST, 0) - self.http.c.setopt(pycurl.CUSTOMREQUEST, "GET") - return Browser.load(self, *args, **kwargs) - -""" -Class to support adyoulike captcha service -""" -class AdYouLike(): - ADYOULIKE_INPUT_PATTERN = r'Adyoulike.create\((.*?)\);' - ADYOULIKE_CALLBACK = r'Adyoulike.g._jsonp_5579316662423138' - ADYOULIKE_CHALLENGE_PATTERN = ADYOULIKE_CALLBACK + r'\((.*?)\)' - - def __init__(self, plugin, engine = "adyoulike"): - self.plugin = plugin - self.engine = engine - - def challenge(self, html): - adyoulike_data_string = None - found = re.search(self.ADYOULIKE_INPUT_PATTERN, html) - if found: - adyoulike_data_string = found.group(1) - else: - self.plugin.fail("Can't read AdYouLike input data") - - ayl_data = json_loads(adyoulike_data_string) #{"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"},"all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} - - res = self.plugin.load(r'http://api-ayl.appspot.com/challenge?key=%(ayl_key)s&env=%(ayl_env)s&callback=%(callback)s' % {"ayl_key": ayl_data[self.engine]["key"], "ayl_env": ayl_data["all"]["env"], "callback": self.ADYOULIKE_CALLBACK}) - - found = re.search(self.ADYOULIKE_CHALLENGE_PATTERN, res) - challenge_string = None - if found: - challenge_string = found.group(1) - else: - self.plugin.fail("Invalid AdYouLike challenge") - challenge_data = json_loads(challenge_string) - - return ayl_data, challenge_data - - def result(self, ayl, challenge): - """ - Adyoulike.g._jsonp_5579316662423138({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}},"site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[],"VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike","iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\" height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256,"token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60},"big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}},"tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) - """ - response = None - try: - instructions_visual = challenge["translations"][ayl["all"]["lang"]]["instructions_visual"] - found = re.search(u".*«(.*)».*", instructions_visual) - if found: - response = found.group(1).strip() - else: - self.plugin.fail("Can't parse instructions visual") - except KeyError: - self.plugin.fail("No instructions visual") - - #TODO: Supports captcha - - if not response: - self.plugin.fail("AdYouLike result failed") - - return {"_ayl_captcha_engine" : self.engine, - "_ayl_env" : ayl["all"]["env"], - "_ayl_tid" : challenge["tid"], - "_ayl_token_challenge" : challenge["token"], - "_ayl_response": response } - -class DlFreeFr(SimpleHoster): - __name__ = "DlFreeFr" - __type__ = "hoster" - __pattern__ = r"http://dl\.free\.fr/([a-zA-Z0-9]+|getfile\.pl\?file=/[a-zA-Z0-9]+)" - __version__ = "0.24" - __description__ = """dl.free.fr download hoster""" - __author_name__ = ("the-razer", "zoidberg", "Toilal") - __author_mail__ = ("daniel_ AT gmx DOT net", "zoidberg@mujmail.cz", "toilal.dev@gmail.com") - - FILE_NAME_PATTERN = r"Fichier:</td>\s*<td[^>]*>(?P<N>[^>]*)</td>" - FILE_SIZE_PATTERN = r"Taille:</td>\s*<td[^>]*>(?P<S>[\d.]+[KMG])o" - FILE_OFFLINE_PATTERN = r"Erreur 404 - Document non trouv|Fichier inexistant|Le fichier demandé n'a pas été trouvé" - #FILE_URL_PATTERN = r'href="(?P<url>http://.*?)">Télécharger ce fichier' - - def setup(self): - self.limitDL = 5 - self.resumeDownload = True - self.chunkLimit = 1 - - def init(self): - factory = self.core.requestFactory - self.req = CustomBrowser(factory.bucket, factory.getOptions()) - - def process(self, pyfile): - self.req.setCookieJar(None) - - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - valid_url = pyfile.url - headers = self.load(valid_url, just_header = True) - - self.html = None - if headers.get('code') == 302: - valid_url = headers.get('location') - headers = self.load(valid_url, just_header = True) - - if headers.get('code') == 200: - content_type = headers.get('content-type') - if content_type and content_type.startswith("text/html"): - # Undirect acces to requested file, with a web page providing it (captcha) - self.html = self.load(valid_url) - self.handleFree() - else: - # Direct access to requested file for users using free.fr as Internet Service Provider. - self.download(valid_url, disposition=True) - elif headers.get('code') == 404: - self.offline() - else: - self.fail("Invalid return code: " + str(headers.get('code'))) - - def handleFree(self): - action, inputs = self.parseHtmlForm('action="getfile.pl"') - - adyoulike = AdYouLike(self) - ayl, challenge = adyoulike.challenge(self.html) - result = adyoulike.result(ayl, challenge) - inputs.update(result) - - self.load("http://dl.free.fr/getfile.pl", post = inputs) - headers = self.getLastHeaders() - if headers.get("code") == 302 and headers.has_key("set-cookie") and headers.has_key("location"): - found = re.search("(.*?)=(.*?); path=(.*?); domain=(.*?)", headers.get("set-cookie")) - cj = CookieJar(__name__) - if found: - cj.setCookie(found.group(4), found.group(1), found.group(2), found.group(3)) - else: - self.fail("Cookie error") - location = headers.get("location") - self.req.setCookieJar(cj) - self.download(location, disposition=True); - else: - self.fail("Invalid response") - - def getLastHeaders(self): - #parse header - header = {"code": self.req.code} - for line in self.req.http.header.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - return header - -getInfo = create_getInfo(DlFreeFr) diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py deleted file mode 100644 index 96e3d93d2..000000000 --- a/module/plugins/hoster/EasybytezCom.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class EasybytezCom(XFileSharingPro): - __name__ = "EasybytezCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w+).*" - __version__ = "0.14" - __description__ = """easybytez.com""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>[^"]+)"' - FILE_SIZE_PATTERN = r'You have requested <font color="red">[^<]+</font> \((?P<S>[^<]+)\)</font>' - FILE_INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>' - FILE_OFFLINE_PATTERN = r'<h1>File not available</h1>' - - DIRECT_LINK_PATTERN = r'(http://(\w+\.(easybytez|zingload)\.com|\d+\.\d+\.\d+\.\d+)/files/\d+/\w+/[^"<]+)' - OVR_DOWNLOAD_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' - OVR_KILL_LINK_PATTERN = r'<h2>Delete Link</h2>\s*<textarea[^>]*>([^<]+)' - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<Center><b>)(.*?)</' - - HOSTER_NAME = "easybytez.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - - -getInfo = create_getInfo(EasybytezCom) diff --git a/module/plugins/hoster/EdiskCz.py b/module/plugins/hoster/EdiskCz.py deleted file mode 100644 index a253be0d9..000000000 --- a/module/plugins/hoster/EdiskCz.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class EdiskCz(SimpleHoster): - __name__ = "EdiskCz" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?edisk.(cz|sk|eu)/(stahni|sk/stahni|en/download)/.*" - __version__ = "0.21" - __description__ = """Edisk.cz""" - __author_name__ = ("zoidberg") - - URL_PATTERN = r'<form name = "formular" action = "([^"]+)" method = "post">' - FILE_INFO_PATTERN = r'<span class="fl" title="(?P<N>[^"]+)">\s*.*?\((?P<S>[0-9.]*) (?P<U>[kKMG])i?B\)</h1></span>' - ACTION_PATTERN = r'/en/download/(\d+/.*\.html)' - DLLINK_PATTERN = r'http://.*edisk.cz.*\.html' - FILE_OFFLINE_PATTERN = r'<h3>This file does not exist due to one of the following:</h3><ul><li>' - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - url = re.sub("/(stahni|sk/stahni)/", "/en/download/", pyfile.url) - - self.logDebug('URL:' + url) - - found = re.search(self.ACTION_PATTERN, url) - if found is None: self.parseError("ACTION") - action = found.group(1) - - self.html = self.load(url, decode=True) - self.getFileInfo() - - self.html = self.load(re.sub("/en/download/", "/en/download-slow/", url)) - - url = self.load(re.sub("/en/download/", "/x-download/", url), post={ - "action": action - }) - - if not re.match(self.DLLINK_PATTERN, url): - self.fail("Unexpected server response") - - self.download(url) - -getInfo = create_getInfo(EdiskCz)
\ No newline at end of file diff --git a/module/plugins/hoster/EgoFilesCom.py b/module/plugins/hoster/EgoFilesCom.py deleted file mode 100644 index 7e6673210..000000000 --- a/module/plugins/hoster/EgoFilesCom.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*-
-
-############################################################################
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU Affero General Public License as #
-# published by the Free Software Foundation, either version 3 of the #
-# License, or (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU Affero General Public License for more details. #
-# #
-# You should have received a copy of the GNU Affero General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-############################################################################
-
-# Test link (random.bin):
-# http://egofiles.com/mOZfMI1WLZ6HBkGG/random.bin
-
-import re
-
-from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo
-from module.plugins.ReCaptcha import ReCaptcha
-
-
-class EgoFilesCom(SimpleHoster):
- __name__ = "EgoFilesCom"
- __type__ = "hoster"
- __pattern__ = r"https?://(www\.)?egofiles.com/(\w+)"
- __version__ = "0.13"
- __description__ = """Egofiles.com Download Hoster"""
- __author_name__ = ("stickell")
- __author_mail__ = ("l.stickell@yahoo.it")
-
- FILE_INFO_PATTERN = r'<div class="down-file">\s+(?P<N>[^\t]+)\s+<div class="file-properties">\s+(File size|Rozmiar): (?P<S>[\w.]+) (?P<U>\w+) \|'
- FILE_OFFLINE_PATTERN = r'(File size|Rozmiar): 0 KB'
- WAIT_TIME_PATTERN = r'For next free download you have to wait <strong>((?P<m>\d*)m)? ?((?P<s>\d+)s)?</strong>'
- DIRECT_LINK_PATTERN = r'<a href="(?P<link>[^"]+)">Download ></a>'
- RECAPTCHA_KEY = '6LeXatQSAAAAAHezcjXyWAni-4t302TeYe7_gfvX'
-
- def init(self):
- self.file_info = {}
- # Set English language
- self.load("https://egofiles.com/ajax/lang.php?lang=en", just_header=True)
-
- def process(self, pyfile):
- if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.handlePremium()
- else:
- self.handleFree()
-
- def handleFree(self):
- self.html = self.load(self.pyfile.url, decode=True)
- self.getFileInfo()
-
- # Wait time between free downloads
- if 'For next free download you have to wait' in self.html:
- m = re.search(self.WAIT_TIME_PATTERN, self.html).groupdict('0')
- waittime = int(m['m']) * 60 + int(m['s'])
- self.setWait(waittime, True)
- self.wait()
-
- downloadURL = ''
- recaptcha = ReCaptcha(self)
- for i in xrange(5):
- challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY)
- post_data = {'recaptcha_challenge_field': challenge,
- 'recaptcha_response_field': response}
- self.html = self.load(self.pyfile.url, post=post_data, decode=True)
- m = re.search(self.DIRECT_LINK_PATTERN, self.html)
- if not m:
- self.logInfo('Wrong captcha')
- self.invalidCaptcha()
- elif hasattr(m, 'group'):
- downloadURL = m.group('link')
- self.correctCaptcha()
- break
- else:
- self.fail('Unknown error - Plugin may be out of date')
-
- if not downloadURL:
- self.fail("No Download url retrieved/all captcha attempts failed")
-
- self.download(downloadURL, disposition=True)
-
- def handlePremium(self):
- header = self.load(self.pyfile.url, just_header=True)
- if header.has_key('location'):
- self.logDebug('DIRECT LINK from header: ' + header['location'])
- self.download(header['location'])
- else:
- self.html = self.load(self.pyfile.url, decode=True)
- self.getFileInfo()
- m = re.search(r'<a href="(?P<link>[^"]+)">Download ></a>', self.html)
- if not m:
- self.parseError('Unable to detect direct download url')
- else:
- self.logDebug('DIRECT URL from html: ' + m.group('link'))
- self.download(m.group('link'), disposition=True)
-
-
-getInfo = create_getInfo(EgoFilesCom)
diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py deleted file mode 100644 index 5224dfd9f..000000000 --- a/module/plugins/hoster/EuroshareEu.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class EuroshareEu(SimpleHoster): - __name__ = "EuroshareEu" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?euroshare.(eu|sk|cz|hu|pl)/file/.*" - __version__ = "0.25" - __description__ = """Euroshare.eu""" - __author_name__ = ("zoidberg") - - FILE_INFO_PATTERN = r'<span style="float: left;"><strong>(?P<N>.+?)</strong> \((?P<S>.+?)\)</span>' - FILE_OFFLINE_PATTERN = ur'<h2>S.bor sa nena.iel</h2>|Poşadovaná stránka neexistuje!' - - FREE_URL_PATTERN = r'<a href="(/file/\d+/[^/]*/download/)"><div class="downloadButton"' - ERR_PARDL_PATTERN = r'<h2>Prebieha s.ahovanie</h2>|<p>Naraz je z jednej IP adresy mo.n. s.ahova. iba jeden s.bor' - ERR_NOT_LOGGED_IN_PATTERN = r'href="/customer-zone/login/"' - - FILE_URL_REPLACEMENTS = [(r"(http://[^/]*\.)(sk|cz|hu|pl)/", r"\1eu/")] - - def setup(self): - self.multiDL = self.resumeDownload = self.premium - self.req.setOption("timeout", 120) - - def handlePremium(self): - if self.ERR_NOT_LOGGED_IN_PATTERN in self.html: - self.account.relogin(self.user) - self.retry(reason="User not logged in") - - self.download(self.pyfile.url.rstrip('/') + "/download/") - - check = self.checkDownload({"login": re.compile(self.ERR_NOT_LOGGED_IN_PATTERN), - "json": re.compile(r'\{"status":"error".*?"message":"(.*?)"') - }) - if check == "login" or (check == "json" and self.lastCheck.group(1) == "Access token expired"): - self.account.relogin(self.user) - self.retry(reason="Access token expired") - elif check == "json": - self.fail(self.lastCheck.group(1)) - - def handleFree(self): - if re.search(self.ERR_PARDL_PATTERN, self.html) is not None: - self.longWait(300, 12) - - found = re.search(self.FREE_URL_PATTERN, self.html) - if found is None: - self.parseError("Parse error (URL)") - parsed_url = "http://euroshare.eu%s" % found.group(1) - self.logDebug("URL", parsed_url) - self.download(parsed_url, disposition=True) - - check = self.checkDownload({"multi_dl": re.compile(self.ERR_PARDL_PATTERN)}) - if check == "multi_dl": - self.longWait(300, 12) - -getInfo = create_getInfo(EuroshareEu)
\ No newline at end of file diff --git a/module/plugins/hoster/ExtabitCom.py b/module/plugins/hoster/ExtabitCom.py deleted file mode 100644 index fd91bb023..000000000 --- a/module/plugins/hoster/ExtabitCom.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads - - -class ExtabitCom(SimpleHoster): - __name__ = "ExtabitCom" - __type__ = "hoster" - __pattern__ = r"http://(\w+\.)*extabit\.com/(file|go|fid)/(?P<ID>\w+)" - __version__ = "0.3" - __description__ = """Extabit.com""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<th>File:</th>\s*<td class="col-fileinfo">\s*<div title="(?P<N>[^"]+)">' - FILE_SIZE_PATTERN = r'<th>Size:</th>\s*<td class="col-fileinfo">(?P<S>[^<]+)</td>' - FILE_OFFLINE_PATTERN = r'<h1>File not found</h1>' - TEMP_OFFLINE_PATTERN = r">(File is temporary unavailable|No download mirror)<" - - DOWNLOAD_LINK_PATTERN = r'"(http://guest\d+\.extabit\.com/[a-z0-9]+/.*?)"' - - def handleFree(self): - if r">Only premium users can download this file" in self.html: - self.fail("Only premium users can download this file") - - m = re.search(r"Next free download from your ip will be available in <b>(\d+)\s*minutes", self.html) - if m: - self.setWait(int(m.group(1)) * 60, True) - self.wait() - elif "The daily downloads limit from your IP is exceeded" in self.html: - self.setWait(3600, True) - self.wait() - - self.logDebug("URL: " + self.req.http.lastEffectiveURL) - m = re.match(self.__pattern__, self.req.http.lastEffectiveURL) - fileID = m.group('ID') if m else self.file_info('ID') - - m = re.search(r'recaptcha/api/challenge\?k=(\w+)', self.html) - if m: - recaptcha = ReCaptcha(self) - captcha_key = m.group(1) - - for i in range(5): - get_data = {"type": "recaptcha"} - get_data["challenge"], get_data["capture"] = recaptcha.challenge(captcha_key) - response = json_loads(self.load("http://extabit.com/file/%s/" % fileID, get=get_data)) - if "ok" in response: - self.correctCaptcha() - break - else: - self.invalidCaptcha() - else: - self.fail("Invalid captcha") - else: - self.parseError('Captcha') - - if not "href" in response: self.parseError('JSON') - - self.html = self.load("http://extabit.com/file/%s%s" % (fileID, response['href'])) - m = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if not m: - self.parseError('Download URL') - url = m.group(1) - self.logDebug("Download URL: " + url) - self.download(url) - - -getInfo = create_getInfo(ExtabitCom) diff --git a/module/plugins/hoster/FastshareCz.py b/module/plugins/hoster/FastshareCz.py deleted file mode 100644 index 1dbf9fe8f..000000000 --- a/module/plugins/hoster/FastshareCz.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -# Test links (random.bin): -# http://www.fastshare.cz/2141189/random.bin - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns - - -class FastshareCz(SimpleHoster): - __name__ = "FastshareCz" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?fastshare.cz/\d+/.+" - __version__ = "0.15" - __description__ = """FastShare.cz""" - __author_name__ = ("zoidberg", "stickell") - - FILE_INFO_PATTERN = r'<h1 class="dwp">(?P<N>[^<]+)</h1>\s*<div class="fileinfo">\s*(?:Velikost|Size)\s*: (?P<S>[^,]+),' - FILE_OFFLINE_PATTERN = ur'<td align=center>Tento soubor byl smazán' - FILE_URL_REPLACEMENTS = [('#.*','')] - - FREE_URL_PATTERN = ur'<form method=post action=(/free/.*?)><b>Stáhnout FREE.*?<img src="([^"]*)">' - PREMIUM_URL_PATTERN = r'(http://data\d+\.fastshare\.cz/download\.php\?id=\d+\&[^\s\"\'<>]+)' - NOT_ENOUGH_CREDIC_PATTERN = "Nem.te dostate.n. kredit pro sta.en. tohoto souboru" - - def process(self, pyfile): - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - self.req.setOption("timeout", 120) - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.handlePremium() - else: - self.html = self.load(pyfile.url, decode = not self.SH_BROKEN_ENCODING, cookies = self.SH_COOKIES) - self.getFileInfo() - self.handleFree() - - def handleFree(self): - if u">100% FREE slotů je plnÜch.<" in self.html: - self.setWait(60, False) - self.wait() - self.retry(120, "No free slots") - - found = re.search(self.FREE_URL_PATTERN, self.html) - if not found: self.parseError("Free URL") - action, captcha_src = found.groups() - captcha = self.decryptCaptcha("http://www.fastshare.cz/" + captcha_src) - self.download("http://www.fastshare.cz/" + action, post = {"code": captcha, "submit": u"stáhnout"}) - - check = self.checkDownload({ - "paralell_dl": "<title>FastShare.cz</title>|<script>alert\('Pres FREE muzete stahovat jen jeden soubor najednou.'\)" - }) - self.logDebug(self.req.lastEffectiveURL, self.req.lastURL, self.req.code) - - if check == "paralell_dl": - self.setWait(600, True) - self.wait() - self.retry(6, "Paralell download") - - def handlePremium(self): - header = self.load(self.pyfile.url, just_header=True) - if 'location' in header: - url = header['location'] - else: - self.html = self.load(self.pyfile.url) - self.getFileInfo() - if self.NOT_ENOUGH_CREDIC_PATTERN in self.html: - self.logWarning('Not enough traffic left') - self.resetAccount() - - found = re.search(self.PREMIUM_URL_PATTERN, self.html) - if not found: self.parseError("Premium URL") - url = found.group(1) - - self.logDebug("PREMIUM URL: %s" % url) - self.download(url) - - check = self.checkDownload({"credit": re.compile(self.NOT_ENOUGH_CREDIC_PATTERN)}) - if check == "credit": - self.resetAccount() - -getInfo = create_getInfo(FastshareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/FileApeCom.py b/module/plugins/hoster/FileApeCom.py deleted file mode 100644 index 1f933e776..000000000 --- a/module/plugins/hoster/FileApeCom.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Hoster import Hoster - -class FileApeCom(Hoster): - __name__ = "FileApeCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?fileape\.com/(index\.php\?act=download\&id=|dl/)\w+" - __version__ = "0.1" - __description__ = """FileApe Download Hoster""" - __author_name__ = ("espes") - - def setup(self): - self.multiDL = False - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - - self.html = self.load(self.pyfile.url) - - if "This file is either temporarily unavailable or does not exist" in self.html: - self.offline() - - self.html = self.load(self.pyfile.url+"&g=1") - - continueMatch = re.search(r"window\.location = '(http://.*?)'", self.html) - if not continueMatch: - continueMatch = re.search(r"'(http://fileape\.com/\?act=download&t=[A-Za-z0-9_-]+)'", self.html) - if continueMatch: - continuePage = continueMatch.group(1) - else: - self.fail("Plugin Defect") - - wait = 60 - waitMatch = re.search("id=\"waitnumber\" style=\"font-size:2em; text-align:center; width:33px; height:33px;\">(\\d+)</span>", self.html) - if waitMatch: - wait = int(waitMatch.group(1)) - self.setWait(wait+3) - self.wait() - - self.html = self.load(continuePage) - linkMatch = \ - re.search(r"<div style=\"text-align:center; font-size: 30px;\"><a href=\"(http://.*?)\"", self.html) - if not linkMatch: - linkMatch = re.search(r"\"(http://tx\d+\.fileape\.com/[a-z]+/.*?)\"", self.html) - if linkMatch: - link = linkMatch.group(1) - else: - self.fail("Plugin Defect") - - pyfile.name = link.rpartition('/')[2] - - self.download(link) - - check = self.checkDownload({"exp": "Download ticket expired"}) - if check == "exp": - self.log.info("Ticket expired, retrying...") - self.retry()
\ No newline at end of file diff --git a/module/plugins/hoster/FilebeerInfo.py b/module/plugins/hoster/FilebeerInfo.py deleted file mode 100644 index 216ecfbca..000000000 --- a/module/plugins/hoster/FilebeerInfo.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.internal.DeadHoster import DeadHoster, create_getInfo - - -class FilebeerInfo(DeadHoster): - __name__ = "FilebeerInfo" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?filebeer\.info/(?!\d*~f)(?P<ID>\w+).*" - __version__ = "0.03" - __description__ = """Filebeer.info plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - -getInfo = create_getInfo(FilebeerInfo)
\ No newline at end of file diff --git a/module/plugins/hoster/FilecloudIo.py b/module/plugins/hoster/FilecloudIo.py deleted file mode 100644 index 4a096e400..000000000 --- a/module/plugins/hoster/FilecloudIo.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError -from module.common.json_layer import json_loads -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getURL - -class FilecloudIo(SimpleHoster): - __name__ = "FilecloudIo" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*(?:filecloud\.io|ifile\.it|mihd\.net)/(?P<ID>\w+).*" - __version__ = "0.01" - __description__ = """Filecloud.io (formerly Ifile.it) plugin - free account only""" - __author_name__ = ("zoidberg") - - FILE_SIZE_PATTERN = r'{var __ab1 = (?P<S>\d+);}' - FILE_NAME_PATTERN = r'id="aliasSpan">(?P<N>.*?) <' - FILE_OFFLINE_PATTERN = r'l10n.(FILES__DOESNT_EXIST|REMOVED)' - TEMP_OFFLINE_PATTERN = r'l10n.FILES__WARNING' - - UKEY_PATTERN = r"'ukey'\s*:'(\w+)'," - AB1_PATTERN = r"if\( __ab1 == '(\w+)' \)" - ERROR_MSG_PATTERN = r"var __error_msg\s*=\s*l10n\.(.*?);" - DOWNLOAD_LINK_PATTERN = r'"(http://s\d+.filecloud.io/%s/\d+/.*?)"' - RECAPTCHA_KEY_PATTERN = r"var __recaptcha_public\s*=\s*'([^']+)';" - RECAPTCHA_KEY = '6Lf5OdISAAAAAEZObLcx5Wlv4daMaASRov1ysDB1' - - def setup(self): - self.resumeDownload = self.multiDL = True - self.chunkLimit = 1 - - def handleFree(self): - data = {"ukey": self.file_info['ID']} - - found = re.search(self.AB1_PATTERN, self.html) - if not found: - raise PluginParseError("__AB1") - data["__ab1"] = found.group(1) - - if not self.account: - self.fail("User not logged in") - elif not self.account.logged_in: - recaptcha = ReCaptcha(self) - captcha_challenge, captcha_response = recaptcha.challenge(self.RECAPTCHA_KEY) - self.account.form_data = {"recaptcha_challenge_field" : captcha_challenge, - "recaptcha_response_field" : captcha_response} - self.account.relogin(self.user) - self.retry(max_tries = 2) - - json_url = "http://filecloud.io/download-request.json" - response = self.load(json_url, post = data) - self.logDebug(response) - response = json_loads(response) - - if "error" in response and response["error"]: - self.fail(response) - - self.logDebug(response) - if response["captcha"]: - recaptcha = ReCaptcha(self) - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - captcha_key = found.group(1) if found else self.RECAPTCHA_KEY - data["ctype"] = "recaptcha" - - for i in range(5): - data["recaptcha_challenge"], data["recaptcha_response"] = recaptcha.challenge(captcha_key) - - json_url = "http://filecloud.io/download-request.json" - response = self.load(json_url, post = data) - self.logDebug(response) - response = json_loads(response) - - if "retry" in response and response["retry"]: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: - self.fail("Incorrect captcha") - - if response["dl"]: - self.html = self.load('http://filecloud.io/download.html') - found = re.search(self.DOWNLOAD_LINK_PATTERN % self.file_info['ID'], self.html) - if not found: - raise PluginParseError("Download URL") - download_url = found.group(1) - self.logDebug("Download URL: %s" % download_url) - - if "size" in self.file_info and self.file_info['size']: - self.check_data = {"size": int(self.file_info['size'])} - self.download(download_url) - else: - self.fail("Unexpected server response") - -getInfo = create_getInfo(FilecloudIo)
\ No newline at end of file diff --git a/module/plugins/hoster/FilefactoryCom.py b/module/plugins/hoster/FilefactoryCom.py deleted file mode 100644 index fdde1f9d7..000000000 --- a/module/plugins/hoster/FilefactoryCom.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -# Test links (random.bin): -# http://www.filefactory.com/file/ymxkmdud2o3/n/random.bin - -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster -from module.network.RequestFactory import getURL -from module.utils import parseFileSize - - -def getInfo(urls): - file_info = list() - list_ids = dict() - - # Create a dict id:url. Will be used to retrieve original url - for url in urls: - m = re.search(FilefactoryCom.__pattern__, url) - list_ids[m.group('id')] = url - - # WARN: There could be a limit of urls for request - post_data = {'func': 'links', 'links': '\n'.join(urls)} - rep = getURL('http://www.filefactory.com/tool/links.php', post=post_data, decode=True) - - # Online links - for m in re.finditer( - r'innerText">\s*<h1 class="name">(?P<N>.+) \((?P<S>[\w.]+) (?P<U>\w+)\)</h1>\s*<p>http://www.filefactory.com/file/(?P<ID>\w+).*</p>\s*<p class="hidden size">', - rep): - file_info.append((m.group('N'), parseFileSize(m.group('S'), m.group('U')), 2, list_ids[m.group('ID')])) - - # Offline links - for m in re.finditer( - r'innerText">\s*<h1>(http://www.filefactory.com/file/(?P<ID>\w+)/)</h1>\s*<p>\1</p>\s*<p class="errorResponse">Error: file not found</p>', - rep): - file_info.append((list_ids[m.group('ID')], 0, 1, list_ids[m.group('ID')])) - - return file_info - - -class FilefactoryCom(SimpleHoster): - __name__ = "FilefactoryCom" - __type__ = "hoster" - __pattern__ = r"https?://(?:www\.)?filefactory\.com/file/(?P<id>[a-zA-Z0-9]+)" - __version__ = "0.41" - __description__ = """Filefactory.Com File Download Hoster""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - DIRECT_LINK_PATTERN = r'<section id="downloadLink">\s*<p class="textAlignCenter">\s*<a href="([^"]+)">[^<]+</a>\s*</p>\s*</section>' - - def process(self, pyfile): - if not re.match(self.__pattern__ + r'/n/.+', pyfile.url): # Not in standard format - header = self.load(pyfile.url, just_header=True) - if 'location' in header: - self.pyfile.url = 'http://www.filefactory.com' + header['location'] - - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.handlePremium() - else: - self.handleFree() - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode=True) - if "Currently only Premium Members can download files larger than" in self.html: - self.fail("File too large for free download") - elif "All free download slots on this server are currently in use" in self.html: - self.retry(50, 900, "All free slots are busy") - - # Load the page that contains the direct link - url = re.search(r"document\.location\.host \+\s*'(.+)';", self.html) - if not url: - self.parseError('Unable to detect free link') - url = 'http://www.filefactory.com' + url.group(1) - self.html = self.load(url, decode=True) - - # Free downloads wait time - waittime = re.search(r'id="startWait" value="(\d+)"', self.html) - if not waittime: - self.parseError('Unable to detect wait time') - self.setWait(int(waittime.group(1))) - self.wait() - - # Parse the direct link and download it - direct = re.search(r'data-href-direct="(.*)" class="button', self.html) - if not direct: - self.parseError('Unable to detect free direct link') - direct = direct.group(1) - self.logDebug('DIRECT LINK: ' + direct) - self.download(direct, disposition=True) - - check = self.checkDownload({"multiple": "You are currently downloading too many files at once.", - "error": '<div id="errorMessage">'}) - - if check == "multiple": - self.logDebug("Parallel downloads detected; waiting 15 minutes") - self.retry(wait_time=15 * 60, reason='Parallel downloads') - elif check == "error": - self.fail("Unknown error") - - def handlePremium(self): - header = self.load(self.pyfile.url, just_header=True) - if 'location' in header: - url = header['location'].strip() - if not url.startswith("http://"): - url = "http://www.filefactory.com" + url - elif 'content-disposition' in header: - url = self.pyfile.url - else: - html = self.load(self.pyfile.url) - found = re.search(self.DIRECT_LINK_PATTERN, html) - if found: - url = found.group(1) - else: - self.parseError('Unable to detect premium direct link') - - self.logDebug('DIRECT PREMIUM LINK: ' + url) - self.download(url, disposition=True) diff --git a/module/plugins/hoster/FilejungleCom.py b/module/plugins/hoster/FilejungleCom.py deleted file mode 100644 index fd833eef2..000000000 --- a/module/plugins/hoster/FilejungleCom.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -from module.plugins.hoster.FileserveCom import FileserveCom, checkFile -from module.plugins.Plugin import chunks - -class FilejungleCom(FileserveCom): - __name__ = "FilejungleCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?filejungle\.com/f/(?P<id>[^/]+).*" - __version__ = "0.51" - __description__ = """Filejungle.com plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - URLS = ['http://www.filejungle.com/f/', 'http://www.filejungle.com/check_links.php', 'http://www.filejungle.com/checkReCaptcha.php'] - LINKCHECK_TR = r'<li>\s*(<div class="col1">.*?)</li>' - LINKCHECK_TD = r'<div class="(?:col )?col\d">(?:<[^>]*>| )*([^<]*)' - - LONG_WAIT_PATTERN = r'<h1>Please wait for (\d+) (\w+)\s*to download the next file\.</h1>' - -def getInfo(urls): - for chunk in chunks(urls, 100): yield checkFile(FilejungleCom, chunk)
\ No newline at end of file diff --git a/module/plugins/hoster/FilepostCom.py b/module/plugins/hoster/FilepostCom.py deleted file mode 100644 index bc47856e5..000000000 --- a/module/plugins/hoster/FilepostCom.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg - - changelog: - 0.27 - 2012-08-12 - hgg - fix "global name 'js_answer' is not defined" bug - fix captcha bug #1 (failed on non-english "captcha wrong" errors) -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads -from time import time - -class FilepostCom(SimpleHoster): - __name__ = "FilepostCom" - __type__ = "hoster" - __pattern__ = r"https?://(?:www\.)?(?:filepost\.com/files|fp.io)/([^/]+).*" - __version__ = "0.27" - __description__ = """Filepost.com plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<input type="text" id="url" value=\'<a href[^>]*>(?P<N>[^>]+?) - (?P<S>[0-9\.]+ [kKMG]i?B)</a>\' class="inp_text"/>' - #FILE_INFO_PATTERN = r'<h1>(?P<N>[^<]+)</h1>\s*<div class="ul">\s*<ul>\s*<li><span>Size:</span> (?P<S>[0-9.]+) (?P<U>[kKMG])i?B</li>' - FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>|<div class="file_info file_info_deleted">' - RECAPTCHA_KEY_PATTERN = r"Captcha.init\({\s*key:\s*'([^']+)'" - FLP_TOKEN_PATTERN = r"set_store_options\({token: '([^']+)'" - - def handleFree(self): - # Find token and captcha key - file_id = re.search(self.__pattern__, self.pyfile.url).group(1) - - found = re.search(self.FLP_TOKEN_PATTERN, self.html) - if not found: self.parseError("Token") - flp_token = found.group(1) - - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - if not found: self.parseError("Captcha key") - captcha_key = found.group(1) - - # Get wait time - get_dict = {'SID' : self.req.cj.getCookie('SID'), 'JsHttpRequest' : str(int(time()*10000)) + '-xml'} - post_dict = {'action' : 'set_download', 'token' : flp_token, 'code' : file_id} - wait_time = int(self.getJsonResponse(get_dict, post_dict, 'wait_time')) - - if wait_time > 0: - self.setWait(wait_time) - self.wait() - - post_dict = {"token" : flp_token, "code" : file_id, "file_pass" : ''} - - if 'var is_pass_exists = true;' in self.html: - # Solve password - for file_pass in self.getPassword().splitlines(): - get_dict['JsHttpRequest'] = str(int(time()*10000)) + '-xml' - post_dict['file_pass'] = file_pass - self.logInfo("Password protected link, trying " + file_pass) - - download_url = self.getJsonResponse(get_dict, post_dict, 'link') - if download_url: - break - - else: self.fail("No or incorrect password") - - else: - # Solve recaptcha - recaptcha = ReCaptcha(self) - - for pokus in range(5): - get_dict['JsHttpRequest'] = str(int(time()*10000)) + '-xml' - if pokus: - post_dict["recaptcha_challenge_field"], post_dict["recaptcha_response_field"] = recaptcha.challenge(captcha_key) - self.logDebug(u"RECAPTCHA: %s : %s : %s" % (captcha_key, post_dict["recaptcha_challenge_field"], post_dict["recaptcha_response_field"])) - - download_url = self.getJsonResponse(get_dict, post_dict, 'link') - if download_url: - if pokus: self.correctCaptcha() - break - elif pokus: - self.invalidCaptcha() - - else: self.fail("Invalid captcha") - - # Download - self.download(download_url) - - def getJsonResponse(self, get_dict, post_dict, field): - json_response = json_loads(self.load('https://filepost.com/files/get/', get = get_dict, post = post_dict)) - self.logDebug(json_response) - - if not 'js' in json_response: self.parseError('JSON %s 1' % field) - - # i changed js_answer to json_response['js'] since js_answer is nowhere set. - # i don't know the JSON-HTTP specs in detail, but the previous author - # accessed json_response['js']['error'] as well as js_answer['error']. - # see the two lines commented out with "# ~?". - if 'error' in json_response['js']: - if json_response['js']['error'] == 'download_delay': - self.retry(json_response['js']['params']['next_download']) - # ~? self.retry(js_answer['params']['next_download']) - elif 'Wrong file password' in json_response['js']['error']: - return None - elif 'You entered a wrong CAPTCHA code' in json_response['js']['error']: - return None - elif 'CAPTCHA Code nicht korrekt' in json_response['js']['error']: - return None - elif 'CAPTCHA' in json_response['js']['error']: - self.logDebug('error response is unknown, but mentions CAPTCHA -> return None') - return None - else: - self.fail(json_response['js']['error']) - # ~? self.fail(js_answer['error']) - - if not 'answer' in json_response['js'] or not field in json_response['js']['answer']: - self.parseError('JSON %s 2' % field) - - return json_response['js']['answer'][field] - -getInfo = create_getInfo(FilepostCom) diff --git a/module/plugins/hoster/FilerNet.py b/module/plugins/hoster/FilerNet.py deleted file mode 100644 index 9693723f9..000000000 --- a/module/plugins/hoster/FilerNet.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -# Test links (random.bin): -# http://filer.net/get/ivgf5ztw53et3ogd -# http://filer.net/get/hgo14gzcng3scbvv - -import pycurl -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.ReCaptcha import ReCaptcha - - -class FilerNet(SimpleHoster): - __name__ = "FilerNet" - __type__ = "hoster" - __pattern__ = r"https?://(www\.)?filer\.net/get/(\w+)" - __version__ = "0.01" - __description__ = """Filer.net Download Hoster""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_INFO_PATTERN = r'<h1 class="page-header">Free Download (?P<N>\S+) <small>(?P<S>[\w.]+) (?P<U>\w+)</small></h1>' - FILE_OFFLINE_PATTERN = r'Nicht gefunden' - RECAPTCHA_KEY = '6LcFctISAAAAAAgaeHgyqhNecGJJRnxV1m_vAz3V' - - def process(self, pyfile): - self.req.setOption("timeout", 120) - self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) - - # Wait between downloads - m = re.search(r'musst du <span id="time">(\d+)</span> Sekunden warten', self.html) - if m: - waittime = int(m.group(1)) - self.retry(3, waittime, 'Wait between free downloads') - - self.getFileInfo() - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.handlePremium() - else: - self.handleFree() - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode=True) - - inputs = self.parseHtmlForm(input_names='token')[1] - if 'token' not in inputs: - self.parseError('Unable to detect token') - token = inputs['token'] - self.logDebug('Token: ' + token) - - self.html = self.load(self.pyfile.url, post={'token': token}, decode=True) - - inputs = self.parseHtmlForm(input_names='hash')[1] - if 'hash' not in inputs: - self.parseError('Unable to detect hash') - hash_data = inputs['hash'] - self.logDebug('Hash: ' + hash_data) - - downloadURL = '' - recaptcha = ReCaptcha(self) - for i in xrange(5): - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) - post_data = {'recaptcha_challenge_field': challenge, - 'recaptcha_response_field': response, - 'hash': hash_data} - - # Workaround for 0.4.9 just_header issue. In 0.5 clean the code using just_header - self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.load(self.pyfile.url, post=post_data) - self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) - - if 'location' in self.req.http.header: - location = re.search(r'location: (\S+)', self.req.http.header).group(1) - downloadURL = 'http://filer.net' + location - self.correctCaptcha() - break - else: - self.logInfo('Wrong captcha') - self.invalidCaptcha() - - if not downloadURL: - self.fail("No Download url retrieved/all captcha attempts failed") - - self.download(downloadURL, disposition=True) - - -getInfo = create_getInfo(FilerNet) diff --git a/module/plugins/hoster/FilerioCom.py b/module/plugins/hoster/FilerioCom.py deleted file mode 100644 index 7be0fa4f6..000000000 --- a/module/plugins/hoster/FilerioCom.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class FilerioCom(XFileSharingPro): - __name__ = "FilerioCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*(filerio\.(in|com)|filekeen\.com)/\w{12}" - __version__ = "0.02" - __description__ = """FileRio.in hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_OFFLINE_PATTERN = '<b>"File Not Found"</b>|File has been removed due to Copyright Claim' - HOSTER_NAME = "filerio.in" - FILE_URL_REPLACEMENTS = [(r'http://.*?/','http://filerio.in/')] - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - -getInfo = create_getInfo(FilerioCom)
\ No newline at end of file diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py deleted file mode 100644 index ee4ea4953..000000000 --- a/module/plugins/hoster/FilesMailRu.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster, chunks -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - for chunk in chunks(urls, 10): - for url in chunk: - src = getURL(url) - if r'<div class="errorMessage mb10">' in src: - result.append((url, 0, 1, url)) - elif r'Page cannot be displayed' in src: - result.append((url, 0, 1, url)) - else: - try: - url_pattern = '<a href="(.+?)" onclick="return Act\(this\, \'dlink\'\, event\)">(.+?)</a>' - file_name = re.search(url_pattern, src).group(0).split(', event)">')[1].split('</a>')[0] - result.append((file_name, 0, 2, url)) - except: - pass - - - # status 1=OFFLINE, 2=OK, 3=UNKNOWN - # result.append((#name,#size,#status,#url)) - yield result - -class FilesMailRu(Hoster): - __name__ = "FilesMailRu" - __type__ = "hoster" - __pattern__ = r"http://files\.mail\.ru/.*" - __version__ = "0.3" - __description__ = """Files.Mail.Ru One-Klick Hoster""" - __author_name__ = ("oZiRiz") - __author_mail__ = ("ich@oziriz.de") - - - def setup(self): - if not self.account: - self.multiDL = False - self.chunkLimit = 1 - - def process(self, pyfile): - self.html = self.load(pyfile.url) - self.url_pattern = '<a href="(.+?)" onclick="return Act\(this\, \'dlink\'\, event\)">(.+?)</a>' - - #marks the file as "offline" when the pattern was found on the html-page''' - if r'<div class="errorMessage mb10">' in self.html: - self.offline() - - elif r'Page cannot be displayed' in self.html: - self.offline() - - #the filename that will be showed in the list (e.g. test.part1.rar)''' - pyfile.name = self.getFileName() - - #prepare and download''' - if not self.account: - self.prepare() - self.download(self.getFileUrl()) - self.myPostProcess() - else: - self.download(self.getFileUrl()) - self.myPostProcess() - - def prepare(self): - '''You have to wait some seconds. Otherwise you will get a 40Byte HTML Page instead of the file you expected''' - self.setWait(10) - self.wait() - return True - - def getFileUrl(self): - '''gives you the URL to the file. Extracted from the Files.mail.ru HTML-page stored in self.html''' - file_url = re.search(self.url_pattern, self.html).group(0).split('<a href="')[1].split('" onclick="return Act')[0] - return file_url - - - def getFileName(self): - '''gives you the Name for each file. Also extracted from the HTML-Page''' - file_name = re.search(self.url_pattern, self.html).group(0).split(', event)">')[1].split('</a>')[0] - return file_name - - def myPostProcess(self): - # searches the file for HTMl-Code. Sometimes the Redirect - # doesn't work (maybe a curl Problem) and you get only a small - # HTML file and the Download is marked as "finished" - # then the download will be restarted. It's only bad for these - # who want download a HTML-File (it's one in a million ;-) ) - # - # The maximum UploadSize allowed on files.mail.ru at the moment is 100MB - # so i set it to check every download because sometimes there are downloads - # that contain the HTML-Text and 60MB ZEROs after that in a xyzfile.part1.rar file - # (Loading 100MB in to ram is not an option) - check = self.checkDownload({"html": "<meta name="}, read_size=50000) - if check == "html": - self.log.info(_("There was HTML Code in the Downloaded File("+ self.pyfile.name +")...redirect error? The Download will be restarted.")) - self.retry() diff --git a/module/plugins/hoster/FileserveCom.py b/module/plugins/hoster/FileserveCom.py deleted file mode 100644 index 04c297661..000000000 --- a/module/plugins/hoster/FileserveCom.py +++ /dev/null @@ -1,211 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads -from module.utils import parseFileSize -from module.plugins.Plugin import chunks - -def checkFile(plugin, urls): - html = getURL(plugin.URLS[1], post = {"urls": "\n".join(urls)}, decode=True) - - file_info = [] - for li in re.finditer(plugin.LINKCHECK_TR, html, re.DOTALL): - try: - cols = re.findall(plugin.LINKCHECK_TD, li.group(1)) - if cols: - file_info.append(( - cols[1] if cols[1] != '--' else cols[0], - parseFileSize(cols[2]) if cols[2] != '--' else 0, - 2 if cols[3].startswith('Available') else 1, - cols[0])) - except Exception, e: - continue - - return file_info - -class FileserveCom(Hoster): - __name__ = "FileserveCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?fileserve\.com/file/(?P<id>[^/]+).*" - __version__ = "0.51" - __description__ = """Fileserve.Com File Download Hoster""" - __author_name__ = ("jeix", "mkaay", "paul king", "zoidberg") - __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "", "zoidberg@mujmail.cz") - - URLS = ['http://www.fileserve.com/file/', 'http://www.fileserve.com/link-checker.php', 'http://www.fileserve.com/checkReCaptcha.php'] - LINKCHECK_TR = r'<tr>\s*(<td>http://www.fileserve\.com/file/.*?)</tr>' - LINKCHECK_TD = r'<td>(?:<[^>]*>| )*([^<]*)' - - CAPTCHA_KEY_PATTERN = r"var reCAPTCHA_publickey='(?P<key>[^']+)'" - LONG_WAIT_PATTERN = r'<li class="title">You need to wait (\d+) (\w+) to start another download\.</li>' - LINK_EXPIRED_PATTERN = "Your download link has expired" - DAILY_LIMIT_PATTERN = "Your daily download limit has been reached" - NOT_LOGGED_IN_PATTERN = '<form (name="loginDialogBoxForm"|id="login_form")|<li><a href="/login.php">Login</a></li>' - - # shares code with FilejungleCom and UploadstationCom - - def setup(self): - self.resumeDownload = self.multiDL = True if self.premium else False - - self.file_id = re.search(self.__pattern__, self.pyfile.url).group('id') - self.url = "%s%s" % (self.URLS[0], self.file_id) - self.logDebug("File ID: %s URL: %s" % (self.file_id, self.url)) - - def process(self, pyfile): - pyfile.name, pyfile.size, status, self.url = checkFile(self, [self.url])[0] - if status != 2: self.offline() - self.logDebug("File Name: %s Size: %d" % (pyfile.name, pyfile.size)) - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - def handleFree(self): - self.html = self.load(self.url) - action = self.load(self.url, post={"checkDownload": "check"}, decode=True) - action = json_loads(action) - self.logDebug(action) - - if "fail" in action: - if action["fail"] == "timeLimit": - self.html = self.load(self.url, - post={"checkDownload": "showError", - "errorType": "timeLimit"}, - decode=True) - - self.doLongWait(re.search(self.LONG_WAIT_PATTERN, self.html)) - - elif action["fail"] == "parallelDownload": - self.logWarning(_("Parallel download error, now waiting 60s.")) - self.retry(wait_time=60, reason="parallelDownload") - - else: - self.fail("Download check returned %s" % action["fail"]) - - elif "success" in action: - if action["success"] == "showCaptcha": - self.doCaptcha() - self.doTimmer() - elif action["success"] == "showTimmer": - self.doTimmer() - - else: - self.fail("Unknown server response") - - # show download link - response = self.load(self.url, post={"downloadLink": "show"}, decode=True) - self.logDebug("show downloadLink response : %s" % response) - if "fail" in response: - self.fail("Couldn't retrieve download url") - - # this may either download our file or forward us to an error page - self.download(self.url, post = {"download": "normal"}) - self.logDebug(self.req.http.lastEffectiveURL) - - check = self.checkDownload({"expired": self.LINK_EXPIRED_PATTERN, - "wait": re.compile(self.LONG_WAIT_PATTERN), - "limit": self.DAILY_LIMIT_PATTERN}) - - if check == "expired": - self.logDebug("Download link was expired") - self.retry() - elif check == "wait": - self.doLongWait(self.lastCheck) - elif check == "limit": - #download limited reached for today (not a exact time known) - self.setWait(180 * 60, True) # wait 3 hours - self.wait() - self.retry(max_tries=0) - - self.thread.m.reconnecting.wait(3) # Ease issue with later downloads appearing to be in parallel - - def doTimmer(self): - response = self.load(self.url, - post={"downloadLink": "wait"}, - decode=True) - self.logDebug("wait response : %s" % response[:80]) - - if "fail" in response: - self.fail("Failed getting wait time") - - if self.__name__ == "FilejungleCom": - found = re.search(r'"waitTime":(\d+)', response) - if not found: self.fail("Cannot get wait time") - wait_time = int(found.group(1)) - else: - wait_time = int(response) + 3 - - self.setWait(wait_time) - self.wait() - - def doCaptcha(self): - captcha_key = re.search(self.CAPTCHA_KEY_PATTERN, self.html).group("key") - recaptcha = ReCaptcha(self) - - for i in range(5): - challenge, code = recaptcha.challenge(captcha_key) - - response = json_loads(self.load(self.URLS[2], - post={'recaptcha_challenge_field': challenge, - 'recaptcha_response_field': code, - 'recaptcha_shortencode_field': self.file_id})) - self.logDebug("reCaptcha response : %s" % response) - if not response["success"]: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: self.fail("Invalid captcha") - - def doLongWait(self, m): - wait_time = (int(m.group(1)) * {'seconds':1, 'minutes':60, 'hours':3600}[m.group(2)]) if m else 720 - self.setWait(wait_time, True) - self.wait() - self.retry() - - def handlePremium(self): - premium_url = None - if self.__name__ == "FileserveCom": - #try api download - response = self.load("http://app.fileserve.com/api/download/premium/", - post = {"username": self.user, - "password": self.account.getAccountData(self.user)["password"], - "shorten": self.file_id}, - decode = True) - if response: - response = json_loads(response) - if response['error_code'] == "302": premium_url = response['next'] - elif response['error_code'] in ["305", "500"]: self.tempOffline() - elif response['error_code'] in ["403", "605"]: self.resetAccount() - elif response['error_code'] in ["606", "607", "608"]: self.offline() - else: self.logError(response['error_code'], response['error_message']) - - self.download(premium_url or self.pyfile.url) - - if not premium_url: - check = self.checkDownload({"login": re.compile(self.NOT_LOGGED_IN_PATTERN)}) - - if check == "login": - self.account.relogin(self.user) - self.retry(reason=_("Not logged in.")) - -def getInfo(urls): - for chunk in chunks(urls, 100): yield checkFile(FileserveCom, chunk)
\ No newline at end of file diff --git a/module/plugins/hoster/FileshareInUa.py b/module/plugins/hoster/FileshareInUa.py deleted file mode 100644 index 9700b2d0a..000000000 --- a/module/plugins/hoster/FileshareInUa.py +++ /dev/null @@ -1,78 +0,0 @@ -from urllib import urlencode -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.utils import parseFileSize - -class FileshareInUa(Hoster): - __name__ = "FileshareInUa" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?fileshare.in.ua/[A-Za-z0-9]+" - __version__ = "0.01" - __description__ = """fileshare.in.ua hoster plugin""" - __author_name__ = ("fwannmacher") - __author_mail__ = ("felipe@warhammerproject.com") - - HOSTER_NAME = "fileshare.in.ua" - PATTERN_FILENAME = r'<h3 class="b-filename">(.*?)</h3>' - PATTERN_FILESIZE = r'<b class="b-filesize">(.*?)</b>' - PATTERN_OFFLINE = "This file doesn't exist, or has been removed." - - def setup(self): - self.resumeDownload = True - self.multiDL = True - - def process(self, pyfile): - self.pyfile = pyfile - self.html = self.load(pyfile.url, decode=True) - - if not self._checkOnline(): - self.offline() - - self.pyfile.name = self._getName() - - self.link = self._getLink() - - if not self.link.startswith('http://'): - self.link = "http://fileshare.in.ua" + self.link - - self.download(self.link) - - def _checkOnline(self): - if re.search(self.PATTERN_OFFLINE, self.html): - return False - else: - return True - - def _getName(self): - name = re.search(self.PATTERN_FILENAME, self.html) - if name is None: - self.fail("%s: Plugin broken." % self.__name__) - - return name.group(1) - - def _getLink(self): - return re.search("<a href=\"(/get/.+)\" class=\"b-button m-blue m-big\" >", self.html).group(1) - -def getInfo(urls): - result = [] - - for url in urls: - html = getURL(url) - - if re.search(FileshareInUa.PATTERN_OFFLINE, html): - result.append((url, 0, 1, url)) - else: - name = re.search(FileshareInUa.PATTERN_FILENAME, html) - - if name is None: - result.append((url, 0, 1, url)) - continue - - name = name.group(1) - size = re.search(FileshareInUa.PATTERN_FILESIZE, html) - size = parseFileSize(size.group(1)) - - result.append((name, size, 3, url)) - - yield result diff --git a/module/plugins/hoster/FlyFilesNet.py b/module/plugins/hoster/FlyFilesNet.py deleted file mode 100644 index 0ffb76191..000000000 --- a/module/plugins/hoster/FlyFilesNet.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import urllib - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.RequestFactory import getURL - -class FlyFilesNet(SimpleHoster): - __name__ = "FlyFilesNet" - __version__ = "0.1" - __type__ = "hoster" - __pattern__ = r'http://flyfiles\.net/.*' - - SESSION_PATTERN = r'flyfiles\.net/(.*)/.*' - FILE_NAME_PATTERN = r'flyfiles\.net/.*/(.*)' - - def process(self, pyfile): - - pyfile.name = re.search(self.FILE_NAME_PATTERN, pyfile.url).group(1) - pyfile.name = urllib.unquote_plus(pyfile.name) - - session = re.search(self.SESSION_PATTERN, pyfile.url).group(1) - - url = "http://flyfiles.net" - - # get download URL - parsed_url = getURL(url, post={"getDownLink": session}, cookies=True) - self.logDebug("Parsed URL: %s" % parsed_url) - - if parsed_url == '#downlink|' or parsed_url == "#downlink|#": - self.logWarning("Could not get the download URL. Please wait 10 minutes.") - self.setWait(600, True) # wait 10 minutes - self.wait() - self.retry() - - download_url = parsed_url.replace('#downlink|','') - - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) diff --git a/module/plugins/hoster/FourSharedCom.py b/module/plugins/hoster/FourSharedCom.py deleted file mode 100644 index 518ae2ae6..000000000 --- a/module/plugins/hoster/FourSharedCom.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -import re - -class FourSharedCom(SimpleHoster): - __name__ = "FourSharedCom" - __type__ = "hoster" - __pattern__ = r"https?://(www\.)?4shared(\-china)?\.com/(account/)?(download|get|file|document|photo|video|audio|mp3|office|rar|zip|archive|music)/.+?/.*" - __version__ = "0.29" - __description__ = """4Shared Download Hoster""" - __author_name__ = ("jeix", "zoidberg") - __author_mail__ = ("jeix@hasnomail.de", "zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<meta name="title" content="(?P<N>.+?)"' - FILE_SIZE_PATTERN = '<span title="Size: (?P<S>[0-9,.]+) (?P<U>[kKMG])i?B">' - FILE_OFFLINE_PATTERN = 'The file link that you requested is not valid\.|This file was deleted.' - FILE_NAME_REPLACEMENTS = [(r"&#(\d+).", lambda m: unichr(int(m.group(1))))] - FILE_SIZE_REPLACEMENTS = [(",", "")] - - DOWNLOAD_BUTTON_PATTERN = 'id="btnLink" href="(.*?)"' - FID_PATTERN = 'name="d3fid" value="(.*?)"' - DOWNLOAD_URL_PATTERN = r'name="d3link" value="(.*?)"' - - def handleFree(self): - if not self.account: - self.fail("User not logged in") - - found = re.search(self.DOWNLOAD_BUTTON_PATTERN, self.html) - if found: - link = found.group(1) - else: - link = re.sub(r'/(download|get|file|document|photo|video|audio)/', r'/get/', self.pyfile.url) - - self.html = self.load(link) - - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError('Download link') - link = found.group(1) - - try: - found = re.search(self.FID_PATTERN, self.html) - response = self.load('http://www.4shared.com/web/d2/getFreeDownloadLimitInfo?fileId=%s' % found.group(1)) - self.logDebug(response) - except: - pass - - self.setWait(20) - self.wait() - self.download(link) - -getInfo = create_getInfo(FourSharedCom)
\ No newline at end of file diff --git a/module/plugins/hoster/FreakshareCom.py b/module/plugins/hoster/FreakshareCom.py deleted file mode 100644 index 56e02cbdc..000000000 --- a/module/plugins/hoster/FreakshareCom.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from module.plugins.Hoster import Hoster
-from module.plugins.ReCaptcha import ReCaptcha
-
-class FreakshareCom(Hoster):
- __name__ = "FreakshareCom"
- __type__ = "hoster"
- __pattern__ = r"http://(?:www\.)?freakshare\.(net|com)/files/\S*?/"
- __version__ = "0.37"
- __description__ = """Freakshare.com Download Hoster"""
- __author_name__ = ("sitacuisses","spoob","mkaay", "Toilal")
- __author_mail__ = ("sitacuisses@yahoo.de","spoob@pyload.org","mkaay@mkaay.de", "toilal.dev@gmail.com")
-
- def setup(self):
- self.html = None
- self.wantReconnect = False
- self.multiDL = False
- self.req_opts = []
-
- def process(self, pyfile):
- self.pyfile = pyfile
-
- pyfile.url = pyfile.url.replace("freakshare.net/","freakshare.com/")
-
- if self.account:
- self.html = self.load(pyfile.url, cookies=False)
- pyfile.name = self.get_file_name()
- self.download(pyfile.url)
-
- else:
- self.prepare()
- self.get_file_url()
-
- self.download(self.pyfile.url, post=self.req_opts)
-
-
- check = self.checkDownload({"bad": "bad try",
- "paralell": "> Sorry, you cant download more then 1 files at time. <",
- "empty": "Warning: Unknown: Filename cannot be empty",
- "wrong_captcha": "Wrong Captcha!"})
-
- if check == "bad":
- self.fail("Bad Try.")
- if check == "paralell":
- self.setWait(300, True)
- self.wait()
- self.retry()
- if check == "empty":
- self.fail("File not downloadable")
- if check == "wrong_captcha":
- self.invalidCaptcha()
- self.retry()
-
- def prepare(self):
- pyfile = self.pyfile
-
- self.wantReconnect = False
-
- self.download_html()
-
- if not self.file_exists():
- self.offline()
-
- self.setWait( self.get_waiting_time() )
-
- pyfile.name = self.get_file_name()
- pyfile.size = self.get_file_size()
-
- self.wait()
-
- return True
-
- def download_html(self):
- self.load("http://freakshare.com/index.php", {"language": "EN"}); # Set english language in server session
- self.html = self.load(self.pyfile.url)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- if self.html is None:
- self.download_html()
- if not self.wantReconnect:
- self.req_opts = self.get_download_options() # get the Post options for the Request
- #file_url = self.pyfile.url
- #return file_url
- else:
- self.offline()
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
- if not self.wantReconnect:
- file_name = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">([^ ]+)", self.html)
- if file_name is not None:
- file_name = file_name.group(1)
- else:
- file_name = self.pyfile.url
- return file_name
- else:
- return self.pyfile.url
-
- def get_file_size(self):
- size = 0
- if self.html is None:
- self.download_html()
- if not self.wantReconnect:
- file_size_check = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">[^ ]+ - ([^ ]+) (\w\w)yte", self.html)
- if file_size_check is not None:
- units = float(file_size_check.group(1).replace(",", ""))
- pow = {'KB': 1, 'MB': 2, 'GB': 3}[file_size_check.group(2)]
- size = int(units * 1024 ** pow)
-
- return size
-
- def get_waiting_time(self):
- if self.html is None:
- self.download_html()
-
- if "Your Traffic is used up for today" in self.html:
- self.wantReconnect = True
- return 24*3600
-
- timestring = re.search('\s*var\s(?:downloadWait|time)\s=\s(\d*)[.\d]*;', self.html)
- if timestring:
- return int(timestring.group(1)) + 1 #add 1 sec as tenths of seconds are cut off
- else:
- return 60
-
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
- if re.search(r"This file does not exist!", self.html) is not None:
- return False
- else:
- return True
-
- def get_download_options(self):
- re_envelope = re.search(r".*?value=\"Free\sDownload\".*?\n*?(.*?<.*?>\n*)*?\n*\s*?</form>", self.html).group(0) #get the whole request
- to_sort = re.findall(r"<input\stype=\"hidden\"\svalue=\"(.*?)\"\sname=\"(.*?)\"\s\/>", re_envelope)
- request_options = dict((n, v) for (v, n) in to_sort)
-
- herewego = self.load(self.pyfile.url, None, request_options) # the actual download-Page
-
- # comment this in, when it doesnt work
- # with open("DUMP__FS_.HTML", "w") as fp:
- # fp.write(herewego)
-
- to_sort = re.findall(r"<input\stype=\".*?\"\svalue=\"(\S*?)\".*?name=\"(\S*?)\"\s.*?\/>", herewego)
- request_options = dict((n, v) for (v, n) in to_sort)
-
- # comment this in, when it doesnt work as well
- #print "\n\n%s\n\n" % ";".join(["%s=%s" % x for x in to_sort])
-
- challenge = re.search(r"http://api\.recaptcha\.net/challenge\?k=([0-9A-Za-z]+)", herewego)
-
- if challenge:
- re_captcha = ReCaptcha(self)
- request_options["recaptcha_challenge_field"], request_options["recaptcha_response_field"] \
- = re_captcha.challenge(challenge.group(1))
-
- return request_options
diff --git a/module/plugins/hoster/FreevideoCz.py b/module/plugins/hoster/FreevideoCz.py deleted file mode 100644 index 19eb77470..000000000 --- a/module/plugins/hoster/FreevideoCz.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - - html = getURL(url) - if re.search(FreevideoCz.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - result.append((url, 0, 2, url)) - yield result - -class FreevideoCz(Hoster): - __name__ = "FreevideoCz" - __type__ = "hoster" - __pattern__ = r"http://www.freevideo.cz/vase-videa/(.*)\.html" - __version__ = "0.2" - __description__ = """freevideo.cz""" - __author_name__ = ("zoidberg") - - URL_PATTERN = r'clip: {\s*url: "([^"]+)"' - FILE_OFFLINE_PATTERN = r'<h2 class="red-corner-full">Str.nka nebyla nalezena</h2>' - - def setup(self): - self.multiDL = True - self.resumeDownload = True - - def process(self, pyfile): - - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.URL_PATTERN, self.html) - if found is None: self.fail("Parse error (URL)") - download_url = found.group(1) - - pyfile.name = re.search(self.__pattern__, pyfile.url).group(1) + ".mp4" - - self.download(download_url) diff --git a/module/plugins/hoster/FshareVn.py b/module/plugins/hoster/FshareVn.py deleted file mode 100644 index 926781b40..000000000 --- a/module/plugins/hoster/FshareVn.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL -import re -from time import strptime, mktime, gmtime - -def getInfo(urls): - for url in urls: - html = getURL('http://www.fshare.vn/check_link.php', post = { - "action" : "check_link", - "arrlinks" : url - }, decode = True) - - file_info = parseFileInfo(FshareVn, url, html) - - yield file_info - -def doubleDecode(m): - return m.group(1).decode('raw_unicode_escape') - -class FshareVn(SimpleHoster): - __name__ = "FshareVn" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?fshare.vn/file/.*" - __version__ = "0.16" - __description__ = """FshareVn Download Hoster""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<p>(?P<N>[^<]+)<\\/p>[\\trn\s]*<p>(?P<S>[0-9,.]+)\s*(?P<U>[kKMG])i?B<\\/p>' - FILE_OFFLINE_PATTERN = r'<div class=\\"f_left file_w\\"|<\\/p>\\t\\t\\t\\t\\r\\n\\t\\t<p><\\/p>\\t\\t\\r\\n\\t\\t<p>0 KB<\\/p>' - FILE_NAME_REPLACEMENTS = [("(.*)", doubleDecode)] - DOWNLOAD_URL_PATTERN = r'action="(http://download.*?)[#"]' - VIP_URL_PATTERN = r'<form action="([^>]+)" method="get" name="frm_download">' - WAIT_PATTERN = ur'Lượt tải xuá»ng kế tiếp là :\s*(.*?)\s*<' - - def process(self, pyfile): - self.html = self.load('http://www.fshare.vn/check_link.php', post = { - "action": "check_link", - "arrlinks": pyfile.url - }, decode = True) - self.getFileInfo() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - self.checkDownloadedFile() - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode = True) - - self.checkErrors() - - action, inputs = self.parseHtmlForm('frm_download') - self.url = self.pyfile.url + action - - if not inputs: self.parseError('FORM') - elif 'link_file_pwd_dl' in inputs: - for password in self.getPassword().splitlines(): - self.logInfo('Password protected link, trying "%s"' % password) - inputs['link_file_pwd_dl'] = password - self.html = self.load(self.url, post=inputs, decode=True) - if not 'name="link_file_pwd_dl"' in self.html: - break - else: - self.fail("No or incorrect password") - else: - self.html = self.load(self.url, post=inputs, decode=True) - - self.checkErrors() - - found = re.search(r'var count = (\d+)', self.html) - self.setWait(int(found.group(1)) if found else 30) - - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError('FREE DL URL') - self.url = found.group(1) - self.logDebug("FREE DL URL: %s" % self.url) - - self.wait() - self.download(self.url) - - def handlePremium(self): - self.download(self.pyfile.url) - - def checkErrors(self): - if '/error.php?' in self.req.lastEffectiveURL or u"Liên kết bạn chá»n khÃŽng tá»n" in self.html: - self.offline() - - found = re.search(self.WAIT_PATTERN, self.html) - if found: - self.logInfo("Wait until %s ICT" % found.group(1)) - wait_until = mktime(strptime(found.group(1), "%d/%m/%Y %H:%M")) - self.setWait(wait_until - mktime(gmtime()) - 7 * 3600, True) - self.wait() - self.retry() - elif '<ul class="message-error">' in self.html: - self.logError("Unknown error occured or wait time not parsed") - self.retry(30, 120, "Unknown error") - - def checkDownloadedFile(self): - # check download - check = self.checkDownload({ - "not_found": ("<head><title>404 Not Found</title></head>") - }) - - if check == "not_found": - self.fail("File not found on server") diff --git a/module/plugins/hoster/Ftp.py b/module/plugins/hoster/Ftp.py deleted file mode 100644 index 3bda3b32e..000000000 --- a/module/plugins/hoster/Ftp.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: jeix
- @author: mkaay
-"""
-from urlparse import urlparse, urljoin
-from urllib import quote, unquote
-import pycurl, re
-
-from module.plugins.Hoster import Hoster
-from module.network.HTTPRequest import BadHeader
-
-class Ftp(Hoster):
- __name__ = "Ftp"
- __version__ = "0.41"
- __pattern__ = r'(ftps?|sftp)://(.*?:.*?@)?.*?/.*' # ftp://user:password@ftp.server.org/path/to/file
- __type__ = "hoster"
- __description__ = """A Plugin that allows you to download from an from an ftp directory"""
- __author_name__ = ("jeix", "mkaay", "zoidberg")
- __author_mail__ = ("jeix@hasnomail.com", "mkaay@mkaay.de", "zoidberg@mujmail.cz")
-
- def setup(self):
- self.chunkLimit = -1
- self.resumeDownload = True
-
- def process(self, pyfile):
- parsed_url = urlparse(pyfile.url)
- netloc = parsed_url.netloc
-
- pyfile.name = parsed_url.path.rpartition('/')[2]
- try:
- pyfile.name = unquote(str(pyfile.name)).decode('utf8')
- except:
- pass
-
- if not "@" in netloc:
- servers = [ x['login'] for x in self.account.getAllAccounts() ] if self.account else []
-
- if netloc in servers:
- self.logDebug("Logging on to %s" % netloc)
- self.req.addAuth(self.account.accounts[netloc]["password"])
- else:
- for pwd in pyfile.package().password.splitlines():
- if ":" in pwd:
- self.req.addAuth(pwd.strip())
- break
-
- self.req.http.c.setopt(pycurl.NOBODY, 1)
-
- try:
- response = self.load(pyfile.url)
- except pycurl.error, e:
- self.fail("Error %d: %s" % e.args)
-
- self.req.http.c.setopt(pycurl.NOBODY, 0)
- self.logDebug(self.req.http.header)
-
- found = re.search(r"Content-Length:\s*(\d+)", response)
- if found:
- pyfile.size = int(found.group(1))
- self.download(pyfile.url)
- else:
- #Naive ftp directory listing
- if re.search(r'^25\d.*?"', self.req.http.header, re.M):
- pyfile.url = pyfile.url.rstrip('/')
- pkgname = "/".join((pyfile.package().name,urlparse(pyfile.url).path.rpartition('/')[2]))
- pyfile.url += '/'
- self.req.http.c.setopt(48, 1) # CURLOPT_DIRLISTONLY
- response = self.load(pyfile.url, decode = False)
- links = [ pyfile.url + quote(x) for x in response.splitlines() ]
- self.logDebug("LINKS", links)
- self.core.api.addPackage(pkgname, links, 1)
- #self.core.files.addLinks(links, pyfile.package().id)
- else:
- self.fail("Unexpected server response")
-
-
\ No newline at end of file diff --git a/module/plugins/hoster/GamefrontCom.py b/module/plugins/hoster/GamefrontCom.py deleted file mode 100644 index 34fda09d2..000000000 --- a/module/plugins/hoster/GamefrontCom.py +++ /dev/null @@ -1,80 +0,0 @@ -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.utils import parseFileSize - -class GamefrontCom(Hoster): - __name__ = "GamefrontCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?gamefront.com/files/[A-Za-z0-9]+" - __version__ = "0.02" - __description__ = """gamefront.com hoster plugin""" - __author_name__ = ("fwannmacher") - __author_mail__ = ("felipe@warhammerproject.com") - - HOSTER_NAME = "gamefront.com" - PATTERN_FILENAME = r'<title>(.*?) | Game Front' - PATTERN_FILESIZE = r'<dt>File Size:</dt>[\n\s]*<dd>(.*?)</dd>' - PATTERN_OFFLINE = "This file doesn't exist, or has been removed." - - def setup(self): - self.resumeDownload = True - self.multiDL = False - - def process(self, pyfile): - self.pyfile = pyfile - self.html = self.load(pyfile.url, decode=True) - - if not self._checkOnline(): - self.offline() - - self.pyfile.name = self._getName() - - self.link = self._getLink() - - if not self.link.startswith('http://'): - self.link = "http://www.gamefront.com/" + self.link - - self.download(self.link) - - def _checkOnline(self): - if re.search(self.PATTERN_OFFLINE, self.html): - return False - else: - return True - - def _getName(self): - name = re.search(self.PATTERN_FILENAME, self.html) - if name is None: - self.fail("%s: Plugin broken." % self.__name__) - - return name.group(1) - - def _getLink(self): - self.html2 = self.load("http://www.gamefront.com/" + re.search("(files/service/thankyou\\?id=[A-Za-z0-9]+)", self.html).group(1)) - self.link = re.search("<a href=\"(http://media[0-9]+\.gamefront.com/.*)\">click here</a>", self.html2) - - return self.link.group(1).replace("&", "&") - -def getInfo(urls): - result = [] - - for url in urls: - html = getURL(url) - - if re.search(GamefrontCom.PATTERN_OFFLINE, html): - result.append((url, 0, 1, url)) - else: - name = re.search(GamefrontCom.PATTERN_FILENAME, html) - - if name is None: - result.append((url, 0, 1, url)) - continue - - name = name.group(1) - size = re.search(GamefrontCom.PATTERN_FILESIZE, html) - size = parseFileSize(size.group(1)) - - result.append((name, size, 3, url)) - - yield result
\ No newline at end of file diff --git a/module/plugins/hoster/GigapetaCom.py b/module/plugins/hoster/GigapetaCom.py deleted file mode 100644 index 28ba35abe..000000000 --- a/module/plugins/hoster/GigapetaCom.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from random import randint -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from pycurl import FOLLOWLOCATION - -class GigapetaCom(SimpleHoster): - __name__ = "GigapetaCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?gigapeta\.com/dl/\w+" - __version__ = "0.01" - __description__ = """GigaPeta.com plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - SH_COOKIES = [("http://gigapeta.com", "lang", "us")] - FILE_NAME_PATTERN = r'<img src=".*" alt="file" />-->\s*(?P<N>.*?)\s*</td>' - FILE_SIZE_PATTERN = r'<th>\s*Size\s*</th>\s*<td>\s*(?P<S>.*?)\s*</td>' - FILE_OFFLINE_PATTERN = r'<div id="page_error">' - - def handleFree(self): - captcha_key = str(randint(1,100000000)) - captcha_url = "http://gigapeta.com/img/captcha.gif?x=%s" % captcha_key - - self.req.http.c.setopt(FOLLOWLOCATION, 0) - - for i in range(5): - self.checkErrors() - - captcha = self.decryptCaptcha(captcha_url) - self.html = self.load(self.pyfile.url, post = { - "captcha_key": captcha_key, - "captcha": captcha, - "download": "Download"}) - - found = re.search(r"Location\s*:\s*(.*)", self.req.http.header, re.I) - if found: - download_url = found.group(1) - break - elif "Entered figures don`t coincide with the picture" in self.html: - self.invalidCaptcha() - else: - self.fail("No valid captcha code entered") - - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - - def checkErrors(self): - if "All threads for IP" in self.html: - self.logDebug("Your IP is already downloading a file - wait and retry") - self.setWait(300, True) - self.wait() - self.retry() - -getInfo = create_getInfo(GigapetaCom)
\ No newline at end of file diff --git a/module/plugins/hoster/HellshareCz.py b/module/plugins/hoster/HellshareCz.py deleted file mode 100644 index aa494e34e..000000000 --- a/module/plugins/hoster/HellshareCz.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from math import ceil -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - - -class HellshareCz(SimpleHoster): - __name__ = "HellshareCz" - __type__ = "hoster" - __pattern__ = r"(http://(?:.*\.)*hellshare\.(?:cz|com|sk|hu|pl)/[^?]*/\d+).*" - __version__ = "0.82" - __description__ = """Hellshare.cz - premium only""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<h1 id="filename"[^>]*>(?P<N>[^<]+)</h1>' - FILE_SIZE_PATTERN = r'<strong id="FileSize_master">(?P<S>[0-9.]*) (?P<U>[kKMG])i?B</strong>' - FILE_OFFLINE_PATTERN = r'<h1>File not found.</h1>' - SHOW_WINDOW_PATTERN = r'<a href="([^?]+/(\d+)/\?do=(fileDownloadButton|relatedFileDownloadButton-\2)-showDownloadWindow)"' - - def setup(self): - self.resumeDownload = self.multiDL = True if self.account else False - self.chunkLimit = 1 - - def process(self, pyfile): - if not self.account: self.fail("User not logged in") - pyfile.url = re.search(self.__pattern__, pyfile.url).group(1) - self.html = self.load(pyfile.url, decode = True) - self.getFileInfo() - if not self.checkTrafficLeft(): - self.fail("Not enough traffic left for user %s." % self.user) - - found = re.search(self.SHOW_WINDOW_PATTERN, self.html) - if not found: self.parseError('SHOW WINDOW') - self.url = "http://www.hellshare.com" + found.group(1) - self.logDebug("DOWNLOAD URL: " + self.url) - - self.download(self.url) - -getInfo = create_getInfo(HellshareCz) diff --git a/module/plugins/hoster/HellspyCz.py b/module/plugins/hoster/HellspyCz.py deleted file mode 100644 index 9858c82b7..000000000 --- a/module/plugins/hoster/HellspyCz.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class HellspyCz(SimpleHoster): - __name__ = "HellspyCz" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*(?:hellspy\.(?:cz|com|sk|hu|pl)|sciagaj.pl)(/\S+/\d+)/?.*" - __version__ = "0.27" - __description__ = """HellSpy.cz""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_SIZE_PATTERN = r'<span class="filesize right">(?P<S>[0-9.]+)\s*<span>(?P<U>[kKMG])i?B' - FILE_NAME_PATTERN = r'<h1 title="(?P<N>.*?)"' - FILE_OFFLINE_PATTERN = r'<h2>(404 - Page|File) not found</h2>' - FILE_URL_REPLACEMENTS = [(__pattern__, r"http://www.hellspy.com\1")] - - CREDIT_LEFT_PATTERN = r'<strong>Credits: </strong>\s*(\d+)' - DOWNLOAD_AGAIN_PATTERN = r'<a id="button-download-start"[^>]*title="You can download the file without deducting your credit.">' - DOWNLOAD_URL_PATTERN = r"launchFullDownload\('([^']+)'" - - def setup(self): - self.resumeDownload = self.multiDL = True - self.chunkLimit = 1 - - def handleFree(self): - self.fail("Only premium users can download from HellSpy.cz") - - def handlePremium(self): - # set PHPSESSID cookie - cj = self.account.getAccountCookies(self.user) - cj.setCookie(".hellspy.com", "PHPSESSID", self.account.phpsessid) - self.logDebug("PHPSESSID: " + cj.getCookie("PHPSESSID")) - - info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i credits left" % (self.user, info["trafficleft"]/1024)) - - if self.pyfile.size / 1024 > info["trafficleft"]: - self.logWarning("Not enough credit left to download file") - - # get premium download URL and download - self.html = self.load(self.pyfile.url + "?download=1") - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError("Download URL") - url = found.group(1) - self.logDebug("Download URL: " + url) - self.download(url) - - info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i credits left" % (self.user, info["trafficleft"]/1024)) - -getInfo = create_getInfo(HellspyCz)
\ No newline at end of file diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py deleted file mode 100644 index 2caf1ffbc..000000000 --- a/module/plugins/hoster/HotfileCom.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha - -from module.network.RequestFactory import getURL -from module.utils import chunks - -def getInfo(urls): - api_url_base = "http://api.hotfile.com/" - - for chunk in chunks(urls, 90): - api_param_file = {"action":"checklinks","links": ",".join(chunk),"fields":"id,status,name,size"} #api only supports old style links - src = getURL(api_url_base, post=api_param_file, decode=True) - result = [] - for i, res in enumerate(src.split("\n")): - if not res: - continue - fields = res.split(",") - - if fields[1] in ("1", "2"): - status = 2 - else: - status = 1 - - result.append((fields[2], int(fields[3]), status, chunk[i])) - yield result - -class HotfileCom(Hoster): - __name__ = "HotfileCom" - __type__ = "hoster" - __pattern__ = r"https?://(www.)?hotfile\.com/dl/\d+/[0-9a-zA-Z]+/" - __version__ = "0.36" - __description__ = """Hotfile.com Download Hoster""" - __author_name__ = ("sitacuisses","spoob","mkaay","JoKoT3") - __author_mail__ = ("sitacuisses@yhoo.de","spoob@pyload.org","mkaay@mkaay.de","jokot3@gmail.com") - - FILE_OFFLINE_PATTERN = r'File is removed' - - def setup(self): - self.html = [None, None] - self.wantReconnect = False - self.htmlwithlink = None - self.url = None - - if self.premium: - self.multiDL = True - self.resumeDownload = True - self.chunkLimit = -1 - else: - self.multiDL = False - self.chunkLimit = 1 - - def apiCall(self, method, post, login=False): - if not self.account and login: - return - elif self.account and login: - return self.account.apiCall(method, post, self.user) - post.update({"action": method}) - return self.load("http://api.hotfile.com/", post=post, decode=True) - - def process(self, pyfile): - self.wantReconnect = False - - args = {"links":self.pyfile.url, "fields":"id,status,name,size,sha1"} - resp = self.apiCall("checklinks", args) - self.api_data = {} - for k, v in zip(args["fields"].split(","), resp.strip().split(",")): - self.api_data[k] = v - - if self.api_data["status"] == "0": - self.offline() - - pyfile.name = self.api_data["name"] - - if not self.premium: - self.downloadHTML() - - if self.FILE_OFFLINE_PATTERN in self.html[0]: - self.offline() - - self.setWait(self.getWaitTime()) - self.wait() - - self.freeDownload() - else: - dl = self.account.apiCall("getdirectdownloadlink", {"link":self.pyfile.url}, self.user) - #dl = unquote(dl).strip() <- Made problems - dl = dl.strip() - self.download(dl) - - def downloadHTML(self): - self.html[0] = self.load(self.pyfile.url, get={"lang":"en"}) - - def freeDownload(self): - - form_content = re.search(r"<form style=.*(\n<.*>\s*)*?[\n\t]?<tr>", self.html[0]) - if form_content is None: - print self.html[0] - self.fail("Form not found in HTML. Can not proceed.") - - form_content = form_content.group(0) - form_posts = dict(re.findall(r"<input\stype=hidden\sname=(\S*)\svalue=(\S*)>", form_content)) - - self.html[1] = self.load(self.pyfile.url, post=form_posts) - - challenge = re.search(r"http://api\.recaptcha\.net/challenge\?k=([0-9A-Za-z]+)", self.html[1]) - - if challenge: - re_captcha = ReCaptcha(self) - challenge, result = re_captcha.challenge(challenge.group(1)) - - url = re.search(r'<form action="(/dl/[^"]+)', self.html[1] ) - - self.html[1] = self.load("http://hotfile.com"+url.group(1), post={"action": "checkcaptcha", - "recaptcha_challenge_field" : challenge, - "recaptcha_response_field": result}) - - if "Wrong Code. Please try again." in self.html[1]: - self.freeDownload() - return - - file_url = re.search(r'a href="(http://hotfile\.com/get/\S*)"', self.html[1]).group(1) - self.download(file_url) - - def getWaitTime(self): - free_limit_pattern = re.compile(r"timerend=d\.getTime\(\)\+(\d+);") - matches = free_limit_pattern.findall(self.html[0]) - if matches: - wait_time = (sum([int(match) for match in matches])/1000) or 60 - if wait_time > 300: - self.wantReconnect = True - return wait_time + 1 - else: - self.fail("Don't know how long to wait. Cannot proceed.")
\ No newline at end of file diff --git a/module/plugins/hoster/IFileWs.py b/module/plugins/hoster/IFileWs.py deleted file mode 100644 index 160fe641c..000000000 --- a/module/plugins/hoster/IFileWs.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class IFileWs(XFileSharingPro): - __name__ = "IFileWs" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?ifile\.ws/\w+(/.+)?" - __version__ = "0.01" - __description__ = """Ifile.ws hoster plugin""" - __author_name__ = ("z00nx") - __author_mail__ = ("z00nx0@gmail.com") - - FILE_INFO_PATTERN = '<h1\s+style="display:inline;">(?P<N>[^<]+)</h1>\s+\[(?P<S>[^]]+)\]' - FILE_OFFLINE_PATTERN = 'File Not Found|The file was removed by administrator' - HOSTER_NAME = "ifile.ws" - LONG_WAIT_PATTERN = "(?P<M>\d(?=\s+minutes)).*(?P<S>\d+(?=\s+seconds))" - - -getInfo = create_getInfo(IFileWs) diff --git a/module/plugins/hoster/IcyFilesCom.py b/module/plugins/hoster/IcyFilesCom.py deleted file mode 100644 index 34737e560..000000000 --- a/module/plugins/hoster/IcyFilesCom.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: godofdream -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - for url in urls: - html = getURL(url, decode=True) - if re.search(IcyFilesCom.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - name = re.search(IcyFilesCom.FILE_NAME_PATTERN, html) - size = re.search(IcyFilesCom.SIZE_PATTERN, html) - if name is not None: - name = name.group(1) - size = (int(size.group(1)) * 1000000) - result.append((name, size, 2, url)) - yield result - - -class IcyFilesCom(Hoster): - __name__ = "IcyFilesCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?icyfiles\.com/(.*)" - __version__ = "0.05" - __description__ = """IcyFiles.com plugin - free only""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - FILE_NAME_PATTERN = r'<div id="file">(.*?)</div>' - SIZE_PATTERN = r'<li>(\d+) <span>Size/mb' - FILE_OFFLINE_PATTERN = r'The requested File cant be found' - WAIT_LONGER_PATTERN = r'All download tickets are in use\. please try it again in a few seconds' - WAIT_PATTERN = r'<div class="counter">(\d+)</div>' - TOOMUCH_PATTERN = r'Sorry dude, you have downloaded too much\. Please wait (\d+) seconds' - - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - # check if offline - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - # All Downloadtickets in use - timmy = re.search(self.WAIT_LONGER_PATTERN, self.html) - if timmy: - self.logDebug("waitforfreeslot") - self.waitForFreeSlot() - # Wait the waittime - timmy = re.search(self.WAIT_PATTERN, self.html) - if timmy: - self.logDebug("waiting", timmy.group(1)) - self.setWait(int(timmy.group(1)) + 2, False) - self.wait() - # Downloaded to much - timmy = re.search(self.TOOMUCH_PATTERN, self.html) - if timmy: - self.logDebug("too much", timmy.group(1)) - self.setWait(int(timmy.group(1)), True) - self.wait() - # Find Name - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (NAME)") - pyfile.name = found.group(1) - # Get the URL - url = pyfile.url - found = re.search(self.__pattern__, url) - if found is None: - self.fail("Parse error (URL)") - download_url = "http://icyfiles.com/download.php?key=" + found.group(1) - self.download(download_url) - # check download - check = self.checkDownload({ - "notfound": re.compile(r"^<head><title>404 Not Found</title>$"), - "skippedcountdown": re.compile(r"^Dont skip the countdown$"), - "waitforfreeslots": re.compile(self.WAIT_LONGER_PATTERN), - "downloadedtoomuch": re.compile(self.TOOMUCH_PATTERN) - }) - if check == "skippedcountdown": - self.fail("Countdown error") - elif check == "notfound": - self.fail("404 Not found") - elif check == "waitforfreeslots": - self.waitForFreeSlot() - elif check == "downloadedtoomuch": - self.retry() - - def waitForFreeSlot(self): - self.retry(60, 60, "Wait for free slot") diff --git a/module/plugins/hoster/IfileIt.py b/module/plugins/hoster/IfileIt.py deleted file mode 100644 index bf394f340..000000000 --- a/module/plugins/hoster/IfileIt.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.common.json_layer import json_loads -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getURL - -class IfileIt(SimpleHoster): - __name__ = "IfileIt" - __type__ = "hoster" - __pattern__ = r"^unmatchable$" - __version__ = "0.27" - __description__ = """Ifile.it""" - __author_name__ = ("zoidberg") - - #EVAL_PATTERN = r'(eval\(function\(p,a,c,k,e,d\).*)' - #DEC_PATTERN = r"requestBtn_clickEvent[^}]*url:\s*([^,]+)" - DOWNLOAD_LINK_PATTERN = r'</span> If it doesn\'t, <a target="_blank" href="([^"]+)">' - RECAPTCHA_KEY_PATTERN = r"var __recaptcha_public\s*=\s*'([^']+)';" - FILE_INFO_PATTERN = r'<span style="cursor: default;[^>]*>\s*(?P<N>.*?)\s* \s*<strong>\s*(?P<S>[0-9.]+)\s*(?P<U>[kKMG])i?B\s*</strong>\s*</span>' - FILE_OFFLINE_PATTERN = r'<span style="cursor: default;[^>]*>\s* \s*<strong>\s*</strong>\s*</span>' - TEMP_OFFLINE_PATTERN = r'<span class="msg_red">Downloading of this file is temporarily disabled</span>' - - def handleFree(self): - ukey = re.search(self.__pattern__, self.pyfile.url).group(1) - json_url = 'http://ifile.it/new_download-request.json' - post_data = {"ukey" : ukey, "ab": "0"} - - json_response = json_loads(self.load(json_url, post = post_data)) - self.logDebug(json_response) - if json_response['status'] == 3: - self.offline() - - if json_response["captcha"]: - captcha_key = re.search(self.RECAPTCHA_KEY_PATTERN, self.html).group(1) - recaptcha = ReCaptcha(self) - post_data["ctype"] = "recaptcha" - - for i in range(5): - post_data["recaptcha_challenge"], post_data["recaptcha_response"] = recaptcha.challenge(captcha_key) - json_response = json_loads(self.load(json_url, post = post_data)) - self.logDebug(json_response) - - if json_response["retry"]: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: - self.fail("Incorrect captcha") - - if not "ticket_url" in json_response: - self.parseError("Download URL") - - self.download(json_response["ticket_url"]) - -getInfo = create_getInfo(IfileIt)
\ No newline at end of file diff --git a/module/plugins/hoster/IfolderRu.py b/module/plugins/hoster/IfolderRu.py deleted file mode 100644 index 6accbc524..000000000 --- a/module/plugins/hoster/IfolderRu.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from urllib import quote -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.RequestFactory import getURL - -class IfolderRu(SimpleHoster): - __name__ = "IfolderRu" - __type__ = "hoster" - __pattern__ = r"http://(?:[^.]*\.)?(?:ifolder\.ru|rusfolder\.(?:com|net|ru))/(?:files/)?(?P<ID>\d+).*" - __version__ = "0.37" - __description__ = """rusfolder.com / ifolder.ru""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_SIZE_REPLACEMENTS = [(u'Ðб', 'KB'), (u'Ðб', 'MB'), (u'Ðб', 'GB')] - FILE_NAME_PATTERN = ur'(?:<div><span>)?ÐазваМОе:(?:</span>)? <b>(?P<N>[^<]+)</b><(?:/div|br)>' - FILE_SIZE_PATTERN = ur'(?:<div><span>)?РазЌеÑ:(?:</span>)? <b>(?P<S>[^<]+)</b><(?:/div|br)>' - FILE_OFFLINE_PATTERN = ur'<p>Ѐайл ÐœÐŸÐŒÐµÑ <b>[^<]*</b> (Ме МайЎеМ|ÑЎалеМ) !!!</p>' - - SESSION_ID_PATTERN = r'<a href=(http://ints.(?:rusfolder.com|ifolder.ru)/ints/sponsor/\?bi=\d*&session=([^&]+)&u=[^>]+)>' - INTS_SESSION_PATTERN = r'\(\'ints_session\'\);\s*if\(tag\)\{tag.value = "([^"]+)";\}' - HIDDEN_INPUT_PATTERN = r"var v = .*?name='([^']+)' value='1'" - DOWNLOAD_LINK_PATTERN = r'<a id="download_file_href" href="([^"]+)"' - WRONG_CAPTCHA_PATTERN = ur'<font color=Red>МевеÑÐœÑй кПЎ,<br>ввеЎОÑе еÑе Ñаз</font><br>' - - def setup(self): - self.resumeDownload = self.multiDL = True if self.account else False - self.chunkLimit = 1 - - def process(self, pyfile): - file_id = re.search(self.__pattern__, pyfile.url).group('ID') - self.html = self.load("http://rusfolder.com/%s" % file_id, cookies=True, decode=True) - self.getFileInfo() - - url = re.search('<a href="(http://ints\..*?=)"', self.html).group(1) - self.html = self.load(url, cookies=True, decode=True) - - url, session_id = re.search(self.SESSION_ID_PATTERN, self.html).groups() - self.html = self.load(url, cookies=True, decode=True) - - url = "http://ints.rusfolder.com/ints/frame/?session=%s" % session_id - self.html = self.load(url, cookies=True) - - self.setWait(31, False) - self.wait() - - captcha_url = "http://ints.rusfolder.com/random/images/?session=%s" % session_id - for i in range(5): - self.html = self.load(url, cookies=True) - action, inputs = self.parseHtmlForm('ID="Form1"') - inputs['ints_session'] = re.search(self.INTS_SESSION_PATTERN, self.html).group(1) - inputs[re.search(self.HIDDEN_INPUT_PATTERN, self.html).group(1)] = '1' - inputs['confirmed_number'] = self.decryptCaptcha(captcha_url, cookies = True) - inputs['action'] = '1' - self.logDebug(inputs) - - self.html = self.load(url, decode = True, cookies = True, post = inputs) - if self.WRONG_CAPTCHA_PATTERN in self.html: - self.invalidCaptcha() - else: - break; - else: - self.fail("Invalid captcha") - - #self.html = self.load("http://rusfolder.com/%s?ints_code=%s" % (file_id, session_id), decode=True, cookies = True) - - download_url = re.search(self.DOWNLOAD_LINK_PATTERN, self.html).group(1) - self.correctCaptcha() - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - -getInfo = create_getInfo(IfolderRu)
\ No newline at end of file diff --git a/module/plugins/hoster/JumbofilesCom.py b/module/plugins/hoster/JumbofilesCom.py deleted file mode 100644 index 9e8adb512..000000000 --- a/module/plugins/hoster/JumbofilesCom.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.utils import html_unescape - -class JumbofilesCom(SimpleHoster): - __name__ = "JumbofilesCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*jumbofiles.com/(\w{12}).*" - __version__ = "0.02" - __description__ = """JumboFiles.com hoster plugin""" - __author_name__ = ("godofdream") - __author_mail__ = ("soilfiction@gmail.com") - - FILE_INFO_PATTERN = '<TR><TD>(?P<N>[^<]+?)\s*<small>\((?P<S>[\d.]+)\s*(?P<U>[KMG][bB])\)</small></TD></TR>' - FILE_OFFLINE_PATTERN = 'Not Found or Deleted / Disabled due to inactivity or DMCA' - DIRECT_LINK_PATTERN = '<meta http-equiv="refresh" content="10;url=(.+)">' - - def setup(self): - self.resumeDownload = True - self.multiDL = True - - def handleFree(self): - ukey = re.search(self.__pattern__, self.pyfile.url).group(1) - post_data = {"id" : ukey, "op": "download3", "rand": ""} - html = self.load(self.pyfile.url, post = post_data, decode=True) - url = re.search(self.DIRECT_LINK_PATTERN, html).group(1) - self.logDebug("Download " + url) - self.download(url) - -getInfo = create_getInfo(JumbofilesCom) diff --git a/module/plugins/hoster/LetitbitNet.py b/module/plugins/hoster/LetitbitNet.py deleted file mode 100644 index c63272f27..000000000 --- a/module/plugins/hoster/LetitbitNet.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.common.json_layer import json_loads -from module.plugins.ReCaptcha import ReCaptcha - - -class LetitbitNet(SimpleHoster): - __name__ = "LetitbitNet" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*(letitbit|shareflare).net/download/.*" - __version__ = "0.20" - __description__ = """letitbit.net""" - __author_name__ = ("zoidberg", "z00nx") - __author_mail__ = ("zoidberg@mujmail.cz", "z00nx0@gmail.com") - - CHECK_URL_PATTERN = r"ajax_check_url\s*=\s*'((http://[^/]+)[^']+)';" - SECONDS_PATTERN = r"seconds\s*=\s*(\d+);" - CAPTCHA_CONTROL_FIELD = r"recaptcha_control_field\s=\s'(?P<value>[^']+)'" - FILE_INFO_PATTERN = r'<span[^>]*>File:.*?<span[^>]*>(?P<N>[^&]+).*</span>.*?\[(?P<S>[^\]]+)\]</span>' - FILE_OFFLINE_PATTERN = r'>File not found<' - - DOMAIN = "http://letitbit.net" - FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "letitbit.net")] - RECAPTCHA_KEY = "6Lc9zdMSAAAAAF-7s2wuQ-036pLRbM0p8dDaQdAM" - - def setup(self): - self.resumeDownload = True - #TODO confirm that resume works - - def handleFree(self): - action, inputs = self.parseHtmlForm('id="ifree_form"') - if not action: - self.parseError("page 1 / ifree_form") - self.pyfile.size = float(inputs['sssize']) - self.logDebug(action, inputs) - inputs['desc'] = "" - - self.html = self.load(self.DOMAIN + action, post=inputs, cookies=True) - - """ - action, inputs = self.parseHtmlForm('id="d3_form"') - if not action: self.parseError("page 2 / d3_form") - #self.logDebug(action, inputs) - - self.html = self.load(action, post = inputs, cookies = True) - - try: - ajax_check_url, captcha_url = re.search(self.CHECK_URL_PATTERN, self.html).groups() - found = re.search(self.SECONDS_PATTERN, self.html) - seconds = int(found.group(1)) if found else 60 - self.setWait(seconds+1) - self.wait() - except Exception, e: - self.logError(e) - self.parseError("page 3 / js") - """ - - found = re.search(self.SECONDS_PATTERN, self.html) - seconds = int(found.group(1)) if found else 60 - self.logDebug("Seconds found", seconds) - found = re.search(self.CAPTCHA_CONTROL_FIELD, self.html) - recaptcha_control_field = found.group(1) - self.logDebug("ReCaptcha control field found", recaptcha_control_field) - self.setWait(seconds + 1) - self.wait() - - response = self.load("%s/ajax/download3.php" % self.DOMAIN, post=" ", cookies=True) - if response != '1': - self.parseError('Unknown response - ajax_check_url') - self.logDebug(response) - - recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) - post_data = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": response, "recaptcha_control_field": recaptcha_control_field} - self.logDebug("Post data to send", post_data) - response = self.load('%s/ajax/check_recaptcha.php' % self.DOMAIN, post=post_data, cookies=True) - self.logDebug(response) - if not response: - self.invalidCaptcha() - if response == "error_free_download_blocked": - self.logInfo("Daily limit reached, waiting 24 hours") - self.setWait(24 * 60 * 60) - self.wait() - if response == "error_wrong_captcha": - self.logInfo("Wrong Captcha") - self.invalidCaptcha() - self.retry() - elif response.startswith('['): - urls = json_loads(response) - elif response.startswith('http://'): - urls = [response] - else: - self.parseError("Unknown response - captcha check") - - self.correctCaptcha() - - for download_url in urls: - try: - self.logDebug("Download URL", download_url) - self.download(download_url) - break - except Exception, e: - self.logError(e) - else: - self.fail("Download did not finish correctly") - -getInfo = create_getInfo(LetitbitNet) diff --git a/module/plugins/hoster/LoadTo.py b/module/plugins/hoster/LoadTo.py deleted file mode 100644 index babf354a9..000000000 --- a/module/plugins/hoster/LoadTo.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: halfman -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - - html = getURL(url, decode=True) - if re.search(LoadTo.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - name = re.search(LoadTo.FILE_NAME_PATTERN, html) - size = re.search(LoadTo.SIZE_PATTERN, html) - if name is not None and size is not None: - name = name.group(1) - size = size.group(1) - result.append((name, size, 2, url)) - yield result - -class LoadTo(Hoster): - __name__ = "LoadTo" - __type__ = "hoster" - __pattern__ = r"http://(www.*?\.)?load\.to/.{7,10}?/.*" - __version__ = "0.11" - __description__ = """load.to""" - __author_name__ = ("halfman") - __author_mail__ = ("Pulpan3@gmail.com") - - FILE_NAME_PATTERN = r'<div class="toolarge"><h1>(.+?)</h1></div>' - URL_PATTERN = r'<form method="post" action="(.+?)"' - SIZE_PATTERN = r'<div class="download_table_right">(\d+) Bytes</div>' - FILE_OFFLINE_PATTERN = r'Can\'t find file. Please check URL.<br />' - WAIT_PATTERN = r'type="submit" value="Download \((\d+)\)"' - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (NAME)") - pyfile.name = found.group(1) - - found = re.search(self.URL_PATTERN, self.html) - if found is None: - self.fail("Parse error (URL)") - download_url = found.group(1) - - timmy = re.search(self.WAIT_PATTERN, self.html) - if timmy: - self.setWait(timmy.group(1)) - self.wait() - - self.req.setOption("timeout", 120) - self.download(download_url)
\ No newline at end of file diff --git a/module/plugins/hoster/LuckyShareNet.py b/module/plugins/hoster/LuckyShareNet.py deleted file mode 100644 index a1e866089..000000000 --- a/module/plugins/hoster/LuckyShareNet.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*-
-
-import re
-from module.lib.bottle import json_loads
-
-from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo
-from module.plugins.ReCaptcha import ReCaptcha
-
-
-class LuckyShareNet(SimpleHoster):
- __name__ = "LuckyShareNet"
- __type__ = "hoster"
- __pattern__ = r"https?://(www\.)?luckyshare.net/(?P<ID>\d{10,})"
- __version__ = "0.02"
- __description__ = """LuckyShare.net Download Hoster"""
- __author_name__ = ("stickell")
- __author_mail__ = ("l.stickell@yahoo.it")
-
- FILE_INFO_PATTERN = r"<h1 class='file_name'>(?P<N>\S+)</h1>\s*<span class='file_size'>Filesize: (?P<S>[\d.]+)(?P<U>\w+)</span>"
- FILE_OFFLINE_PATTERN = 'There is no such file available'
- RECAPTCHA_KEY = '6LdivsgSAAAAANWh-d7rPE1mus4yVWuSQIJKIYNw'
-
- def parseJson(self, rep):
- if 'AJAX Error' in rep:
- html = self.load(self.pyfile.url, decode=True)
- m = re.search(r"waitingtime = (\d+);", html)
- if m:
- waittime = int(m.group(1))
- self.logDebug('You have to wait %d seconds between free downloads' % waittime)
- self.retry(wait_time=waittime)
- else:
- self.parseError('Unable to detect wait time between free downloads')
- elif 'Hash expired' in rep:
- self.retry(reason='Hash expired')
- return json_loads(rep)
-
- # TODO: There should be a filesize limit for free downloads
- # TODO: Some files could not be downloaded in free mode
- def handleFree(self):
- file_id = re.search(self.__pattern__, self.pyfile.url).group('ID')
- self.logDebug('File ID: ' + file_id)
- rep = self.load(r"http://luckyshare.net/download/request/type/time/file/" + file_id, decode=True)
- self.logDebug('JSON: ' + rep)
- json = self.parseJson(rep)
-
- self.setWait(int(json['time']))
- self.wait()
-
- recaptcha = ReCaptcha(self)
- for i in xrange(5):
- challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY)
- rep = self.load(r"http://luckyshare.net/download/verify/challenge/%s/response/%s/hash/%s" %
- (challenge, response, json['hash']), decode=True)
- self.logDebug('JSON: ' + rep)
- if 'link' in rep:
- json.update(self.parseJson(rep))
- self.correctCaptcha()
- break
- elif 'Verification failed' in rep:
- self.logInfo('Wrong captcha')
- self.invalidCaptcha()
- else:
- self.parseError('Unable to get downlaod link')
-
- if not json['link']:
- self.fail("No Download url retrieved/all captcha attempts failed")
-
- self.logDebug('Direct URL: ' + json['link'])
- self.download(json['link'])
-
-
-getInfo = create_getInfo(LuckyShareNet)
diff --git a/module/plugins/hoster/MediafireCom.py b/module/plugins/hoster/MediafireCom.py deleted file mode 100644 index 1e856c41d..000000000 --- a/module/plugins/hoster/MediafireCom.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.plugins.internal.CaptchaService import SolveMedia -from module.network.RequestFactory import getURL - - -def replace_eval(js_expr): - return js_expr.replace(r'eval("', '').replace(r"\'", r"'").replace(r'\"', r'"') - - -def checkHTMLHeader(url): - try: - for i in range(3): - header = getURL(url, just_header=True) - for line in header.splitlines(): - line = line.lower() - if 'location' in line: - url = line.split(':', 1)[1].strip() - if 'error.php?errno=320' in url: - return url, 1 - if not url.startswith('http://'): url = 'http://www.mediafire.com' + url - break - elif 'content-disposition' in line: - return url, 2 - else: - break - except: - return url, 3 - - return url, 0 - - -def getInfo(urls): - for url in urls: - location, status = checkHTMLHeader(url) - if status: - file_info = (url, 0, status, url) - else: - file_info = parseFileInfo(MediafireCom, url, getURL(url, decode=True)) - yield file_info - - -class MediafireCom(SimpleHoster): - __name__ = "MediafireCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*mediafire\.com/(file/|(view/?|download.php)?\?)(\w{11}|\w{15})($|/)" - __version__ = "0.79" - __description__ = """Mediafire.com plugin - free only""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - DOWNLOAD_LINK_PATTERN = r'<div class="download_link"[^>]*(?:z-index:(?P<zindex>\d+))?[^>]*>\s*<a href="(?P<href>http://[^"]+)"' - JS_KEY_PATTERN = r"DoShow\('mfpromo1'\);[^{]*{((\w+)='';.*?)eval\(\2\);" - JS_ZMODULO_PATTERN = r"\('z-index'\)\) \% (\d+)\)\);" - SOLVEMEDIA_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.noscript\?k=([^"]+)' - PAGE1_ACTION_PATTERN = r'<link rel="canonical" href="([^"]+)"/>' - PASSWORD_PATTERN = r'<form name="form_password"' - - FILE_NAME_PATTERN = r'<META NAME="description" CONTENT="(?P<N>[^"]+)"/>' - FILE_INFO_PATTERN = r"oFileSharePopup\.ald\('(?P<ID>[^']*)','(?P<N>[^']*)','(?P<S>[^']*)','','(?P<sha256>[^']*)'\)" - FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>' - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - pyfile.url = re.sub(r'/view/?\?', '/?', pyfile.url) - - self.url, result = checkHTMLHeader(pyfile.url) - self.logDebug('Location (%d): %s' % (result, self.url)) - - if result == 0: - self.html = self.load(self.url, decode=True) - self.checkCaptcha() - self.multiDL = True - self.check_data = self.getFileInfo() - - if self.account: - self.handlePremium() - else: - self.handleFree() - elif result == 1: - self.offline() - else: - self.multiDL = True - self.download(self.url, disposition=True) - - def handleFree(self): - passwords = self.getPassword().splitlines() - while self.PASSWORD_PATTERN in self.html: - if len(passwords): - password = passwords.pop(0) - self.logInfo("Password protected link, trying " + password) - self.html = self.load(self.url, post={"downloadp": password}) - else: - self.fail("No or incorrect password") - - found = re.search(r'kNO = "(http://.*?)";', self.html) - if not found: self.parseError("Download URL") - download_url = found.group(1) - self.logDebug("DOWNLOAD LINK:", download_url) - - self.download(download_url) - - def checkCaptcha(self): - for i in xrange(5): - found = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if found: - captcha_key = found.group(1) - solvemedia = SolveMedia(self) - captcha_challenge, captcha_response = solvemedia.challenge(captcha_key) - self.html = self.load(self.url, post={"adcopy_challenge": captcha_challenge, - "adcopy_response": captcha_response}, decode=True) - else: - break - else: - self.fail("No valid recaptcha solution received") diff --git a/module/plugins/hoster/MegaNz.py b/module/plugins/hoster/MegaNz.py deleted file mode 100644 index e5be4eeb7..000000000 --- a/module/plugins/hoster/MegaNz.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import random -from array import array -from os import remove -from base64 import standard_b64decode - -from Crypto.Cipher import AES -from Crypto.Util import Counter - -from module.common.json_layer import json -from module.plugins.Hoster import Hoster - -#def getInfo(urls): -# pass - -class MegaNz(Hoster): - __name__ = "MegaNz" - __type__ = "hoster" - __pattern__ = r"https?://([a-z0-9]+\.)?mega\.co\.nz/#!([a-zA-Z0-9!_\-]+)" - __version__ = "0.12" - __description__ = """mega.co.nz hoster plugin""" - __author_name__ = ("RaNaN", ) - __author_mail__ = ("ranan@pyload.org", ) - - API_URL = "https://g.api.mega.co.nz/cs?id=%d" - FILE_SUFFIX = ".crypted" - - def b64_decode(self, data): - data = data.replace("-", "+").replace("_", "/") - return standard_b64decode(data + '=' * (-len(data) % 4)) - - def getCipherKey(self, key): - """ Construct the cipher key from the given data """ - a = array("I", key) - key_array = array("I", [a[0] ^ a[4], a[1] ^ a[5], a[2] ^ a[6], a[3] ^ a[7]]) - return key_array - - def callApi(self, **kwargs): - """ Dispatch a call to the api, see https://mega.co.nz/#developers """ - # generate a session id, no idea where to obtain elsewhere - uid = random.randint(10 << 9, 10 ** 10) - - resp = self.load(self.API_URL % uid, post=json.dumps([kwargs])) - self.logDebug("Api Response: " + resp) - return json.loads(resp) - - def decryptAttr(self, data, key): - - cbc = AES.new(self.getCipherKey(key), AES.MODE_CBC, "\0" * 16) - attr = cbc.decrypt(self.b64_decode(data)) - self.logDebug("Decrypted Attr: " + attr) - if not attr.startswith("MEGA"): - self.fail(_("Decryption failed")) - - # Data is padded, 0-bytes must be stripped - return json.loads(attr.replace("MEGA", "").rstrip("\0").strip()) - - def decryptFile(self, key): - """ Decrypts the file at lastDownload` """ - - # upper 64 bit of counter start - n = key[16:24] - - # convert counter to long and shift bytes - ctr = Counter.new(128, initial_value=long(n.encode("hex"),16) << 64) - cipher = AES.new(self.getCipherKey(key), AES.MODE_CTR, counter=ctr) - - self.pyfile.setStatus("decrypting") - f = open(self.lastDownload, "rb") - df = open(self.lastDownload.rsplit(self.FILE_SUFFIX)[0], "wb") - - # TODO: calculate CBC-MAC for checksum - - size = 2 ** 15 # buffer size, 32k - while True: - buf = f.read(size) - if not buf: break - - df.write(cipher.decrypt(buf)) - - f.close() - df.close() - remove(self.lastDownload) - - def process(self, pyfile): - - key = None - - # match is guaranteed because plugin was chosen to handle url - node = re.search(self.__pattern__, pyfile.url).group(2) - if "!" in node: - node, key = node.split("!") - - self.logDebug("File id: %s | Key: %s" % (node, key)) - - if not key: - self.fail(_("No file key provided in the URL")) - - # g is for requesting a download url - # this is similar to the calls in the mega js app, documentation is very bad - dl = self.callApi(a="g", g=1, p=node, ssl=1)[0] - - if "e" in dl: - e = dl["e"] - # ETEMPUNAVAIL (-18): Resource temporarily not available, please try again later - if e == -18: - self.retry() - else: - self.fail(_("Error code:") + e) - - # TODO: map other error codes, e.g - # EACCESS (-11): Access violation (e.g., trying to write to a read-only share) - - key = self.b64_decode(key) - attr = self.decryptAttr(dl["at"], key) - - pyfile.name = attr["n"] + self.FILE_SUFFIX - - self.download(dl["g"]) - self.decryptFile(key) - - # Everything is finished and final name can be set - pyfile.name = attr["n"] diff --git a/module/plugins/hoster/MegasharesCom.py b/module/plugins/hoster/MegasharesCom.py deleted file mode 100644 index 3fac633bc..000000000 --- a/module/plugins/hoster/MegasharesCom.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from time import time -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class MegasharesCom(SimpleHoster): - __name__ = "MegasharesCom" - __type__ = "hoster" - __pattern__ = r"http://(\w+\.)?megashares.com/.*" - __version__ = "0.21" - __description__ = """megashares.com plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = '<h1 class="black xxl"[^>]*title="(?P<N>[^"]+)">' - FILE_SIZE_PATTERN = '<strong><span class="black">Filesize:</span></strong> (?P<S>[0-9.]+) (?P<U>[kKMG])i?B<br />' - DOWNLOAD_URL_PATTERN = '<div id="show_download_button_%d"[^>]*>\s*<a href="([^"]+)">' - PASSPORT_LEFT_PATTERN = 'Your Download Passport is: <[^>]*>(\w+).*\s*You have\s*<[^>]*>\s*([0-9.]+) ([kKMG]i?B)' - PASSPORT_RENEW_PATTERN = 'Your download passport will renew in\s*<strong>(\d+)</strong>:<strong>(\d+)</strong>:<strong>(\d+)</strong>' - REACTIVATE_NUM_PATTERN = r'<input[^>]*id="random_num" value="(\d+)" />' - REACTIVATE_PASSPORT_PATTERN = r'<input[^>]*id="passport_num" value="(\w+)" />' - REQUEST_URI_PATTERN = r'var request_uri = "([^"]+)";' - NO_SLOTS_PATTERN = r'<dd class="red">All download slots for this link are currently filled' - FILE_OFFLINE_PATTERN = r'<dd class="red">(Invalid Link Request|Link has been deleted)' - - def setup(self): - self.resumeDownload = True - self.multiDL = True if self.premium else False - - def handlePremium(self): - self.handleDownload(True) - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode=True) - - if self.NO_SLOTS_PATTERN in self.html: - self.retry(wait_time = 300) - - self.getFileInfo() - #if self.pyfile.size > 576716800: self.fail("This file is too large for free download") - - # Reactivate passport if needed - found = re.search(self.REACTIVATE_PASSPORT_PATTERN, self.html) - if found: - passport_num = found.group(1) - request_uri = re.search(self.REQUEST_URI_PATTERN, self.html).group(1) - - for i in range(5): - random_num = re.search(self.REACTIVATE_NUM_PATTERN, self.html).group(1) - - verifyinput = self.decryptCaptcha("http://megashares.com/index.php?secgfx=gfx&random_num=%s" % random_num) - self.logInfo("Reactivating passport %s: %s %s" % (passport_num, random_num, verifyinput)) - - url = "http://d01.megashares.com%s&rs=check_passport_renewal" % request_uri + \ - "&rsargs[]=%s&rsargs[]=%s&rsargs[]=%s" % (verifyinput, random_num, passport_num) + \ - "&rsargs[]=replace_sec_pprenewal&rsrnd=%s" % str(int(time()*1000)) - self.logDebug(url) - response = self.load(url) - - if 'Thank you for reactivating your passport.' in response: - self.correctCaptcha() - self.retry(0) - else: - self.invalidCaptcha() - else: self.fail("Failed to reactivate passport") - - # Check traffic left on passport - found = re.search(self.PASSPORT_LEFT_PATTERN, self.html) - if not found: self.fail('Passport not found') - self.logInfo("Download passport: %s" % found.group(1)) - data_left = float(found.group(2)) * 1024 ** {'KB': 1, 'MB': 2, 'GB': 3}[found.group(3)] - self.logInfo("Data left: %s %s (%d MB needed)" % (found.group(2), found.group(3), self.pyfile.size / 1048576)) - - if not data_left: - found = re.search(self.PASSPORT_RENEW_PATTERN, self.html) - renew = (found.group(1) + 60 * (found.group(2) + 60 * found.group(3))) if found else 600 - self.retry(renew, 15, "Unable to get passport") - - self.handleDownload(False) - - def handleDownload(self, premium = False): - # Find download link; - found = re.search(self.DOWNLOAD_URL_PATTERN % (1 if premium else 2), self.html) - msg = '%s download URL' % ('Premium' if premium else 'Free') - if not found: self.parseError(msg) - - download_url = found.group(1) - self.logDebug("%s: %s" % (msg, download_url)) - self.download(download_url) - -getInfo = create_getInfo(MegasharesCom)
\ No newline at end of file diff --git a/module/plugins/hoster/MovReelCom.py b/module/plugins/hoster/MovReelCom.py deleted file mode 100644 index 6f5f1d3f1..000000000 --- a/module/plugins/hoster/MovReelCom.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.utils import html_unescape -from module.network.RequestFactory import getURL - -class MovReelCom(SimpleHoster): - __name__ = "MovReelCom" - __type__ = "hoster" - __pattern__ = r"http://movreel.com/.*" - __version__ = "1.00" - __description__ = """MovReel.com hoster plugin""" - __author_name__ = ("JorisV83") - __author_mail__ = ("jorisv83-pyload@yahoo.com") - - FILE_INFO_PATTERN = r'You have requested <font color="red">http://movreel.com/.*/(?P<N>.+?)</font>.*\((?P<S>[\d.]+) (?P<U>..)\)</font>' - FILE_OFFLINE_PATTERN = r'<b>File Not Found</b>' - - def setup(self): - self.resumeDownload = True - self.multiDL = False - - def handleFree(self): - - # Define search patterns - op_pattern = '<input type="hidden" name="op" value="(.*)">' - id_pattern = '<input type="hidden" name="id" value="(.*)">' - fn_pattern = '<input type="hidden" name="fname" value="(.*)">' - re_pattern = '<input type="hidden" name="referer" value="(.*)">' - ul_pattern = '<input type="hidden" name="usr_login" value="(.*)">' - rand_pattern = '<input type="hidden" name="rand" value="(.*)">' - link_pattern = "var file_link = '(.*)';" - downlimit_pattern = '<br><p class="err">You have reached the download-limit: .*</p>' - - # Get HTML source - self.logDebug("Getting first HTML source") - html = self.load(self.pyfile.url) - self.logDebug(" > Done") - - op_val = re.search(op_pattern, html).group(1) - id_val = re.search(id_pattern, html).group(1) - fn_val = re.search(fn_pattern, html).group(1) - re_val = re.search(re_pattern, html).group(1) - ul_val = re.search(ul_pattern, html).group(1) - - # Debug values - self.logDebug(" > Op " + op_val) - self.logDebug(" > Id " + id_val) - self.logDebug(" > Fname " + fn_val) - self.logDebug(" > Referer " + re_val) - self.logDebug(" > User Login " + ul_val) - - # Create post data - post_data = {"op" : op_val, "usr_login" : ul_val, "id" : id_val, "fname" : fn_val, "referer" : re_val, "method_free" : "+Free+Download"} - - # Post and get new HTML source - self.logDebug("Getting second HTML source") - html = self.load(self.pyfile.url, post = post_data, decode=True) - self.logDebug(" > Done") - - # Check download limit - if re.search(downlimit_pattern, html) is not None: - self.retry(3, 7200, "Download limit reached, wait 2h") - - # Retrieve data - if re.search(op_pattern, html) is not None: - op_val = re.search(op_pattern, html).group(1) - else: - self.retry(3, 10, "Second html: no op found!!") - - if re.search(id_pattern, html) is not None: - id_val = re.search(id_pattern, html).group(1) - else: - self.retry(3, 10, "Second html: no id found!!") - - if re.search(rand_pattern, html) is not None: - rand_val = re.search(rand_pattern, html).group(1) - else: - self.retry(3, 10, "Second html: no rand found!!") - - re_val = self.pyfile.url - - # Debug values - self.logDebug(" > Op " + op_val) - self.logDebug(" > Id " + id_val) - self.logDebug(" > Rand " + rand_val) - self.logDebug(" > Referer " + re_val) - - # Create post data - post_data = {"op" : op_val, "id" : id_val, "rand" : rand_val, "referer" : re_val, "method_free" : "+Free+Download", "method_premium" : "", "down_direct" : "1"} - - # Post and get new HTML source - self.logDebug("Getting third HTML source") - html = self.load(self.pyfile.url, post = post_data, decode=True) - self.logDebug(" > Done") - - # Get link value - if re.search(link_pattern, html) is not None: - link_val = re.search(link_pattern, html).group(1) - self.logDebug(" > Link " + link_val) - self.download(link_val) - else: - self.logDebug("No link found!!") - self.retry(3, 10, "No link found!!") - -getInfo = create_getInfo(MovReelCom)
\ No newline at end of file diff --git a/module/plugins/hoster/MultiDebridCom.py b/module/plugins/hoster/MultiDebridCom.py deleted file mode 100644 index ca98e8a0e..000000000 --- a/module/plugins/hoster/MultiDebridCom.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- - -############################################################################ -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU Affero General Public License as # -# published by the Free Software Foundation, either version 3 of the # -# License, or (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU Affero General Public License for more details. # -# # -# You should have received a copy of the GNU Affero General Public License # -# along with this program. If not, see <http://www.gnu.org/licenses/>. # -############################################################################ - -import re - -from module.plugins.Hoster import Hoster -from module.common.json_layer import json_loads - - -class MultiDebridCom(Hoster): - __name__ = "MultiDebridCom" - __version__ = "0.01" - __type__ = "hoster" - __pattern__ = r"http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/dl/" - __description__ = """Multi-debrid.com hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - def init(self): - self.chunkLimit = -1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: - self.logError("Please enter your Multi-debrid.com account or deactivate this plugin") - self.fail("No Multi-debrid.com account provided") - - self.logDebug("Original URL: %s" % pyfile.url) - if re.match(self.__pattern__, pyfile.url): - new_url = pyfile.url - else: - page = self.req.load('http://multi-debrid.com/api.php', - get={'user': self.user, 'pass': self.account.getAccountData(self.user)['password'], - 'link': pyfile.url}) - self.logDebug("JSON data: " + page) - page = json_loads(page) - if page['status'] != 'ok': - self.fail('Unable to unrestrict link') - new_url = page['link'] - - self.logDebug("Unrestricted URL: " + new_url) - - self.download(new_url, disposition=True) diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py deleted file mode 100644 index af7aa94cf..000000000 --- a/module/plugins/hoster/MultishareCz.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from random import random -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class MultishareCz(SimpleHoster): - __name__ = "MultishareCz" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?multishare.cz/stahnout/(?P<ID>\d+).*" - __version__ = "0.40" - __description__ = """MultiShare.cz""" - __author_name__ = ("zoidberg") - - FILE_INFO_PATTERN = ur'(?:<li>Název|Soubor): <strong>(?P<N>[^<]+)</strong><(?:/li><li|br)>Velikost: <strong>(?P<S>[^<]+)</strong>' - FILE_OFFLINE_PATTERN = ur'<h1>Stáhnout soubor</h1><p><strong>PoşadovanÜ soubor neexistuje.</strong></p>' - FILE_SIZE_REPLACEMENTS = [(' ', '')] - - def process(self, pyfile): - msurl = re.match(self.__pattern__, pyfile.url) - if msurl: - self.fileID = msurl.group('ID') - self.html = self.load(pyfile.url, decode = True) - self.getFileInfo() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - else: - self.handleOverriden() - - def handleFree(self): - self.download("http://www.multishare.cz/html/download_free.php?ID=%s" % self.fileID) - - def handlePremium(self): - if not self.checkTrafficLeft(): - self.logWarning("Not enough credit left to download file") - self.resetAccount() - - self.download("http://www.multishare.cz/html/download_premium.php?ID=%s" % self.fileID) - self.checkTrafficLeft() - - def handleOverriden(self): - if not self.premium: - self.fail("Only premium users can download from other hosters") - - self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post = {"link": self.pyfile.url}, decode = True) - self.getFileInfo() - - if not self.checkTrafficLeft(): - self.fail("Not enough credit left to download file") - - url = "http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random()*10000*random()) - params = {"u_ID" : self.acc_info["u_ID"], "u_hash" : self.acc_info["u_hash"], "link" : self.pyfile.url} - self.logDebug(url, params) - self.download(url, get = params) - self.checkTrafficLeft() - -getInfo = create_getInfo(MultishareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/MyvideoDe.py b/module/plugins/hoster/MyvideoDe.py deleted file mode 100644 index f2d2082a7..000000000 --- a/module/plugins/hoster/MyvideoDe.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster -from module.unescape import unescape - -class MyvideoDe(Hoster): - __name__ = "MyvideoDe" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?myvideo.de/watch/" - __version__ = "0.9" - __description__ = """Myvideo.de Video Download Hoster""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") - - def setup(self): - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - self.download_html() - pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - self.html = self.load(self.pyfile.url) - - def get_file_url(self): - videoId = re.search(r"addVariable\('_videoid','(.*)'\);p.addParam\('quality'", self.html).group(1) - videoServer = re.search("rel='image_src' href='(.*)thumbs/.*' />", self.html).group(1) - file_url = videoServer + videoId + ".flv" - return file_url - - def get_file_name(self): - file_name_pattern = r"<h1 class='globalHd'>(.*)</h1>" - return unescape(re.search(file_name_pattern, self.html).group(1).replace("/", "") + '.flv') - - def file_exists(self): - self.download_html() - self.load(str(self.pyfile.url), cookies=False, just_header=True) - if self.req.lastEffectiveURL == "http://www.myvideo.de/": - return False - return True diff --git a/module/plugins/hoster/NarodRu.py b/module/plugins/hoster/NarodRu.py deleted file mode 100644 index 335860de9..000000000 --- a/module/plugins/hoster/NarodRu.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from random import random -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class NarodRu(SimpleHoster): - __name__ = "NarodRu" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?narod(\.yandex)?\.ru/(disk|start/[0-9]+\.\w+-narod\.yandex\.ru)/(?P<ID>\d+)/.+" - __version__ = "0.1" - __description__ = """Narod.ru""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<dt class="name">(?:<[^<]*>)*(?P<N>[^<]+)</dt>' - FILE_SIZE_PATTERN = r'<dd class="size">(?P<S>\d[^<]*)</dd>' - FILE_OFFLINE_PATTERN = r'<title>404</title>|Ѐайл ÑЎалеМ Ñ ÑеÑвОÑа|ÐакПМÑОлÑÑ ÑÑПк Ñ
ÑÐ°ÐœÐµÐœÐžÑ Ñайла\.' - - FILE_SIZE_REPLACEMENTS = [(u'ÐÐ', 'KB'), (u'ÐÐ', 'MB'), (u'ÐÐ', 'GB')] - FILE_URL_REPLACEMENTS = [("narod.yandex.ru/", "narod.ru/"), (r"/start/[0-9]+\.\w+-narod\.yandex\.ru/([0-9]{6,15})/\w+/(\w+)", r"/disk/\1/\2")] - - CAPTCHA_PATTERN = r'<number url="(.*?)">(\w+)</number>' - DOWNLOAD_LINK_PATTERN = r'<a class="h-link" rel="yandex_bar" href="(.+?)">' - - def handleFree(self): - for i in range(5): - self.html = self.load('http://narod.ru/disk/getcapchaxml/?rnd=%d' % int(random() * 777)) - found = re.search(self.CAPTCHA_PATTERN, self.html) - if not found: self.parseError('Captcha') - post_data = {"action": "sendcapcha"} - captcha_url, post_data['key'] = found.groups() - post_data['rep'] = self.decryptCaptcha(captcha_url) - - self.html = self.load(self.pyfile.url, post = post_data, decode = True) - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if found: - url = 'http://narod.ru' + found.group(1) - self.correctCaptcha() - break - elif u'<b class="error-msg"><strong>ÐÑОблОÑÑ?</strong>' in self.html: - self.invalidCaptcha() - else: - self.parseError('Download link') - else: - self.fail("No valid captcha code entered") - - self.logDebug('Download link: ' + url) - self.download(url) - -getInfo = create_getInfo(NarodRu)
\ No newline at end of file diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py deleted file mode 100644 index 39338c88d..000000000 --- a/module/plugins/hoster/NetloadIn.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import sleep, time - - -from module.utils import chunks -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - ## returns list of tuples (name, size (in bytes), status (see FileDatabase), url) - - - apiurl = "http://api.netload.in/info.php?auth=Zf9SnQh9WiReEsb18akjvQGqT0I830e8&bz=1&md5=1&file_id=" - id_regex = re.compile(NetloadIn.__pattern__) - urls_per_query = 80 - - for chunk in chunks(urls, urls_per_query): - ids = "" - for url in chunk: - match = id_regex.search(url) - if match: - ids = ids + match.group(1) +";" - - api = getURL(apiurl+ids, decode = True) - - if api is None or len(api) < 10: - print "Netload prefetch: failed " - return - if api.find("unknown_auth") >= 0: - print "Netload prefetch: Outdated auth code " - return - - result = [] - - for i, r in enumerate(api.splitlines()): - try: - tmp = r.split(";") - try: - size = int(tmp[2]) - except: - size = 0 - result.append( (tmp[1], size, 2 if tmp[3] == "online" else 1, chunk[i] ) ) - except: - print "Netload prefetch: Error while processing response: " - print r - - yield result - -class NetloadIn(Hoster): - __name__ = "NetloadIn" - __type__ = "hoster" - __pattern__ = r"https?://.*netload\.in/(?:datei(.*?)(?:\.htm|/)|index.php?id=10&file_id=)" - __version__ = "0.42" - __description__ = """Netload.in Download Hoster""" - __author_name__ = ("spoob", "RaNaN", "Gregy") - __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "gregy@gregy.cz") - - def setup(self): - self.multiDL = False - if self.premium: - self.multiDL = True - self.chunkLimit = -1 - self.resumeDownload = True - - def process(self, pyfile): - self.url = pyfile.url - self.prepare() - self.pyfile.setStatus("downloading") - self.proceed(self.url) - - def prepare(self): - self.download_api_data() - - if self.api_data and self.api_data["filename"]: - self.pyfile.name = self.api_data["filename"] - - if self.premium: - self.log.debug("Netload: Use Premium Account") - return True - - if self.download_html(): - return True - else: - self.fail("Failed") - return False - - def download_api_data(self, n=0): - url = self.url - id_regex = re.compile(self.__pattern__) - match = id_regex.search(url) - - if match: - #normalize url - self.url = 'http://www.netload.in/datei%s.htm' % match.group(1) - self.logDebug("URL: %s" % self.url) - else: - self.api_data = False - return - - apiurl = "http://api.netload.in/info.php" - src = self.load(apiurl, cookies=False, get={"file_id": match.group(1), "auth": "Zf9SnQh9WiReEsb18akjvQGqT0I830e8", "bz": "1", "md5": "1"}, decode = True).strip() - if not src and n <= 3: - sleep(0.2) - self.download_api_data(n+1) - return - - self.log.debug("Netload: APIDATA: "+src) - self.api_data = {} - if src and ";" in src and src not in ("unknown file_data", "unknown_server_data", "No input file specified."): - lines = src.split(";") - self.api_data["exists"] = True - self.api_data["fileid"] = lines[0] - self.api_data["filename"] = lines[1] - self.api_data["size"] = lines[2] - self.api_data["status"] = lines[3] - if self.api_data["status"] == "online": - self.api_data["checksum"] = lines[4].strip() - else: - self.api_data = False #check manually since api data is useless sometimes - - if lines[0] == lines[1] and lines[2] == "0": #useless api data - self.api_data = False - else: - self.api_data = False - - def final_wait(self, page): - wait_time = self.get_wait_time(page) - self.setWait(wait_time) - self.log.debug("Netload: final wait %d seconds" % wait_time) - self.wait() - self.url = self.get_file_url(page) - - def download_html(self): - self.log.debug("Netload: Entering download_html") - page = self.load(self.url, decode=True) - t = time() + 30 - - if "/share/templates/download_hddcrash.tpl" in page: - self.log.error("Netload HDD Crash") - self.fail(_("File temporarily not available")) - - if not self.api_data: - self.log.debug("API Data may be useless, get details from html page") - - if "* The file was deleted" in page: - self.offline() - - name = re.search(r'class="dl_first_filename">([^<]+)', page, re.MULTILINE) - # the found filename is not truncated - if name: - name = name.group(1).strip() - if not name.endswith(".."): - self.pyfile.name = name - - captchawaited = False - for i in range(10): - - if not page: - page = self.load(self.url) - t = time() + 30 - - if "/share/templates/download_hddcrash.tpl" in page: - self.log.error("Netload HDD Crash") - self.fail(_("File temporarily not available")) - - self.log.debug("Netload: try number %d " % i) - - if ">Your download is being prepared.<" in page: - self.log.debug("Netload: We will prepare your download") - self.final_wait(page) - return True - if ">An access request has been made from IP address <" in page: - wait = self.get_wait_time(page) - if wait == 0: - self.log.debug("Netload: Wait was 0 setting 30") - wait = 30 - self.log.info(_("Netload: waiting between downloads %d s." % wait)) - self.wantReconnect = True - self.setWait(wait) - self.wait() - - return self.download_html() - - - self.log.debug("Netload: Trying to find captcha") - - try: - url_captcha_html = "http://netload.in/" + re.search('(index.php\?id=10&.*&captcha=1)', page).group(1).replace("amp;", "") - except: - page = None - continue - - try: - page = self.load(url_captcha_html, cookies=True) - captcha_url = "http://netload.in/" + re.search('(share/includes/captcha.php\?t=\d*)', page).group(1) - except: - self.log.debug("Netload: Could not find captcha, try again from beginning") - captchawaited = False - continue - - file_id = re.search('<input name="file_id" type="hidden" value="(.*)" />', page).group(1) - if not captchawaited: - wait = self.get_wait_time(page) - if i == 0: self.pyfile.waitUntil = time() # don't wait contrary to time on web site - else: self.pyfile.waitUntil = t - self.log.info(_("Netload: waiting for captcha %d s.") % (self.pyfile.waitUntil - time())) - #self.setWait(wait) - self.wait() - captchawaited = True - - captcha = self.decryptCaptcha(captcha_url) - page = self.load("http://netload.in/index.php?id=10", post={"file_id": file_id, "captcha_check": captcha}, cookies=True) - - return False - - - def get_file_url(self, page): - try: - file_url_pattern = r"<a class=\"Orange_Link\" href=\"(http://.+)\".?>Or click here" - attempt = re.search(file_url_pattern, page) - if attempt is not None: - return attempt.group(1) - else: - self.log.debug("Netload: Backup try for final link") - file_url_pattern = r"<a href=\"(.+)\" class=\"Orange_Link\">Click here" - attempt = re.search(file_url_pattern, page) - return "http://netload.in/"+attempt.group(1) - except: - self.log.debug("Netload: Getting final link failed") - return None - - def get_wait_time(self, page): - wait_seconds = int(re.search(r"countdown\((.+),'change\(\)'\)", page).group(1)) / 100 - return wait_seconds - - - def proceed(self, url): - self.log.debug("Netload: Downloading..") - - self.download(url, disposition=True) - - check = self.checkDownload({"empty": re.compile(r"^$"), "offline": re.compile("The file was deleted")}) - - if check == "empty": - self.logInfo(_("Downloaded File was empty")) - self.retry() - elif check == "offline": - self.offline() - diff --git a/module/plugins/hoster/NovafileCom.py b/module/plugins/hoster/NovafileCom.py deleted file mode 100644 index dfd18761c..000000000 --- a/module/plugins/hoster/NovafileCom.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class NovafileCom(XFileSharingPro): - __name__ = "NovafileCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*novafile\.com/\w{12}" - __version__ = "0.01" - __description__ = """novafile.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_SIZE_PATTERN = r'<div class="size">(?P<S>.+?)</div>' - #FILE_OFFLINE_PATTERN = '<b>"File Not Found"</b>|File has been removed due to Copyright Claim' - FORM_PATTERN = r'name="F\d+"' - ERROR_PATTERN = r'class="alert[^"]*alert-separate"[^>]*>\s*(?:<p>)?(.*?)\s*</' - DIRECT_LINK_PATTERN = r'<a href="(http://s\d+\.novafile\.com/.*?)" class="btn btn-green">Download File</a>' - - HOSTER_NAME = "novafile.com" - - def setup(self): - self.multiDL = False - -getInfo = create_getInfo(NovafileCom)
\ No newline at end of file diff --git a/module/plugins/hoster/NowDownloadEu.py b/module/plugins/hoster/NowDownloadEu.py deleted file mode 100644 index 126ca3d89..000000000 --- a/module/plugins/hoster/NowDownloadEu.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from random import random -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.utils import fixup - -class NowDownloadEu(SimpleHoster): - __name__ = "NowDownloadEu" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?nowdownload\.(eu|co)/dl/(?P<ID>[a-z0-9]+)" - __version__ = "0.02" - __description__ = """NowDownloadEu""" - __author_name__ = ("godofdream") - FILE_INFO_PATTERN = r'Downloading</span> <br> (?P<N>.*) (?P<S>[0-9,.]+) (?P<U>[kKMG])i?B </h4>' - FILE_OFFLINE_PATTERN = r'(This file does not exist!)' - FILE_TOKEN_PATTERN = r'"(/api/token\.php\?token=[a-z0-9]+)"' - FILE_CONTINUE_PATTERN = r'"(/dl2/[a-z0-9]+/[a-z0-9]+)"' - FILE_WAIT_PATTERN = r'\.countdown\(\{until: \+(\d+),' - FILE_DOWNLOAD_LINK = r'"(http://f\d+\.nowdownload\.eu/dl/[a-z0-9]+/[a-z0-9]+/[^<>"]*?)"' - - FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup), (r'<[^>]*>', '')] - - def setup(self): - self.wantReconnect = False - self.multiDL = True - self.chunkLimit = -1 - self.resumeDownload = True - - def handleFree(self): - tokenlink = re.search(self.FILE_TOKEN_PATTERN, self.html) - continuelink = re.search(self.FILE_CONTINUE_PATTERN, self.html) - if (not tokenlink) or (not continuelink): self.fail('Plugin out of Date') - - wait = 60 - found = re.search(self.FILE_WAIT_PATTERN, self.html) - if found: wait = int(found.group(1)) - - self.html = self.load("http://www.nowdownload.eu" + str(tokenlink.group(1))) - self.setWait(wait) - self.wait() - - self.html = self.load("http://www.nowdownload.eu" + str(continuelink.group(1))) - - url = re.search(self.FILE_DOWNLOAD_LINK, self.html) - if not url: self.fail('Download Link not Found (Plugin out of Date?)') - self.logDebug('Download link: ' + str(url.group(1))) - self.download(str(url.group(1))) - -getInfo = create_getInfo(NowDownloadEu) diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py deleted file mode 100644 index c7c3384e9..000000000 --- a/module/plugins/hoster/OneFichierCom.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class OneFichierCom(SimpleHoster): - __name__ = "OneFichierCom" - __type__ = "hoster" - __pattern__ = r"(http://(\w+)\.((1fichier|d(es)?fichiers|pjointe)\.(com|fr|net|org)|(cjoint|mesfichiers|piecejointe|oi)\.(org|net)|tenvoi\.(com|org|net)|dl4free\.com|alterupload\.com|megadl.fr))" - __version__ = "0.47" - __description__ = """1fichier.com download hoster""" - __author_name__ = ("fragonib", "the-razer", "zoidberg","imclem") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es", "daniel_ AT gmx DOT net", "zoidberg@mujmail.cz","imclem on github") - - FILE_NAME_PATTERN = r'">File name :</th>\s*<td>(?P<N>[^<]+)</td>' - FILE_SIZE_PATTERN = r'<th>File size :</th>\s*<td>(?P<S>[^<]+)</td>' - FILE_OFFLINE_PATTERN = r'The (requested)? file (could not be found|has been deleted)' - FILE_URL_REPLACEMENTS = [(r'(http://[^/]*).*', r'\1/en/')] - - DOWNLOAD_LINK_PATTERN = r'<br/> <br/> <br/> \s+<a href="(?P<url>http://.*?)"' - PASSWORD_PROTECTED_TOKEN = "protected by password" - WAITING_PATTERN = "Warning ! Without premium status, you can download only one file at a time and you must wait up to (\d+) minutes between each downloads." - def process(self, pyfile): - found = re.search(self.__pattern__, pyfile.url) - file_id = found.group(2) - url = "http://%s.%s/en/" % (found.group(2), found.group(3)) - self.html = self.load(url, decode = True) - - found = re.search(self.WAITING_PATTERN, self.html) - if found: - self.waitAndRetry(int(found.group(1)) * 60) - - self.getFileInfo() - - url, inputs = self.parseHtmlForm('action="http://%s' % file_id) - if not url or not inputs: - self.parseError("Download link not found") - - # Check for protection - if "pass" in inputs: - inputs['pass'] = self.getPassword() - - self.download(url, post = inputs) - - # Check download - self.checkDownloadedFile() - - def checkDownloadedFile(self): - check = self.checkDownload({"wait": self.WAITING_PATTERN}) - if check == "wait": - self.waitAndRetry(int(self.lastcheck.group(1)) * 60) - - def waitAndRetry(self, wait_time): - self.setWait(wait_time, True) - self.wait() - self.retry() - -getInfo = create_getInfo(OneFichierCom) diff --git a/module/plugins/hoster/PornhostCom.py b/module/plugins/hoster/PornhostCom.py deleted file mode 100644 index a4124c4a4..000000000 --- a/module/plugins/hoster/PornhostCom.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from module.plugins.Hoster import Hoster
-
-class PornhostCom(Hoster):
- __name__ = "PornhostCom"
- __type__ = "hoster"
- __pattern__ = r'http://[\w\.]*?pornhost\.com/([0-9]+/[0-9]+\.html|[0-9]+)'
- __version__ = "0.2"
- __description__ = """Pornhost.com Download Hoster"""
- __author_name__ = ("jeix")
- __author_mail__ = ("jeix@hasnomail.de")
-
- def process(self, pyfile):
- self.download_html()
- if not self.file_exists():
- self.offline()
-
- pyfile.name = self.get_file_name()
- self.download(self.get_file_url())
-
-
- ### old interface
- def download_html(self):
- url = self.pyfile.url
- self.html = self.load(url)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- if self.html is None:
- self.download_html()
-
- file_url = re.search(r'download this file</label>.*?<a href="(.*?)"', self.html)
- if not file_url:
- file_url = re.search(r'"(http://dl[0-9]+\.pornhost\.com/files/.*?/.*?/.*?/.*?/.*?/.*?\..*?)"', self.html)
- if not file_url:
- file_url = re.search(r'width: 894px; height: 675px">.*?<img src="(.*?)"', self.html)
- if not file_url:
- file_url = re.search(r'"http://file[0-9]+\.pornhost\.com/[0-9]+/.*?"', self.html) # TODO: fix this one since it doesn't match
-
- file_url = file_url.group(1).strip()
-
- return file_url
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
-
- name = re.search(r'<title>pornhost\.com - free file hosting with a twist - gallery(.*?)</title>', self.html)
- if not name:
- name = re.search(r'id="url" value="http://www\.pornhost\.com/(.*?)/"', self.html)
- if not name:
- name = re.search(r'<title>pornhost\.com - free file hosting with a twist -(.*?)</title>', self.html)
- if not name:
- name = re.search(r'"http://file[0-9]+\.pornhost\.com/.*?/(.*?)"', self.html)
-
- name = name.group(1).strip() + ".flv"
-
- return name
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
-
- if re.search(r'gallery not found', self.html) is not None \
- or re.search(r'You will be redirected to', self.html) is not None:
- return False
- else:
- return True
-
-
diff --git a/module/plugins/hoster/PornhubCom.py b/module/plugins/hoster/PornhubCom.py deleted file mode 100644 index e1ed612b9..000000000 --- a/module/plugins/hoster/PornhubCom.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from module.plugins.Hoster import Hoster
-
-class PornhubCom(Hoster):
- __name__ = "PornhubCom"
- __type__ = "hoster"
- __pattern__ = r'http://[\w\.]*?pornhub\.com/view_video\.php\?viewkey=[\w\d]+'
- __version__ = "0.5"
- __description__ = """Pornhub.com Download Hoster"""
- __author_name__ = ("jeix")
- __author_mail__ = ("jeix@hasnomail.de")
-
- def process(self, pyfile):
- self.download_html()
- if not self.file_exists():
- self.offline()
-
- pyfile.name = self.get_file_name()
- self.download(self.get_file_url())
-
- def download_html(self):
- url = self.pyfile.url
- self.html = self.load(url)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- if self.html is None:
- self.download_html()
-
- url = "http://www.pornhub.com//gateway.php"
- video_id = self.pyfile.url.split('=')[-1]
- # thanks to jD team for this one v
- post_data = "\x00\x03\x00\x00\x00\x01\x00\x0c\x70\x6c\x61\x79\x65\x72\x43\x6f\x6e\x66\x69\x67\x00\x02\x2f\x31\x00\x00\x00\x44\x0a\x00\x00\x00\x03\x02\x00"
- post_data += chr(len(video_id))
- post_data += video_id
- post_data += "\x02\x00\x02\x2d\x31\x02\x00\x20"
- post_data += "add299463d4410c6d1b1c418868225f7"
-
- content = self.req.load(url, post=str(post_data))
-
- new_content = ""
- for x in content:
- if ord(x) < 32 or ord(x) > 176:
- new_content += '#'
- else:
- new_content += x
-
- content = new_content
-
- file_url = re.search(r'flv_url.*(http.*?)##post_roll', content).group(1)
-
- return file_url
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
-
- match = re.search(r'<title[^>]+>([^<]+) - ', self.html)
- if match:
- name = match.group(1)
- else:
- matches = re.findall('<h1>(.*?)</h1>', self.html)
- if len(matches) > 1:
- name = matches[1]
- else:
- name = matches[0]
-
- return name + '.flv'
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
-
- if re.search(r'This video is no longer in our database or is in conversion', self.html) is not None:
- return False
- else:
- return True
diff --git a/module/plugins/hoster/Premium4Me.py b/module/plugins/hoster/Premium4Me.py deleted file mode 100644 index 5dd907b9f..000000000 --- a/module/plugins/hoster/Premium4Me.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from urllib import quote
-from os.path import exists
-from os import remove
-
-from module.plugins.Hoster import Hoster
-from module.utils import fs_encode
-
-
-class Premium4Me(Hoster):
- __name__ = "Premium4Me"
- __version__ = "0.05"
- __type__ = "hoster"
-
- __pattern__ = r"http://premium4.me/.*"
- __description__ = """premium4.me hoster plugin"""
- __author_name__ = ("RaNaN", "zoidberg", "stickell")
- __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz", "l.stickell@yahoo.it")
-
- def setup(self):
- self.resumeDownload = True
- self.chunkLimit = 1
-
- def process(self, pyfile):
- if not self.account:
- self.logError(_("Please enter your premium4.me account or deactivate this plugin"))
- self.fail("No premium4.me account provided")
-
- self.logDebug("premium4.me: Old URL: %s" % pyfile.url)
-
- tra = self.getTraffic()
-
- #raise timeout to 2min
- self.req.setOption("timeout", 120)
-
- self.download(
- "http://premium4.me/api/getfile.php?authcode=%s&link=%s" % (self.account.authcode, quote(pyfile.url, "")),
- disposition=True)
-
- check = self.checkDownload({"nopremium": "No premium account available"})
-
- if check == "nopremium":
- self.retry(3, 60, 'No premium account available')
-
- err = ''
- if self.req.http.code == '420':
- # Custom error code send - fail
- lastDownload = fs_encode(self.lastDownload)
-
- if exists(lastDownload):
- f = open(lastDownload, "rb")
- err = f.read(256).strip()
- f.close()
- remove(lastDownload)
- else:
- err = 'File does not exist'
-
- trb = self.getTraffic()
- self.logInfo("Filesize: %d, Traffic used %d, traffic left %d" % (pyfile.size, tra - trb, trb))
-
- if err: self.fail(err)
-
- def getTraffic(self):
- try:
- traffic = int(self.load("http://premium4.me/api/traffic.php?authcode=%s" % self.account.authcode))
- except:
- traffic = 0
- return traffic
diff --git a/module/plugins/hoster/PremiumizeMe.py b/module/plugins/hoster/PremiumizeMe.py deleted file mode 100644 index 4ae59d198..000000000 --- a/module/plugins/hoster/PremiumizeMe.py +++ /dev/null @@ -1,50 +0,0 @@ -from module.plugins.Hoster import Hoster
-
-from module.common.json_layer import json_loads
-
-class PremiumizeMe(Hoster):
- __name__ = "PremiumizeMe"
- __version__ = "0.11"
- __type__ = "hoster"
- __description__ = """Premiumize.Me hoster plugin"""
-
- # Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady create the regex patterns for us using getHosters in our PremiumizeMe hook.
- __pattern__ = None
-
- __author_name__ = ("Florian Franzen")
- __author_mail__ = ("FlorianFranzen@gmail.com")
-
- def process(self, pyfile):
- # Check account
- if not self.account or not self.account.canUse():
- self.logError(_("Please enter a valid premiumize.me account or deactivate this plugin"))
- self.fail("No valid premiumize.me account provided")
-
- # In some cases hostsers do not supply us with a filename at download, so we are going to set a fall back filename (e.g. for freakshare or xfileshare)
- self.pyfile.name = self.pyfile.name.split('/').pop() # Remove everthing before last slash
-
- # Correction for automatic assigned filename: Removing html at end if needed
- suffix_to_remove = ["html", "htm", "php", "php3", "asp", "shtm", "shtml", "cfml", "cfm"]
- temp = self.pyfile.name.split('.')
- if temp.pop() in suffix_to_remove:
- self.pyfile.name = ".".join(temp)
-
- # Get account data
- (user, data) = self.account.selectAccount()
-
- # Get rewritten link using the premiumize.me api v1 (see https://secure.premiumize.me/?show=api)
- answer = self.load("https://api.premiumize.me/pm-api/v1.php?method=directdownloadlink¶ms[login]=%s¶ms[pass]=%s¶ms[link]=%s" % (user, data['password'], self.pyfile.url))
- data = json_loads(answer)
-
- # Check status and decide what to do
- status = data['status']
- if status == 200:
- self.download(data['result']['location'], disposition=True)
- elif status == 400:
- self.fail("Invalid link")
- elif status == 404:
- self.offline()
- elif status >= 500:
- self.tempOffline()
- else:
- self.fail(data['statusmessage'])
diff --git a/module/plugins/hoster/PutlockerCom.py b/module/plugins/hoster/PutlockerCom.py deleted file mode 100644 index b2016472d..000000000 --- a/module/plugins/hoster/PutlockerCom.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: jeix -""" - -# http://www.putlocker.com/file/83C174C844583CF7 - -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster - - -class PutlockerCom(SimpleHoster): - __name__ = "PutlockerCom" - __type__ = "hoster" - __pattern__ = r'http://(www\.)?putlocker\.com/(file|embed)/[A-Z0-9]+' - __version__ = "0.25" - __description__ = """Putlocker.Com""" - __author_name__ = ("jeix", "stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_OFFLINE_PATTERN = r"This file doesn't exist, or has been removed." - FILE_INFO_PATTERN = r'site-content">\s*<h1>(?P<N>.+)<strong>\( (?P<S>[^)]+) \)</strong></h1>' - - def handleFree(self): - self.pyfile.url = re.sub(r'http://putlocker\.com', r'http://www.putlocker.com', self.pyfile.url) - - self.html = self.load(self.pyfile.url, decode=True) - - link = self._getLink() - if not link.startswith('http://'): - link = "http://www.putlocker.com" + link - self.download(link, disposition=True) - - def _getLink(self): - hash_data = re.search(r'<input type="hidden" value="([a-z0-9]+)" name="hash">', self.html) - if not hash_data: - self.parseError('Unable to detect hash') - - post_data = {"hash": hash_data.group(1), "confirm": "Continue+as+Free+User"} - self.html = self.load(self.pyfile.url, post=post_data) - if ">You have exceeded the daily stream limit for your country\\. You can wait until tomorrow" in self.html or \ - "(>This content server has been temporarily disabled for upgrades|Try again soon\\. You can still download it below\\.<)" in self.html: - self.retry(wait_time=2 * 60 * 60, reason="Download limit exceeded or server disabled") - - patterns = (r'(/get_file\.php\?id=[A-Z0-9]+&key=[A-Za-z0-9=]+&original=1)', - r"(/get_file\.php\?download=[A-Z0-9]+&key=[a-z0-9]+)", - r"(/get_file\.php\?download=[A-Z0-9]+&key=[a-z0-9]+&original=1)", - r'<a href="/gopro\.php">Tired of ads and waiting\? Go Pro!</a>[\t\n\rn ]+</div>[\t\n\rn ]+<a href="(/.*?)"') - for pattern in patterns: - link = re.search(pattern, self.html) - if link: - break - else: - link = re.search(r"playlist: '(/get_file\.php\?stream=[A-Za-z0-9=]+)'", self.html) - if link: - self.html = self.load("http://www.putlocker.com" + link.group(1)) - link = re.search(r'media:content url="(http://.*?)"', self.html) - if not link: - link = re.search("\"(http://media\\-b\\d+\\.putlocker\\.com/download/\\d+/.*?)\"", self.html) - else: - self.parseError('Unable to detect a download link') - - return link.group(1).replace("&", "&") diff --git a/module/plugins/hoster/QuickshareCz.py b/module/plugins/hoster/QuickshareCz.py deleted file mode 100644 index 4932c4702..000000000 --- a/module/plugins/hoster/QuickshareCz.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from pycurl import FOLLOWLOCATION - -class QuickshareCz(SimpleHoster): - __name__ = "QuickshareCz" - __type__ = "hoster" - __pattern__ = r"http://.*quickshare.cz/stahnout-soubor/.*" - __version__ = "0.54" - __description__ = """Quickshare.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<th width="145px">Název:</th>\s*<td style="word-wrap:break-word;">(?P<N>[^<]+)</td>' - FILE_SIZE_PATTERN = r'<th>Velikost:</th>\s*<td>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</td>' - FILE_OFFLINE_PATTERN = r'<script type="text/javascript">location.href=\'/chyba\';</script>' - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode = True) - self.getFileInfo() - - # parse js variables - self.jsvars = dict((x, y.strip("'")) for x,y in re.findall(r"var (\w+) = ([0-9.]+|'[^']*')", self.html)) - self.logDebug(self.jsvars) - pyfile.name = self.jsvars['ID3'] - - # determine download type - free or premium - if self.premium: - if 'UU_prihlasen' in self.jsvars: - if self.jsvars['UU_prihlasen'] == '0': - self.logWarning('User not logged in') - self.relogin(self.user) - self.retry() - elif float(self.jsvars['UU_kredit']) < float(self.jsvars['kredit_odecet']): - self.logWarning('Not enough credit left') - self.premium = False - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - check = self.checkDownload({"err": re.compile(r"\AChyba!")}, max_size=100) - if check == "err": - self.fail("File not found or plugin defect") - - def handleFree(self): - # get download url - download_url = '%s/download.php' % self.jsvars['server'] - data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID3', 'ID4')) - self.logDebug("FREE URL1:" + download_url, data) - - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.load(download_url, post=data) - self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) - - found = re.search("Location\s*:\s*(.*)", self.header, re.I) - if not found: self.fail('File not found') - download_url = found.group(1) - self.logDebug("FREE URL2:" + download_url) - - # check errors - found = re.search(r'/chyba/(\d+)', download_url) - if found: - if found.group(1) == '1': - self.retry(max_tries=60, wait_time=120, reason="This IP is already downloading") - elif found.group(1) == '2': - self.retry(max_tries=60, wait_time=60, reason="No free slots available") - else: - self.fail('Error %d' % found.group(1)) - - # download file - self.download(download_url) - - def handlePremium(self): - download_url = '%s/download_premium.php' % self.jsvars['server'] - data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ('ID1', 'ID2', 'ID4', 'ID5')) - self.logDebug("PREMIUM URL:" + download_url, data) - self.download(download_url, get=data) - -getInfo = create_getInfo(QuickshareCz) diff --git a/module/plugins/hoster/RapidgatorNet.py b/module/plugins/hoster/RapidgatorNet.py deleted file mode 100644 index d2ca77e0f..000000000 --- a/module/plugins/hoster/RapidgatorNet.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from pycurl import HTTPHEADER -from random import random - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia, AdsCaptcha -from module.common.json_layer import json_loads -from module.network.HTTPRequest import BadHeader - -class RapidgatorNet(SimpleHoster): - __name__ = "RapidgatorNet" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?(rapidgator.net)/file/(\w+)" - __version__ = "0.17" - __description__ = """rapidgator.net""" - __author_name__ = ("zoidberg", "chrox", "stickell") - - API_URL = 'http://rapidgator.net/api/file' - - FILE_INFO_PATTERN = r'Downloading:(\s*<[^>]*>)*\s*(?P<N>.*?)(\s*<[^>]*>)*\s*File size:\s*<strong>(?P<S>.*?)</strong>' - FILE_OFFLINE_PATTERN = r'<title>File not found</title>' - - JSVARS_PATTERN = r"\s+var\s*(startTimerUrl|getDownloadUrl|captchaUrl|fid|secs)\s*=\s*'?(.*?)'?;" - DOWNLOAD_LINK_PATTERN = r"return '(http[^']+)';\s*}\s*}\s*}\);" - RECAPTCHA_KEY_PATTERN = r'"http://api.recaptcha.net/challenge?k=(.*?)"' - ADSCAPTCHA_SRC_PATTERN = r'(http://api.adscaptcha.com/Get.aspx[^"\']*)' - SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' - - def setup(self): - self.resumeDownload = False - self.multiDL = False - self.sid = None - self.chunkLimit = 1 - self.req.setOption("timeout", 120) - - def process(self, pyfile): - if self.account: - self.sid = self.account.getAccountData(self.user).get('SID', None) - - if self.sid: - self.handlePremium() - else: - self.handleFree() - - def getAPIResponse(self, cmd): - try: - json = self.load('%s/%s' % (self.API_URL, cmd), - get = {'sid': self.sid, - 'url': self.pyfile.url}, decode = True) - self.logDebug('API:%s' % cmd, json, "SID: %s" % self.sid) - json = json_loads(json) - status = json['response_status'] - msg = json['response_details'] - except BadHeader, e: - self.logError('API:%s' % cmd, e, "SID: %s" % self.sid) - status = e.code - msg = e - - if status == 200: - return json['response'] - elif status == 423: - self.account.empty(self.user) - self.retry() - else: - self.account.relogin(self.user) - self.retry(wait_time=60) - - def handlePremium(self): - #self.logDebug("ACCOUNT_DATA", self.account.getAccountData(self.user)) - self.api_data = self.getAPIResponse('info') - self.api_data['md5'] = self.api_data['hash'] - self.pyfile.name = self.api_data['filename'] - self.pyfile.size = self.api_data['size'] - url = self.getAPIResponse('download')['url'] - self.multiDL = True - self.download(url) - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode = True) - self.getFileInfo() - - if "You can download files up to 500 MB in free mode" in self.html \ - or "This file can be downloaded by premium only" in self.html: - self.fail("Premium account needed for download") - - self.checkWait() - - jsvars = dict(re.findall(self.JSVARS_PATTERN, self.html)) - self.logDebug(jsvars) - - self.req.http.lastURL = self.pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - - url = "http://rapidgator.net%s?fid=%s" % (jsvars.get('startTimerUrl', '/download/AjaxStartTimer'), jsvars["fid"]) - jsvars.update(self.getJsonResponse(url)) - - self.setWait(int(jsvars.get('secs', 30)) + 1, False) - self.wait() - - url = "http://rapidgator.net%s?sid=%s" % (jsvars.get('getDownloadUrl', '/download/AjaxGetDownload'), jsvars["sid"]) - jsvars.update(self.getJsonResponse(url)) - - self.req.http.lastURL = self.pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) - - url = "http://rapidgator.net%s" % jsvars.get('captchaUrl', '/download/captcha') - self.html = self.load(url) - found = re.search(self.ADSCAPTCHA_SRC_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = AdsCaptcha(self) - else: - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = ReCaptcha(self) - - else: - found = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = SolveMedia(self) - else: - self.parseError("Captcha") - - for i in range(5): - self.checkWait() - captcha_challenge, captcha_response = captcha.challenge(captcha_key) - - self.html = self.load(url, post={ - "DownloadCaptchaForm[captcha]": "", - "adcopy_challenge": captcha_challenge, - "adcopy_response": captcha_response - }) - - if 'The verification code is incorrect' in self.html: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: - self.fail("No valid captcha solution received") - - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if not found: - self.parseError("download link") - download_url = found.group(1) - self.logDebug(download_url) - self.download(download_url) - - def checkWait(self): - found = re.search(r"(?:Delay between downloads must be not less than|Try again in)\s*(\d+)\s*(hour|min)", self.html) - if found: - wait_time = int(found.group(1)) * {"hour": 60, "min": 1}[found.group(2)] - else: - found = re.search(r"You have reached your (daily|hourly) downloads limit", self.html) - if found: - wait_time = 60 - else: - return - - self.logDebug("Waiting %d minutes" % wait_time) - self.setWait(wait_time * 60, True) - self.wait() - self.retry(max_tries = 24) - - def getJsonResponse(self, url): - response = self.load(url, decode = True) - if not response.startswith('{'): - self.retry() - self.logDebug(url, response) - return json_loads(response) - -getInfo = create_getInfo(RapidgatorNet) diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py deleted file mode 100644 index 6aacd684e..000000000 --- a/module/plugins/hoster/RapidshareCom.py +++ /dev/null @@ -1,225 +0,0 @@ - -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# v1.36 -# * fixed call checkfiles subroutine -# v1.35 -# * fixed rs-urls in handleFree(..) and freeWait(..) -# * removed getInfo(..) function as it was not used anywhere (in this file) -# * removed some (old?) comment blocks - -import re - -from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster - -def getInfo(urls): - ids = "" - names = "" - - p = re.compile(RapidshareCom.__pattern__) - - for url in urls: - r = p.search(url) - if r.group("name"): - ids+= ","+r.group("id") - names+= ","+r.group("name") - elif r.group("name_new"): - ids+= ","+r.group("id_new") - names+= ","+r.group("name_new") - - url = "http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=checkfiles&files=%s&filenames=%s" % (ids[1:], names[1:]) - - api = getURL(url) - result = [] - i = 0 - for res in api.split(): - tmp = res.split(",") - if tmp[4] in ("0", "4", "5"): status = 1 - elif tmp[4] == "1": status = 2 - else: status = 3 - - result.append( (tmp[1], tmp[2], status, urls[i]) ) - i += 1 - - yield result - - -class RapidshareCom(Hoster): - __name__ = "RapidshareCom" - __type__ = "hoster" - __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))" - __version__ = "1.38" - __description__ = """Rapidshare.com Download Hoster""" - __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] - __author_name__ = ("spoob", "RaNaN", "mkaay") - __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") - - def setup(self): - self.html = None - self.no_download = True - self.api_data = None - self.offset = 0 - self.dl_dict = {} - - self.id = None - self.name = None - - self.chunkLimit = -1 if self.premium else 1 - self.multiDL = self.resumeDownload = self.premium - - def process(self, pyfile): - self.url = self.pyfile.url - self.prepare() - - def prepare(self): - m = re.search(self.__pattern__, self.url) - - if m.group("name"): - self.id = m.group("id") - self.name = m.group("name") - else: - self.id = m.group("id_new") - self.name = m.group("name_new") - - self.download_api_data() - if self.api_data["status"] == "1": - self.pyfile.name = self.get_file_name() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - elif self.api_data["status"] == "2": - self.log.info(_("Rapidshare: Traffic Share (direct download)")) - self.pyfile.name = self.get_file_name() - - self.download(self.pyfile.url, get={"directstart":1}) - - elif self.api_data["status"] in ("0","4","5"): - self.offline() - elif self.api_data["status"] == "3": - self.tempOffline() - else: - self.fail("Unknown response code.") - - def handleFree(self): - - while self.no_download: - self.dl_dict = self.freeWait() - - #tmp = "#!download|%(server)s|%(id)s|%(name)s|%(size)s" - download = "http://%(host)s/cgi-bin/rsapi.cgi?sub=download&editparentlocation=0&bin=1&fileid=%(id)s&filename=%(name)s&dlauth=%(auth)s" % self.dl_dict - - self.log.debug("RS API Request: %s" % download) - self.download(download, ref=False) - - check = self.checkDownload({"ip" : "You need RapidPro to download more files from your IP address", - "auth" : "Download auth invalid"}) - if check == "ip": - self.setWait(60) - self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) - self.wait() - self.handleFree() - elif check == "auth": - self.log.info(_("Invalid Auth Code, download will be restarted")) - self.offset += 5 - self.handleFree() - - def handlePremium(self): - info = self.account.getAccountInfo(True) - self.log.debug("%s: Use Premium Account" % self.__name__) - url = self.api_data["mirror"] - self.download(url, get={"directstart":1}) - - - def download_api_data(self, force=False): - """ - http://images.rapidshare.com/apidoc.txt - """ - if self.api_data and not force: - return - api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi" - api_param_file = {"sub": "checkfiles", "incmd5": "1", "files": self.id, "filenames": self.name} - src = self.load(api_url_base, cookies=False, get=api_param_file).strip() - self.log.debug("RS INFO API: %s" % src) - if src.startswith("ERROR"): - return - fields = src.split(",") - """ - status codes: - 0=File not found - 1=File OK (Anonymous downloading) - 3=Server down - 4=File marked as illegal - 5=Anonymous file locked, because it has more than 10 downloads already - 50+n=File OK (TrafficShare direct download type "n" without any logging.) - 100+n=File OK (TrafficShare direct download type "n" with logging. Read our privacy policy to see what is logged.) - """ - self.api_data = {"fileid": fields[0], "filename": fields[1], "size": int(fields[2]), "serverid": fields[3], - "status": fields[4], "shorthost": fields[5], "checksum": fields[6].strip().lower()} - - if int(self.api_data["status"]) > 100: - self.api_data["status"] = str(int(self.api_data["status"]) - 100) - elif int(self.api_data["status"]) > 50: - self.api_data["status"] = str(int(self.api_data["status"]) - 50) - - self.api_data["mirror"] = "http://rs%(serverid)s%(shorthost)s.rapidshare.com/files/%(fileid)s/%(filename)s" % self.api_data - - def freeWait(self): - """downloads html with the important information - """ - self.no_download = True - - id = self.id - name = self.name - - prepare = "https://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=download&fileid=%(id)s&filename=%(name)s&try=1&cbf=RSAPIDispatcher&cbid=1" % {"name": name, "id" : id} - - self.log.debug("RS API Request: %s" % prepare) - result = self.load(prepare, ref=False) - self.log.debug("RS API Result: %s" % result) - - between_wait = re.search("You need to wait (\d+) seconds", result) - - if "You need RapidPro to download more files from your IP address" in result: - self.setWait(60) - self.log.info(_("Already downloading from this ip address, waiting 60 seconds")) - self.wait() - elif "Too many users downloading from this server right now" in result or "All free download slots are full" in result: - self.setWait(120) - self.log.info(_("RapidShareCom: No free slots")) - self.wait() - elif "This file is too big to download it for free" in result: - self.fail(_("You need a premium account for this file")) - elif "Filename invalid." in result: - self.fail(_("Filename reported invalid")) - elif between_wait: - self.setWait(int(between_wait.group(1))) - self.wantReconnect = True - self.wait() - else: - self.no_download = False - - tmp, info = result.split(":") - data = info.split(",") - - dl_dict = {"id": id, - "name": name, - "host": data[0], - "auth": data[1], - "server": self.api_data["serverid"], - "size": self.api_data["size"] - } - self.setWait(int(data[2])+2+self.offset) - self.wait() - - return dl_dict - - - def get_file_name(self): - if self.api_data["filename"]: - return self.api_data["filename"] - return self.url.split("/")[-1]
\ No newline at end of file diff --git a/module/plugins/hoster/RarefileNet.py b/module/plugins/hoster/RarefileNet.py deleted file mode 100644 index a0f5930b5..000000000 --- a/module/plugins/hoster/RarefileNet.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -from module.utils import html_unescape - - -class RarefileNet(XFileSharingPro): - __name__ = "RarefileNet" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*rarefile.net/\w{12}" - __version__ = "0.03" - __description__ = """Rarefile.net hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<td><font color="red">(?P<N>.*?)</font></td>' - FILE_SIZE_PATTERN = r'<td>Size : (?P<S>.+?) ' - DIRECT_LINK_PATTERN = r'<a href="(?P<link>[^"]+)">(?P=link)</a>' - HOSTER_NAME = "rarefile.net" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - - def handleCaptcha(self, inputs): - captcha_div = re.search(r'<b>Enter code.*?<div.*?>(.*?)</div>', self.html, re.S).group(1) - self.logDebug(captcha_div) - numerals = re.findall('<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - -getInfo = create_getInfo(RarefileNet) diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py deleted file mode 100644 index 3c796232e..000000000 --- a/module/plugins/hoster/RealdebridCom.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from time import time -from urllib import quote, unquote -from random import randrange - -from module.utils import parseFileSize, remove_chars -from module.common.json_layer import json_loads -from module.plugins.Hoster import Hoster - -class RealdebridCom(Hoster): - __name__ = "RealdebridCom" - __version__ = "0.49" - __type__ = "hoster" - - __pattern__ = r"https?://.*real-debrid\..*" - __description__ = """Real-Debrid.com hoster plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - if not name or name.endswith(".."): #incomplete filename, append random stuff - name += "%s.tmp" % randrange(100,999) - return name - - def init(self): - self.tries = 0 - self.chunkLimit = 3 - self.resumeDownload = True - - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your Real-debrid account or deactivate this plugin")) - self.fail("No Real-debrid account provided") - - self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url) - if re.match(self.__pattern__, pyfile.url): - new_url = pyfile.url - else: - password = self.getPassword().splitlines() - if not password: password = "" - else: password = password[0] - - url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(pyfile.url, ""), password, int(time()*1000)) - page = self.load(url) - data = json_loads(page) - - self.logDebug("Returned Data: %s" % data) - - if data["error"] != 0: - if data["message"] == "Your file is unavailable on the hoster.": - self.offline() - else: - self.logWarning(data["message"]) - self.tempOffline() - else: - if self.pyfile.name is not None and self.pyfile.name.endswith('.tmp') and data["file_name"]: - self.pyfile.name = data["file_name"] - self.pyfile.size = parseFileSize(data["file_size"]) - new_url = data['generated_links'][0][-1] - - if self.getConfig("https"): - new_url = new_url.replace("http://", "https://") - else: - new_url = new_url.replace("https://", "http://") - - self.log.debug("Real-Debrid: New URL: %s" % new_url) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): - #only use when name wasnt already set - pyfile.name = self.getFilename(new_url) - - self.download(new_url, disposition=True) - - check = self.checkDownload( - {"error": "<title>An error occured while processing your request</title>"}) - - if check == "error": - #usual this download can safely be retried - self.retry(reason="An error occured while generating link.", wait_time=60) - diff --git a/module/plugins/hoster/RedtubeCom.py b/module/plugins/hoster/RedtubeCom.py deleted file mode 100644 index 9ffafd905..000000000 --- a/module/plugins/hoster/RedtubeCom.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from module.plugins.Hoster import Hoster
-from module.unescape import unescape
-
-class RedtubeCom(Hoster):
- __name__ = "RedtubeCom"
- __type__ = "hoster"
- __pattern__ = r'http://[\w\.]*?redtube\.com/\d+'
- __version__ = "0.2"
- __description__ = """Redtube.com Download Hoster"""
- __author_name__ = ("jeix")
- __author_mail__ = ("jeix@hasnomail.de")
-
- def process(self, pyfile):
- self.download_html()
- if not self.file_exists():
- self.offline()
-
- pyfile.name = self.get_file_name()
- self.download(self.get_file_url())
-
- def download_html(self):
- url = self.pyfile.url
- self.html = self.load(url)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- if self.html is None:
- self.download_html()
-
- file_url = unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1))
-
- return file_url
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
-
- name = re.search('<title>(.*?)- RedTube - Free Porn Videos</title>', self.html).group(1).strip() + ".flv"
- return name
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
-
- if re.search(r'This video has been removed.', self.html) is not None:
- return False
- else:
- return True
-
diff --git a/module/plugins/hoster/RehostTo.py b/module/plugins/hoster/RehostTo.py deleted file mode 100644 index 141dcb8c8..000000000 --- a/module/plugins/hoster/RehostTo.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from urllib import quote, unquote -from module.plugins.Hoster import Hoster - -class RehostTo(Hoster): - __name__ = "RehostTo" - __version__ = "0.11" - __type__ = "hoster" - __pattern__ = r"https?://.*rehost.to\..*" - __description__ = """rehost.com hoster plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - def getFilename(self, url): - return unquote(url.rsplit("/", 1)[1]) - - def setup(self): - self.chunkLimit = 1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: - self.log.error(_("Please enter your rehost.to account or deactivate this plugin")) - self.fail("No rehost.to account provided") - - data = self.account.getAccountInfo(self.user) - long_ses = data["long_ses"] - - self.log.debug("Rehost.to: Old URL: %s" % pyfile.url) - new_url = "http://rehost.to/process_download.php?user=cookie&pass=%s&dl=%s" % (long_ses, quote(pyfile.url, "")) - - #raise timeout to 2min - self.req.setOption("timeout", 120) - - self.download(new_url, disposition=True)
\ No newline at end of file diff --git a/module/plugins/hoster/ReloadCc.py b/module/plugins/hoster/ReloadCc.py deleted file mode 100644 index 7dc6d9bb6..000000000 --- a/module/plugins/hoster/ReloadCc.py +++ /dev/null @@ -1,103 +0,0 @@ -from module.plugins.Hoster import Hoster - -from module.common.json_layer import json_loads - -from module.network.HTTPRequest import BadHeader - -class ReloadCc(Hoster): - __name__ = "ReloadCc" - __version__ = "0.4" - __type__ = "hoster" - __description__ = """Reload.Cc hoster plugin""" - - # Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady create the regex patterns for us using getHosters in our ReloadCc hook. - __pattern__ = None - - __author_name__ = ("Reload Team") - __author_mail__ = ("hello@reload.cc") - - def process(self, pyfile): - # Check account - if not self.account or not self.account.canUse(): - self.logError("Please enter a valid reload.cc account or deactivate this plugin") - self.fail("No valid reload.cc account provided") - - # In some cases hostsers do not supply us with a filename at download, so we are going to set a fall back filename (e.g. for freakshare or xfileshare) - self.pyfile.name = self.pyfile.name.split('/').pop() # Remove everthing before last slash - - # Correction for automatic assigned filename: Removing html at end if needed - suffix_to_remove = ["html", "htm", "php", "php3", "asp", "shtm", "shtml", "cfml", "cfm"] - temp = self.pyfile.name.split('.') - if temp.pop() in suffix_to_remove: - self.pyfile.name = ".".join(temp) - - # Get account data - (user, data) = self.account.selectAccount() - - query_params = dict( - via='pyload', - v=1, - user=user, - uri=self.pyfile.url - ) - - try: - query_params.update(dict(hash=self.account.infos[user]['pwdhash'])) - except Exception: - query_params.update(dict(pwd=data['password'])) - - try: - answer = self.load("http://api.reload.cc/dl", get=query_params) - except BadHeader, e: - if e.code == 400: - self.fail("The URI is not supported by Reload.cc.") - elif e.code == 401: - self.fail("Wrong username or password") - elif e.code == 402: - self.fail("Your account is inactive. A payment is required for downloading!") - elif e.code == 403: - self.fail("Your account is disabled. Please contact the Reload.cc support!") - elif e.code == 409: - self.logWarning("The hoster seems to be a limited hoster and you've used your daily traffic for this hoster: %s" % self.pyfile.url) - # Wait for 6 hours and retry up to 4 times => one day - self.retry(max_retries=4, wait_time=(3600 * 6), reason="Limited hoster traffic limit exceeded") - elif e.code == 429: - self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again - elif e.code == 503: - self.retry(wait_time=600, reason="Reload.cc is currently in maintenance mode! Please check again later.") # Retry in 10 minutes - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") - return - - data = json_loads(answer) - - # Check status and decide what to do - status = data.get('status', None) - if status == "ok": - conn_limit = data.get('msg', 0) - # API says these connections are limited - # Make sure this limit is used - the download will fail if not - if conn_limit > 0: - try: - self.limitDL = int(conn_limit) - except ValueError: - self.limitDL = 1 - else: - self.limitDL = 0 - - try: - self.download(data['link'], disposition=True) - except BadHeader, e: - if e.code == 404: - self.fail("File Not Found") - elif e.code == 412: - self.fail("File access password is wrong") - elif e.code == 417: - self.fail("Password required for file access") - elif e.code == 429: - self.retry(max_retries=5, wait_time=120, reason="Too many concurrent connections") # Too many connections, wait 2 minutes and try again - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") - return - else: - self.fail("Internal error within Reload.cc. Please contact the Reload.cc support for further information.") diff --git a/module/plugins/hoster/RyushareCom.py b/module/plugins/hoster/RyushareCom.py deleted file mode 100644 index 7bfe4e8fe..000000000 --- a/module/plugins/hoster/RyushareCom.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -import re - - -class RyushareCom(XFileSharingPro): - __name__ = "RyushareCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?ryushare.com/\w{11,}" - __version__ = "0.11" - __description__ = """ryushare.com hoster plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") - - HOSTER_NAME = "ryushare.com" - - WAIT_PATTERN = r'(?:You have to|Please) wait (?:(?P<min>\d+) minutes, )?(?:<span id="[^"]+">)?(?P<sec>\d+)(?:</span>)? seconds' - DIRECT_LINK_PATTERN = r'<a href="([^"]+)">Click here to download</a>' - - def setup(self): - self.resumeDownload = self.multiDL = True - if not self.premium: - self.limitDL = 2 - # Up to 3 chunks allowed in free downloads. Unknown for premium - self.chunkLimit = 3 - - def getDownloadLink(self): - self.html = self.load(self.pyfile.url) - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) - if inputs.has_key('method_premium'): - del inputs['method_premium'] - - self.html = self.load(self.pyfile.url, post = inputs) - action, inputs = self.parseHtmlForm('F1') - - for i in xrange(10): - self.logInfo('Attempt to detect direct link #%d' % i) - - # Wait - if 'You have reached the download-limit!!!' in self.html: - self.setWait(3600, True) - else: - m = re.search(self.WAIT_PATTERN, self.html).groupdict('0') - waittime = int(m['min']) * 60 + int(m['sec']) - self.setWait(waittime) - self.wait() - - self.html = self.load(self.pyfile.url, post = inputs) - if 'Click here to download' in self.html: - m = re.search(self.DIRECT_LINK_PATTERN, self.html) - return m.group(1) - - self.parseError('No direct link within 10 retries') - -getInfo = create_getInfo(RyushareCom) diff --git a/module/plugins/hoster/SecureUploadEu.py b/module/plugins/hoster/SecureUploadEu.py deleted file mode 100644 index b9a900d96..000000000 --- a/module/plugins/hoster/SecureUploadEu.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SecureUploadEu(XFileSharingPro): - __name__ = "SecureUploadEu" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?secureupload\.eu/(\w){12}(/\w+)" - __version__ = "0.01" - __description__ = """SecureUpload.eu hoster plugin""" - __author_name__ = ("z00nx") - __author_mail__ = ("z00nx0@gmail.com") - - HOSTER_NAME = "secureupload.eu" - FILE_INFO_PATTERN = '<h3>Downloading (?P<N>[^<]+) \((?P<S>[^<]+)\)</h3>' - FILE_OFFLINE_PATTERN = 'The file was removed|File Not Found' - -getInfo = create_getInfo(SecureUploadEu) diff --git a/module/plugins/hoster/SendmywayCom.py b/module/plugins/hoster/SendmywayCom.py deleted file mode 100644 index fcbac850a..000000000 --- a/module/plugins/hoster/SendmywayCom.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SendmywayCom(XFileSharingPro): - __name__ = "SendmywayCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?sendmyway.com/\w{12}" - __version__ = "0.01" - __description__ = """SendMyWay hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<p class="file-name" ><.*?>\s*(?P<N>.+)' - FILE_SIZE_PATTERN = r'<small>\((?P<S>\d+) bytes\)</small>' - HOSTER_NAME = "sendmyway.com" - -getInfo = create_getInfo(SendmywayCom) diff --git a/module/plugins/hoster/SendspaceCom.py b/module/plugins/hoster/SendspaceCom.py deleted file mode 100644 index 22abaff56..000000000 --- a/module/plugins/hoster/SendspaceCom.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class SendspaceCom(SimpleHoster): - __name__ = "SendspaceCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?sendspace.com/file/.*" - __version__ = "0.13" - __description__ = """sendspace.com plugin - free only""" - __author_name__ = ("zoidberg") - - DOWNLOAD_URL_PATTERN = r'<a id="download_button" href="([^"]+)"' - FILE_NAME_PATTERN = r'<h2 class="bgray">\s*<(?:b|strong)>(?P<N>[^<]+)</' - FILE_SIZE_PATTERN = r'<div class="file_description reverse margin_center">\s*<b>File Size:</b>\s*(?P<S>[0-9.]+)(?P<U>[kKMG])i?B\s*</div>' - FILE_OFFLINE_PATTERN = r'<div class="msg error" style="cursor: default">Sorry, the file you requested is not available.</div>' - CAPTCHA_PATTERN = r'<td><img src="(/captchas/captcha.php?captcha=([^"]+))"></td>' - USER_CAPTCHA_PATTERN = r'<td><img src="/captchas/captcha.php?user=([^"]+))"></td>' - - def handleFree(self): - params = {} - for i in range(3): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if found: - if params.has_key('captcha_hash'): self.correctCaptcha() - download_url = found.group(1) - break - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if found: - if params.has_key('captcha_hash'): self.invalidCaptcha() - captcha_url1 = "http://www.sendspace.com/" + found.group(1) - found = re.search(self.USER_CAPTCHA_PATTERN, self.html) - captcha_url2 = "http://www.sendspace.com/" + found.group(1) - params = {'captcha_hash' : found.group(2), - 'captcha_submit': 'Verify', - 'captcha_answer': self.decryptCaptcha(captcha_url1) + " " + self.decryptCaptcha(captcha_url2) - } - else: - params = {'download': "Regular Download"} - - self.logDebug(params) - self.html = self.load(self.pyfile.url, post = params) - else: - self.fail("Download link not found") - - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - -create_getInfo(SendspaceCom)
\ No newline at end of file diff --git a/module/plugins/hoster/Share4webCom.py b/module/plugins/hoster/Share4webCom.py deleted file mode 100644 index ef9c2acf8..000000000 --- a/module/plugins/hoster/Share4webCom.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.hoster.UnibytesCom import UnibytesCom -from module.plugins.internal.SimpleHoster import create_getInfo - -class Share4webCom(UnibytesCom): - __name__ = "Share4webCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?share4web\.com/get/\w+" - __version__ = "0.1" - __description__ = """Share4web.com""" - __author_name__ = ("zoidberg") - - DOMAIN = 'http://www.share4web.com' - -getInfo = create_getInfo(UnibytesCom)
\ No newline at end of file diff --git a/module/plugins/hoster/Share76Com.py b/module/plugins/hoster/Share76Com.py deleted file mode 100644 index db850cb73..000000000 --- a/module/plugins/hoster/Share76Com.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class Share76Com(XFileSharingPro): - __name__ = "Share76Com" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?share76.com/\w{12}" - __version__ = "0.03" - __description__ = """share76.com hoster plugin""" - __author_name__ = ("me") - - FILE_INFO_PATTERN = r'<h2>\s*File:\s*<font[^>]*>(?P<N>[^>]+)</font>\s*\[<font[^>]*>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</font>\]</h2>' - HOSTER_NAME = "share76.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - self.chunkLimit = 1 - -getInfo = create_getInfo(Share76Com) diff --git a/module/plugins/hoster/ShareFilesCo.py b/module/plugins/hoster/ShareFilesCo.py deleted file mode 100644 index ee44b0a1f..000000000 --- a/module/plugins/hoster/ShareFilesCo.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo -import re - -class ShareFilesCo(XFileSharingPro): - __name__ = "ShareFilesCo" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?sharefiles\.co/\w{12}" - __version__ = "0.01" - __description__ = """Sharefiles.co hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - HOSTER_NAME = "sharefiles.co" - - def startDownload(self, link): - link = link.strip() - if link.startswith('http://adf.ly'): - link = re.sub('http://adf.ly/\d+/', '', link) - if self.captcha: self.correctCaptcha() - self.logDebug('DIRECT LINK: %s' % link) - self.download(link) - -getInfo = create_getInfo(ShareFilesCo) diff --git a/module/plugins/hoster/ShareRapidCom.py b/module/plugins/hoster/ShareRapidCom.py deleted file mode 100644 index 5a08fed1f..000000000 --- a/module/plugins/hoster/ShareRapidCom.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from pycurl import HTTPHEADER -from module.network.RequestFactory import getRequest, getURL -from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.common.json_layer import json_loads - -def checkFile(url): - response = getURL("http://share-rapid.com/checkfiles.php", post = {"files": url}, decode = True) - info = json_loads(response) - - if "error" in info: - if info['error'] == False: - info['name'] = info['filename'] - info['status'] = 2 - elif info['msg'] == "Not found": - info['status'] = 1 #offline - elif info['msg'] == "Service Unavailable": - info['status'] = 6 #temp.offline - - return info - -def getInfo(urls): - for url in urls: - info = checkFile(url) - if "filename" in info: - yield info['name'], info['size'], info['status'], url - else: - file_info = (url, 0, 3, url) - h = getRequest() - try: - h.c.setopt(HTTPHEADER, ["Accept: text/html"]) - html = h.load(url, cookies = True, decode = True) - file_info = parseFileInfo(ShareRapidCom, url, html) - finally: - h.close() - yield file_info - -class ShareRapidCom(SimpleHoster): - __name__ = "ShareRapidCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/stahuj/(\w+)" - __version__ = "0.52" - __description__ = """Share-rapid.com plugin - premium only""" - __author_name__ = ("MikyWoW", "zoidberg") - __author_mail__ = ("MikyWoW@seznam.cz", "zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<h1[^>]*><span[^>]*>(?:<a[^>]*>)?(?P<N>[^<]+)' - FILE_SIZE_PATTERN = r'<td class="i">Velikost:</td>\s*<td class="h"><strong>\s*(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</strong></td>' - FILE_OFFLINE_PATTERN = ur'Nastala chyba 404|Soubor byl smazán' - - DOWNLOAD_URL_PATTERN = r'<a href="([^"]+)" title="Stahnout">([^<]+)</a>' - ERR_LOGIN_PATTERN = ur'<div class="error_div"><strong>Stahovánà je pÅÃstupné pouze pÅihlášenÃœm uÅŸivatelům' - ERR_CREDIT_PATTERN = ur'<div class="error_div"><strong>Stahovánà zdarma je moÅŸné jen pÅes náš' - - FILE_URL_REPLACEMENTS = [(__pattern__, r'http://share-rapid.com/stahuj/\1')] - - def setup(self): - self.chunkLimit = 1 - self.resumeDownload = True - - def process(self, pyfile): - if not self.account: self.fail("User not logged in") - - self.info = checkFile(pyfile.url) - self.logDebug(self.info) - - pyfile.status = self.info['status'] - - if pyfile.status == 2: - pyfile.name = self.info['name'] - pyfile.size = self.info['size'] - elif pyfile.status == 1: - self.offline() - elif pyfile.status == 6: - self.tempOffline() - else: - self.fail("Unexpected file status") - - url = "http://share-rapid.com/stahuj/%s" % self.info['filepath'] - try: - self.html = self.load(url, decode=True) - except BadHeader, e: - self.account.relogin(self.user) - self.retry(3, 0, str(e)) - - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if found is not None: - link = found.group(1) - self.logDebug("Premium link: %s" % link) - - self.check_data = {"size": pyfile.size} - self.download(link) - else: - if re.search(self.ERR_LOGIN_PATTERN, self.html): - self.relogin(self.user) - self.retry(3,0,"User login failed") - elif re.search(self.ERR_CREDIT_PATTERN, self.html): - self.fail("Not enough credit left") - else: - self.fail("Download link not found") diff --git a/module/plugins/hoster/SharebeesCom.py b/module/plugins/hoster/SharebeesCom.py deleted file mode 100644 index f5bacc5b0..000000000 --- a/module/plugins/hoster/SharebeesCom.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - - -class SharebeesCom(XFileSharingPro): - __name__ = "SharebeesCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?sharebees.com/\w{12}" - __version__ = "0.01" - __description__ = """ShareBees hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<p class="file-name" ><.*?>\s*(?P<N>.+)' - FILE_SIZE_PATTERN = r'<small>\((?P<S>\d+) bytes\)</small>' - FORM_PATTERN = 'F1' - HOSTER_NAME = "sharebees.com" - -getInfo = create_getInfo(SharebeesCom) diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py deleted file mode 100644 index e1867168b..000000000 --- a/module/plugins/hoster/ShareonlineBiz.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from base64 import b64decode -import hashlib -import random -from time import time, sleep - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks -from module.plugins.ReCaptcha import ReCaptcha as _ReCaptcha - -def getInfo(urls): - api_url_base = "http://api.share-online.biz/linkcheck.php" - - for chunk in chunks(urls, 90): - api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links - src = getURL(api_url_base, post=api_param_file, decode=True) - result = [] - for i, res in enumerate(src.split("\n")): - if not res: - continue - fields = res.split(";") - - if fields[1] == "OK": - status = 2 - elif fields[1] in ("DELETED", "NOT FOUND"): - status = 1 - else: - status = 3 - - result.append((fields[2], int(fields[3]), status, chunk[i])) - yield result - -#suppress ocr plugin -class ReCaptcha(_ReCaptcha): - def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage"%server, get={"c":challenge}, cookies=True, forceUser=True, imgtype="jpg") - -class ShareonlineBiz(Hoster): - __name__ = "ShareonlineBiz" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?(share\-online\.biz|egoshare\.com)/(download.php\?id\=|dl/)[\w]+" - __version__ = "0.36" - __description__ = """Shareonline.biz Download Hoster""" - __author_name__ = ("spoob", "mkaay", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz") - - ERROR_INFO_PATTERN = r'<p class="b">Information:</p>\s*<div>\s*<strong>(.*?)</strong>' - - def setup(self): - # range request not working? - # api supports resume, only one chunk - # website isn't supporting resuming in first place - self.file_id = re.search(r"(id\=|/dl/)([a-zA-Z0-9]+)", self.pyfile.url).group(2) - self.pyfile.url = "http://www.share-online.biz/dl/" + self.file_id - - self.resumeDownload = self.premium - self.multiDL = False - #self.chunkLimit = 1 - - self.check_data = None - - def process(self, pyfile): - if self.premium: - self.handleAPIPremium() - #web-download fallback removed - didn't work anyway - else: - self.handleFree() - - """ - check = self.checkDownload({"failure": re.compile(self.ERROR_INFO_PATTERN)}) - if check == "failure": - try: - self.retry(reason = self.lastCheck.group(1).decode("utf8")) - except: - self.retry(reason = "Unknown error") - """ - - if self.api_data: - self.check_data = {"size": int(self.api_data['size']), "md5": self.api_data['md5']} - - def downloadAPIData(self): - api_url_base = "http://api.share-online.biz/linkcheck.php?md5=1" - api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links - src = self.load(api_url_base, cookies=False, post=api_param_file, decode=True) - - fields = src.split(";") - self.api_data = {"fileid": fields[0], - "status": fields[1]} - if not self.api_data["status"] == "OK": - self.offline() - self.api_data["filename"] = fields[2] - self.api_data["size"] = fields[3] # in bytes - self.api_data["md5"] = fields[4].strip().lower().replace("\n\n", "") # md5 - - def handleFree(self): - self.downloadAPIData() - self.pyfile.name = self.api_data["filename"] - self.pyfile.size = int(self.api_data["size"]) - - self.html = self.load(self.pyfile.url, cookies = True) #refer, stuff - self.setWait(3) - self.wait() - - self.html = self.load("%s/free/" % self.pyfile.url, post={"dl_free":"1", "choice": "free"}, decode = True) - self.checkErrors() - - found = re.search(r'var wait=(\d+);', self.html) - - recaptcha = ReCaptcha(self) - for i in range(5): - challenge, response = recaptcha.challenge("6LdatrsSAAAAAHZrB70txiV5p-8Iv8BtVxlTtjKX") - self.setWait(int(found.group(1)) if found else 30) - response = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time() * 1000)), post = { - 'dl_free': '1', - 'recaptcha_challenge_field': challenge, - 'recaptcha_response_field': response}) - - if not response == '0': - break - - else: self.fail("No valid captcha solution received") - - download_url = response.decode("base64") - self.logDebug(download_url) - if not download_url.startswith("http://"): - self.parseError("download url") - - self.wait() - self.download(download_url) - # check download - check = self.checkDownload({ - "cookie": re.compile(r'<div id="dl_failure"'), - "fail": re.compile(r"<title>Share-Online") - }) - if check == "cookie": - self.retry(5, 60, "Cookie failure") - elif check == "fail": - self.retry(5, 300, "Download failed") - - def checkErrors(self): - found = re.search(r"/failure/(.*?)/1", self.req.lastEffectiveURL) - if found: - err = found.group(1) - found = re.search(self.ERROR_INFO_PATTERN, self.html) - msg = found.group(1) if found else "" - self.logError(err, msg or "Unknown error occurred") - - if err in ('freelimit', 'size', 'proxy'): - self.fail(msg or "Premium account needed") - if err in ('invalid'): - self.fail(msg or "File not available") - elif err in ('server'): - self.setWait(600, False) - elif err in ('expired'): - self.setWait(30, False) - else: - self.setWait(300, True) - - self.wait() - self.retry(max_tries=25, reason = msg) - - def handleAPIPremium(self): #should be working better - self.account.getAccountInfo(self.user, True) - src = self.load("http://api.share-online.biz/account.php", - {"username": self.user, "password": self.account.accounts[self.user]["password"], "act": "download", "lid": self.file_id}) - - self.api_data = dlinfo = {} - for line in src.splitlines(): - key, value = line.split(": ") - dlinfo[key.lower()] = value - - self.logDebug(dlinfo) - if not dlinfo["status"] == "online": - self.offline() - - self.pyfile.name = dlinfo["name"] - self.pyfile.size = int(dlinfo["size"]) - - dlLink = dlinfo["url"] - if dlLink == "server_under_maintenance": - self.tempoffline() - else: - self.multiDL = True - self.download(dlLink) - - def checksum(self, local_file): - if self.api_data and "md5" in self.api_data and self.api_data["md5"]: - h = hashlib.md5() - f = open(local_file, "rb") - h.update(f.read()) - f.close() - hexd = h.hexdigest() - if hexd == self.api_data["md5"]: - return True, 0 - else: - return False, 1 - else: - self.logWarning("MD5 checksum missing") - return True, 5 diff --git a/module/plugins/hoster/ShareplaceCom.py b/module/plugins/hoster/ShareplaceCom.py deleted file mode 100644 index 7f0dee0e5..000000000 --- a/module/plugins/hoster/ShareplaceCom.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-import urllib
-from module.plugins.Hoster import Hoster
-
-class ShareplaceCom(Hoster):
- __name__ = "ShareplaceCom"
- __type__ = "hoster"
- __pattern__ = r"(http://)?(www\.)?shareplace\.(com|org)/\?[a-zA-Z0-9]+"
- __version__ = "0.1"
- __description__ = """Shareplace.com Download Hoster"""
- __author_name__ = ("ACCakut, based on YourfilesTo by jeix and skydancer")
- __author_mail__ = ("none")
-
- def setup(self):
- self.html = None
- self.multiDL = True
-
- def process(self,pyfile):
- self.pyfile = pyfile
- self.prepare()
- self.download(self.get_file_url())
-
- def prepare(self):
- if not self.file_exists():
- self.offline()
-
- self.pyfile.name = self.get_file_name()
-
- wait_time = self.get_waiting_time()
- self.setWait(wait_time)
- self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time))
- self.wait()
-
- def get_waiting_time(self):
- if self.html is None:
- self.download_html()
-
- #var zzipitime = 15;
- m = re.search(r'var zzipitime = (\d+);', self.html)
- if m:
- sec = int(m.group(1))
- else:
- sec = 0
-
- return sec
-
- def download_html(self):
- url = re.sub("shareplace.com\/\?", "shareplace.com//index1.php/?a=", self.pyfile.url)
- self.html = self.load(url, decode=True)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- url = re.search(r"var beer = '(.*?)';", self.html)
- if url:
- url = url.group(1)
- url = urllib.unquote(url.replace("http://http:/", "").replace("vvvvvvvvv", "").replace("lllllllll", "").replace("teletubbies", ""))
- self.logDebug("URL: %s" % url)
- return url
- else:
- self.fail("absolute filepath could not be found. offline? ")
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
-
- return re.search("<title>\s*(.*?)\s*</title>", self.html).group(1)
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
-
- if re.search(r"HTTP Status 404", self.html) is not None:
- return False
- else:
- return True
-
-
-
diff --git a/module/plugins/hoster/ShragleCom.py b/module/plugins/hoster/ShragleCom.py deleted file mode 100644 index 99f9f2366..000000000 --- a/module/plugins/hoster/ShragleCom.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from pycurl import FOLLOWLOCATION - -from module.plugins.Hoster import Hoster -from module.plugins.internal.SimpleHoster import parseHtmlForm -from module.plugins.ReCaptcha import ReCaptcha -from module.network.RequestFactory import getURL - -API_KEY = "078e5ca290d728fd874121030efb4a0d" - -def parseFileInfo(self, url): - file_id = re.match(self.__pattern__, url).group('ID') - - data = getURL( - "http://www.cloudnator.com/api.php?key=%s&action=getStatus&fileID=%s" % (API_KEY, file_id), - decode = True - ).split() - - if len(data) == 4: - name, size, md5, status = data - size = int(size) - - if hasattr(self, "check_data"): - self.checkdata = {"size": size, "md5": md5} - - return name, size, 2 if status == "0" else 1, url - else: - return url, 0, 1, url - -def getInfo(urls): - for url in urls: - file_info = parseFileInfo(ShragleCom, url) - yield file_info - -class ShragleCom(Hoster): - __name__ = "ShragleCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www.)?(cloudnator|shragle).com/files/(?P<ID>.*?)/" - __version__ = "0.21" - __description__ = """Cloudnator.com (Shragle.com) Download PLugin""" - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - def setup(self): - self.html = None - self.multiDL = False - self.check_data = None - - def process(self, pyfile): - #get file status and info - self.pyfile.name, self.pyfile.size, status = parseFileInfo(self, pyfile.url)[:3] - if status != 2: - self.offline() - - self.handleFree() - - def handleFree(self): - self.html = self.load(self.pyfile.url) - - #get wait time - found = re.search('\s*var\sdownloadWait\s=\s(\d+);', self.html) - self.setWait(int(found.group(1)) if found else 30) - - #parse download form - action, inputs = parseHtmlForm('id="download', self.html) - - #solve captcha - found = re.search('recaptcha/api/(?:challenge|noscript)?k=(.+?)', self.html) - captcha_key = found.group(1) if found else "6LdEFb0SAAAAAAwM70vnYo2AkiVkCx-xmfniatHz" - - recaptcha = ReCaptcha(self) - - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - self.wait() - - #validate - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.html = self.load(action, post = inputs) - - found = re.search(r"Location\s*:\s*(\S*)", self.req.http.header, re.I) - if found: - self.correctCaptcha() - download_url = found.group(1) - else: - if "Sicherheitscode falsch" in self.html: - self.invalidCaptcha() - self.retry(max_tries = 5, reason = "Invalid captcha") - else: - self.fail("Invalid session") - - #download - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.download(download_url) - - check = self.checkDownload({ - "ip_blocked": re.compile(r'<div class="error".*IP.*loading') - }) - if check == "ip_blocked": - self.setWait(1800, True) - self.wait() - self.retry() - - diff --git a/module/plugins/hoster/SpeedLoadOrg.py b/module/plugins/hoster/SpeedLoadOrg.py deleted file mode 100644 index 32e7baf13..000000000 --- a/module/plugins/hoster/SpeedLoadOrg.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class SpeedLoadOrg(XFileSharingPro): - __name__ = "SpeedLoadOrg" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?speedload\.org/(?P<ID>\w+)" - __version__ = "1.01" - __description__ = """Speedload.org hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_NAME_PATTERN = r'Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>' - FILE_SIZE_PATTERN = r'Size:</b></td><td>[\w. ]+<small>\((?P<S>\d+) bytes\)</small>' - - HOSTER_NAME = "speedload.org" - - def handlePremium(self): - self.download(self.pyfile.url, post = self.getPostParameters()) - -getInfo = create_getInfo(SpeedLoadOrg) diff --git a/module/plugins/hoster/SpeedfileCz.py b/module/plugins/hoster/SpeedfileCz.py deleted file mode 100644 index bfd316dfa..000000000 --- a/module/plugins/hoster/SpeedfileCz.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class SpeedfileCz(SimpleHoster): - __name__ = "SpeedFileCz" - __type__ = "hoster" - __pattern__ = r"http://speedfile.cz/.*" - __version__ = "0.31" - __description__ = """speedfile.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<meta property="og:title" content="(?P<N>[^"]+)" />' - FILE_SIZE_PATTERN = r'<strong><big>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B' - URL_PATTERN = r'<a id="request" class="caps" href="([^"]+)" rel="nofollow">' - FILE_OFFLINE_PATTERN = r'<title>Speedfile \| 404' - WAIT_PATTERN = r'"requestedAt":(\d+),"allowedAt":(\d+),"adUri"' - - def setup(self): - self.multiDL = False - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (NAME)") - pyfile.name = found.group(1) - - found = re.search(self.URL_PATTERN, self.html) - if found is None: - self.fail("Parse error (URL)") - download_url = "http://speedfile.cz/" + found.group(1) - - self.html = self.load(download_url) - self.logDebug(self.html) - found = re.search(self.WAIT_PATTERN, self.html) - if found is None: - self.fail("Parse error (WAIT)") - self.setWait(int(found.group(2)) - int(found.group(1))) - self.wait() - - self.download(download_url) - -create_getInfo(SpeedfileCz)
\ No newline at end of file diff --git a/module/plugins/hoster/StreamCz.py b/module/plugins/hoster/StreamCz.py deleted file mode 100644 index ca1033502..000000000 --- a/module/plugins/hoster/StreamCz.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - - html = getURL(url) - if re.search(StreamCz.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - result.append((url, 0, 2, url)) - yield result - -class StreamCz(Hoster): - __name__ = "StreamCz" - __type__ = "hoster" - __pattern__ = r"http://www.stream.cz/[^/]+/\d+.*" - __version__ = "0.1" - __description__ = """stream.cz""" - __author_name__ = ("zoidberg") - - FILE_OFFLINE_PATTERN = r'<h1 class="commonTitle">Str.nku nebylo mo.n. nal.zt \(404\)</h1>' - FILE_NAME_PATTERN = r'<link rel="video_src" href="http://www.stream.cz/\w+/(\d+)-([^"]+)" />' - CDN_PATTERN = r'<param name="flashvars" value="[^"]*&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' - - def setup(self): - self.multiDL = True - self.resumeDownload = True - - def process(self, pyfile): - - self.html = self.load(pyfile.url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html): - self.offline() - - found = re.search(self.CDN_PATTERN, self.html) - if found is None: self.fail("Parse error (CDN)") - cdn = found.groupdict() - self.logDebug(cdn) - for cdnkey in ("cdnHD", "cdnHQ", "cdnLQ"): - if cdn.has_key(cdnkey) and cdn[cdnkey] > '': - cdnid = cdn[cdnkey] - break - else: - self.fail("Stream URL not found") - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: self.fail("Parse error (NAME)") - pyfile.name = "%s-%s.%s.mp4" % (found.group(2), found.group(1), cdnkey[-2:]) - - download_url = "http://cdn-dispatcher.stream.cz/?id=" + cdnid - self.logInfo("STREAM (%s): %s" % (cdnkey[-2:], download_url)) - self.download(download_url) diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py deleted file mode 100644 index b429d5510..000000000 --- a/module/plugins/hoster/TurbobitNet.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Copyright (C) 2012 pyLoad team - Copyright (C) 2012 JD-Team support@jdownloader.org - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -import random -from urllib import quote -from binascii import hexlify, unhexlify -from Crypto.Cipher import ARC4 -import time - -from module.network.RequestFactory import getURL -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp -from module.plugins.ReCaptcha import ReCaptcha - -from pycurl import HTTPHEADER - -class TurbobitNet(SimpleHoster): - __name__ = "TurbobitNet" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*" - __version__ = "0.08" - __description__ = """Turbobit.net plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r"<span class='file-icon1[^>]*>(?P<N>[^<]+)</span>\s*\((?P<S>[^\)]+)\)\s*</h1>" #long filenames are shortened - FILE_NAME_PATTERN = r'<meta name="keywords" content="\s+(?P<N>[^,]+)' #full name but missing on page2 - FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File was not found' - FILE_URL_REPLACEMENTS = [(r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*", "http://turbobit.net/\g<ID>.html")] - SH_COOKIES = [("turbobit.net", "user_lang", "en")] - - CAPTCHA_KEY_PATTERN = r'src="http://api\.recaptcha\.net/challenge\?k=([^"]+)"' - DOWNLOAD_URL_PATTERN = r'(?P<url>/download/redirect/[^"\']+)' - LIMIT_WAIT_PATTERN = r'<div id="time-limit-text">\s*.*?<span id=\'timeout\'>(\d+)</span>' - CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' - - def handleFree(self): - self.url = "http://turbobit.net/download/free/%s" % self.file_info['ID'] - self.html = self.load(self.url) - - rtUpdate = self.getRtUpdate() - - self.solveCaptcha() - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.url = self.getDownloadUrl(rtUpdate) - - self.wait() - self.html = self.load(self.url) - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) - self.downloadFile() - - def solveCaptcha(self): - for i in range(5): - found = re.search(self.LIMIT_WAIT_PATTERN, self.html) - if found: - wait_time = int(found.group(1)) - self.setWait(wait_time, wait_time > 60) - self.wait() - self.retry() - - action, inputs = self.parseHtmlForm("action='#'") - if not inputs: self.parseError("captcha form") - self.logDebug(inputs) - - if inputs['captcha_type'] == 'recaptcha': - recaptcha = ReCaptcha(self) - found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) - captcha_key = found.group(1) if found else '6LcTGLoSAAAAAHCWY9TTIrQfjUlxu6kZlTYP50_c' - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - else: - found = re.search(self.CAPTCHA_SRC_PATTERN, self.html) - if not found: self.parseError('captcha') - captcha_url = found.group(1) - inputs['captcha_response'] = self.decryptCaptcha(captcha_url) - - self.logDebug(inputs) - self.html = self.load(self.url, post = inputs) - - if not "<div class='download-timer-header'>" in self.html: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - else: self.fail("Invalid captcha") - - def getRtUpdate(self): - rtUpdate = self.getStorage("rtUpdate") - if not rtUpdate: - if self.getStorage("version") != self.__version__ or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): - # that's right, we are even using jdownloader updates - rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") - rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) - # but we still need to fix the syntax to work with other engines than rhino - rtUpdate = re.sub(r'for each\(var (\w+) in(\[[^\]]+\])\)\{',r'zza=\2;for(var zzi=0;zzi<zza.length;zzi++){\1=zza[zzi];',rtUpdate) - rtUpdate = re.sub(r"for\((\w+)=",r"for(var \1=", rtUpdate) - - self.logDebug("rtUpdate") - self.setStorage("rtUpdate", rtUpdate) - self.setStorage("timestamp", timestamp()) - self.setStorage("version", self.__version__) - else: - self.logError("Unable to download, wait for update...") - self.tempOffline() - - return rtUpdate - - def getDownloadUrl(self, rtUpdate): - self.req.http.lastURL = self.url - - found = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) - url = "http://turbobit.net%s%s" % (found.groups() if found else ('/files/timeout.js?ver=', ''.join(random.choice('0123456789ABCDEF') for x in range(32)))) - fun = self.load(url) - - self.setWait(65, False) - - for b in [1,3]: - self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % (self.file_info['ID'], b, quote(fun), rtUpdate) - - try: - out = self.js.eval(self.jscode) - self.logDebug("URL", self.js.engine, out) - if out.startswith('/download/'): - return "http://turbobit.net%s" % out.strip() - except Exception, e: - self.logError(e) - else: - if self.retries >= 2: - # retry with updated js - self.delStorage("rtUpdate") - self.retry() - - def decrypt(self, data): - cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) - return unhexlify(cipher.encrypt(unhexlify(data))) - - def getLocalTimeString(self): - lt = time.localtime() - tz = time.altzone if lt.tm_isdst else time.timezone - return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) - - def handlePremium(self): - self.logDebug("Premium download as user %s" % self.user) - self.downloadFile() - - def downloadFile(self): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError("download link") - self.url = "http://turbobit.net" + found.group('url') - self.logDebug(self.url) - self.download(self.url) - -getInfo = create_getInfo(TurbobitNet) diff --git a/module/plugins/hoster/TurbouploadCom.py b/module/plugins/hoster/TurbouploadCom.py deleted file mode 100644 index 6e81c6319..000000000 --- a/module/plugins/hoster/TurbouploadCom.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.DeadHoster import DeadHoster as EasybytezCom, create_getInfo -#from module.plugins.internal.SimpleHoster import create_getInfo -#from module.plugins.hoster.EasybytezCom import EasybytezCom - -class TurbouploadCom(EasybytezCom): - __name__ = "TurbouploadCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w+).*" - __version__ = "0.02" - __description__ = """turboupload.com""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - # shares code with EasybytezCom - - DIRECT_LINK_PATTERN = r'<a href="(http://turboupload.com/files/[^"]+)">\1</a>' - - def handleFree(self): - self.html = self.load(self.pyfile.url, post = self.getPostParameters(), ref = True, cookies = True) - found = re.search(self.DIRECT_LINK_PATTERN, self.html) - if not found: self.parseError('Download Link') - url = found.group(1) - self.logDebug('URL: ' + url) - self.download(url) - -getInfo = create_getInfo(TurbouploadCom)
\ No newline at end of file diff --git a/module/plugins/hoster/TusfilesNet.py b/module/plugins/hoster/TusfilesNet.py deleted file mode 100644 index 517df8561..000000000 --- a/module/plugins/hoster/TusfilesNet.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class TusfilesNet(XFileSharingPro): - __name__ = "TusfilesNet" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?tusfiles\.net/\w{12}" - __version__ = "0.01" - __description__ = """Tusfiles.net hoster plugin""" - __author_name__ = ("stickell") - __author_mail__ = ("l.stickell@yahoo.it") - - FILE_INFO_PATTERN = r'<li>(?P<N>[^<]+)</li>\s+<li><b>Size:</b> <small>(?P<S>[\d.]+) (?P<U>\w+)</small></li>' - FILE_OFFLINE_PATTERN = r'The file you were looking for could not be found' - - HOSTER_NAME = "tusfiles.net" - -getInfo = create_getInfo(TusfilesNet) diff --git a/module/plugins/hoster/TwoSharedCom.py b/module/plugins/hoster/TwoSharedCom.py deleted file mode 100644 index 8401e0cb0..000000000 --- a/module/plugins/hoster/TwoSharedCom.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -import re - -class TwoSharedCom(SimpleHoster): - __name__ = "TwoSharedCom" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?2shared.com/(account/)?(download|get|file|document|photo|video|audio)/.*" - __version__ = "0.11" - __description__ = """2Shared Download Hoster""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<h1>(?P<N>.*)</h1>' - FILE_SIZE_PATTERN = r'<span class="dtitle">File size:</span>\s*(?P<S>[0-9,.]+) (?P<U>[kKMG])i?B' - FILE_OFFLINE_PATTERN = r'The file link that you requested is not valid\.|This file was deleted\.' - DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" - - def setup(self): - self.resumeDownload = self.multiDL = True - - def handleFree(self): - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError('Download link') - link = found.group(1) - self.logDebug("Download URL %s" % link) - - self.download(link) - -getInfo = create_getInfo(TwoSharedCom) -
\ No newline at end of file diff --git a/module/plugins/hoster/UlozTo.py b/module/plugins/hoster/UlozTo.py deleted file mode 100644 index 5c38fdaad..000000000 --- a/module/plugins/hoster/UlozTo.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -def convertDecimalPrefix(m): - # decimal prefixes used in filesize and traffic - return ("%%.%df" % {'k':3,'M':6,'G':9}[m.group(2)] % float(m.group(1))).replace('.','') - -class UlozTo(SimpleHoster): - __name__ = "UlozTo" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj.cz|zachowajto.pl)/(?:live/)?(?P<id>\w+/[^/?]*)" - __version__ = "0.92" - __description__ = """uloz.to""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<a href="#download" class="jsShowDownload">(?P<N>[^<]+)</a>' - FILE_SIZE_PATTERN = r'<span id="fileSize">.*?(?P<S>[0-9.]+\s[kMG]?B)</span>' - FILE_INFO_PATTERN = r'<p>File <strong>(?P<N>[^<]+)</strong> is password protected</p>' - FILE_OFFLINE_PATTERN = r'<title>404 - Page not found</title>|<h1 class="h1">File (has been deleted|was banned)</h1>' - FILE_SIZE_REPLACEMENTS = [('([0-9.]+)\s([kMG])B', convertDecimalPrefix)] - FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "www.ulozto.net")] - - PASSWD_PATTERN = r'<div class="passwordProtectedFile">' - VIPLINK_PATTERN = r'<a href="[^"]*\?disclaimer=1" class="linkVip">' - FREE_URL_PATTERN = r'<div class="freeDownloadForm"><form action="([^"]+)"' - PREMIUM_URL_PATTERN = r'<div class="downloadForm"><form action="([^"]+)"' - - def setup(self): - self.multiDL = self.premium - self.resumeDownload = True - - def process(self, pyfile): - pyfile.url = re.sub(r"(?<=http://)([^/]+)", "www.ulozto.net", pyfile.url) - self.html = self.load(pyfile.url, decode = True, cookies = True) - - passwords = self.getPassword().splitlines() - while self.PASSWD_PATTERN in self.html: - if passwords: - password = passwords.pop(0) - self.logInfo("Password protected link, trying " + password) - self.html = self.load(pyfile.url, get = {"do": "passwordProtectedForm-submit"}, - post={"password": password, "password_send": 'Send'}, cookies=True) - else: - self.fail("No or incorrect password") - - if re.search(self.VIPLINK_PATTERN, self.html): - self.html = self.load(pyfile.url, get={"disclaimer": "1"}) - - self.file_info = self.getFileInfo() - - if self.premium and self.checkTrafficLeft(): - self.handlePremium() - else: - self.handleFree() - - self.doCheckDownload() - - def handleFree(self): - action, inputs = self.parseHtmlForm('id="frm-downloadDialog-freeDownloadForm"') - if not action or not inputs: - self.parseError("free download form") - - # get and decrypt captcha - captcha_id_field = captcha_text_field = None - - for key in inputs.keys(): - found = re.match("captcha.*(id|text|value)", key) - if found: - if found.group(1) == "id": - captcha_id_field = key - else: - captcha_text_field = key - - if not captcha_id_field or not captcha_text_field: - self.parseError("CAPTCHA form changed") - - """ - captcha_id = self.getStorage("captcha_id") - captcha_text = self.getStorage("captcha_text") - - if not captcha_id or not captcha_text: - """ - captcha_id = inputs[captcha_id_field] - captcha_text = self.decryptCaptcha("http://img.uloz.to/captcha/%s.png" % captcha_id) - - self.log.debug(' CAPTCHA ID:' + captcha_id + ' CAPTCHA TEXT:' + captcha_text) - - """ - self.setStorage("captcha_id", captcha_id) - self.setStorage("captcha_text", captcha_text) - """ - self.multiDL = True - - inputs.update({captcha_id_field: captcha_id, captcha_text_field: captcha_text}) - - self.download("http://www.ulozto.net" + action, post=inputs, cookies=True, disposition=True) - - def handlePremium(self): - self.download(self.pyfile.url + "?do=directDownload", disposition=True) - #parsed_url = self.findDownloadURL(premium=True) - #self.download(parsed_url, post={"download": "Download"}) - - def findDownloadURL(self, premium=False): - msg = "%s link" % ("Premium" if premium else "Free") - found = re.search(self.PREMIUM_URL_PATTERN if premium else self.FREE_URL_PATTERN, self.html) - if not found: self.parseError(msg) - parsed_url = "http://www.ulozto.net" + found.group(1) - self.logDebug("%s: %s" % (msg, parsed_url)) - return parsed_url - - def doCheckDownload(self): - check = self.checkDownload({ - "wrong_captcha": re.compile(r'<ul class="error">\s*<li>Error rewriting the text.</li>'), - "offline": re.compile(self.FILE_OFFLINE_PATTERN), - "passwd": self.PASSWD_PATTERN, - "server_error": 'src="http://img.ulozto.cz/error403/vykricnik.jpg"', #paralell dl, server overload etc. - "not_found": "<title>UloÅŸ.to</title>" - }) - - if check == "wrong_captcha": - self.delStorage("captcha_id") - self.delStorage("captcha_text") - self.invalidCaptcha() - self.retry(reason="Wrong captcha code") - elif check == "offline": - self.offline() - elif check == "passwd": - self.fail("Wrong password") - elif check == "server_error": - self.logError("Server error, try downloading later") - self.multiDL = False - self.setWait(3600, True) - self.wait() - self.retry() - elif check == "not_found": - self.fail("Server error - file not downloadable") - -getInfo = create_getInfo(UlozTo) diff --git a/module/plugins/hoster/UloziskoSk.py b/module/plugins/hoster/UloziskoSk.py deleted file mode 100644 index c607e7a5b..000000000 --- a/module/plugins/hoster/UloziskoSk.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError - -class UloziskoSk(SimpleHoster): - __name__ = "UloziskoSk" - __type__ = "hoster" - __pattern__ = r"http://(\w*\.)?ulozisko.sk/.*" - __version__ = "0.23" - __description__ = """Ulozisko.sk""" - __author_name__ = ("zoidberg") - - URL_PATTERN = r'<form name = "formular" action = "([^"]+)" method = "post">' - ID_PATTERN = r'<input type = "hidden" name = "id" value = "([^"]+)" />' - FILE_NAME_PATTERN = r'<div class="down1">(?P<N>[^<]+)</div>' - FILE_SIZE_PATTERN = ur'VeÄŸkosÅ¥ súboru: <strong>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</strong><br />' - CAPTCHA_PATTERN = r'<img src="(/obrazky/obrazky.php\?fid=[^"]+)" alt="" />' - FILE_OFFLINE_PATTERN = ur'<span class = "red">ZadanÃœ súbor neexistuje z jedného z nasledujúcich dÃŽvodov:</span>' - IMG_PATTERN = ur'<strong>PRE ZVÃÄÅ ENIE KLIKNITE NA OBRÃZOK</strong><br /><a href = "([^"]+)">' - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - self.getFileInfo() - - found = re.search(self.IMG_PATTERN, self.html) - if found: - url = "http://ulozisko.sk" + found.group(1) - self.download(url) - else: - self.handleFree() - - def handleFree(self): - found = re.search(self.URL_PATTERN, self.html) - if found is None: raise PluginParseError('URL') - parsed_url = 'http://www.ulozisko.sk' + found.group(1) - - found = re.search(self.ID_PATTERN, self.html) - if found is None: raise PluginParseError('ID') - id = found.group(1) - - self.logDebug('URL:' + parsed_url + ' ID:' + id) - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if found is None: raise PluginParseError('CAPTCHA') - captcha_url = 'http://www.ulozisko.sk' + found.group(1) - - captcha = self.decryptCaptcha(captcha_url, cookies=True) - - self.logDebug('CAPTCHA_URL:' + captcha_url + ' CAPTCHA:' + captcha) - - self.download(parsed_url, post={ - "antispam": captcha, - "id": id, - "name": self.pyfile.name, - "but": "++++STIAHNI+S%DABOR++++" - }) - -getInfo = create_getInfo(UloziskoSk) diff --git a/module/plugins/hoster/UnibytesCom.py b/module/plugins/hoster/UnibytesCom.py deleted file mode 100644 index 3c8552271..000000000 --- a/module/plugins/hoster/UnibytesCom.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from pycurl import FOLLOWLOCATION -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class UnibytesCom(SimpleHoster): - __name__ = "UnibytesCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?unibytes\.com/[a-zA-Z0-9-._ ]{11}B" - __version__ = "0.1" - __description__ = """UniBytes.com""" - __author_name__ = ("zoidberg") - - FILE_INFO_PATTERN = r'<span[^>]*?id="fileName"[^>]*>(?P<N>[^>]+)</span>\s*\((?P<S>\d.*?)\)' - DOMAIN = 'http://www.unibytes.com' - - WAIT_PATTERN = r'Wait for <span id="slowRest">(\d+)</span> sec' - DOWNLOAD_LINK_PATTERN = r'<a href="([^"]+)">Download</a>' - - def handleFree(self): - action, post_data = self.parseHtmlForm('id="startForm"') - self.req.http.c.setopt(FOLLOWLOCATION, 0) - - for i in range(8): - self.logDebug(action, post_data) - self.html = self.load(self.DOMAIN + action, post = post_data) - - found = re.search(r'location:\s*(\S+)', self.req.http.header, re.I) - if found: - url = found.group(1) - break - - if '>Somebody else is already downloading using your IP-address<' in self.html: - self.setWait(600, True) - self.wait() - self.retry() - - if post_data['step'] == 'last': - found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) - if found: - url = found.group(1) - self.correctCaptcha() - break - else: - self.invalidCaptcha() - - last_step = post_data['step'] - action, post_data = self.parseHtmlForm('id="stepForm"') - - if last_step == 'timer': - found = re.search(self.WAIT_PATTERN, self.html) - self.setWait(int(found.group(1)) if found else 60, False) - self.wait() - elif last_step in ('captcha', 'last'): - post_data['captcha'] = self.decryptCaptcha(self.DOMAIN + '/captcha.jpg') - else: - self.fail("No valid captcha code entered") - - self.logDebug('Download link: ' + url) - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.download(url) - -getInfo = create_getInfo(UnibytesCom)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py deleted file mode 100644 index d24682e4d..000000000 --- a/module/plugins/hoster/UploadStationCom.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*-
-from module.plugins.hoster.FileserveCom import FileserveCom, checkFile
-from module.plugins.Plugin import chunks
-
-class UploadStationCom(FileserveCom):
- __name__ = "UploadStationCom"
- __type__ = "hoster"
- __pattern__ = r"http://(?:www\.)?uploadstation\.com/file/(?P<id>[A-Za-z0-9]+)"
- __version__ = "0.51"
- __description__ = """UploadStation.Com File Download Hoster"""
- __author_name__ = ("fragonib", "zoidberg")
- __author_mail__ = ("fragonib[AT]yahoo[DOT]es", "zoidberg@mujmail.cz")
-
- URLS = ['http://www.uploadstation.com/file/', 'http://www.uploadstation.com/check-links.php', 'http://www.uploadstation.com/checkReCaptcha.php']
- LINKCHECK_TR = r'<div class="details (?:white|grey)">(.*?)\t{9}</div>'
- LINKCHECK_TD = r'<div class="(?:col )?col\d">(?:<[^>]*>| )*([^<]*)'
-
- LONG_WAIT_PATTERN = r'<h1>You have to wait (\d+) (\w+) to download the next file\.</h1>'
-
-def getInfo(urls):
- for chunk in chunks(urls, 100): yield checkFile(UploadStationCom, chunk)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py deleted file mode 100644 index 5aa08891e..000000000 --- a/module/plugins/hoster/UploadedTo.py +++ /dev/null @@ -1,238 +0,0 @@ -# -*- coding: utf-8 -*- - -# Test links (random.bin): -# http://ul.to/044yug9o -# http://ul.to/gzfhd0xs - -import re -from time import sleep - -from module.utils import html_unescape, parseFileSize, chunks - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL -from module.plugins.ReCaptcha import ReCaptcha - -key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') - - -def getID(url): - """ returns id from file url""" - m = re.match(UploadedTo.__pattern__, url) - return m.group('ID') - - -def getAPIData(urls): - post = {"apikey": key} - - idMap = {} - - for i, url in enumerate(urls): - id = getID(url) - post["id_%s" % i] = id - idMap[id] = url - - for i in xrange(5): - api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1') - if api != "can't find request": - break - else: - sleep(3) - - result = {} - - if api: - for line in api.splitlines(): - data = line.split(",", 4) - if data[1] in idMap: - result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]]) - - return result - - -def parseFileInfo(self, url='', html=''): - if not html and hasattr(self, "html"): html = self.html - name, size, status, found, fileid = url, 0, 3, None, None - - if re.search(self.FILE_OFFLINE_PATTERN, html): - # File offline - status = 1 - else: - found = re.search(self.FILE_INFO_PATTERN, html) - if found: - name, fileid = html_unescape(found.group('N')), found.group('ID') - size = parseFileSize(found.group('S')) - status = 2 - - return name, size, status, fileid - - -def getInfo(urls): - for chunk in chunks(urls, 80): - result = [] - - api = getAPIData(chunk) - - for data in api.itervalues(): - if data[0] == "online": - result.append((html_unescape(data[2]), data[1], 2, data[4])) - - elif data[0] == "offline": - result.append((data[4], 0, 1, data[4])) - - yield result - - -class UploadedTo(Hoster): - __name__ = "UploadedTo" - __type__ = "hoster" - __pattern__ = r"https?://[\w\.-]*?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P<ID>\w+)" - __version__ = "0.70" - __description__ = """Uploaded.net Download Hoster""" - __author_name__ = ("spoob", "mkaay", "zoidberg", "netpok", "stickell") - __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz", "netpok@gmail.com", "l.stickell@yahoo.it") - - FILE_INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> \s*<small[^>]*>(?P<S>[^<]+)</small>' - FILE_OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>' - DL_LIMIT_PATTERN = "You have reached the max. number of possible free downloads for this hour" - - def setup(self): - self.html = None - self.multiDL = False - self.resumeDownload = False - self.url = False - self.chunkLimit = 1 # critical problems with more chunks - if self.account: - self.premium = self.account.getAccountInfo(self.user)["premium"] - if self.premium: - self.multiDL = True - self.resumeDownload = True - - self.fileID = getID(self.pyfile.url) - self.pyfile.url = "http://uploaded.net/file/%s" % self.fileID - - def process(self, pyfile): - self.load("http://uploaded.net/language/en", just_header=True) - - api = getAPIData([pyfile.url]) - - # TODO: fallback to parse from site, because api sometimes delivers wrong status codes - - if not api: - self.logWarning("No response for API call") - - self.html = unicode(self.load(pyfile.url, decode=False), 'iso-8859-1') - name, size, status, self.fileID = parseFileInfo(self) - self.logDebug(name, size, status, self.fileID) - if status == 1: - self.offline() - elif status == 2: - pyfile.name, pyfile.size = name, size - else: - self.fail('Parse error - file info') - elif api == 'Access denied': - self.fail(_("API key invalid")) - - else: - if self.fileID not in api: - self.offline() - - self.data = api[self.fileID] - if self.data[0] != "online": - self.offline() - - pyfile.name = html_unescape(self.data[2]) - - # self.pyfile.name = self.get_file_name() - - if self.premium: - self.handlePremium() - else: - self.handleFree() - - - def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) - self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name": self.__name__, - "left": info["trafficleft"] / 1024 / 1024}) - if int(self.data[1]) / 1024 > info["trafficleft"]: - self.log.info(_("%s: Not enough traffic left" % self.__name__)) - self.account.empty(self.user) - self.resetAccount() - self.fail(_("Traffic exceeded")) - - header = self.load("http://uploaded.net/file/%s" % self.fileID, just_header=True) - if "location" in header: - #Direct download - print "Direct Download: " + header['location'] - self.download(header['location']) - else: - #Indirect download - self.html = self.load("http://uploaded.net/file/%s" % self.fileID) - found = re.search(r'<div class="tfree".*\s*<form method="post" action="(.*?)"', self.html) - if not found: - self.fail("Download URL not found. Try to enable direct downloads.") - url = found.group(1) - print "Premium URL: " + url - self.download(url, post={}) - - def handleFree(self): - self.html = self.load(self.pyfile.url, decode=True) - - if 'var free_enabled = false;' in self.html: - self.logError("Free-download capacities exhausted.") - self.retry(24, 300) - - found = re.search(r"Current waiting period: <span>(\d+)</span> seconds", self.html) - if not found: - self.fail("File not downloadable for free users") - self.setWait(int(found.group(1))) - - js = self.load("http://uploaded.net/js/download.js", decode=True) - - challengeId = re.search(r'Recaptcha\.create\("([^"]+)', js) - - url = "http://uploaded.net/io/ticket/captcha/%s" % self.fileID - downloadURL = "" - - for i in range(5): - #self.req.lastURL = str(self.url) - re_captcha = ReCaptcha(self) - challenge, result = re_captcha.challenge(challengeId.group(1)) - options = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": result} - self.wait() - - result = self.load(url, post=options) - self.logDebug("result: %s" % result) - - if "limit-size" in result: - self.fail("File too big for free download") - elif "limit-slot" in result: # Temporary restriction so just wait a bit - self.setWait(30 * 60, True) - self.wait() - self.retry() - elif "limit-parallel" in result: - self.fail("Cannot download in parallel") - elif self.DL_LIMIT_PATTERN in result: # limit-dl - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() - elif 'err:"captcha"' in result: - self.logError("ul.net captcha is disabled") - self.invalidCaptcha() - elif "type:'download'" in result: - self.correctCaptcha() - downloadURL = re.search("url:'([^']+)", result).group(1) - break - else: - self.fail("Unknown error '%s'") - - if not downloadURL: - self.fail("No Download url retrieved/all captcha attempts failed") - - self.download(downloadURL, disposition=True) - check = self.checkDownload({"limit-dl": self.DL_LIMIT_PATTERN}) - if check == "limit-dl": - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() diff --git a/module/plugins/hoster/UploadheroCom.py b/module/plugins/hoster/UploadheroCom.py deleted file mode 100644 index 502f849af..000000000 --- a/module/plugins/hoster/UploadheroCom.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo - -class UploadheroCom(SimpleHoster): - __name__ = "UploadheroCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?uploadhero\.com?/dl/\w+" - __version__ = "0.13" - __description__ = """UploadHero.com plugin""" - __author_name__ = ("mcmyst", "zoidberg") - __author_mail__ = ("mcmyst@hotmail.fr", "zoidberg@mujmail.cz") - - SH_COOKIES = [("http://uploadhero.com", "lang", "en")] - FILE_NAME_PATTERN = r'<div class="nom_de_fichier">(?P<N>.*?)</div>' - FILE_SIZE_PATTERN = r'Taille du fichier : </span><strong>(?P<S>.*?)</strong>' - FILE_OFFLINE_PATTERN = r'<p class="titre_dl_2">|<div class="raison"><strong>Le lien du fichier ci-dessus n\'existe plus.' - - DOWNLOAD_URL_PATTERN = r'<a href="([^"]+)" id="downloadnow"' - - IP_BLOCKED_PATTERN = r'href="(/lightbox_block_download.php\?min=.*?)"' - IP_WAIT_PATTERN = r'<span id="minutes">(\d+)</span>.*\s*<span id="seconds">(\d+)</span>' - - CAPTCHA_PATTERN = r'"(/captchadl\.php\?[a-z0-9]+)"' - FREE_URL_PATTERN = r'var magicomfg = \'<a href="(http://[^<>"]*?)"|"(http://storage\d+\.uploadhero\.com/\?d=[A-Za-z0-9]+/[^<>"/]+)"' - - def handleFree(self): - self.checkErrors() - - found = re.search(self.CAPTCHA_PATTERN, self.html) - if not found: self.parseError("Captcha URL") - captcha_url = "http://uploadhero.com" + found.group(1) - - for i in range(5): - captcha = self.decryptCaptcha(captcha_url) - self.html = self.load(self.pyfile.url, get = {"code": captcha}) - found = re.search(self.FREE_URL_PATTERN, self.html) - if found: - self.correctCaptcha() - download_url = found.group(1) or found.group(2) - break - else: - self.invalidCaptcha() - else: - self.fail("No valid captcha code entered") - - self.download(download_url) - - def handlePremium(self): - self.log.debug("%s: Use Premium Account" % self.__name__) - self.html = self.load(self.pyfile.url) - link = re.search(self.DOWNLOAD_URL_PATTERN, self.html).group(1) - self.log.debug("Downloading link : '%s'" % link) - self.download(link) - - def checkErrors(self): - found = re.search(self.IP_BLOCKED_PATTERN, self.html) - if found: - self.html = self.load("http://uploadhero.com%s" % found.group(1)) - - found = re.search(self.IP_WAIT_PATTERN, self.html) - wait_time = (int(found.group(1)) * 60 + int(found.group(2))) if found else 300 - self.setWait(wait_time, True) - self.wait() - self.retry() - -getInfo = create_getInfo(UploadheroCom) diff --git a/module/plugins/hoster/UploadingCom.py b/module/plugins/hoster/UploadingCom.py deleted file mode 100644 index 4a157a787..000000000 --- a/module/plugins/hoster/UploadingCom.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: jeix
-"""
-
-import re
-from pycurl import HTTPHEADER
-from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp
-from module.common.json_layer import json_loads
-
-class UploadingCom(SimpleHoster):
- __name__ = "UploadingCom"
- __type__ = "hoster"
- __pattern__ = r"http://(?:www\.)?uploading\.com/files/(?:get/)?(?P<ID>[\w\d]+)"
- __version__ = "0.32"
- __description__ = """Uploading.Com File Download Hoster"""
- __author_name__ = ("jeix", "mkaay", "zoidberg")
- __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "zoidberg@mujmail.cz")
-
- FILE_NAME_PATTERN = r'<title>Download (?P<N>.*?) for free on uploading.com</title>'
- FILE_SIZE_PATTERN = r'<span>File size: (?P<S>.*?)</span>'
- FILE_OFFLINE_PATTERN = r'<h2.*?>The requested file is not found</h2>'
-
- def process(self, pyfile):
- # set lang to english
- self.req.cj.setCookie("uploading.com", "lang", "1")
- self.req.cj.setCookie("uploading.com", "language", "1")
- self.req.cj.setCookie("uploading.com", "setlang", "en")
- self.req.cj.setCookie("uploading.com", "_lang", "en")
-
- if not "/get/" in self.pyfile.url:
- self.pyfile.url = self.pyfile.url.replace("/files", "/files/get")
-
- self.html = self.load(pyfile.url, decode = True)
- self.file_info = self.getFileInfo()
-
- if self.premium:
- self.handlePremium()
- else:
- self.handleFree()
-
- def handlePremium(self):
- postData = {'action': 'get_link',
- 'code': self.file_info['ID'],
- 'pass': 'undefined'}
-
- self.html = self.load('http://uploading.com/files/get/?JsHttpRequest=%d-xml' % timestamp(), post=postData)
- url = re.search(r'"link"\s*:\s*"(.*?)"', self.html)
- if url:
- url = url.group(1).replace("\\/", "/")
- self.download(url)
-
- raise Exception("Plugin defect.")
-
- def handleFree(self):
- found = re.search('<h2>((Daily )?Download Limit)</h2>', self.html)
- if found:
- self.pyfile.error = found.group(1)
- self.logWarning(self.pyfile.error)
- self.retry(max_tries=6, wait_time = 21600 if found.group(2) else 900, reason = self.pyfile.error)
-
- ajax_url = "http://uploading.com/files/get/?ajax"
- self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"])
- self.req.http.lastURL = self.pyfile.url
-
- response = json_loads(self.load(ajax_url, post = {'action': 'second_page', 'code': self.file_info['ID']}))
- if 'answer' in response and 'wait_time' in response['answer']:
- wait_time = int(response['answer']['wait_time'])
- self.log.info("%s: Waiting %d seconds." % (self.__name__, wait_time))
- self.setWait(wait_time)
- self.wait()
- else:
- self.pluginParseError("AJAX/WAIT")
-
- response = json_loads(self.load(ajax_url, post = {'action': 'get_link', 'code': self.file_info['ID'], 'pass': 'false'}))
- if 'answer' in response and 'link' in response['answer']:
- url = response['answer']['link']
- else:
- self.pluginParseError("AJAX/URL")
-
- self.html = self.load(url)
- found = re.search(r'<form id="file_form" action="(.*?)"', self.html)
- if found:
- url = found.group(1)
- else:
- self.pluginParseError("URL")
-
- self.download(url)
-
- check = self.checkDownload({"html" : re.compile("\A<!DOCTYPE html PUBLIC")})
- if check == "html":
- self.logWarning("Redirected to a HTML page, wait 10 minutes and retry")
- self.setWait(600, True)
- self.wait()
-
-getInfo = create_getInfo(UploadingCom)
\ No newline at end of file diff --git a/module/plugins/hoster/UptoboxCom.py b/module/plugins/hoster/UptoboxCom.py deleted file mode 100644 index 60a93c1e5..000000000 --- a/module/plugins/hoster/UptoboxCom.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from module.plugins.hoster.XFileSharingPro import XFileSharingPro, create_getInfo - -class UptoboxCom(XFileSharingPro): - __name__ = "UptoboxCom" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*?uptobox.com/\w{12}" - __version__ = "0.06" - __description__ = """Uptobox.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'<h2>\s*Download File\s*<span[^>]*>(?P<N>[^>]+)</span></h2>\s*[^\(]*\((?P<S>[^\)]+)\)</h2>' - FILE_OFFLINE_PATTERN = r'<center>File Not Found</center>' - HOSTER_NAME = "uptobox.com" - - def setup(self): - self.resumeDownload = self.multiDL = self.premium - self.chunkLimit = 1 - -getInfo = create_getInfo(UptoboxCom)
\ No newline at end of file diff --git a/module/plugins/hoster/VeehdCom.py b/module/plugins/hoster/VeehdCom.py deleted file mode 100644 index 23048b831..000000000 --- a/module/plugins/hoster/VeehdCom.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import re -from module.plugins.Hoster import Hoster - -class VeehdCom(Hoster): - __name__ = 'VeehdCom' - __type__ = 'hoster' - __pattern__ = r'http://veehd\.com/video/\d+_\S+' - __config__ = [ - ('filename_spaces', 'bool', "Allow spaces in filename", 'False'), - ('replacement_char', 'str', "Filename replacement character", '_'), - ] - __version__ = '0.2' - __description__ = """Veehd.com Download Hoster""" - __author_name__ = ('cat') - __author_mail__ = ('cat@pyload') - - def _debug(self, msg): - self.log.debug('[%s] %s' % (self.__name__, msg)) - - def setup(self): - self.html = None - self.multiDL = True - self.req.canContinue = True - - def process(self, pyfile): - self.download_html() - if not self.file_exists(): - self.offline() - - pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self._debug("Requesting page: %s" % (repr(url),)) - self.html = self.load(url) - - def file_exists(self): - if self.html is None: - self.download_html() - - if '<title>Veehd</title>' in self.html: - return False - return True - - def get_file_name(self): - if self.html is None: - self.download_html() - - match = re.search(r'<title[^>]*>([^<]+) on Veehd</title>', self.html) - if not match: - self.fail("video title not found") - name = match.group(1) - - # replace unwanted characters in filename - if self.getConf('filename_spaces'): - pattern = '[^0-9A-Za-z\.\ ]+' - else: - pattern = '[^0-9A-Za-z\.]+' - - name = re.sub(pattern, self.getConf('replacement_char'), - name) - return name + '.avi' - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - match = re.search(r'<embed type="video/divx" ' - r'src="(http://([^/]*\.)?veehd\.com/dl/[^"]+)"', - self.html) - if not match: - self.fail("embedded video url not found") - file_url = match.group(1) - - return file_url diff --git a/module/plugins/hoster/WarserverCz.py b/module/plugins/hoster/WarserverCz.py deleted file mode 100644 index b256f8d1b..000000000 --- a/module/plugins/hoster/WarserverCz.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -#similar to coolshare.cz (down) - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.HTTPRequest import BadHeader -from module.utils import html_unescape - -class WarserverCz(SimpleHoster): - __name__ = "WarserverCz" - __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?warserver.cz/stahnout/(?P<ID>\d+)/.+" - __version__ = "0.12" - __description__ = """Warserver.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<h1.*?>(?P<N>[^<]+)</h1>' - FILE_SIZE_PATTERN = r'<li>Velikost: <strong>(?P<S>[^<]+)</strong>' - FILE_OFFLINE_PATTERN = r'<h1>Soubor nenalezen</h1>' - - PREMIUM_URL_PATTERN = r'href="(http://[^/]+/dwn-premium.php.*?)"' - DOMAIN = "http://csd01.coolshare.cz" - - DOMAIN = "http://s01.warserver.cz" - - def handleFree(self): - try: - self.download("%s/dwn-free.php?fid=%s" % (self.DOMAIN, self.file_info['ID'])) - except BadHeader, e: - self.logError(e) - if e.code == 403: - self.longWait(60,60) - else: raise - self.checkDownloadedFile() - - def handlePremium(self): - found = re.search(self.PREMIUM_URL_PATTERN, self.html) - if not found: self.parseError("Premium URL") - url = html_unescape(found.group(1)) - self.logDebug("Premium URL: " + url) - if not url.startswith("http://"): self.resetAccount() - self.download(url) - self.checkDownloadedFile() - - def checkDownloadedFile(self): - check = self.checkDownload({ - "offline": ">404 Not Found<" - }) - - if check == "offline": - self.offline() - -getInfo = create_getInfo(WarserverCz)
\ No newline at end of file diff --git a/module/plugins/hoster/WebshareCz.py b/module/plugins/hoster/WebshareCz.py deleted file mode 100644 index 195e65a93..000000000 --- a/module/plugins/hoster/WebshareCz.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.HTTPRequest import BadHeader - -class WebshareCz(SimpleHoster): - __name__ = "WebshareCz" - __type__ = "hoster" - __pattern__ = r"http://(\w+\.)?webshare.cz/(stahnout/)?(?P<ID>\w{10})-.+" - __version__ = "0.12" - __description__ = """WebShare.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<h3>Stahujete soubor: </h3>\s*<div class="textbox">(?P<N>[^<]+)</div>' - FILE_SIZE_PATTERN = r'<h3>Velikost souboru je: </h3>\s*<div class="textbox">(?P<S>[^<]+)</div>' - FILE_OFFLINE_PATTERN = r'<h3>Soubor ".*?" nebyl nalezen.</h3>' - - DOWNLOAD_LINK_PATTERN = r'id="download_link" href="(?P<url>.*?)"' - - def setup(self): - self.multiDL = True - - def handleFree(self): - url_a = re.search(r"(var l.*)", self.html).group(1) - url_b = re.search(r"(var keyStr.*)", self.html).group(1) - url = self.js.eval("%s\n%s\ndec(l)" % (url_a, url_b)) - - self.logDebug('Download link: ' + url) - self.download(url) - -getInfo = create_getInfo(WebshareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/WrzucTo.py b/module/plugins/hoster/WrzucTo.py deleted file mode 100644 index 4a5e89f22..000000000 --- a/module/plugins/hoster/WrzucTo.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from pycurl import HTTPHEADER - -class WrzucTo(SimpleHoster): - __name__ = "WrzucTo" - __type__ = "hoster" - __pattern__ = r"http://(?:\w+\.)*?wrzuc\.to/([a-zA-Z0-9]+(\.wt|\.html)|(\w+/?linki/[a-zA-Z0-9]+))" - __version__ = "0.01" - __description__ = """Wrzuc.to plugin - free only""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - SH_COOKIES = [("http://www.wrzuc.to", "language", "en")] - FILE_SIZE_PATTERN = r'class="info">\s*<tr>\s*<td>(?P<S>.*?)</td>' - FILE_NAME_PATTERN = r'id="file_info">\s*<strong>(?P<N>.*?)</strong>' - - def setup(self): - self.multiDL = True - - def handleFree(self): - data = dict(re.findall(r'(md5|file): "(.*?)"', self.html)) - if len(data) != 2: self.parseError('File ID') - - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.req.http.lastURL = self.pyfile.url - self.load("http://www.wrzuc.to/ajax/server/prepair", post = {"md5": data['md5']}) - - self.req.http.lastURL = self.pyfile.url - self.html = self.load("http://www.wrzuc.to/ajax/server/download_link", post = {"file": data['file']}) - - data.update(re.findall(r'"(download_link|server_id)":"(.*?)"', self.html)) - if len(data) != 4: self.parseError('Download URL') - - download_url = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) - self.logDebug("Download URL: %s" % download_url) - self.download(download_url) - -getInfo = create_getInfo(WrzucTo) - diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py deleted file mode 100644 index 1a0eb442b..000000000 --- a/module/plugins/hoster/WuploadCom.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import string -from urllib import unquote - -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha -from module.plugins.Plugin import chunks - -from module.network.RequestFactory import getURL -from module.common.json_layer import json_loads - - -def getInfo(urls): - for chunk in chunks(urls, 20): - result = [] - ids = dict() - for url in chunk: - id = getId(url) - if id: - ids[id] = url - else: - result.append((None, 0, 1, url)) - - if len(ids) > 0: - check_url = "http://api.wupload.com/link?method=getInfo&format=json&ids=" + ",".join(ids.keys()) - response = json_loads(getURL(check_url).decode("utf8", "ignore")) - for item in response["FSApi_Link"]["getInfo"]["response"]["links"]: - if item["status"] != "AVAILABLE": - result.append((None, 0, 1, ids[str(item["id"])])) - else: - result.append((unquote(item["filename"]), item["size"], 2, ids[str(item["id"])])) - yield result - - -def getId(url): - match = re.search(WuploadCom.FILE_ID_PATTERN, url) - if match: - return string.replace(match.group("id"), "/", "-") - else: - return None - - -class WuploadCom(Hoster): - __name__ = "WuploadCom" - __type__ = "hoster" - __pattern__ = r"http://[\w\.]*?wupload\..*?/file/(([a-z][0-9]+/)?[0-9]+)(/.*)?" - __version__ = "0.20" - __description__ = """Wupload com""" - __author_name__ = ("jeix", "paulking") - __author_mail__ = ("jeix@hasnomail.de", "") - - API_ADDRESS = "http://api.wupload.com" - URL_DOMAIN_PATTERN = r'(?P<prefix>.*?)(?P<domain>.wupload\..+?)(?P<suffix>/.*)' - FILE_ID_PATTERN = r'/file/(?P<id>([a-z][0-9]+/)?[0-9]+)(/.*)?' - FILE_LINK_PATTERN = r'<p><a href="(http://.+?\.wupload\..+?)"><span>Download Now' - WAIT_TIME_PATTERN = r'countDownDelay = (?P<wait>\d+)' - WAIT_TM_PATTERN = r"name='tm' value='(.*?)' />" - WAIT_TM_HASH_PATTERN = r"name='tm_hash' value='(.*?)' />" - CAPTCHA_TYPE1_PATTERN = r'Recaptcha.create\("(.*?)",' - CAPTCHA_TYPE2_PATTERN = r'id="recaptcha_image"><img style="display: block;" src="(.+)image?c=(.+?)"' - - def init(self): - if self.account: - self.premium = self.account.getAccountInfo(self.user)["premium"] - if not self.premium: - self.chunkLimit = 1 - self.multiDL = False - - def process(self, pyfile): - self.pyfile = pyfile - - self.pyfile.url = self.checkFile(self.pyfile.url) - - if self.premium: - self.downloadPremium() - else: - self.downloadFree() - - def checkFile(self, url): - id = getId(url) - self.logDebug("file id is %s" % id) - if id: - # Use the api to check the current status of the file and fixup data - check_url = self.API_ADDRESS + "/link?method=getInfo&format=json&ids=%s" % id - result = json_loads(self.load(check_url, decode=True)) - item = result["FSApi_Link"]["getInfo"]["response"]["links"][0] - self.logDebug("api check returns %s" % item) - - if item["status"] != "AVAILABLE": - self.offline() - if item["is_password_protected"] != 0: - self.fail("This file is password protected") - - # ignored this check due to false api information - #if item["is_premium_only"] != 0 and not self.premium: - # self.fail("need premium account for file") - - self.pyfile.name = unquote(item["filename"]) - - # Fix the url and resolve the domain to the correct regional variation - url = item["url"] - urlparts = re.search(self.URL_DOMAIN_PATTERN, url) - if urlparts: - url = urlparts.group("prefix") + self.getDomain() + urlparts.group("suffix") - self.logDebug("localised url is %s" % url) - return url - else: - self.fail("Invalid URL") - - def getDomain(self): - result = json_loads( - self.load(self.API_ADDRESS + "/utility?method=getWuploadDomainForCurrentIp&format=json", decode=True)) - self.log.debug("%s: response to get domain %s" % (self.__name__, result)) - return result["FSApi_Utility"]["getWuploadDomainForCurrentIp"]["response"] - - def downloadPremium(self): - self.logDebug("Premium download") - - api = self.API_ADDRESS + "/link?method=getDownloadLink&u=%%s&p=%%s&ids=%s" % getId(self.pyfile.url) - - result = json_loads(self.load(api % (self.user, self.account.getAccountData(self.user)["password"]))) - links = result["FSApi_Link"]["getDownloadLink"]["response"]["links"] - - #wupload seems to return list and no dicts - if type(links) == dict: - info = links.values()[0] - else: - info = links[0] - - if "status" in info and info["status"] == "NOT_AVAILABLE": - self.tempOffline() - - self.download(info["url"]) - - def downloadFree(self): - self.logDebug("Free download") - # Get initial page - self.html = self.load(self.pyfile.url) - url = self.pyfile.url + "?start=1" - self.html = self.load(url) - self.handleErrors() - - finalUrl = re.search(self.FILE_LINK_PATTERN, self.html) - - if not finalUrl: - self.doWait(url) - - chall = re.search(self.CAPTCHA_TYPE1_PATTERN, self.html) - chall2 = re.search(self.CAPTCHA_TYPE2_PATTERN, self.html) - if chall or chall2: - for i in range(5): - re_captcha = ReCaptcha(self) - if chall: - self.logDebug("Captcha type1") - challenge, result = re_captcha.challenge(chall.group(1)) - else: - self.logDebug("Captcha type2") - server = chall2.group(1) - challenge = chall2.group(2) - result = re_captcha.result(server, challenge) - - postData = {"recaptcha_challenge_field": challenge, - "recaptcha_response_field": result} - - self.html = self.load(url, post=postData) - self.handleErrors() - chall = re.search(self.CAPTCHA_TYPE1_PATTERN, self.html) - chall2 = re.search(self.CAPTCHA_TYPE2_PATTERN, self.html) - - if chall or chall2: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - - finalUrl = re.search(self.FILE_LINK_PATTERN, self.html) - - if not finalUrl: - self.fail("Couldn't find free download link") - - self.logDebug("got download url %s" % finalUrl.group(1)) - self.download(finalUrl.group(1)) - - def doWait(self, url): - # If the current page requires us to wait then wait and move to the next page as required - - # There maybe more than one wait period. The extended wait if download limits have been exceeded (in which case we try reconnect) - # and the short wait before every download. Visually these are the same, the difference is that one includes a code to allow - # progress to the next page - - waitSearch = re.search(self.WAIT_TIME_PATTERN, self.html) - while waitSearch: - wait = int(waitSearch.group("wait")) - if wait > 300: - self.wantReconnect = True - - self.setWait(wait) - self.logDebug("Waiting %d seconds." % wait) - self.wait() - - tm = re.search(self.WAIT_TM_PATTERN, self.html) - tm_hash = re.search(self.WAIT_TM_HASH_PATTERN, self.html) - - if tm and tm_hash: - tm = tm.group(1) - tm_hash = tm_hash.group(1) - self.html = self.load(url, post={"tm": tm, "tm_hash": tm_hash}) - self.handleErrors() - break - else: - self.html = self.load(url) - self.handleErrors() - waitSearch = re.search(self.WAIT_TIME_PATTERN, self.html) - - def handleErrors(self): - if "This file is available for premium users only." in self.html: - self.fail("need premium account for file") - - if "The file that you're trying to download is larger than" in self.html: - self.fail("need premium account for file") - - if "Free users may only download 1 file at a time" in self.html: - self.fail("only 1 file at a time for free users") - - if "Free user can not download files" in self.html: - self.fail("need premium account for file") - - if "Download session in progress" in self.html: - self.fail("already downloading") - - if "This file is password protected" in self.html: - self.fail("This file is password protected") - - if "An Error Occurred" in self.html: - self.fail("A server error occured.") - - if "This file was deleted" in self.html: - self.offline() diff --git a/module/plugins/hoster/X7To.py b/module/plugins/hoster/X7To.py deleted file mode 100644 index 79adf2a3f..000000000 --- a/module/plugins/hoster/X7To.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*-
-import re
-
-from module.plugins.Hoster import Hoster
-
-from module.network.RequestFactory import getURL
-
-def getInfo(urls):
- yield [(url, 0, 1, url) for url in urls]
-
-
-class X7To(Hoster):
- __name__ = "X7To"
- __type__ = "hoster"
- __pattern__ = r"http://(?:www.)?x7.to/"
- __version__ = "0.3"
- __description__ = """X7.To File Download Hoster"""
- __author_name__ = ("ernieb")
- __author_mail__ = ("ernieb")
-
- FILE_INFO_PATTERN=r'<meta name="description" content="Download: (.*?) \(([0-9,.]+) (KB|MB|GB)\)'
-
- def init(self):
- if self.premium:
- self.multiDL = False
- self.resumeDownload = False
- self.chunkLimit = 1
- else:
- self.multiDL = False
-
- self.file_id = re.search(r"http://x7.to/([a-zA-Z0-9]+)", self.pyfile.url).group(1)
- self.logDebug("file id is %s" % self.file_id)
- self.pyfile.url = "http://x7.to/" + self.file_id
-
- def process(self, pyfile):
- self.fail("Hoster not longer available")
-
- def handlePremium(self):
- # check if over limit first
- overLimit = re.search(r'<a onClick="cUser.buyTraffic\(\)" id="DL">', self.html)
- if overLimit:
- self.logDebug("over limit, falling back to free")
- self.handleFree()
- else:
- realurl = re.search(r'<a href="(http://stor.*?)" id="DL">', self.html)
- if realurl:
- realurl = realurl.group(1)
- self.logDebug("premium url found %s" % realurl)
- else:
- self.logDebug("premium link not found")
- self.download(realurl)
-
- def handleFree(self):
- # find file id
- file_id = re.search(r"var dlID = '(.*?)'", self.html)
- if not file_id:
- self.fail("Free download id not found")
-
- file_url = "http://x7.to/james/ticket/dl/" + file_id.group(1)
- self.logDebug("download id %s" % file_id.group(1))
-
- self.html = self.load(file_url, ref=False, decode=True)
-
- # deal with errors
- if "limit-dl" in self.html:
- self.logDebug("Limit reached ... waiting")
- self.setWait(900,True)
- self.wait()
- self.retry()
-
- if "limit-parallel" in self.html:
- self.fail("Cannot download in parallel")
-
- # no waiting required, go to download
- waitCheck = re.search(r"wait:(\d*),", self.html)
- if waitCheck:
- waitCheck = int(waitCheck.group(1))
- self.setWait(waitCheck)
- self.wait()
-
- urlCheck = re.search(r"url:'(.*?)'", self.html)
- url = None
- if urlCheck:
- url = urlCheck.group(1)
- self.logDebug("free url found %s" % url)
-
- if url:
- try:
- self.download(url)
- except:
- self.logDebug("downloading url failed: %s" % url)
- else:
- self.fail("Free download url found")
diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py deleted file mode 100644 index 1120a2a8b..000000000 --- a/module/plugins/hoster/XFileSharingPro.py +++ /dev/null @@ -1,314 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from random import random -from urllib import unquote -from urlparse import urlparse -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError -from module.plugins.ReCaptcha import ReCaptcha -from module.plugins.internal.CaptchaService import SolveMedia, AdsCaptcha -from module.utils import html_unescape - -class XFileSharingPro(SimpleHoster): - """ - Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom... - Some hosters may work straight away when added to __pattern__ - However, most of them will NOT work because they are either down or running a customized version - """ - __name__ = "XFileSharingPro" - __type__ = "hoster" - __pattern__ = r"^unmatchable$" - __version__ = "0.17" - __description__ = """XFileSharingPro common hoster base""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>[^"]+)"' - FILE_SIZE_PATTERN = r'You have requested <font color="red">[^<]+</font> \((?P<S>[^<]+)\)</font>' - FILE_INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>' - FILE_OFFLINE_PATTERN = r'<(b|h[1-6])>File Not Found</(b|h[1-6])>' - - WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>' - LONG_WAIT_PATTERN = r'(?P<H>\d+(?=\s*hour))?.*?(?P<M>\d+(?=\s*minute))?.*?(?P<S>\d+(?=\s*second))?' - OVR_DOWNLOAD_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' - OVR_KILL_LINK_PATTERN = r'<h2>Delete Link</h2>\s*<textarea[^>]*>([^<]+)' - CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' - RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"' - CAPTCHA_DIV_PATTERN = r'<b>Enter code.*?<div.*?>(.*?)</div>' - SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"' - ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)</' - - def setup(self): - if self.__name__ == "XFileSharingPro": - self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - self.multiDL = True - else: - self.resumeDownload = self.multiDL = self.premium - - self.chunkLimit = 1 - - def process(self, pyfile): - self.prepare() - - if not re.match(self.__pattern__, self.pyfile.url): - if self.premium: - self.handleOverriden() - else: - self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME) - else: - try: - self.html = self.load(pyfile.url, cookies = False, decode = True) - self.file_info = self.getFileInfo() - except PluginParseError: - self.file_info = None - - self.location = self.getDirectDownloadLink() - - if not self.file_info: - pyfile.name = html_unescape(unquote(urlparse(self.location if self.location else pyfile.url).path.split("/")[-1])) - - if self.location: - self.startDownload(self.location) - elif self.premium: - self.handlePremium() - else: - self.handleFree() - - def prepare(self): - """ Initialize important variables """ - if not hasattr(self, "HOSTER_NAME"): - self.HOSTER_NAME = re.search(self.__pattern__, self.pyfile.url).group(1) - if not hasattr(self, "DIRECT_LINK_PATTERN"): - self.DIRECT_LINK_PATTERN = r'(http://([^/]*?%s|\d+\.\d+\.\d+\.\d+)(:\d+/d/|/files/\d+/\w+/)[^"\'<]+)' % self.HOSTER_NAME - - self.captcha = self.errmsg = None - self.passwords = self.getPassword().splitlines() - - def getDirectDownloadLink(self): - """ Get download link for premium users with direct download enabled """ - self.req.http.lastURL = self.pyfile.url - - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.html = self.load(self.pyfile.url, cookies = True, decode = True) - self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) - - location = None - found = re.search("Location\s*:\s*(.*)", self.header, re.I) - if found and re.match(self.DIRECT_LINK_PATTERN, found.group(1)): - location = found.group(1).strip() - - return location - - def handleFree(self): - url = self.getDownloadLink() - self.logDebug("Download URL: %s" % url) - self.startDownload(url) - - def getDownloadLink(self): - for i in range(5): - self.logDebug("Getting download link: #%d" % i) - data = self.getPostParameters() - - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.html = self.load(self.pyfile.url, post = data, ref = True, decode = True) - self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) - - found = re.search("Location\s*:\s*(.*)", self.header, re.I) - if found: - break - - found = re.search(self.DIRECT_LINK_PATTERN, self.html, re.S) - if found: - break - - else: - if self.errmsg and 'captcha' in self.errmsg: - self.fail("No valid captcha code entered") - else: - self.fail("Download link not found") - - return found.group(1) - - def handlePremium(self): - self.html = self.load(self.pyfile.url, post = self.getPostParameters()) - found = re.search(self.DIRECT_LINK_PATTERN, self.html) - if not found: self.parseError('DIRECT LINK') - self.startDownload(found.group(1)) - - def handleOverriden(self): - #only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_NAME) - action, inputs = self.parseHtmlForm('') - upload_id = "%012d" % int(random()*10**12) - action += upload_id + "&js_on=1&utype=prem&upload_type=url" - inputs['tos'] = '1' - inputs['url_mass'] = self.pyfile.url - inputs['up1oad_type'] = 'url' - - self.logDebug(self.HOSTER_NAME, action, inputs) - #wait for file to upload to easybytez.com - self.req.http.c.setopt(LOW_SPEED_TIME, 600) - self.html = self.load(action, post = inputs) - - action, inputs = self.parseHtmlForm('F1') - if not inputs: self.parseError('TEXTAREA') - self.logDebug(self.HOSTER_NAME, inputs) - if inputs['st'] == 'OK': - self.html = self.load(action, post = inputs) - elif inputs['st'] == 'Can not leech file': - self.retry(max_tries=20, wait_time=180, reason=inputs['st']) - else: - self.fail(inputs['st']) - - #get easybytez.com link for uploaded file - found = re.search(self.OVR_DOWNLOAD_LINK_PATTERN, self.html) - if not found: self.parseError('DIRECT LINK (OVR)') - self.pyfile.url = found.group(1) - self.retry() - - def startDownload(self, link): - link = link.strip() - if self.captcha: self.correctCaptcha() - self.logDebug('DIRECT LINK: %s' % link) - self.download(link) - - def checkErrors(self): - found = re.search(self.ERROR_PATTERN, self.html) - if found: - self.errmsg = found.group(1) - self.logWarning(re.sub(r"<.*?>"," ",self.errmsg)) - - if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in re.findall('(\d+)\s*(hour|minute|second)?', self.errmsg)]) - self.setWait(wait_time, True) - self.wait() - elif 'captcha' in self.errmsg: - self.invalidCaptcha() - elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail("File can be downloaded by premium users only") - elif 'limit' in self.errmsg: - self.setWait(3600, True) - self.wait() - self.retry(25) - elif 'countdown' in self.errmsg or 'Expired session' in self.errmsg: - self.retry(3) - elif 'maintenance' in self.errmsg: - self.tempOffline() - elif 'download files up to' in self.errmsg: - self.fail("File too large for free download") - else: - self.fail(self.errmsg) - - else: - self.errmsg = None - - return self.errmsg - - def getPostParameters(self): - for i in range(3): - if not self.errmsg: self.checkErrors() - - if hasattr(self,"FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) - - if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry() - else: - self.parseError("Form not found") - - self.logDebug(self.HOSTER_NAME, inputs) - - if 'op' in inputs and inputs['op'] in ('download2', 'download3'): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail("No or invalid passport") - - if not self.premium: - found = re.search(self.WAIT_PATTERN, self.html) - if found: - wait_time = int(found.group(1)) + 1 - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: self.wait() - - self.errmsg = None - return inputs - - else: - inputs['referer'] = self.pyfile.url - - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: del inputs['method_free'] - else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: del inputs['method_premium'] - - self.html = self.load(self.pyfile.url, post = inputs, ref = True) - self.errmsg = None - - else: self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) - - def handleCaptcha(self, inputs): - found = re.search(self.RECAPTCHA_URL_PATTERN, self.html) - if found: - recaptcha_key = unquote(found.group(1)) - self.logDebug("RECAPTCHA KEY: %s" % recaptcha_key) - recaptcha = ReCaptcha(self) - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key) - return 1 - else: - found = re.search(self.CAPTCHA_URL_PATTERN, self.html) - if found: - captcha_url = found.group(1) - inputs['code'] = self.decryptCaptcha(captcha_url) - return 2 - else: - found = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) - if found: - captcha_div = found.group(1) - self.logDebug(captcha_div) - numerals = re.findall('<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key = lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - else: - found = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if found: - captcha_key = found.group(1) - captcha = SolveMedia(self) - inputs['adcopy_challenge'], inputs['adcopy_response'] = captcha.challenge(captcha_key) - return 4 - return 0 - -getInfo = create_getInfo(XFileSharingPro) diff --git a/module/plugins/hoster/XHamsterCom.py b/module/plugins/hoster/XHamsterCom.py deleted file mode 100644 index 0779a78e6..000000000 --- a/module/plugins/hoster/XHamsterCom.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster -from urllib import unquote -from module.common.json_layer import json_loads - -def clean_json(json_expr): - json_expr = re.sub('[\n\r]', '', json_expr) - json_expr = re.sub(' +', '', json_expr) - json_expr = re.sub('\'','"', json_expr) - - return json_expr - -class XHamsterCom(Hoster): - __name__ = "XHamsterCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?xhamster\.com/movies/.+" - __version__ = "0.1" - __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] - __description__ = """XHamster.com Video Download Hoster""" - - def setup(self): - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - - if not self.file_exists(): - self.offline() - - if self.getConfig("type"): - self.desired_fmt = self.getConf("type") - - self.pyfile.name = self.get_file_name() + self.desired_fmt - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - flashvar_pattern = re.compile('flashvars = ({.*?});', re.DOTALL) - json_flashvar=flashvar_pattern.search(self.html) - - if json_flashvar is None: - self.fail("Parse error (flashvars)") - - j = clean_json(json_flashvar.group(1)) - flashvars = json_loads(j) - - if flashvars["srv"]: - srv_url = flashvars["srv"] + '/' - else: - self.fail("Parse error (srv_url)") - - if flashvars["url_mode"]: - url_mode = flashvars["url_mode"] - else: - self.fail("Parse error (url_mode)") - - - if self.desired_fmt == ".mp4": - file_url = re.search(r"<a href=\"" + srv_url + "(.+?)\"", self.html) - if file_url is None: - self.fail("Parse error (file_url)") - file_url=file_url.group(1) - long_url = srv_url + file_url - self.logDebug(_("long_url: %s") % long_url) - else: - if flashvars["file"]: - file_url = unquote(flashvars["file"]) - else: - self.fail("Parse error (file_url)") - - if url_mode=='3': - long_url = file_url - self.logDebug(_("long_url: %s") % long_url) - else: - long_url = srv_url + "key=" + file_url - self.logDebug(_("long_url: %s") % long_url) - - return long_url - - - def get_file_name(self): - if self.html is None: - self.download_html() - - file_name_pattern = r"<title>(.*?) - xHamster\.com</title>" - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - file_name_pattern = r"<h1 >(.*)</h1>" - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - file_name_pattern = r"http://[www.]+xhamster\.com/movies/.*/(.*?)\.html?" - file_name = re.search(file_name_pattern, self.pyfile.url) - if file_name is None: - file_name_pattern = r"<div id=\"element_str_id\" style=\"display:none;\">(.*)</div>" - file_name = re.search(file_name_pattern, self.html) - if file_name is None: - return "Unknown" - - return file_name.group(1) - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - if re.search(r"(.*Video not found.*)", self.html) is not None: - return False - else: - return True diff --git a/module/plugins/hoster/XVideosCom.py b/module/plugins/hoster/XVideosCom.py deleted file mode 100644 index b7f3f7b58..000000000 --- a/module/plugins/hoster/XVideosCom.py +++ /dev/null @@ -1,19 +0,0 @@ - -import re -import urllib - -from module.plugins.Hoster import Hoster - -class XVideosCom(Hoster): - __name__ = "XVideos.com" - __version__ = "0.1" - __pattern__ = r"http://www\.xvideos\.com/video([0-9]+)/.*" - __config__ = [] - - def process(self, pyfile): - site = self.load(pyfile.url) - pyfile.name = "%s (%s).flv" %( - re.search(r"<h2>([^<]+)<span", site).group(1), - re.search(self.__pattern__, pyfile.url).group(1), - ) - self.download(urllib.unquote(re.search(r"flv_url=([^&]+)&", site).group(1))) diff --git a/module/plugins/hoster/Xdcc.py b/module/plugins/hoster/Xdcc.py deleted file mode 100644 index 6f0a1b176..000000000 --- a/module/plugins/hoster/Xdcc.py +++ /dev/null @@ -1,229 +0,0 @@ -# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: jeix
-"""
-
-from os.path import join
-from os.path import exists
-from os import makedirs
-import re
-import sys
-import time
-import socket, struct
-from select import select
-from module.utils import save_join
-
-from module.plugins.Hoster import Hoster
-
-
-class Xdcc(Hoster):
- __name__ = "Xdcc"
- __version__ = "0.3"
- __pattern__ = r'xdcc://.*?(/#?.*?)?/.*?/#?\d+/?' # xdcc://irc.Abjects.net/#channel/[XDCC]|Shit/#0004/
- __type__ = "hoster"
- __config__ = [
- ("nick", "str", "Nickname", "pyload"),
- ("ident", "str", "Ident", "pyloadident"),
- ("realname", "str", "Realname", "pyloadreal")
- ]
- __description__ = """A Plugin that allows you to download from an IRC XDCC bot"""
- __author_name__ = ("jeix")
- __author_mail__ = ("jeix@hasnomail.com")
-
- def setup(self):
- self.debug = 0 #0,1,2
- self.timeout = 30
- self.multiDL = False
-
-
-
- def process(self, pyfile):
- # change request type
- self.req = pyfile.m.core.requestFactory.getRequest(self.__name__, type="XDCC")
-
- self.pyfile = pyfile
- for i in range(0,3):
- try:
- nmn = self.doDownload(pyfile.url)
- self.log.debug("%s: Download of %s finished." % (self.__name__, nmn))
- return
- except socket.error, e:
- if hasattr(e, "errno"):
- errno = e.errno
- else:
- errno = e.args[0]
-
- if errno in (10054,):
- self.log.debug("XDCC: Server blocked our ip, retry in 5 min")
- self.setWait(300)
- self.wait()
- continue
-
- self.fail("Failed due to socket errors. Code: %d" % errno)
-
- self.fail("Server blocked our ip, retry again later manually")
-
-
- def doDownload(self, url):
- self.pyfile.setStatus("waiting") # real link
-
- download_folder = self.config['general']['download_folder']
- location = join(download_folder, self.pyfile.package().folder.decode(sys.getfilesystemencoding()))
- if not exists(location):
- makedirs(location)
-
- m = re.search(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url)
- server = m.group(1)
- chan = m.group(2)
- bot = m.group(3)
- pack = m.group(4)
- nick = self.getConf('nick')
- ident = self.getConf('ident')
- real = self.getConf('realname')
-
- temp = server.split(':')
- ln = len(temp)
- if ln == 2:
- host, port = temp
- elif ln == 1:
- host, port = temp[0], 6667
- else:
- self.fail("Invalid hostname for IRC Server (%s)" % server)
-
-
- #######################
- # CONNECT TO IRC AND IDLE FOR REAL LINK
- dl_time = time.time()
-
- sock = socket.socket()
- sock.connect((host, int(port)))
- if nick == "pyload":
- nick = "pyload-%d" % (time.time() % 1000) # last 3 digits
- sock.send("NICK %s\r\n" % nick)
- sock.send("USER %s %s bla :%s\r\n" % (ident, host, real))
- time.sleep(3)
- sock.send("JOIN #%s\r\n" % chan)
- sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))
-
- # IRC recv loop
- readbuffer = ""
- done = False
- retry = None
- m = None
- while True:
-
- # done is set if we got our real link
- if done:
- break
-
- if retry:
- if time.time() > retry:
- retry = None
- dl_time = time.time()
- sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))
-
- else:
- if (dl_time + self.timeout) < time.time(): # todo: add in config
- sock.send("QUIT :byebye\r\n")
- sock.close()
- self.fail("XDCC Bot did not answer")
-
-
- fdset = select([sock], [], [], 0)
- if sock not in fdset[0]:
- continue
-
- readbuffer += sock.recv(1024)
- temp = readbuffer.split("\n")
- readbuffer = temp.pop()
-
- for line in temp:
- if self.debug is 2: print "*> " + unicode(line, errors='ignore')
- line = line.rstrip()
- first = line.split()
-
- if first[0] == "PING":
- sock.send("PONG %s\r\n" % first[1])
-
- if first[0] == "ERROR":
- self.fail("IRC-Error: %s" % line)
-
- msg = line.split(None, 3)
- if len(msg) != 4:
- continue
-
- msg = { \
- "origin":msg[0][1:], \
- "action":msg[1], \
- "target":msg[2], \
- "text" :msg[3][1:] \
- }
-
-
- if nick == msg["target"][0:len(nick)] and "PRIVMSG" == msg["action"]:
- if msg["text"] == "\x01VERSION\x01":
- self.log.debug("XDCC: Sending CTCP VERSION.")
- sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface"))
- elif msg["text"] == "\x01TIME\x01":
- self.log.debug("Sending CTCP TIME.")
- sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time()))
- elif msg["text"] == "\x01LAG\x01":
- pass # don't know how to answer
-
- if not (bot == msg["origin"][0:len(bot)]
- and nick == msg["target"][0:len(nick)]
- and msg["action"] in ("PRIVMSG", "NOTICE")):
- continue
-
- if self.debug is 1:
- print "%s: %s" % (msg["origin"], msg["text"])
-
- if "You already requested that pack" in msg["text"]:
- retry = time.time() + 300
-
- if "you must be on a known channel to request a pack" in msg["text"]:
- self.fail("Wrong channel")
-
- m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg["text"])
- if m:
- done = True
-
- # get connection data
- ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2)))))
- port = int(m.group(3))
- packname = m.group(1)
-
- if len(m.groups()) > 3:
- self.req.filesize = int(m.group(4))
-
- self.pyfile.name = packname
- filename = save_join(location, packname)
- self.log.info("XDCC: Downloading %s from %s:%d" % (packname, ip, port))
-
- self.pyfile.setStatus("downloading")
- newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress)
- if newname and newname != filename:
- self.log.info("%(name)s saved as %(newname)s" % {"name": self.pyfile.name, "newname": newname})
- filename = newname
-
- # kill IRC socket
- # sock.send("QUIT :byebye\r\n")
- sock.close()
-
- self.lastDownload = filename
- return self.lastDownload
-
diff --git a/module/plugins/hoster/XvidstageCom.py b/module/plugins/hoster/XvidstageCom.py deleted file mode 100644 index 4962c05af..000000000 --- a/module/plugins/hoster/XvidstageCom.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: 4Christopher -""" - -import re -import HTMLParser - -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - - -def setup(self): - self.wantReconnect = False - self.resumeDownload = True - self.multiDL = True - - -def getInfo(urls): - result = [] - - for url in urls: - result.append(parseFileInfo(url, getInfoMode=True)) - yield result - - -def parseFileInfo(url, getInfoMode=False): - html = getURL(url) - info = {"name": url, "size": 0, "status": 3} - try: - info['name'] = re.search(r'(?:Filename|Dateiname):</b></td><td nowrap[^>]*?>(.*?)<', html).group(1) - info['size'] = re.search(r'(?:Size|GröÃe):</b></td><td>.*? <small>\((\d+?) bytes\)', html).group(1) - except: ## The file is offline - info['status'] = 1 - else: - info['status'] = 2 - - if getInfoMode: - return info['name'], info['size'], info['status'], url - else: - return info['name'], info['size'], info['status'], html - - -class XvidstageCom(Hoster): - __name__ = 'XvidstageCom' - __version__ = '0.4' - __pattern__ = r'http://(?:www.)?xvidstage.com/(?P<id>[0-9A-Za-z]+)' - __type__ = 'hoster' - __description__ = """A Plugin that allows you to download files from http://xvidstage.com""" - __author_name__ = ('4Christopher') - __author_mail__ = ('4Christopher@gmx.de') - - - def process(self, pyfile): - pyfile.name, pyfile.size, pyfile.status, self.html = parseFileInfo(pyfile.url) - self.logDebug('Name: %s' % pyfile.name) - if pyfile.status == 1: ## offline - self.offline() - self.id = re.search(self.__pattern__, pyfile.url).group('id') - - wait_sec = int(re.search(r'countdown_str">.+?>(\d+?)<', self.html).group(1)) - self.setWait(wait_sec, reconnect=False) - self.logDebug('Waiting %d seconds before submitting the captcha' % wait_sec) - self.wait() - - rand = re.search(r'<input type="hidden" name="rand" value="(.*?)">', self.html).group(1) - self.logDebug('rand: %s, id: %s' % (rand, self.id)) - self.html = self.req.load(pyfile.url, - post={'op': 'download2', 'id': self.id, 'rand': rand, 'code': self.get_captcha()}) - file_url = re.search(r'<a href="(?P<url>.*?)">(?P=url)</a>', self.html).group('url') - try: - hours_file_available = int( - re.search(r'This direct link will be available for your IP next (?P<hours>\d+?) hours', - self.html).group('hours')) - self.logDebug( - 'You have %d hours to download this file with your current IP address.' % hours_file_available) - except: - self.logDebug('Failed') - self.logDebug('Download file: %s' % file_url) - self.download(file_url) - check = self.checkDownload({'empty': re.compile(r'^$')}) - - if check == 'empty': - self.logInfo('Downloaded File was empty') - # self.retry() - - def get_captcha(self): - ## <span style='position:absolute;padding-left:7px;padding-top:6px;'>1 ⊠- cap_chars = {} - for pad_left, char in re.findall(r"position:absolute;padding-left:(\d+?)px;.*?;'>(.*?)<", self.html): - cap_chars[int(pad_left)] = char - - h = HTMLParser.HTMLParser() - ## Sorting after padding-left - captcha = '' - for pad_left in sorted(cap_chars): - captcha += h.unescape(cap_chars[pad_left]) - - self.logDebug('The captcha is: %s' % captcha) - return captcha diff --git a/module/plugins/hoster/YibaishiwuCom.py b/module/plugins/hoster/YibaishiwuCom.py deleted file mode 100644 index 901225944..000000000 --- a/module/plugins/hoster/YibaishiwuCom.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.common.json_layer import json_loads - -class YibaishiwuCom(SimpleHoster): - __name__ = "YibaishiwuCom" - __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?(?:u\.)?115.com/file/(?P<ID>\w+)" - __version__ = "0.12" - __description__ = """115.com""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r"file_name: '(?P<N>[^']+)'" - FILE_SIZE_PATTERN = r"file_size: '(?P<S>[^']+)'" - FILE_OFFLINE_PATTERN = ur'<h3><i style="color:red;">ååïŒæåç äžååšïŒäžåŠšææçå§ïŒ</i></h3>' - - AJAX_URL_PATTERN = r'(/\?ct=(pickcode|download)[^"\']+)' - - def handleFree(self): - found = re.search(self.AJAX_URL_PATTERN, self.html) - if not found: self.parseError("AJAX URL") - url = found.group(1) - self.logDebug(('FREEUSER' if found.group(2) == 'download' else 'GUEST') + ' URL', url) - - response = json_loads(self.load("http://115.com" + url, decode = False)) - for mirror in (response['urls'] if 'urls' in response else response['data'] if 'data' in response else []): - try: - url = mirror['url'].replace('\\','') - self.logDebug("Trying URL: " + url) - self.download(url) - break - except: - continue - else: self.fail('No working link found') - -getInfo = create_getInfo(YibaishiwuCom) diff --git a/module/plugins/hoster/YoupornCom.py b/module/plugins/hoster/YoupornCom.py deleted file mode 100644 index b17a4ef80..000000000 --- a/module/plugins/hoster/YoupornCom.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from module.plugins.Hoster import Hoster - -class YoupornCom(Hoster): - __name__ = "YoupornCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?youporn\.com/watch/.+" - __version__ = "0.2" - __description__ = """Youporn.com Video Download Hoster""" - __author_name__ = ("willnix") - __author_mail__ = ("willnix@pyload.org") - - def setup(self): - self.html = None - - def process(self, pyfile): - self.pyfile = pyfile - - if not self.file_exists(): - self.offline() - - self.pyfile.name = self.get_file_name() - self.download(self.get_file_url()) - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url, post={"user_choice":"Enter"}, cookies=False) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - if self.html is None: - self.download_html() - - file_url = re.search(r'(http://download\.youporn\.com/download/\d+\?save=1)">', self.html).group(1) - return file_url - - def get_file_name(self): - if self.html is None: - self.download_html() - - file_name_pattern = r"<title>(.*) - Free Porn Videos - YouPorn</title>" - return re.search(file_name_pattern, self.html).group(1).replace("&", "&").replace("/","") + '.flv' - - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - if re.search(r"(.*invalid video_id.*)", self.html) is not None: - return False - else: - return True diff --git a/module/plugins/hoster/YourfilesTo.py b/module/plugins/hoster/YourfilesTo.py deleted file mode 100644 index b67ccb68d..000000000 --- a/module/plugins/hoster/YourfilesTo.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-import urllib
-from module.plugins.Hoster import Hoster
-
-class YourfilesTo(Hoster):
- __name__ = "YourfilesTo"
- __type__ = "hoster"
- __pattern__ = r"(http://)?(www\.)?yourfiles\.(to|biz)/\?d=[a-zA-Z0-9]+"
- __version__ = "0.2"
- __description__ = """Youfiles.to Download Hoster"""
- __author_name__ = ("jeix", "skydancer")
- __author_mail__ = ("jeix@hasnomail.de", "skydancer@hasnomail.de")
-
- def setup(self):
- self.html = None
- self.multiDL = True
-
- def process(self,pyfile):
- self.pyfile = pyfile
- self.prepare()
- self.download(self.get_file_url())
-
- def prepare(self):
- if not self.file_exists():
- self.offline()
-
- self.pyfile.name = self.get_file_name()
-
- wait_time = self.get_waiting_time()
- self.setWait(wait_time)
- self.log.debug("%s: Waiting %d seconds." % (self.__name__,wait_time))
- self.wait()
-
- def get_waiting_time(self):
- if self.html is None:
- self.download_html()
-
- #var zzipitime = 15;
- m = re.search(r'var zzipitime = (\d+);', self.html)
- if m:
- sec = int(m.group(1))
- else:
- sec = 0
-
- return sec
-
- def download_html(self):
- url = self.pyfile.url
- self.html = self.load(url)
-
- def get_file_url(self):
- """ returns the absolute downloadable filepath
- """
- url = re.search(r"var bla = '(.*?)';", self.html)
- if url:
- url = url.group(1)
- url = urllib.unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", ""))
- return url
- else:
- self.fail("absolute filepath could not be found. offline? ")
-
- def get_file_name(self):
- if self.html is None:
- self.download_html()
-
- return re.search("<title>(.*)</title>", self.html).group(1)
-
- def file_exists(self):
- """ returns True or False
- """
- if self.html is None:
- self.download_html()
-
- if re.search(r"HTTP Status 404", self.html) is not None:
- return False
- else:
- return True
-
-
-
diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py deleted file mode 100644 index 70db597cf..000000000 --- a/module/plugins/hoster/YoutubeCom.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import subprocess -import os -import os.path -from urllib import unquote - -from module.utils import html_unescape -from module.plugins.Hoster import Hoster - -def which(program): - """Works exactly like the unix command which - - Courtesy of http://stackoverflow.com/a/377028/675646""" - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -class YoutubeCom(Hoster): - __name__ = "YoutubeCom" - __type__ = "hoster" - __pattern__ = r"https?://(?:[^/]*?)youtube\.com/watch.*?[?&]v=.*" - __version__ = "0.32" - __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"), - ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0), - (".mp4", "bool", "Allow .mp4", True), - (".flv", "bool", "Allow .flv", True), - (".webm", "bool", "Allow .webm", False), - (".3gp", "bool", "Allow .3gp", False), - ("3d", "bool", "Prefer 3D", False)] - __description__ = """Youtube.com Video Download Hoster""" - __author_name__ = ("spoob", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - - # name, width, height, quality ranking, 3D - formats = {5: (".flv", 400, 240, 1, False), - 6: (".flv", 640, 400, 4, False), - 17: (".3gp", 176, 144, 0, False), - 18: (".mp4", 480, 360, 2, False), - 22: (".mp4", 1280, 720, 8, False), - 43: (".webm", 640, 360, 3, False), - 34: (".flv", 640, 360, 4, False), - 35: (".flv", 854, 480, 6, False), - 36: (".3gp", 400, 240, 1, False), - 37: (".mp4", 1920, 1080, 9, False), - 38: (".mp4", 4096, 3072, 10, False), - 44: (".webm", 854, 480, 5, False), - 45: (".webm", 1280, 720, 7, False), - 46: (".webm", 1920, 1080, 9, False), - 82: (".mp4", 640, 360, 3, True), - 83: (".mp4", 400, 240, 1, True), - 84: (".mp4", 1280, 720, 8, True), - 85: (".mp4", 1920, 1080, 9, True), - 100: (".webm", 640, 360, 3, True), - 101: (".webm", 640, 360, 4, True), - 102: (".webm", 1280, 720, 8, True) - } - - def setup(self): - self.resumeDownload = self.multiDL = True - - def process(self, pyfile): - html = self.load(pyfile.url, decode=True) - - if '<h1 id="unavailable-message" class="message">' in html: - self.offline() - - if "We have been receiving a large volume of requests from your network." in html: - self.tempOffline() - - #get config - use3d = self.getConf("3d") - if use3d: - quality = {"sd":82,"hd":84,"fullhd":85,"240p":83,"360p":82,"480p":82,"720p":84,"1080p":85,"3072p":85} - else: - quality = {"sd":18,"hd":22,"fullhd":37,"240p":5,"360p":18,"480p":35,"720p":22,"1080p":37,"3072p":38} - desired_fmt = self.getConf("fmt") - if desired_fmt and desired_fmt not in self.formats: - self.logWarning("FMT %d unknown - using default." % desired_fmt) - desired_fmt = 0 - if not desired_fmt: - desired_fmt = quality.get(self.getConf("quality"), 18) - - #parse available streams - streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1) - streams = [x.split('\u0026') for x in streams.split(',')] - streams = [dict((y.split('=',1)) for y in x) for x in streams] - streams = [(int(x['itag']), "%s&signature=%s" % (unquote(x['url']), x['sig'])) for x in streams] - #self.logDebug("Found links: %s" % streams) - self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams]) - - #build dictionary of supported itags (3D/2D) - allowed = lambda x: self.getConfig(self.formats[x][0]) - streams = [x for x in streams if x[0] in self.formats and allowed(x[0])] - if not streams: - self.fail("No available stream meets your preferences") - fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams) - - self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" % - (desired_fmt, - "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt], - "" if desired_fmt in fmt_dict else "NOT ", - "" if allowed(desired_fmt) else "NOT ") - ) - - #return fmt nearest to quality index - if desired_fmt in fmt_dict and allowed(desired_fmt): - fmt = desired_fmt - else: - sel = lambda x: self.formats[x][3] #select quality index - comp = lambda x, y: abs(sel(x) - sel(y)) - - self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()]) - fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and - sel(x) > sel(y) else y, fmt_dict.keys()) - - self.logDebug("Chosen fmt: %s" % fmt) - url = fmt_dict[fmt] - self.logDebug("URL: %s" % url) - - #set file name - file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv" - file_name_pattern = '<meta name="title" content="(.+?)">' - name = re.search(file_name_pattern, html).group(1).replace("/", "") - pyfile.name = html_unescape(name) - - time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url) - ffmpeg = which("ffmpeg") - if ffmpeg and time: - m, s = time.groups()[1:] - if not m: - m = "0" - - pyfile.name += " (starting at %s:%s)" % (m, s) - pyfile.name += file_suffix - - filename = self.download(url) - - if ffmpeg and time: - inputfile = filename + "_" - os.rename(filename, inputfile) - - subprocess.call([ - ffmpeg, - "-ss", "00:%s:%s" % (m, s), - "-i", inputfile, - "-vcodec", "copy", - "-acodec", "copy", - filename]) - os.remove(inputfile) diff --git a/module/plugins/hoster/ZDF.py b/module/plugins/hoster/ZDF.py deleted file mode 100644 index ea45f4fd8..000000000 --- a/module/plugins/hoster/ZDF.py +++ /dev/null @@ -1,46 +0,0 @@ - -import re -from xml.etree.ElementTree import fromstring - -from module.plugins.Hoster import Hoster - -XML_API = "http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?id=%i" - -class ZDF(Hoster): - # Based on zdfm by Roland Beermann - # http://github.com/enkore/zdfm/ - __name__ = "ZDF Mediathek" - __version__ = "0.7" - __pattern__ = r"http://www\.zdf\.de/ZDFmediathek/[^0-9]*([0-9]+)[^0-9]*" - __config__ = [] - - @staticmethod - def video_key(video): - return ( - int(video.findtext("videoBitrate", "0")), - any(f.text == "progressive" for f in video.iter("facet")), - ) - - @staticmethod - def video_valid(video): - return (video.findtext("url").startswith("http") and video.findtext("url").endswith(".mp4")) - - @staticmethod - def get_id(url): - return int(re.search(r"[^0-9]*([0-9]+)[^0-9]*", url).group(1)) - - def process(self, pyfile): - xml = fromstring(self.load(XML_API % self.get_id(pyfile.url))) - - status = xml.findtext("./status/statuscode") - if status != "ok": - self.fail("Error retrieving manifest.") - - video = xml.find("video") - title = video.findtext("information/title") - - pyfile.name = title - - target_url = sorted((v for v in video.iter("formitaet") if self.video_valid(v)), key=self.video_key)[-1].findtext("url") - - self.download(target_url) diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py deleted file mode 100644 index 8be725d2f..000000000 --- a/module/plugins/hoster/ZeveraCom.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape -from urllib import quote, unquote -from time import sleep - -class ZeveraCom(Hoster): - __name__ = "ZeveraCom" - __version__ = "0.20" - __type__ = "hoster" - __pattern__ = r"http://zevera.com/.*" - __description__ = """zevera.com hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def setup(self): - self.resumeDownload = self.multiDL = True - self.chunkLimit = 1 - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your zevera.com account or deactivate this plugin")) - self.fail("No zevera.com account provided") - - self.logDebug("zevera.com: Old URL: %s" % pyfile.url) - - if self.account.getAPIData(self.req, cmd = "checklink", olink = pyfile.url) != "Alive": - self.fail("Offline or not downloadable - contact Zevera support") - - header = self.account.getAPIData(self.req, just_header = True, cmd="generatedownloaddirect", olink = pyfile.url) - if not "location" in header: - self.fail("Unable to initialize download - contact Zevera support") - - self.download(header['location'], disposition = True) - - check = self.checkDownload({"error" : 'action="ErrorDownload.aspx'}) - if check == "error": - self.fail("Error response received - contact Zevera support") - - """ - # BitAPI not used - defunct, probably abandoned by Zevera - - api_url = "http://zevera.com/API.ashx" - - def process(self, pyfile): - if not self.account: - self.logError(_("Please enter your zevera.com account or deactivate this plugin")) - self.fail("No zevera.com account provided") - - self.logDebug("zevera.com: Old URL: %s" % pyfile.url) - - last_size = retries = 0 - olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8')) - - for i in range(100): - self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink) - self.checkAPIErrors(self.retData) - - if self.retData['FileInfo']['StatusID'] == 100: - break - elif self.retData['FileInfo']['StatusID'] == 99: - self.fail('Failed to initialize download (99)') - else: - if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size: - if retries >= 6: - self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] ) - retries += 1 - else: - retries = 0 - - last_size = self.retData['FileInfo']['Progress']['BytesReceived'] - - self.setWait(self.retData['Update_Wait']) - self.wait() - - pyfile.name = self.retData['FileInfo']['RealFileName'] - pyfile.size = self.retData['FileInfo']['FileSizeInBytes'] - - self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID']) - self.checkAPIErrors(self.retData) - - self.download(self.api_url, get = { - 'cmd': "open_stream", - 'login': self.account.loginname, - 'pass': self.account.password, - 'FileID': self.retData['FileInfo']['FileID'], - 'startBytes': 0 - } - ) - - def checkAPIErrors(self, retData): - if not retData: - self.fail('Unknown API response') - - if retData['ErrorCode']: - self.logError(retData['ErrorCode'], retData['ErrorMessage']) - #self.fail('ERROR: ' + retData['ErrorMessage']) - - if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']: - self.logWarning("Not enough data left to download the file") - - def crazyDecode(self, ustring): - # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded - # no idea what the proper order of calling these functions would be :-/ - return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8')) - """
\ No newline at end of file diff --git a/module/plugins/hoster/ZippyshareCom.py b/module/plugins/hoster/ZippyshareCom.py deleted file mode 100644 index 84974e7ba..000000000 --- a/module/plugins/hoster/ZippyshareCom.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re, subprocess, tempfile, os -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp -from module.plugins.ReCaptcha import ReCaptcha -from module.common.json_layer import json_loads - -class ZippyshareCom(SimpleHoster): - __name__ = "ZippyshareCom" - __type__ = "hoster" - __pattern__ = r"(?P<HOST>http://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(?P<KEY>\d+)" - __version__ = "0.37" - __description__ = """Zippyshare.com Download Hoster""" - __author_name__ = ("spoob", "zoidberg") - __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - __config__ = [("swfdump_path", "string", "Path to swfdump", "")] - - FILE_NAME_PATTERN = r'>Name:</font>\s*<font [^>]*>(?P<N>[^<]+)</font><br />' - FILE_SIZE_PATTERN = r'>Size:</font>\s*<font [^>]*>(?P<S>[0-9.,]+) (?P<U>[kKMG]+)i?B</font><br />' - FILE_INFO_PATTERN = r'document\.getElementById\(\'dlbutton\'\)\.href = "[^;]*/(?P<N>[^"]+)";' - FILE_OFFLINE_PATTERN = r'>File does not exist on this server</div>' - - DOWNLOAD_URL_PATTERN = r"<script type=\"text/javascript\">([^<]*?)document\.getElementById\('dlbutton'\).href = ([^;]+);" - SEED_PATTERN = r'swfobject.embedSWF\("([^"]+)".*?seed: (\d+)' - CAPTCHA_KEY_PATTERN = r'Recaptcha.create\("([^"]+)"' - CAPTCHA_SHORTENCODE_PATTERN = r"shortencode: '([^']+)'" - CAPTCHA_DOWNLOAD_PATTERN = r"document.location = '([^']+)'" - - LAST_KNOWN_VALUES = (9, 2374755) #time = (seed * multiply) % modulo - - def setup(self): - self.html = None - self.wantReconnect = False - self.multiDL = True - - def handleFree(self): - url = self.get_file_url() - if not url: self.fail("Download URL not found.") - self.logDebug("Download URL %s" % url) - self.download(url, cookies = True) - - check = self.checkDownload({ - "swf_values": re.compile(self.SEED_PATTERN) - }) - - if check == "swf_values": - swf_sts = self.getStorage("swf_sts") - if not swf_sts: - self.setStorage("swf_sts", 2) - self.setStorage("swf_stamp", 0) - elif swf_sts == '1': - self.setStorage("swf_sts", 2) - - self.retry(max_tries = 1) - - def get_file_url(self): - """ returns the absolute downloadable filepath - """ - url = multiply = modulo = None - - found = re.search(self.DOWNLOAD_URL_PATTERN, self.html, re.S) - if found: - #Method #1: JS eval - url = self.js.eval("\n".join(found.groups())) - else: - #Method #2: SWF eval - seed_search = re.search(self.SEED_PATTERN, self.html) - if seed_search: - swf_url, file_seed = seed_search.groups() - - swf_sts = self.getStorage("swf_sts") - swf_stamp = int(self.getStorage("swf_stamp") or 0) - swf_version = self.getStorage("version") - self.logDebug("SWF", swf_sts, swf_stamp, swf_version) - - if not swf_sts: - self.logDebug('Using default values') - multiply, modulo = self.LAST_KNOWN_VALUES - elif swf_sts == "1": - self.logDebug('Using stored values') - multiply = self.getStorage("multiply") - modulo = self.getStorage("modulo") - elif swf_sts == "2": - if swf_version < self.__version__: - self.logDebug('Reverting to default values') - self.setStorage("swf_sts", "") - self.setStorage("version", self.__version__) - multiply, modulo = self.LAST_KNOWN_VALUES - elif (swf_stamp + 3600000) < timestamp(): - swfdump = self.get_swfdump_path() - if swfdump: - multiply, modulo = self.get_swf_values(self.file_info['HOST'] + swf_url, swfdump) - else: - self.logWarning("Swfdump not found. Install swftools to bypass captcha.") - - if multiply and modulo: - self.logDebug("TIME = (%s * %s) %s" % (file_seed, multiply, modulo)) - url = "/download?key=%s&time=%d" % (self.file_info['KEY'], (int(file_seed) * int(multiply)) % int(modulo)) - - if not url: - #Method #3: Captcha - url = self.do_recaptcha() - - return self.file_info['HOST'] + url - - def get_swf_values(self, swf_url, swfdump): - self.logDebug('Parsing values from %s' % swf_url) - multiply = modulo = None - - fd, fpath = tempfile.mkstemp() - try: - swf_data = self.load(swf_url) - os.write(fd, swf_data) - - p = subprocess.Popen([swfdump, '-a', fpath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if err: - self.logError(err) - else: - m_str = re.search(r'::break.*?{(.*?)}', out, re.S).group(1) - multiply = re.search(r'pushbyte (\d+)', m_str).group(1) - modulo = re.search(r'pushint (\d+)', m_str).group(1) - finally: - os.close(fd) - os.remove(fpath) - - if multiply and modulo: - self.setStorage("multiply", multiply) - self.setStorage("modulo", modulo) - self.setStorage("swf_sts", 1) - self.setStorage("version", self.__version__) - else: - self.logError("Parsing SWF failed: swfdump not installed or plugin out of date") - self.setStorage("swf_sts", 2) - - self.setStorage("swf_stamp", timestamp()) - - return multiply, modulo - - def get_swfdump_path(self): - # used for detecting if swfdump is installed - def is_exe(ppath): - return os.path.isfile(ppath) and os.access(ppath, os.X_OK) - - program = self.getConfig("swfdump_path") or "swfdump" - swfdump = None - ppath, pname = os.path.split(program) - if ppath: - if is_exe(program): - swfdump = program - else: - for ppath in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(ppath, program) - if is_exe(exe_file): - swfdump = exe_file - - # return path to the executable or None if not found - return swfdump - - def do_recaptcha(self): - self.logDebug('Trying to solve captcha') - captcha_key = re.search(self.CAPTCHA_KEY_PATTERN, self.html).group(1) - shortencode = re.search(self.CAPTCHA_SHORTENCODE_PATTERN, self.html).group(1) - url = re.search(self.CAPTCHA_DOWNLOAD_PATTERN, self.html).group(1) - - recaptcha = ReCaptcha(self) - - for i in range(5): - challenge, code = recaptcha.challenge(captcha_key) - - response = json_loads(self.load(self.file_info['HOST'] + '/rest/captcha/test', - post={'challenge': challenge, - 'response': code, - 'shortencode': shortencode})) - self.logDebug("reCaptcha response : %s" % response) - if response == True: - self.correctCaptcha - break - else: - self.invalidCaptcha() - else: self.fail("Invalid captcha") - - return url - -getInfo = create_getInfo(ZippyshareCom)
\ No newline at end of file diff --git a/module/plugins/hoster/__init__.py b/module/plugins/hoster/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/hoster/__init__.py +++ /dev/null diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py deleted file mode 100644 index 3cd635eff..000000000 --- a/module/plugins/internal/AbstractExtractor.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -class ArchiveError(Exception): - pass - -class CRCError(Exception): - pass - -class WrongPassword(Exception): - pass - -class AbtractExtractor: - @staticmethod - def checkDeps(): - """ Check if system satisfies dependencies - :return: boolean - """ - return True - - @staticmethod - def getTargets(files_ids): - """ Filter suited targets from list of filename id tuple list - :param files_ids: List of file paths - :return: List of targets, id tuple list - """ - raise NotImplementedError - - - def __init__(self, m, file, out, fullpath, overwrite, renice): - """Initialize extractor for specific file - - :param m: ExtractArchive addon plugin - :param file: Absolute file path - :param out: Absolute path to destination directory - :param fullpath: Extract to fullpath - :param overwrite: Overwrite existing archives - :param renice: Renice value - """ - self.m = m - self.file = file - self.out = out - self.fullpath = fullpath - self.overwrite = overwrite - self.renice = renice - self.files = [] # Store extracted files here - - - def init(self): - """ Initialize additional data structures """ - pass - - - def checkArchive(self): - """Check if password is needed. Raise ArchiveError if integrity is - questionable. - - :return: boolean - :raises ArchiveError - """ - return False - - def checkPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean - """ - return True - - def extract(self, progress, password=None): - """Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :param password password to use - :raises WrongPassword - :raises CRCError - :raises ArchiveError - :return: - """ - raise NotImplementedError - - def getDeleteFiles(self): - """Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete - """ - raise NotImplementedError - - def getExtractedFiles(self): - """Populate self.files at some point while extracting""" - return self.files
\ No newline at end of file diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py deleted file mode 100644 index b912436a7..000000000 --- a/module/plugins/internal/CaptchaService.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re - -class CaptchaService(): - __version__ = "0.02" - - def __init__(self, plugin): - self.plugin = plugin - -class ReCaptcha(): - def __init__(self, plugin): - self.plugin = plugin - - def challenge(self, id): - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k":id}, cookies=True) - - try: - challenge = re.search("challenge : '(.*?)',", js).group(1) - server = re.search("server : '(.*?)',", js).group(1) - except: - self.plugin.fail("recaptcha error") - result = self.result(server,challenge) - - return challenge, result - - def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage"%server, get={"c":challenge}, cookies=True, forceUser=True, imgtype="jpg") - -class AdsCaptcha(CaptchaService): - def challenge(self, src): - js = self.plugin.req.load(src, cookies=True) - - try: - challenge = re.search("challenge: '(.*?)',", js).group(1) - server = re.search("server: '(.*?)',", js).group(1) - except: - self.plugin.fail("adscaptcha error") - result = self.result(server,challenge) - - return challenge, result - - def result(self, server, challenge): - return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={"cid": challenge, "dummy": random()}, cookies=True, imgtype="jpg") - -class SolveMedia(CaptchaService): - def __init__(self,plugin): - self.plugin = plugin - - def challenge(self, src): - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript?k=%s" % src, cookies=True) - try: - challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="([^"]+)">', html).group(1) - except: - self.plugin.fail("solvmedia error") - result = self.result(challenge) - - return challenge, result - - def result(self,challenge): - return self.plugin.decryptCaptcha("http://api.solvemedia.com/papi/media?c=%s" % challenge,imgtype="gif")
\ No newline at end of file diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py deleted file mode 100644 index e180e2384..000000000 --- a/module/plugins/internal/DeadHoster.py +++ /dev/null @@ -1,18 +0,0 @@ -from module.plugins.Hoster import Hoster as _Hoster - -def create_getInfo(plugin): - def getInfo(urls): - yield [('#N/A: ' + url, 0, 1, url) for url in urls] - return getInfo - -class DeadHoster(_Hoster): - __name__ = "DeadHoster" - __type__ = "hoster" - __pattern__ = r"" - __version__ = "0.11" - __description__ = """Hoster is no longer available""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def setup(self): - self.fail("Hoster is no longer available")
\ No newline at end of file diff --git a/module/plugins/internal/NetloadInOCR.py b/module/plugins/internal/NetloadInOCR.py deleted file mode 100644 index e50978701..000000000 --- a/module/plugins/internal/NetloadInOCR.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -from OCR import OCR - -class NetloadInOCR(OCR): - __version__ = 0.1 - - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.clean(3) - self.clean(3) - self.run_tesser(True, True, False, False) - - self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers - - return self.result_captcha - -if __name__ == '__main__': - import urllib - ocr = NetloadInOCR() - urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") - - print ocr.get_captcha('captcha.png') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py deleted file mode 100644 index 9f8b7ef8c..000000000 --- a/module/plugins/internal/OCR.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### -from __future__ import with_statement -import os -from os.path import join -from os.path import abspath -import logging -import subprocess -#import tempfile - -import Image -import TiffImagePlugin -import PngImagePlugin -import GifImagePlugin -import JpegImagePlugin - - -class OCR(object): - __version__ = 0.1 - - def __init__(self): - self.logger = logging.getLogger("log") - - def load_image(self, image): - self.image = Image.open(image) - self.pixels = self.image.load() - self.result_captcha = '' - - def unload(self): - """delete all tmp images""" - pass - - def threshold(self, value): - self.image = self.image.point(lambda a: a * value + 10) - - def run(self, command): - """Run a command""" - - popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - popen.wait() - output = popen.stdout.read() +" | "+ popen.stderr.read() - popen.stdout.close() - popen.stderr.close() - self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) - - def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): - #self.logger.debug("create tmp tif") - - - #tmp = tempfile.NamedTemporaryFile(suffix=".tif") - tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") - tmp.close() - #self.logger.debug("create tmp txt") - #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") - tmpTxt.close() - - self.logger.debug("save tiff") - self.image.save(tmp.name, 'TIFF') - - if os.name == "nt": - tessparams = [join(pypath,"tesseract","tesseract.exe")] - else: - tessparams = ['tesseract'] - - tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] ) - - if subset and (digits or lowercase or uppercase): - #self.logger.debug("create temp subset config") - #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") - tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") - tmpSub.write("tessedit_char_whitelist ") - if digits: - tmpSub.write("0123456789") - if lowercase: - tmpSub.write("abcdefghijklmnopqrstuvwxyz") - if uppercase: - tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") - tmpSub.write("\n") - tessparams.append("nobatch") - tessparams.append(abspath(tmpSub.name)) - tmpSub.close() - - self.logger.debug("run tesseract") - self.run(tessparams) - self.logger.debug("read txt") - - try: - with open(tmpTxt.name, 'r') as f: - self.result_captcha = f.read().replace("\n", "") - except: - self.result_captcha = "" - - self.logger.debug(self.result_captcha) - try: - os.remove(tmp.name) - os.remove(tmpTxt.name) - if subset and (digits or lowercase or uppercase): - os.remove(tmpSub.name) - except: - pass - - def get_captcha(self, name): - raise NotImplementedError - - def to_greyscale(self): - if self.image.mode != 'L': - self.image = self.image.convert('L') - - self.pixels = self.image.load() - - def eval_black_white(self, limit): - self.pixels = self.image.load() - w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if self.pixels[x, y] > limit: - self.pixels[x, y] = 255 - else: - self.pixels[x, y] = 0 - - def clean(self, allowed): - pixels = self.pixels - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 255: continue - # no point in processing white pixels since we only want to remove black pixel - count = 0 - - try: - if pixels[x-1, y-1] != 255: count += 1 - if pixels[x-1, y] != 255: count += 1 - if pixels[x-1, y + 1] != 255: count += 1 - if pixels[x, y + 1] != 255: count += 1 - if pixels[x + 1, y + 1] != 255: count += 1 - if pixels[x + 1, y] != 255: count += 1 - if pixels[x + 1, y-1] != 255: count += 1 - if pixels[x, y-1] != 255: count += 1 - except: - pass - - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white - if count < allowed: - pixels[x, y] = 1 - - # second pass: this time set all 1's to 255 (white) - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 1: pixels[x, y] = 255 - - self.pixels = pixels - - def derotate_by_average(self): - """rotate by checking each angle and guess most suitable""" - - w, h = self.image.size - pixels = self.pixels - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 155 - - highest = {} - counts = {} - - for angle in range(-45, 45): - - tmpimage = self.image.rotate(angle) - - pixels = tmpimage.load() - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - - count = {} - - for x in xrange(w): - count[x] = 0 - for y in xrange(h): - if pixels[x, y] == 155: - count[x] += 1 - - sum = 0 - cnt = 0 - - for x in count.values(): - if x != 0: - sum += x - cnt += 1 - - avg = sum / cnt - counts[angle] = cnt - highest[angle] = 0 - for x in count.values(): - if x > highest[angle]: - highest[angle] = x - - highest[angle] = highest[angle] - avg - - hkey = 0 - hvalue = 0 - - for key, value in highest.iteritems(): - if value > hvalue: - hkey = key - hvalue = value - - self.image = self.image.rotate(hkey) - pixels = self.image.load() - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - if pixels[x, y] == 155: - pixels[x, y] = 0 - - self.pixels = pixels - - def split_captcha_letters(self): - captcha = self.image - started = False - letters = [] - width, height = captcha.size - bottomY, topY = 0, height - pixels = captcha.load() - - for x in xrange(width): - black_pixel_in_col = False - for y in xrange(height): - if pixels[x, y] != 255: - if not started: - started = True - firstX = x - lastX = x - - if y > bottomY: bottomY = y - if y < topY: topY = y - if x > lastX: lastX = x - - black_pixel_in_col = True - - if black_pixel_in_col == False and started == True: - rect = (firstX, topY, lastX, bottomY) - new_captcha = captcha.crop(rect) - - w, h = new_captcha.size - if w > 5 and h > 5: - letters.append(new_captcha) - - started = False - bottomY, topY = 0, height - - return letters - - def correct(self, values, var=None): - - if var: - result = var - else: - result = self.result_captcha - - for key, item in values.iteritems(): - - if key.__class__ == str: - result = result.replace(key, item) - else: - for expr in key: - result = result.replace(expr, item) - - if var: - return result - else: - self.result_captcha = result - - -if __name__ == '__main__': - ocr = OCR() - ocr.load_image("B.jpg") - ocr.to_greyscale() - ocr.eval_black_white(140) - ocr.derotate_by_average() - ocr.run_tesser() - print "Tesseract", ocr.result_captcha - ocr.image.save("derotated.jpg") - diff --git a/module/plugins/internal/ShareonlineBizOCR.py b/module/plugins/internal/ShareonlineBizOCR.py deleted file mode 100644 index c5c2e92e8..000000000 --- a/module/plugins/internal/ShareonlineBizOCR.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. -# -### -from OCR import OCR - -class ShareonlineBizOCR(OCR): - __version__ = 0.1 - - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.image = self.image.resize((160, 50)) - self.pixels = self.image.load() - self.threshold(1.85) - #self.eval_black_white(240) - #self.derotate_by_average() - - letters = self.split_captcha_letters() - - final = "" - for letter in letters: - self.image = letter - self.run_tesser(True, True, False, False) - final += self.result_captcha - - return final - - #tesseract at 60% - -if __name__ == '__main__': - import urllib - ocr = ShareonlineBizOCR() - urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") - print ocr.get_captcha('captcha.jpeg') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py deleted file mode 100644 index d935bf1da..000000000 --- a/module/plugins/internal/SimpleCrypter.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re - -from module.plugins.Crypter import Crypter -from module.utils import html_unescape - - -class SimpleCrypter(Crypter): - __name__ = "SimpleCrypter" - __version__ = "0.04" - __pattern__ = None - __type__ = "crypter" - __description__ = """Base crypter plugin""" - __author_name__ = ("stickell", "zoidberg") - __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz") - """ - These patterns should be defined by each crypter: - - LINK_PATTERN: group(1) must be a download link - example: <div class="link"><a href="(http://speedload.org/\w+) - - TITLE_PATTERN: (optional) the group defined by 'title' should be the title - example: <title>Files of: (?P<title>[^<]+) folder</title> - """ - - def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - package_name, folder_name = self.getPackageNameAndFolder() - - package_links = re.findall(self.LINK_PATTERN, self.html) - self.logDebug('Package has %d links' % len(package_links)) - - if package_links: - self.packages = [(package_name, package_links, folder_name)] - else: - self.fail('Could not extract any links') - - def getPackageNameAndFolder(self): - if hasattr(self, 'TITLE_PATTERN'): - m = re.search(self.TITLE_PATTERN, self.html) - if m: - name = folder = html_unescape(m.group('title').strip()) - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - return name, folder - - name = self.pyfile.package().name - folder = self.pyfile.package().folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - return name, folder diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py deleted file mode 100644 index 7b1d7323a..000000000 --- a/module/plugins/internal/SimpleHoster.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" -from urlparse import urlparse -import re -from time import time - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape, fixup, parseFileSize -from module.network.RequestFactory import getURL -from module.network.CookieJar import CookieJar - -def replace_patterns(string, ruleslist): - for r in ruleslist: - rf, rt = r - string = re.sub(rf, rt, string) - #self.logDebug(rf, rt, string) - return string - -def set_cookies(cj, cookies): - for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) == 3: - domain, name, value = cookie - cj.setCookie(domain, name, value) - -def parseHtmlTagAttrValue(attr_name, tag): - m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) - return m.group(2) if m else None - -def parseHtmlForm(attr_str, html, input_names=None): - for form in re.finditer(r"(?P<tag><form[^>]*%s[^>]*>)(?P<content>.*?)</?(form|body|html)[^>]*>" % attr_str, html, re.S | re.I): - inputs = {} - action = parseHtmlTagAttrValue("action", form.group('tag')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('content'), re.S | re.I): - name = parseHtmlTagAttrValue("name", inputtag.group(1)) - if name: - value = parseHtmlTagAttrValue("value", inputtag.group(1)) - if value is None: - inputs[name] = inputtag.group(3) or '' - else: - inputs[name] = value - - if isinstance(input_names, dict): - # check input attributes - for key, val in input_names.items(): - if key in inputs: - if isinstance(val, basestring) and inputs[key] == val: - continue - elif isinstance(val, tuple) and inputs[key] in val: - continue - elif hasattr(val, "search") and re.match(val, inputs[key]): - continue - break # attibute value does not match - else: - break # attibute name does not match - else: - return action, inputs # passed attribute check - else: - # no attribute check - return action, inputs - - return {}, None # no matching form found - -def parseFileInfo(self, url = '', html = ''): - info = {"name" : url, "size" : 0, "status" : 3} - - if hasattr(self, "pyfile"): - url = self.pyfile.url - - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - else: - if not html and hasattr(self, "html"): html = self.html - if isinstance(self.SH_BROKEN_ENCODING, (str, unicode)): - html = unicode(html, self.SH_BROKEN_ENCODING) - if hasattr(self, "html"): self.html = html - - if hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): - # File offline - info['status'] = 1 - else: - online = False - try: - info.update(re.match(self.__pattern__, url).groupdict()) - except: - pass - - for pattern in ("FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): - try: - info.update(re.search(getattr(self, pattern), html).groupdict()) - online = True - except AttributeError: - continue - - if online: - # File online, return name and size - info['status'] = 2 - if 'N' in info: - info['name'] = replace_patterns(info['N'], self.FILE_NAME_REPLACEMENTS) - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) - info['size'] = parseFileSize(size) - elif isinstance(info['size'], (str, unicode)): - if 'units' in info: info['size'] += info['units'] - info['size'] = parseFileSize(info['size']) - - if hasattr(self, "file_info"): - self.file_info = info - - return info['name'], info['size'], info['status'], url - -def create_getInfo(plugin): - def getInfo(urls): - for url in urls: - cj = CookieJar(plugin.__name__) - if isinstance(plugin.SH_COOKIES, list): set_cookies(cj, plugin.SH_COOKIES) - file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), \ - decode = not plugin.SH_BROKEN_ENCODING, cookies = cj)) - yield file_info - return getInfo - -def timestamp(): - return int(time()*1000) - -class PluginParseError(Exception): - def __init__(self, msg): - Exception.__init__(self) - self.value = 'Parse error (%s) - plugin may be out of date' % msg - def __str__(self): - return repr(self.value) - -class SimpleHoster(Hoster): - __name__ = "SimpleHoster" - __version__ = "0.28" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - """ - These patterns should be defined by each hoster: - FILE_INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>units)' - or FILE_NAME_PATTERN = r'(?P<N>file_name)' - and FILE_SIZE_PATTERN = r'(?P<S>file_size) (?P<U>units)' - FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintenance' - """ - - FILE_SIZE_REPLACEMENTS = [] - FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - FILE_URL_REPLACEMENTS = [] - - SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account - - def init(self): - self.file_info = {} - - def setup(self): - self.resumeDownload = self.multiDL = True if self.premium else False - if isinstance(self.SH_COOKIES, list): set_cookies(self.req.cj, self.SH_COOKIES) - - def process(self, pyfile): - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - self.req.setOption("timeout", 120) - self.html = self.load(pyfile.url, decode = not self.SH_BROKEN_ENCODING, cookies = self.SH_COOKIES) - self.getFileInfo() - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.handlePremium() - else: - self.handleFree() - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - if type(url) == unicode: url = url.encode('utf8') - return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, just_header=just_header, decode=decode) - - def getFileInfo(self): - self.logDebug("URL: %s" % self.pyfile.url) - if hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() - - name, size, status = parseFileInfo(self)[:3] - - if status == 1: - self.offline() - elif status != 2: - self.logDebug(self.file_info) - self.parseError('File info') - - if name: - self.pyfile.name = name - else: - self.pyfile.name = html_unescape(urlparse(self.pyfile.url).path.split("/")[-1]) - - if size: - self.pyfile.size = size - else: - self.logError("File size not parsed") - - self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) - return self.file_info - - def handleFree(self): - self.fail("Free download not implemented") - - def handlePremium(self): - self.fail("Premium download not implemented") - - def parseError(self, msg): - raise PluginParseError(msg) - - def longWait(self, wait_time = None, max_tries = 3): - if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) - else: - wait_time = 900 - time_str = "(unknown time)" - max_tries = 100 - - self.logInfo("Download limit reached, reconnect or wait %s" % time_str) - - self.setWait(wait_time, True) - self.wait() - self.retry(max_tries = max_tries, reason="Download limit reached") - - def parseHtmlForm(self, attr_str='', input_names=None): - return parseHtmlForm(attr_str, self.html, input_names) - - def checkTrafficLeft(self): - traffic = self.account.getAccountInfo(self.user, True)["trafficleft"] - if traffic == -1: - return True - size = self.pyfile.size / 1024 - self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) - return size <= traffic
\ No newline at end of file diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py deleted file mode 100644 index 7becd663c..000000000 --- a/module/plugins/internal/UnRar.py +++ /dev/null @@ -1,212 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import os -import re -from glob import glob -from subprocess import Popen, PIPE -from string import digits - -from module.utils.fs import save_join, decode, fs_encode -from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError - -class UnRar(AbtractExtractor): - __name__ = "UnRar" - __version__ = "0.13" - - # there are some more uncovered rar formats - re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) - re_partfiles = re.compile(r".*\.(rar|r[0-9]+)", re.I) - re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") - re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) - CMD = "unrar" - - @staticmethod - def checkDeps(): - if os.name == "nt": - UnRar.CMD = save_join(pypath, "UnRAR.exe") - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() - else: - try: - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() - except OSError: - - #fallback to rar - UnRar.CMD = "rar" - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() - - return True - - @staticmethod - def getTargets(files_ids): - result = [] - - for file, id in files_ids: - if not file.endswith(".rar"): continue - - match = UnRar.re_splitfile.findall(file) - if match: - #only add first parts - if int(match[0][1]) == 1: - result.append((file, id)) - else: - result.append((file, id)) - - return result - - - def init(self): - self.passwordProtected = False - self.headerProtected = False #list files will not work without password - self.smallestFile = None #small file to test passwords - self.password = "" #save the correct password - - def checkArchive(self): - p = self.call_unrar("l", "-v", fs_encode(self.file)) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - self.passwordProtected = True - self.headerProtected = True - return True - - # output only used to check if passworded files are present - for name, size, packed in self.re_filelist.findall(out): - if name.startswith("*"): - self.passwordProtected = True - return True - - self.listContent() - if not self.files: - raise ArchiveError("Empty Archive") - - return False - - def checkPassword(self, password): - #at this point we can only verify header protected files - if self.headerProtected: - p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - return False - - return True - - - def extract(self, progress, password=None): - command = "x" if self.fullpath else "e" - - p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) - renice(p.pid, self.renice) - - progress(0) - progressstring = "" - while True: - c = p.stdout.read(1) - # quit loop on eof - if not c: - break - # reading a percentage sign -> set progress and restart - if c == '%': - progress(int(progressstring)) - progressstring = "" - # not reading a digit -> therefore restart - elif c not in digits: - progressstring = "" - # add digit to progressstring - else: - progressstring = progressstring + c - progress(100) - - # retrieve stderr - err = p.stderr.read() - - if "CRC failed" in err and not password and not self.passwordProtected: - raise CRCError - elif "CRC failed" in err: - raise WrongPassword - if err.strip(): #raise error if anything is on stderr - raise ArchiveError(err.strip()) - if p.returncode: - raise ArchiveError("Process terminated") - - if not self.files: - self.password = password - self.listContent() - - - def getDeleteFiles(self): - if ".part" in self.file: - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) - # get files which matches .r* and filter unsuited files out - parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.IGNORECASE)) - return filter(lambda x: self.re_partfiles.match(x), parts) - - def listContent(self): - command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) - out, err = p.communicate() - - if "Cannot open" in err: - raise ArchiveError("Cannot open file") - - if err.strip(): # only log error at this point - self.m.logError(err.strip()) - - result = set() - - for f in decode(out).splitlines(): - f = f.strip() - result.add(save_join(self.out, f)) - - self.files = result - - - def call_unrar(self, command, *xargs, **kwargs): - args = [] - #overwrite flag - args.append("-o+") if self.overwrite else args.append("-o-") - - # assume yes on all queries - args.append("-y") - - #set a password - if "password" in kwargs and kwargs["password"]: - args.append("-p%s" % kwargs["password"]) - else: - args.append("-p-") - - - #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue - call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join([decode(arg) for arg in call])) - - p = Popen(call, stdout=PIPE, stderr=PIPE) - - return p - - -def renice(pid, value): - if os.name != "nt" and value: - try: - Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - except: - print "Renice failed" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py deleted file mode 100644 index 9aa9ac75c..000000000 --- a/module/plugins/internal/UnZip.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import zipfile -import sys - -from module.plugins.internal.AbstractExtractor import AbtractExtractor - -class UnZip(AbtractExtractor): - __name__ = "UnZip" - __version__ = "0.1" - - @staticmethod - def checkDeps(): - return sys.version_info[:2] >= (2, 6) - - @staticmethod - def getTargets(files_ids): - result = [] - - for file, id in files_ids: - if file.endswith(".zip"): - result.append((file, id)) - - return result - - def extract(self, progress, password=None): - z = zipfile.ZipFile(self.file) - self.files = z.namelist() - z.extractall(self.out) - - def getDeleteFiles(self): - return [self.file]
\ No newline at end of file diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py deleted file mode 100644 index 8333c7265..000000000 --- a/module/plugins/internal/XFSPAccount.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from time import mktime, strptime -from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm -from module.utils import parseFileSize - -class XFSPAccount(Account): - __name__ = "XFSPAccount" - __version__ = "0.05" - __type__ = "account" - __description__ = """XFileSharingPro account base""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - MAIN_PAGE = None - - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:</TD><TD><b>([^<]+)</b>' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:</TD><TD><b>([^<]+)</b>' - - def loadAccountInfo(self, user, req): - html = req.load(self.MAIN_PAGE + "?op=my_account", decode = True) - - validuntil = trafficleft = None - premium = True if '>Renew premium<' in html else False - - found = re.search(self.VALID_UNTIL_PATTERN, html) - if found: - premium = True - trafficleft = -1 - try: - self.logDebug(found.group(1)) - validuntil = mktime(strptime(found.group(1), "%d %B %Y")) - except Exception, e: - self.logError(e) - else: - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if found: - trafficleft = found.group(1) - if "Unlimited" in trafficleft: - premium = True - else: - trafficleft = parseFileSize(trafficleft) / 1024 - - return ({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) - - def login(self, user, data, req): - html = req.load('%slogin.html' % self.MAIN_PAGE, decode = True) - - action, inputs = parseHtmlForm('name="FL"', html) - if not inputs: - inputs = {"op": "login", - "redirect": self.MAIN_PAGE} - - inputs.update({"login": user, - "password": data['password']}) - - html = req.load(self.MAIN_PAGE, post = inputs, decode = True) - - if 'Incorrect Login or Password' in html or '>Error<' in html: - self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/internal/__init__.py b/module/plugins/internal/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/internal/__init__.py +++ /dev/null diff --git a/module/remote/ClickAndLoadBackend.py b/module/remote/ClickAndLoadBackend.py deleted file mode 100644 index ad8031587..000000000 --- a/module/remote/ClickAndLoadBackend.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -import re -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from cgi import FieldStorage -from urllib import unquote -from base64 import standard_b64decode -from binascii import unhexlify - -try: - from Crypto.Cipher import AES -except: - pass - -from RemoteManager import BackendBase - -core = None -js = None - -class ClickAndLoadBackend(BackendBase): - def setup(self, host, port): - self.httpd = HTTPServer((host, port), CNLHandler) - global core, js - core = self.m.core - js = core.js - - def serve(self): - while self.enabled: - self.httpd.handle_request() - -class CNLHandler(BaseHTTPRequestHandler): - - def add_package(self, name, urls, queue=0): - print "name", name - print "urls", urls - print "queue", queue - - def get_post(self, name, default=""): - if name in self.post: - return self.post[name] - else: - return default - - def start_response(self, string): - - self.send_response(200) - - self.send_header("Content-Length", len(string)) - self.send_header("Content-Language", "de") - self.send_header("Vary", "Accept-Language, Cookie") - self.send_header("Cache-Control", "no-cache, must-revalidate") - self.send_header("Content-type", "text/html") - self.end_headers() - - def do_GET(self): - path = self.path.strip("/").lower() - #self.wfile.write(path+"\n") - - self.map = [ (r"add$", self.add), - (r"addcrypted$", self.addcrypted), - (r"addcrypted2$", self.addcrypted2), - (r"flashgot", self.flashgot), - (r"crossdomain\.xml", self.crossdomain), - (r"checkSupportForUrl", self.checksupport), - (r"jdcheck.js", self.jdcheck), - (r"", self.flash) ] - - func = None - for r, f in self.map: - if re.match(r"(flash(got)?/?)?"+r, path): - func = f - break - - if func: - try: - resp = func() - if not resp: resp = "success" - resp += "\r\n" - self.start_response(resp) - self.wfile.write(resp) - except Exception,e : - self.send_error(500, str(e)) - else: - self.send_error(404, "Not Found") - - def do_POST(self): - form = FieldStorage( - fp=self.rfile, - headers=self.headers, - environ={'REQUEST_METHOD':'POST', - 'CONTENT_TYPE':self.headers['Content-Type'], - }) - - self.post = {} - for name in form.keys(): - self.post[name] = form[name].value - - return self.do_GET() - - def flash(self): - return "JDownloader" - - def add(self): - package = self.get_post('referer', 'ClickAndLoad Package') - urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) - - self.add_package(package, urls, 0) - - def addcrypted(self): - package = self.get_post('referer', 'ClickAndLoad Package') - dlc = self.get_post('crypted').replace(" ", "+") - - core.upload_container(package, dlc) - - def addcrypted2(self): - package = self.get_post("source", "ClickAndLoad Package") - crypted = self.get_post("crypted") - jk = self.get_post("jk") - - crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) - jk = "%s f()" % jk - jk = js.eval(jk) - Key = unhexlify(jk) - IV = Key - - obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") - - result = filter(lambda x: x != "", result) - - self.add_package(package, result, 0) - - - def flashgot(self): - autostart = int(self.get_post('autostart', 0)) - package = self.get_post('package', "FlashGot") - urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) - - self.add_package(package, urls, autostart) - - def crossdomain(self): - rep = "<?xml version=\"1.0\"?>\n" - rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\n" - rep += "<cross-domain-policy>\n" - rep += "<allow-access-from domain=\"*\" />\n" - rep += "</cross-domain-policy>" - return rep - - def checksupport(self): - pass - - def jdcheck(self): - rep = "jdownloader=true;\n" - rep += "var version='10629';\n" - return rep diff --git a/module/remote/JSONClient.py b/module/remote/JSONClient.py deleted file mode 100644 index a2c07a132..000000000 --- a/module/remote/JSONClient.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from urllib import urlopen, urlencode -from httplib import UNAUTHORIZED, FORBIDDEN - -from json_converter import loads, dumps -from apitypes import Unauthorized, Forbidden - -class JSONClient: - URL = "http://localhost:8001/api" - - def __init__(self, url=None): - self.url = url or self.URL - self.session = None - - def request(self, path, data): - ret = urlopen(self.url + path, urlencode(data)) - if ret.code == 400: - raise loads(ret.read()) - if ret.code == 404: - raise AttributeError("Unknown Method") - if ret.code == 500: - raise Exception("Remote Exception") - if ret.code == UNAUTHORIZED: - raise Unauthorized() - if ret.code == FORBIDDEN: - raise Forbidden() - return ret.read() - - def login(self, username, password): - self.session = loads(self.request("/login", {'username': username, 'password': password})) - return self.session - - def logout(self): - self.call("logout") - self.session = None - - def call(self, func, *args, **kwargs): - # Add the current session - kwargs["session"] = self.session - path = "/" + func + "/" + "/".join(dumps(x) for x in args) - data = dict((k, dumps(v)) for k, v in kwargs.iteritems()) - rep = self.request(path, data) - return loads(rep) - - def __getattr__(self, item): - def call(*args, **kwargs): - return self.call(item, *args, **kwargs) - - return call - -if __name__ == "__main__": - api = JSONClient() - api.login("User", "test") - print api.getServerVersion()
\ No newline at end of file diff --git a/module/remote/RemoteManager.py b/module/remote/RemoteManager.py deleted file mode 100644 index dd567653b..000000000 --- a/module/remote/RemoteManager.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from threading import Thread -from traceback import print_exc - -class BackendBase(Thread): - def __init__(self, manager): - Thread.__init__(self) - self.m = manager - self.core = manager.core - self.enabled = True - self.running = False - - def run(self): - self.running = True - try: - self.serve() - except Exception, e: - self.core.log.error(_("Remote backend error: %s") % e) - if self.core.debug: - print_exc() - finally: - self.running = False - - def setup(self, host, port): - pass - - def checkDeps(self): - return True - - def serve(self): - pass - - def shutdown(self): - pass - - def stop(self): - self.enabled = False# set flag and call shutdowm message, so thread can react - self.shutdown() - - -class RemoteManager(): - available = [] - - def __init__(self, core): - self.core = core - self.backends = [] - - if self.core.remote: - self.available.append("WebSocketBackend") - - - def startBackends(self): - host = self.core.config["remote"]["listenaddr"] - port = self.core.config["remote"]["port"] - - for b in self.available: - klass = getattr(__import__("module.remote.%s" % b, globals(), locals(), [b], -1), b) - backend = klass(self) - if not backend.checkDeps(): - continue - try: - backend.setup(host, port) - self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) - except Exception, e: - self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) - if self.core.debug: - print_exc() - else: - backend.start() - self.backends.append(backend) - - port += 1 diff --git a/module/remote/WSClient.py b/module/remote/WSClient.py deleted file mode 100644 index 793a6ef28..000000000 --- a/module/remote/WSClient.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from websocket import create_connection -from httplib import UNAUTHORIZED, FORBIDDEN - -from json_converter import loads, dumps -from apitypes import Unauthorized, Forbidden - -class WSClient: - URL = "ws://localhost:7227/api" - - def __init__(self, url=None): - self.url = url or self.URL - self.ws = None - - def connect(self): - self.ws = create_connection(self.URL) - - def close(self): - self.ws.close() - - def login(self, username, password): - if not self.ws: self.connect() - return self.call("login", username, password) - - def call(self, func, *args, **kwargs): - if not self.ws: - raise Exception("Not Connected") - - if kwargs: - self.ws.send(dumps([func, args, kwargs])) - else: # omit kwargs - self.ws.send(dumps([func, args])) - - code, result = loads(self.ws.recv()) - if code == 400: - raise result - if code == 404: - raise AttributeError("Unknown Method") - elif code == 500: - raise Exception("Remote Exception: %s" % result) - elif code == UNAUTHORIZED: - raise Unauthorized() - elif code == FORBIDDEN: - raise Forbidden() - - return result - - def __getattr__(self, item): - def call(*args, **kwargs): - return self.call(item, *args, **kwargs) - - return call - -if __name__ == "__main__": - api = WSClient() - api.login("User", "test") - print api.getServerVersion()
\ No newline at end of file diff --git a/module/remote/WebSocketBackend.py b/module/remote/WebSocketBackend.py deleted file mode 100644 index 2d22664c6..000000000 --- a/module/remote/WebSocketBackend.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import logging - -from module.remote.RemoteManager import BackendBase - -from mod_pywebsocket import util -def get_class_logger(o=None): - return logging.getLogger('log') - -# Monkey patch for our logger -util.get_class_logger = get_class_logger - -class WebSocketBackend(BackendBase): - def setup(self, host, port): - - from wsbackend.Server import WebSocketServer, DefaultOptions - from wsbackend.Dispatcher import Dispatcher - from wsbackend.ApiHandler import ApiHandler - from wsbackend.AsyncHandler import AsyncHandler - - options = DefaultOptions() - options.server_host = host - options.port = port - options.dispatcher = Dispatcher() - options.dispatcher.addHandler(ApiHandler.PATH, ApiHandler(self.core.api)) - options.dispatcher.addHandler(AsyncHandler.PATH, AsyncHandler(self.core.api)) - - self.server = WebSocketServer(options) - - - def serve(self): - self.server.serve_forever() diff --git a/module/remote/__init__.py b/module/remote/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/remote/__init__.py +++ /dev/null diff --git a/module/remote/apitypes.py b/module/remote/apitypes.py deleted file mode 100644 index 196491083..000000000 --- a/module/remote/apitypes.py +++ /dev/null @@ -1,536 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -class BaseObject(object): - __slots__ = [] - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, ", ".join("%s=%s" % (k,getattr(self,k)) for k in self.__slots__)) - -class ExceptionObject(Exception): - __slots__ = [] - -class DownloadState: - All = 0 - Finished = 1 - Unfinished = 2 - Failed = 3 - Unmanaged = 4 - -class DownloadStatus: - NA = 0 - Offline = 1 - Online = 2 - Queued = 3 - Paused = 4 - Finished = 5 - Skipped = 6 - Failed = 7 - Starting = 8 - Waiting = 9 - Downloading = 10 - TempOffline = 11 - Aborted = 12 - Decrypting = 13 - Processing = 14 - Custom = 15 - Unknown = 16 - -class FileStatus: - Ok = 0 - Missing = 1 - Remote = 2 - -class InputType: - NA = 0 - Text = 1 - Int = 2 - File = 3 - Folder = 4 - Textbox = 5 - Password = 6 - Bool = 7 - Click = 8 - Select = 9 - Multiple = 10 - List = 11 - Table = 12 - -class Interaction: - All = 0 - Notification = 1 - Captcha = 2 - Query = 4 - -class MediaType: - All = 0 - Other = 1 - Audio = 2 - Image = 4 - Video = 8 - Document = 16 - Archive = 32 - -class PackageStatus: - Ok = 0 - Paused = 1 - Folder = 2 - Remote = 3 - -class Permission: - All = 0 - Add = 1 - Delete = 2 - Modify = 4 - Download = 8 - Accounts = 16 - Interaction = 32 - Plugins = 64 - -class Role: - Admin = 0 - User = 1 - -class AccountInfo(BaseObject): - __slots__ = ['plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'options'] - - def __init__(self, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, options=None): - self.plugin = plugin - self.loginname = loginname - self.owner = owner - self.valid = valid - self.validuntil = validuntil - self.trafficleft = trafficleft - self.maxtraffic = maxtraffic - self.premium = premium - self.activated = activated - self.shared = shared - self.options = options - -class AddonInfo(BaseObject): - __slots__ = ['func_name', 'description', 'value'] - - def __init__(self, func_name=None, description=None, value=None): - self.func_name = func_name - self.description = description - self.value = value - -class AddonService(BaseObject): - __slots__ = ['func_name', 'description', 'arguments', 'media'] - - def __init__(self, func_name=None, description=None, arguments=None, media=None): - self.func_name = func_name - self.description = description - self.arguments = arguments - self.media = media - -class ConfigHolder(BaseObject): - __slots__ = ['name', 'label', 'description', 'long_description', 'items', 'info'] - - def __init__(self, name=None, label=None, description=None, long_description=None, items=None, info=None): - self.name = name - self.label = label - self.description = description - self.long_description = long_description - self.items = items - self.info = info - -class ConfigInfo(BaseObject): - __slots__ = ['name', 'label', 'description', 'category', 'user_context', 'activated'] - - def __init__(self, name=None, label=None, description=None, category=None, user_context=None, activated=None): - self.name = name - self.label = label - self.description = description - self.category = category - self.user_context = user_context - self.activated = activated - -class ConfigItem(BaseObject): - __slots__ = ['name', 'label', 'description', 'input', 'default_value', 'value'] - - def __init__(self, name=None, label=None, description=None, input=None, default_value=None, value=None): - self.name = name - self.label = label - self.description = description - self.input = input - self.default_value = default_value - self.value = value - -class DownloadInfo(BaseObject): - __slots__ = ['url', 'plugin', 'hash', 'status', 'statusmsg', 'error'] - - def __init__(self, url=None, plugin=None, hash=None, status=None, statusmsg=None, error=None): - self.url = url - self.plugin = plugin - self.hash = hash - self.status = status - self.statusmsg = statusmsg - self.error = error - -class DownloadProgress(BaseObject): - __slots__ = ['fid', 'pid', 'speed', 'status'] - - def __init__(self, fid=None, pid=None, speed=None, status=None): - self.fid = fid - self.pid = pid - self.speed = speed - self.status = status - -class EventInfo(BaseObject): - __slots__ = ['eventname', 'event_args'] - - def __init__(self, eventname=None, event_args=None): - self.eventname = eventname - self.event_args = event_args - -class FileDoesNotExists(ExceptionObject): - __slots__ = ['fid'] - - def __init__(self, fid=None): - self.fid = fid - -class FileInfo(BaseObject): - __slots__ = ['fid', 'name', 'package', 'owner', 'size', 'status', 'media', 'added', 'fileorder', 'download'] - - def __init__(self, fid=None, name=None, package=None, owner=None, size=None, status=None, media=None, added=None, fileorder=None, download=None): - self.fid = fid - self.name = name - self.package = package - self.owner = owner - self.size = size - self.status = status - self.media = media - self.added = added - self.fileorder = fileorder - self.download = download - -class Forbidden(ExceptionObject): - pass - -class Input(BaseObject): - __slots__ = ['type', 'data'] - - def __init__(self, type=None, data=None): - self.type = type - self.data = data - -class InteractionTask(BaseObject): - __slots__ = ['iid', 'type', 'input', 'default_value', 'title', 'description', 'plugin'] - - def __init__(self, iid=None, type=None, input=None, default_value=None, title=None, description=None, plugin=None): - self.iid = iid - self.type = type - self.input = input - self.default_value = default_value - self.title = title - self.description = description - self.plugin = plugin - -class InvalidConfigSection(ExceptionObject): - __slots__ = ['section'] - - def __init__(self, section=None): - self.section = section - -class LinkStatus(BaseObject): - __slots__ = ['url', 'name', 'plugin', 'size', 'status', 'packagename'] - - def __init__(self, url=None, name=None, plugin=None, size=None, status=None, packagename=None): - self.url = url - self.name = name - self.plugin = plugin - self.size = size - self.status = status - self.packagename = packagename - -class OnlineCheck(BaseObject): - __slots__ = ['rid', 'data'] - - def __init__(self, rid=None, data=None): - self.rid = rid - self.data = data - -class PackageDoesNotExists(ExceptionObject): - __slots__ = ['pid'] - - def __init__(self, pid=None): - self.pid = pid - -class PackageInfo(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'root', 'owner', 'site', 'comment', 'password', 'added', 'tags', 'status', 'shared', 'packageorder', 'stats', 'fids', 'pids'] - - def __init__(self, pid=None, name=None, folder=None, root=None, owner=None, site=None, comment=None, password=None, added=None, tags=None, status=None, shared=None, packageorder=None, stats=None, fids=None, pids=None): - self.pid = pid - self.name = name - self.folder = folder - self.root = root - self.owner = owner - self.site = site - self.comment = comment - self.password = password - self.added = added - self.tags = tags - self.status = status - self.shared = shared - self.packageorder = packageorder - self.stats = stats - self.fids = fids - self.pids = pids - -class PackageStats(BaseObject): - __slots__ = ['linkstotal', 'linksdone', 'sizetotal', 'sizedone'] - - def __init__(self, linkstotal=None, linksdone=None, sizetotal=None, sizedone=None): - self.linkstotal = linkstotal - self.linksdone = linksdone - self.sizetotal = sizetotal - self.sizedone = sizedone - -class ProgressInfo(BaseObject): - __slots__ = ['plugin', 'name', 'statusmsg', 'eta', 'done', 'total', 'download'] - - def __init__(self, plugin=None, name=None, statusmsg=None, eta=None, done=None, total=None, download=None): - self.plugin = plugin - self.name = name - self.statusmsg = statusmsg - self.eta = eta - self.done = done - self.total = total - self.download = download - -class ServerStatus(BaseObject): - __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', 'notifications', 'paused', 'download', 'reconnect'] - - def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, notifications=None, paused=None, download=None, reconnect=None): - self.speed = speed - self.linkstotal = linkstotal - self.linksqueue = linksqueue - self.sizetotal = sizetotal - self.sizequeue = sizequeue - self.notifications = notifications - self.paused = paused - self.download = download - self.reconnect = reconnect - -class ServiceDoesNotExists(ExceptionObject): - __slots__ = ['plugin', 'func'] - - def __init__(self, plugin=None, func=None): - self.plugin = plugin - self.func = func - -class ServiceException(ExceptionObject): - __slots__ = ['msg'] - - def __init__(self, msg=None): - self.msg = msg - -class TreeCollection(BaseObject): - __slots__ = ['root', 'files', 'packages'] - - def __init__(self, root=None, files=None, packages=None): - self.root = root - self.files = files - self.packages = packages - -class Unauthorized(ExceptionObject): - pass - -class UserData(BaseObject): - __slots__ = ['uid', 'name', 'email', 'role', 'permission', 'folder', 'traffic', 'dllimit', 'dlquota', 'hddquota', 'user', 'templateName'] - - def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, dllimit=None, dlquota=None, hddquota=None, user=None, templateName=None): - self.uid = uid - self.name = name - self.email = email - self.role = role - self.permission = permission - self.folder = folder - self.traffic = traffic - self.dllimit = dllimit - self.dlquota = dlquota - self.hddquota = hddquota - self.user = user - self.templateName = templateName - -class UserDoesNotExists(ExceptionObject): - __slots__ = ['user'] - - def __init__(self, user=None): - self.user = user - -class Iface(object): - def addFromCollector(self, name, paused): - pass - def addLinks(self, pid, links): - pass - def addLocalFile(self, pid, name, path): - pass - def addPackage(self, name, links, password): - pass - def addPackageChild(self, name, links, password, root, paused): - pass - def addPackageP(self, name, links, password, paused): - pass - def addToCollector(self, links): - pass - def addUser(self, username, password): - pass - def callAddon(self, plugin, func, arguments): - pass - def callAddonHandler(self, plugin, func, pid_or_fid): - pass - def checkOnlineStatus(self, urls): - pass - def checkOnlineStatusContainer(self, urls, filename, data): - pass - def checkURLs(self, urls): - pass - def createPackage(self, name, folder, root, password, site, comment, paused): - pass - def deleteCollLink(self, url): - pass - def deleteCollPack(self, name): - pass - def deleteConfig(self, plugin): - pass - def deleteFiles(self, fids): - pass - def deletePackages(self, pids): - pass - def findFiles(self, pattern): - pass - def findPackages(self, tags): - pass - def freeSpace(self): - pass - def generateAndAddPackages(self, links, paused): - pass - def generateDownloadLink(self, fid, timeout): - pass - def generatePackages(self, links): - pass - def getAccountTypes(self): - pass - def getAccounts(self, refresh): - pass - def getAddonHandler(self): - pass - def getAllFiles(self): - pass - def getAllUserData(self): - pass - def getAvailablePlugins(self): - pass - def getCollector(self): - pass - def getConfig(self): - pass - def getConfigValue(self, section, option): - pass - def getCoreConfig(self): - pass - def getFileInfo(self, fid): - pass - def getFileTree(self, pid, full): - pass - def getFilteredFileTree(self, pid, full, state): - pass - def getFilteredFiles(self, state): - pass - def getInteractionTasks(self, mode): - pass - def getLog(self, offset): - pass - def getPackageContent(self, pid): - pass - def getPackageInfo(self, pid): - pass - def getPluginConfig(self): - pass - def getProgressInfo(self): - pass - def getServerStatus(self): - pass - def getServerVersion(self): - pass - def getUserData(self): - pass - def getWSAddress(self): - pass - def hasAddonHandler(self, plugin, func): - pass - def isInteractionWaiting(self, mode): - pass - def loadConfig(self, name): - pass - def login(self, username, password): - pass - def moveFiles(self, fids, pid): - pass - def movePackage(self, pid, root): - pass - def orderFiles(self, fids, pid, position): - pass - def orderPackage(self, pids, position): - pass - def parseURLs(self, html, url): - pass - def pauseServer(self): - pass - def pollResults(self, rid): - pass - def quit(self): - pass - def recheckPackage(self, pid): - pass - def removeAccount(self, account): - pass - def removeUser(self, uid): - pass - def renameCollPack(self, name, new_name): - pass - def restart(self): - pass - def restartFailed(self): - pass - def restartFile(self, fid): - pass - def restartPackage(self, pid): - pass - def saveConfig(self, config): - pass - def searchSuggestions(self, pattern): - pass - def setConfigValue(self, section, option, value): - pass - def setInteractionResult(self, iid, result): - pass - def setPackageFolder(self, pid, path): - pass - def setPassword(self, username, old_password, new_password): - pass - def stopAllDownloads(self): - pass - def stopDownloads(self, fids): - pass - def togglePause(self): - pass - def toggleReconnect(self): - pass - def unpauseServer(self): - pass - def updateAccount(self, plugin, login, password): - pass - def updateAccountInfo(self, account): - pass - def updatePackage(self, pack): - pass - def updateUserData(self, data): - pass - def uploadContainer(self, filename, data): - pass - diff --git a/module/remote/apitypes_debug.py b/module/remote/apitypes_debug.py deleted file mode 100644 index 96673cc99..000000000 --- a/module/remote/apitypes_debug.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -from ttypes import * - -enums = [ - "DownloadState", - "DownloadStatus", - "FileStatus", - "InputType", - "Interaction", - "MediaType", - "PackageStatus", - "Permission", - "Role", -] - -classes = { - 'AccountInfo' : [basestring, basestring, int, bool, int, int, int, bool, bool, bool, (dict, basestring, basestring)], - 'AddonInfo' : [basestring, basestring, basestring], - 'AddonService' : [basestring, basestring, (list, basestring), (None, int)], - 'ConfigHolder' : [basestring, basestring, basestring, basestring, (list, ConfigItem), (None, (list, AddonInfo))], - 'ConfigInfo' : [basestring, basestring, basestring, basestring, bool, (None, bool)], - 'ConfigItem' : [basestring, basestring, basestring, Input, basestring, basestring], - 'DownloadInfo' : [basestring, basestring, basestring, int, basestring, basestring], - 'DownloadProgress' : [int, int, int, int], - 'EventInfo' : [basestring, (list, basestring)], - 'FileDoesNotExists' : [int], - 'FileInfo' : [int, basestring, int, int, int, int, int, int, int, (None, DownloadInfo)], - 'Input' : [int, (None, basestring)], - 'InteractionTask' : [int, int, Input, (None, basestring), basestring, basestring, basestring], - 'InvalidConfigSection' : [basestring], - 'LinkStatus' : [basestring, basestring, basestring, int, int, basestring], - 'OnlineCheck' : [int, (None, (dict, basestring, LinkStatus))], - 'PackageDoesNotExists' : [int], - 'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, bool, int, PackageStats, (list, int), (list, int)], - 'PackageStats' : [int, int, int, int], - 'ProgressInfo' : [basestring, basestring, basestring, int, int, int, (None, DownloadProgress)], - 'ServerStatus' : [int, int, int, int, int, bool, bool, bool, bool], - 'ServiceDoesNotExists' : [basestring, basestring], - 'ServiceException' : [basestring], - 'TreeCollection' : [PackageInfo, (dict, int, FileInfo), (dict, int, PackageInfo)], - 'UserData' : [int, basestring, basestring, int, int, basestring, int, int, basestring, int, int, basestring], - 'UserDoesNotExists' : [basestring], -} - -methods = { - 'addFromCollector': int, - 'addLinks': None, - 'addLocalFile': None, - 'addPackage': int, - 'addPackageChild': int, - 'addPackageP': int, - 'addToCollector': None, - 'addUser': UserData, - 'callAddon': None, - 'callAddonHandler': None, - 'checkOnlineStatus': OnlineCheck, - 'checkOnlineStatusContainer': OnlineCheck, - 'checkURLs': (dict, basestring, list), - 'createPackage': int, - 'deleteCollLink': None, - 'deleteCollPack': None, - 'deleteConfig': None, - 'deleteFiles': None, - 'deletePackages': None, - 'findFiles': TreeCollection, - 'findPackages': TreeCollection, - 'freeSpace': int, - 'generateAndAddPackages': (list, int), - 'generateDownloadLink': basestring, - 'generatePackages': (dict, basestring, list), - 'getAccountTypes': (list, basestring), - 'getAccounts': (list, AccountInfo), - 'getAddonHandler': (dict, basestring, list), - 'getAllFiles': TreeCollection, - 'getAllUserData': (dict, int, UserData), - 'getAvailablePlugins': (list, ConfigInfo), - 'getCollector': (list, LinkStatus), - 'getConfig': (dict, basestring, ConfigHolder), - 'getConfigValue': basestring, - 'getCoreConfig': (list, ConfigInfo), - 'getFileInfo': FileInfo, - 'getFileTree': TreeCollection, - 'getFilteredFileTree': TreeCollection, - 'getFilteredFiles': TreeCollection, - 'getInteractionTasks': (list, InteractionTask), - 'getLog': (list, basestring), - 'getPackageContent': TreeCollection, - 'getPackageInfo': PackageInfo, - 'getPluginConfig': (list, ConfigInfo), - 'getProgressInfo': (list, ProgressInfo), - 'getServerStatus': ServerStatus, - 'getServerVersion': basestring, - 'getUserData': UserData, - 'getWSAddress': basestring, - 'hasAddonHandler': bool, - 'isInteractionWaiting': bool, - 'loadConfig': ConfigHolder, - 'login': bool, - 'moveFiles': bool, - 'movePackage': bool, - 'orderFiles': None, - 'orderPackage': None, - 'parseURLs': (dict, basestring, list), - 'pauseServer': None, - 'pollResults': OnlineCheck, - 'quit': None, - 'recheckPackage': None, - 'removeAccount': None, - 'removeUser': None, - 'renameCollPack': None, - 'restart': None, - 'restartFailed': None, - 'restartFile': None, - 'restartPackage': None, - 'saveConfig': None, - 'searchSuggestions': (list, basestring), - 'setConfigValue': None, - 'setInteractionResult': None, - 'setPackageFolder': bool, - 'setPassword': bool, - 'stopAllDownloads': None, - 'stopDownloads': None, - 'togglePause': bool, - 'toggleReconnect': bool, - 'unpauseServer': None, - 'updateAccount': None, - 'updateAccountInfo': None, - 'updatePackage': None, - 'updateUserData': None, - 'uploadContainer': int, -} diff --git a/module/remote/create_apitypes.py b/module/remote/create_apitypes.py deleted file mode 100644 index d596f07ac..000000000 --- a/module/remote/create_apitypes.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -import inspect -from os.path import abspath, dirname, join - -path = dirname(abspath(__file__)) -root = abspath(join(path, "..", "..")) - -from thrift.Thrift import TType -from thriftgen.pyload import ttypes -from thriftgen.pyload import Pyload - -# TODO: import and add version -# from pyload import CURRENT_VERSION - -type_map = { - TType.BOOL: 'bool', - TType.DOUBLE: 'float', - TType.I16: 'int', - TType.I32: 'int', - TType.I64: 'int', - TType.STRING: 'basestring', - TType.MAP: 'dict', - TType.LIST: 'list', - TType.SET: 'set', - TType.VOID: 'None', - TType.STRUCT: 'BaseObject', - TType.UTF8: 'unicode', -} - -def get_spec(spec, optional=False): - """ analyze the generated spec file and writes information into file """ - if spec[1] == TType.STRUCT: - return spec[3][0].__name__ - elif spec[1] == TType.LIST: - if spec[3][0] == TType.STRUCT: - ttype = spec[3][1][0].__name__ - else: - ttype = type_map[spec[3][0]] - return "(list, %s)" % ttype - elif spec[1] == TType.MAP: - if spec[3][2] == TType.STRUCT: - ttype = spec[3][3][0].__name__ - else: - ttype = type_map[spec[3][2]] - - return "(dict, %s, %s)" % (type_map[spec[3][0]], ttype) - else: - return type_map[spec[1]] - -optional_re = "%d: +optional +[a-z0-9<>_-]+ +%s" - -def main(): - - enums = [] - classes = [] - tf = open(join(path, "pyload.thrift"), "rb").read() - - print "generating apitypes.py" - - for name in dir(ttypes): - klass = getattr(ttypes, name) - - if name in ("TBase", "TExceptionBase") or name.startswith("_") or not (issubclass(klass, ttypes.TBase) or issubclass(klass, ttypes.TExceptionBase)): - continue - - if hasattr(klass, "thrift_spec"): - classes.append(klass) - else: - enums.append(klass) - - - f = open(join(path, "apitypes.py"), "wb") - f.write( - """#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -class BaseObject(object): -\t__slots__ = [] - -\tdef __str__(self): -\t\treturn "<%s %s>" % (self.__class__.__name__, ", ".join("%s=%s" % (k,getattr(self,k)) for k in self.__slots__)) - -class ExceptionObject(Exception): -\t__slots__ = [] - -""") - - dev = open(join(path, "apitypes_debug.py"), "wb") - dev.write("""#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n -from ttypes import *\n -""") - - dev.write("enums = [\n") - - ## generate enums - for enum in enums: - name = enum.__name__ - f.write("class %s:\n" % name) - - for attr in sorted(dir(enum), key=lambda x: getattr(enum, x)): - if attr.startswith("_") or attr in ("read", "write"): continue - f.write("\t%s = %s\n" % (attr, getattr(enum, attr))) - - dev.write('\t"%s",\n' % name) - f.write("\n") - - dev.write("]\n\n") - - dev.write("classes = {\n") - - for klass in classes: - name = klass.__name__ - base = "ExceptionObject" if issubclass(klass, ttypes.TExceptionBase) else "BaseObject" - f.write("class %s(%s):\n" % (name, base)) - - # No attributes, don't write further info - if not klass.__slots__: - f.write("\tpass\n\n") - continue - - f.write("\t__slots__ = %s\n\n" % klass.__slots__) - dev.write("\t'%s' : [" % name) - - #create init - args = ["self"] + ["%s=None" % x for x in klass.__slots__] - specs = [] - - f.write("\tdef __init__(%s):\n" % ", ".join(args)) - for i, attr in enumerate(klass.__slots__): - f.write("\t\tself.%s = %s\n" % (attr, attr)) - - spec = klass.thrift_spec[i+1] - # assert correct order, so the list of types is enough for check - assert spec[2] == attr - # dirty way to check optional attribute, since it is not in the generated code - # can produce false positives, but these are not critical - optional = re.search(optional_re % (i+1, attr), tf, re.I) - if optional: - specs.append("(None, %s)" % get_spec(spec)) - else: - specs.append(get_spec(spec)) - - f.write("\n") - dev.write(", ".join(specs) + "],\n") - - dev.write("}\n\n") - - f.write("class Iface(object):\n") - dev.write("methods = {\n") - - for name in dir(Pyload.Iface): - if name.startswith("_"): continue - - func = inspect.getargspec(getattr(Pyload.Iface, name)) - - f.write("\tdef %s(%s):\n\t\tpass\n" % (name, ", ".join(func.args))) - - spec = getattr(Pyload, "%s_result" % name).thrift_spec - if not spec or not spec[0]: - dev.write("\t'%s': None,\n" % name) - else: - spec = spec[0] - dev.write("\t'%s': %s,\n" % (name, get_spec(spec))) - - f.write("\n") - dev.write("}\n") - - f.close() - dev.close() - -if __name__ == "__main__": - main()
\ No newline at end of file diff --git a/module/remote/create_jstypes.py b/module/remote/create_jstypes.py deleted file mode 100644 index 90afa4c96..000000000 --- a/module/remote/create_jstypes.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from os.path import abspath, dirname, join - -path = dirname(abspath(__file__)) -module = join(path, "..") - -import apitypes -from apitypes_debug import enums - -# generate js enums -def main(): - - print "generating apitypes.js" - - f = open(join(module, 'web', 'app', 'scripts', 'utils', 'apitypes.js'), 'wb') - f.write("""// Autogenerated, do not edit! -/*jslint -W070: false*/ -define([], function() { -\t'use strict'; -\treturn { -""") - - for name in enums: - enum = getattr(apitypes, name) - values = dict([(attr, getattr(enum, attr)) for attr in dir(enum) if not attr.startswith("_")]) - - f.write("\t\t%s: %s,\n" % (name, str(values))) - - f.write("\t};\n});") - f.close() - - -if __name__ == "__main__": - main() diff --git a/module/remote/json_converter.py b/module/remote/json_converter.py deleted file mode 100644 index 50f0309bd..000000000 --- a/module/remote/json_converter.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -try: - from module.common.json_layer import json -except ImportError: - import json - - -import apitypes -from apitypes import BaseObject -from apitypes import ExceptionObject - -# compact json separator -separators = (',', ':') - -# json encoder that accepts api objects -class BaseEncoder(json.JSONEncoder): - - def default(self, o): - if isinstance(o, BaseObject) or isinstance(o, ExceptionObject): - ret = {"@class" : o.__class__.__name__} - for att in o.__slots__: - ret[att] = getattr(o, att) - return ret - - return json.JSONEncoder.default(self, o) - -# more compact representation, only clients with information of the classes can handle it -class BaseEncoderCompact(json.JSONEncoder): - - def default(self, o): - if isinstance(o, BaseObject) or isinstance(o, ExceptionObject): - ret = {"@compact" : [o.__class__.__name__]} - ret["@compact"].extend(getattr(o, attr) for attr in o.__slots__) - return ret - - return json.JSONEncoder.default(self, o) - -def convert_obj(dct): - if '@class' in dct: - cls = getattr(apitypes, dct['@class']) - del dct['@class'] - return cls(**dct) - elif '@compact' in dct: - cls = getattr(apitypes, dct['@compact'][0]) - return cls(*dct['@compact'][1:]) - - return dct - -def dumps(*args, **kwargs): - if 'compact' in kwargs: - kwargs['cls'] = BaseEncoderCompact - del kwargs['compact'] - else: - kwargs['cls'] = BaseEncoder - - kwargs['separators'] = separators - return json.dumps(*args, **kwargs) - - -def loads(*args, **kwargs): - kwargs['object_hook'] = convert_obj - return json.loads(*args, **kwargs)
\ No newline at end of file diff --git a/module/remote/pyload.thrift b/module/remote/pyload.thrift deleted file mode 100644 index 57d7e0a0a..000000000 --- a/module/remote/pyload.thrift +++ /dev/null @@ -1,538 +0,0 @@ -namespace java org.pyload.thrift - -typedef i32 FileID -typedef i32 PackageID -typedef i32 ResultID -typedef i32 InteractionID -typedef i32 UserID -typedef i64 UTCDate -typedef i64 ByteCount -typedef list<string> LinkList -typedef string PluginName -typedef string JSONString - -// NA - Not Available -enum DownloadStatus { - NA, - Offline, - Online, - Queued, - Paused, - Finished, - Skipped, - Failed, - Starting, - Waiting, - Downloading, - TempOffline, - Aborted, - Decrypting, - Processing, - Custom, - Unknown -} - -// Download states, combination of several downloadstatuses -// defined in Api -enum DownloadState { - All, - Finished, - Unfinished, - Failed, - Unmanaged // internal state -} - -enum MediaType { - All = 0 - Other = 1, - Audio = 2, - Image = 4, - Video = 8, - Document = 16, - Archive = 32, -} - -enum FileStatus { - Ok, - Missing, - Remote, // file is available at remote location -} - -enum PackageStatus { - Ok, - Paused, - Folder, - Remote, -} - -// types for user interaction -// some may only be place holder currently not supported -// also all input - output combination are not reasonable, see InteractionManager for further info -// Todo: how about: time, ip, s.o. -enum InputType { - NA, - Text, - Int, - File, - Folder, - Textbox, - Password, - Bool, // confirm like, yes or no dialog - Click, // for positional captchas - Select, // select from list - Multiple, // multiple choice from list of elements - List, // arbitary list of elements - Table // table like data structure -} -// more can be implemented by need - -// this describes the type of the outgoing interaction -// ensure they can be logcial or'ed -enum Interaction { - All = 0, - Notification = 1, - Captcha = 2, - Query = 4, -} - -enum Permission { - All = 0, // requires no permission, but login - Add = 1, // can add packages - Delete = 2, // can delete packages - Modify = 4, // modify some attribute of downloads - Download = 8, // can download from webinterface - Accounts = 16, // can access accounts - Interaction = 32, // can interact with plugins - Plugins = 64 // user can configure plugins and activate addons -} - -enum Role { - Admin = 0, //admin has all permissions implicit - User = 1 -} - -struct Input { - 1: InputType type, - 2: optional JSONString data, -} - -struct DownloadProgress { - 1: FileID fid, - 2: PackageID pid, - 3: ByteCount speed, // per second - 4: DownloadStatus status, -} - -struct ProgressInfo { - 1: PluginName plugin, - 2: string name, - 3: string statusmsg, - 4: i32 eta, // in seconds - 5: ByteCount done, - 6: ByteCount total, // arbitary number, size in case of files - 7: optional DownloadProgress download -} - -// download info for specific file -struct DownloadInfo { - 1: string url, - 2: PluginName plugin, - 3: string hash, - 4: DownloadStatus status, - 5: string statusmsg, - 6: string error, -} - -struct FileInfo { - 1: FileID fid, - 2: string name, - 3: PackageID package, - 4: UserID owner, - 5: ByteCount size, - 6: FileStatus status, - 7: MediaType media, - 8: UTCDate added, - 9: i16 fileorder, - 10: optional DownloadInfo download, -} - -struct PackageStats { - 1: i16 linkstotal, - 2: i16 linksdone, - 3: ByteCount sizetotal, - 4: ByteCount sizedone, -} - -struct PackageInfo { - 1: PackageID pid, - 2: string name, - 3: string folder, - 4: PackageID root, - 5: UserID owner, - 6: string site, - 7: string comment, - 8: string password, - 9: UTCDate added, - 10: list<string> tags, - 11: PackageStatus status, - 12: bool shared, - 13: i16 packageorder, - 14: PackageStats stats, - 15: list<FileID> fids, - 16: list<PackageID> pids, -} - -// thrift does not allow recursive datatypes, so all data is accumulated and mapped with id -struct TreeCollection { - 1: PackageInfo root, - 2: map<FileID, FileInfo> files, - 3: map<PackageID, PackageInfo> packages -} - -// general info about link, used for collector and online results -struct LinkStatus { - 1: string url, - 2: string name, - 3: PluginName plugin, - 4: ByteCount size, // size <= 0 : unknown - 5: DownloadStatus status, - 6: string packagename, -} - -struct ServerStatus { - 1: ByteCount speed, - 2: i16 linkstotal, - 3: i16 linksqueue, - 4: ByteCount sizetotal, - 5: ByteCount sizequeue, - 6: bool notifications, - 7: bool paused, - 8: bool download, - 9: bool reconnect, -} - -struct InteractionTask { - 1: InteractionID iid, - 2: Interaction type, - 3: Input input, - 4: optional JSONString default_value, - 5: string title, - 6: string description, - 7: PluginName plugin, -} - -struct AddonService { - 1: string func_name, - 2: string description, - 3: list<string> arguments, - 4: optional i16 media, -} - -struct AddonInfo { - 1: string func_name, - 2: string description, - 3: JSONString value, -} - -struct ConfigItem { - 1: string name, - 2: string label, - 3: string description, - 4: Input input, - 5: JSONString default_value, - 6: JSONString value, -} - -struct ConfigHolder { - 1: string name, // for plugin this is the PluginName - 2: string label, - 3: string description, - 4: string long_description, - 5: list<ConfigItem> items, - 6: optional list<AddonInfo> info, -} - -struct ConfigInfo { - 1: string name - 2: string label, - 3: string description, - 4: string category, - 5: bool user_context, - 6: optional bool activated, -} - -struct EventInfo { - 1: string eventname, - 2: list<JSONString> event_args, //will contain json objects -} - -struct UserData { - 1: UserID uid, - 2: string name, - 3: string email, - 4: i16 role, - 5: i16 permission, - 6: string folder, - 7: ByteCount traffic - 8: i16 dllimit - 9: string dlquota, - 10: ByteCount hddquota, - 11: UserID user, - 12: string templateName -} - -struct AccountInfo { - 1: PluginName plugin, - 2: string loginname, - 3: UserID owner, - 4: bool valid, - 5: UTCDate validuntil, - 6: ByteCount trafficleft, - 7: ByteCount maxtraffic, - 8: bool premium, - 9: bool activated, - 10: bool shared, - 11: map<string, string> options, -} - -struct OnlineCheck { - 1: ResultID rid, // -1 -> nothing more to get - 2: map<string, LinkStatus> data, // url to result -} - -// exceptions - -exception PackageDoesNotExists { - 1: PackageID pid -} - -exception FileDoesNotExists { - 1: FileID fid -} - -exception UserDoesNotExists { - 1: string user -} - -exception ServiceDoesNotExists { - 1: string plugin - 2: string func -} - -exception ServiceException { - 1: string msg -} - -exception InvalidConfigSection { - 1: string section -} - -exception Unauthorized { -} - -exception Forbidden { -} - - -service Pyload { - - /////////////////////// - // Core Status - /////////////////////// - - string getServerVersion(), - string getWSAddress(), - ServerStatus getServerStatus(), - list<ProgressInfo> getProgressInfo(), - - list<string> getLog(1: i32 offset), - ByteCount freeSpace(), - - void pauseServer(), - void unpauseServer(), - bool togglePause(), - bool toggleReconnect(), - - void quit(), - void restart(), - - /////////////////////// - // Configuration - /////////////////////// - - map<string, ConfigHolder> getConfig(), - string getConfigValue(1: string section, 2: string option), - - // two methods with ambigous classification, could be configuration or addon/plugin related - list<ConfigInfo> getCoreConfig(), - list<ConfigInfo> getPluginConfig(), - list<ConfigInfo> getAvailablePlugins(), - - ConfigHolder loadConfig(1: string name), - - void setConfigValue(1: string section, 2: string option, 3: string value), - void saveConfig(1: ConfigHolder config), - void deleteConfig(1: PluginName plugin), - - /////////////////////// - // Download Preparing - /////////////////////// - - map<PluginName, LinkList> checkURLs(1: LinkList urls), - map<PluginName, LinkList> parseURLs(1: string html, 2: string url), - - // parses results and generates packages - OnlineCheck checkOnlineStatus(1: LinkList urls), - OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) - - // poll results from previously started online check - OnlineCheck pollResults(1: ResultID rid), - - // packagename -> urls - map<string, LinkList> generatePackages(1: LinkList links), - - /////////////////////// - // Download - /////////////////////// - - list<PackageID> generateAndAddPackages(1: LinkList links, 2: bool paused), - - PackageID createPackage(1: string name, 2: string folder, 3: PackageID root, 4: string password, - 5: string site, 6: string comment, 7: bool paused), - - PackageID addPackage(1: string name, 2: LinkList links, 3: string password), - // same as above with paused attribute - PackageID addPackageP(1: string name, 2: LinkList links, 3: string password, 4: bool paused), - - // pid -1 is toplevel - PackageID addPackageChild(1: string name, 2: LinkList links, 3: string password, 4: PackageID root, 5: bool paused), - - PackageID uploadContainer(1: string filename, 2: binary data), - - void addLinks(1: PackageID pid, 2: LinkList links) throws (1: PackageDoesNotExists e), - void addLocalFile(1: PackageID pid, 2: string name, 3: string path) throws (1: PackageDoesNotExists e) - - // these are real file operations and WILL delete files on disk - void deleteFiles(1: list<FileID> fids), - void deletePackages(1: list<PackageID> pids), // delete the whole folder recursive - - // Modify Downloads - - void restartPackage(1: PackageID pid), - void restartFile(1: FileID fid), - void recheckPackage(1: PackageID pid), - void restartFailed(), - void stopDownloads(1: list<FileID> fids), - void stopAllDownloads(), - - /////////////////////// - // Collector - /////////////////////// - - list<LinkStatus> getCollector(), - - void addToCollector(1: LinkList links), - PackageID addFromCollector(1: string name, 2: bool paused), - void renameCollPack(1: string name, 2: string new_name), - void deleteCollPack(1: string name), - void deleteCollLink(1: string url), - - //////////////////////////// - // File Information retrieval - //////////////////////////// - - TreeCollection getAllFiles(), - TreeCollection getFilteredFiles(1: DownloadState state), - - // pid -1 for root, full=False only delivers first level in tree - TreeCollection getFileTree(1: PackageID pid, 2: bool full), - TreeCollection getFilteredFileTree(1: PackageID pid, 2: bool full, 3: DownloadState state), - - // same as above with full=False - TreeCollection getPackageContent(1: PackageID pid), - - PackageInfo getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), - FileInfo getFileInfo(1: FileID fid) throws (1: FileDoesNotExists e), - - TreeCollection findFiles(1: string pattern), - TreeCollection findPackages(1: list<string> tags), - list<string> searchSuggestions(1: string pattern), - - // Modify Files/Packages - - // moving package while downloading is not possible, so they will return bool to indicate success - void updatePackage(1: PackageInfo pack) throws (1: PackageDoesNotExists e), - bool setPackageFolder(1: PackageID pid, 2: string path) throws (1: PackageDoesNotExists e), - - // as above, this will move files on disk - bool movePackage(1: PackageID pid, 2: PackageID root) throws (1: PackageDoesNotExists e), - bool moveFiles(1: list<FileID> fids, 2: PackageID pid) throws (1: PackageDoesNotExists e), - - void orderPackage(1: list<PackageID> pids, 2: i16 position), - void orderFiles(1: list<FileID> fids, 2: PackageID pid, 3: i16 position), - - /////////////////////// - // User Interaction - /////////////////////// - - // mode = interaction types binary ORed - bool isInteractionWaiting(1: i16 mode), - list<InteractionTask> getInteractionTasks(1: i16 mode), - void setInteractionResult(1: InteractionID iid, 2: JSONString result), - - // generate a download link, everybody can download the file until timeout reached - string generateDownloadLink(1: FileID fid, 2: i16 timeout), - - /////////////////////// - // Account Methods - /////////////////////// - - list<AccountInfo> getAccounts(1: bool refresh), - list<string> getAccountTypes(), - void updateAccount(1: PluginName plugin, 2: string login, 3: string password), - void updateAccountInfo(1: AccountInfo account), - void removeAccount(1: AccountInfo account), - - ///////////////////////// - // Auth+User Information - ///////////////////////// - - bool login(1: string username, 2: string password), - // returns own user data - UserData getUserData(), - - // all user, for admins only - map<UserID, UserData> getAllUserData(), - - UserData addUser(1: string username, 2:string password), - - // normal user can only update their own userdata and not all attributes - void updateUserData(1: UserData data), - void removeUser(1: UserID uid), - - // works contextual, admin can change every password - bool setPassword(1: string username, 2: string old_password, 3: string new_password), - - /////////////////////// - // Addon Methods - /////////////////////// - - //map<PluginName, list<AddonInfo>> getAllInfo(), - //list<AddonInfo> getInfoByPlugin(1: PluginName plugin), - - map<PluginName, list<AddonService>> getAddonHandler(), - bool hasAddonHandler(1: PluginName plugin, 2: string func), - - void callAddon(1: PluginName plugin, 2: string func, 3: list<JSONString> arguments) - throws (1: ServiceDoesNotExists e, 2: ServiceException ex), - - // special variant of callAddon that works on the media types, acccepting integer - void callAddonHandler(1: PluginName plugin, 2: string func, 3: PackageID pid_or_fid) - throws (1: ServiceDoesNotExists e, 2: ServiceException ex), - - - //scheduler - - // TODO - -} diff --git a/module/remote/ttypes.py b/module/remote/ttypes.py deleted file mode 100644 index 1f91403d5..000000000 --- a/module/remote/ttypes.py +++ /dev/null @@ -1,534 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -class BaseObject(object): - __slots__ = [] - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, ", ".join("%s=%s" % (k,getattr(self,k)) for k in self.__slots__)) - -class ExceptionObject(Exception): - __slots__ = [] - -class DownloadState: - All = 0 - Finished = 1 - Unfinished = 2 - Failed = 3 - Unmanaged = 4 - -class DownloadStatus: - NA = 0 - Offline = 1 - Online = 2 - Queued = 3 - Paused = 4 - Finished = 5 - Skipped = 6 - Failed = 7 - Starting = 8 - Waiting = 9 - Downloading = 10 - TempOffline = 11 - Aborted = 12 - Decrypting = 13 - Processing = 14 - Custom = 15 - Unknown = 16 - -class FileStatus: - Ok = 0 - Missing = 1 - Remote = 2 - -class Input: - NA = 0 - Text = 1 - Int = 2 - File = 3 - Folder = 4 - Textbox = 5 - Password = 6 - Bool = 7 - Click = 8 - Select = 9 - Multiple = 10 - List = 11 - Table = 12 - -class MediaType: - All = 0 - Other = 1 - Audio = 2 - Image = 4 - Video = 8 - Document = 16 - Archive = 32 - -class Output: - All = 0 - Notification = 1 - Captcha = 2 - Query = 4 - -class PackageStatus: - Ok = 0 - Paused = 1 - Folder = 2 - Remote = 3 - -class Permission: - All = 0 - Add = 1 - Delete = 2 - Modify = 4 - Download = 8 - Accounts = 16 - Interaction = 32 - Plugins = 64 - -class Role: - Admin = 0 - User = 1 - -class AccountInfo(BaseObject): - __slots__ = ['plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'options'] - - def __init__(self, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, options=None): - self.plugin = plugin - self.loginname = loginname - self.owner = owner - self.valid = valid - self.validuntil = validuntil - self.trafficleft = trafficleft - self.maxtraffic = maxtraffic - self.premium = premium - self.activated = activated - self.shared = shared - self.options = options - -class AddonInfo(BaseObject): - __slots__ = ['func_name', 'description', 'value'] - - def __init__(self, func_name=None, description=None, value=None): - self.func_name = func_name - self.description = description - self.value = value - -class AddonService(BaseObject): - __slots__ = ['func_name', 'description', 'arguments', 'media'] - - def __init__(self, func_name=None, description=None, arguments=None, media=None): - self.func_name = func_name - self.description = description - self.arguments = arguments - self.media = media - -class ConfigHolder(BaseObject): - __slots__ = ['name', 'label', 'description', 'long_description', 'items', 'info', 'handler'] - - def __init__(self, name=None, label=None, description=None, long_description=None, items=None, info=None, handler=None): - self.name = name - self.label = label - self.description = description - self.long_description = long_description - self.items = items - self.info = info - self.handler = handler - -class ConfigInfo(BaseObject): - __slots__ = ['name', 'label', 'description', 'category', 'user_context', 'activated'] - - def __init__(self, name=None, label=None, description=None, category=None, user_context=None, activated=None): - self.name = name - self.label = label - self.description = description - self.category = category - self.user_context = user_context - self.activated = activated - -class ConfigItem(BaseObject): - __slots__ = ['name', 'label', 'description', 'type', 'default_value', 'value'] - - def __init__(self, name=None, label=None, description=None, type=None, default_value=None, value=None): - self.name = name - self.label = label - self.description = description - self.type = type - self.default_value = default_value - self.value = value - -class DownloadInfo(BaseObject): - __slots__ = ['url', 'plugin', 'hash', 'status', 'statusmsg', 'error'] - - def __init__(self, url=None, plugin=None, hash=None, status=None, statusmsg=None, error=None): - self.url = url - self.plugin = plugin - self.hash = hash - self.status = status - self.statusmsg = statusmsg - self.error = error - -class DownloadProgress(BaseObject): - __slots__ = ['fid', 'pid', 'speed', 'status'] - - def __init__(self, fid=None, pid=None, speed=None, status=None): - self.fid = fid - self.pid = pid - self.speed = speed - self.status = status - -class EventInfo(BaseObject): - __slots__ = ['eventname', 'event_args'] - - def __init__(self, eventname=None, event_args=None): - self.eventname = eventname - self.event_args = event_args - -class FileDoesNotExists(ExceptionObject): - __slots__ = ['fid'] - - def __init__(self, fid=None): - self.fid = fid - -class FileInfo(BaseObject): - __slots__ = ['fid', 'name', 'package', 'owner', 'size', 'status', 'media', 'added', 'fileorder', 'download'] - - def __init__(self, fid=None, name=None, package=None, owner=None, size=None, status=None, media=None, added=None, fileorder=None, download=None): - self.fid = fid - self.name = name - self.package = package - self.owner = owner - self.size = size - self.status = status - self.media = media - self.added = added - self.fileorder = fileorder - self.download = download - -class Forbidden(ExceptionObject): - pass - -class InteractionTask(BaseObject): - __slots__ = ['iid', 'input', 'data', 'output', 'default_value', 'title', 'description', 'plugin'] - - def __init__(self, iid=None, input=None, data=None, output=None, default_value=None, title=None, description=None, plugin=None): - self.iid = iid - self.input = input - self.data = data - self.output = output - self.default_value = default_value - self.title = title - self.description = description - self.plugin = plugin - -class InvalidConfigSection(ExceptionObject): - __slots__ = ['section'] - - def __init__(self, section=None): - self.section = section - -class LinkStatus(BaseObject): - __slots__ = ['url', 'name', 'plugin', 'size', 'status', 'packagename'] - - def __init__(self, url=None, name=None, plugin=None, size=None, status=None, packagename=None): - self.url = url - self.name = name - self.plugin = plugin - self.size = size - self.status = status - self.packagename = packagename - -class OnlineCheck(BaseObject): - __slots__ = ['rid', 'data'] - - def __init__(self, rid=None, data=None): - self.rid = rid - self.data = data - -class PackageDoesNotExists(ExceptionObject): - __slots__ = ['pid'] - - def __init__(self, pid=None): - self.pid = pid - -class PackageInfo(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'root', 'owner', 'site', 'comment', 'password', 'added', 'tags', 'status', 'shared', 'packageorder', 'stats', 'fids', 'pids'] - - def __init__(self, pid=None, name=None, folder=None, root=None, owner=None, site=None, comment=None, password=None, added=None, tags=None, status=None, shared=None, packageorder=None, stats=None, fids=None, pids=None): - self.pid = pid - self.name = name - self.folder = folder - self.root = root - self.owner = owner - self.site = site - self.comment = comment - self.password = password - self.added = added - self.tags = tags - self.status = status - self.shared = shared - self.packageorder = packageorder - self.stats = stats - self.fids = fids - self.pids = pids - -class PackageStats(BaseObject): - __slots__ = ['linkstotal', 'linksdone', 'sizetotal', 'sizedone'] - - def __init__(self, linkstotal=None, linksdone=None, sizetotal=None, sizedone=None): - self.linkstotal = linkstotal - self.linksdone = linksdone - self.sizetotal = sizetotal - self.sizedone = sizedone - -class ProgressInfo(BaseObject): - __slots__ = ['plugin', 'name', 'statusmsg', 'eta', 'done', 'total', 'download'] - - def __init__(self, plugin=None, name=None, statusmsg=None, eta=None, done=None, total=None, download=None): - self.plugin = plugin - self.name = name - self.statusmsg = statusmsg - self.eta = eta - self.done = done - self.total = total - self.download = download - -class ServerStatus(BaseObject): - __slots__ = ['queuedDownloads', 'totalDownloads', 'speed', 'pause', 'download', 'reconnect'] - - def __init__(self, queuedDownloads=None, totalDownloads=None, speed=None, pause=None, download=None, reconnect=None): - self.queuedDownloads = queuedDownloads - self.totalDownloads = totalDownloads - self.speed = speed - self.pause = pause - self.download = download - self.reconnect = reconnect - -class ServiceDoesNotExists(ExceptionObject): - __slots__ = ['plugin', 'func'] - - def __init__(self, plugin=None, func=None): - self.plugin = plugin - self.func = func - -class ServiceException(ExceptionObject): - __slots__ = ['msg'] - - def __init__(self, msg=None): - self.msg = msg - -class TreeCollection(BaseObject): - __slots__ = ['root', 'files', 'packages'] - - def __init__(self, root=None, files=None, packages=None): - self.root = root - self.files = files - self.packages = packages - -class Unauthorized(ExceptionObject): - pass - -class UserData(BaseObject): - __slots__ = ['uid', 'name', 'email', 'role', 'permission', 'folder', 'traffic', 'dllimit', 'dlquota', 'hddquota', 'user', 'templateName'] - - def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, dllimit=None, dlquota=None, hddquota=None, user=None, templateName=None): - self.uid = uid - self.name = name - self.email = email - self.role = role - self.permission = permission - self.folder = folder - self.traffic = traffic - self.dllimit = dllimit - self.dlquota = dlquota - self.hddquota = hddquota - self.user = user - self.templateName = templateName - -class UserDoesNotExists(ExceptionObject): - __slots__ = ['user'] - - def __init__(self, user=None): - self.user = user - -class Iface(object): - def addFromCollector(self, name, paused): - pass - def addLinks(self, pid, links): - pass - def addLocalFile(self, pid, name, path): - pass - def addPackage(self, name, links, password): - pass - def addPackageChild(self, name, links, password, root, paused): - pass - def addPackageP(self, name, links, password, paused): - pass - def addToCollector(self, links): - pass - def addUser(self, username, password): - pass - def callAddon(self, plugin, func, arguments): - pass - def callAddonHandler(self, plugin, func, pid_or_fid): - pass - def checkOnlineStatus(self, urls): - pass - def checkOnlineStatusContainer(self, urls, filename, data): - pass - def checkURLs(self, urls): - pass - def configurePlugin(self, plugin): - pass - def createPackage(self, name, folder, root, password, site, comment, paused): - pass - def deleteCollLink(self, url): - pass - def deleteCollPack(self, name): - pass - def deleteConfig(self, plugin): - pass - def deleteFiles(self, fids): - pass - def deletePackages(self, pids): - pass - def findFiles(self, pattern): - pass - def findPackages(self, tags): - pass - def freeSpace(self): - pass - def generateAndAddPackages(self, links, paused): - pass - def generateDownloadLink(self, fid, timeout): - pass - def generatePackages(self, links): - pass - def getAccountTypes(self): - pass - def getAccounts(self, refresh): - pass - def getAddonHandler(self): - pass - def getAllFiles(self): - pass - def getAllUserData(self): - pass - def getAutocompletion(self, pattern): - pass - def getAvailablePlugins(self): - pass - def getCollector(self): - pass - def getConfig(self): - pass - def getConfigValue(self, section, option): - pass - def getCoreConfig(self): - pass - def getEvents(self, uuid): - pass - def getFileInfo(self, fid): - pass - def getFileTree(self, pid, full): - pass - def getFilteredFileTree(self, pid, full, state): - pass - def getFilteredFiles(self, state): - pass - def getInteractionTask(self, mode): - pass - def getLog(self, offset): - pass - def getNotifications(self): - pass - def getPackageContent(self, pid): - pass - def getPackageInfo(self, pid): - pass - def getPluginConfig(self): - pass - def getProgressInfo(self): - pass - def getServerStatus(self): - pass - def getServerVersion(self): - pass - def getUserData(self): - pass - def getWSAddress(self): - pass - def hasAddonHandler(self, plugin, func): - pass - def isInteractionWaiting(self, mode): - pass - def login(self, username, password): - pass - def moveFiles(self, fids, pid): - pass - def movePackage(self, pid, root): - pass - def orderFiles(self, fids, pid, position): - pass - def orderPackage(self, pids, position): - pass - def parseURLs(self, html, url): - pass - def pauseServer(self): - pass - def pollResults(self, rid): - pass - def quit(self): - pass - def recheckPackage(self, pid): - pass - def removeAccount(self, plugin, account): - pass - def removeUser(self, uid): - pass - def renameCollPack(self, name, new_name): - pass - def restart(self): - pass - def restartFailed(self): - pass - def restartFile(self, fid): - pass - def restartPackage(self, pid): - pass - def saveConfig(self, config): - pass - def setConfigHandler(self, plugin, iid, value): - pass - def setConfigValue(self, section, option, value): - pass - def setInteractionResult(self, iid, result): - pass - def setPackageFolder(self, pid, path): - pass - def setPassword(self, username, old_password, new_password): - pass - def stopAllDownloads(self): - pass - def stopDownloads(self, fids): - pass - def togglePause(self): - pass - def toggleReconnect(self): - pass - def unpauseServer(self): - pass - def updateAccount(self, plugin, account, password): - pass - def updateAccountInfo(self, account): - pass - def updatePackage(self, pack): - pass - def updateUserData(self, data): - pass - def uploadContainer(self, filename, data): - pass - diff --git a/module/remote/wsbackend/AbstractHandler.py b/module/remote/wsbackend/AbstractHandler.py deleted file mode 100644 index 07cc79c74..000000000 --- a/module/remote/wsbackend/AbstractHandler.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from mod_pywebsocket.msgutil import send_message -from mod_pywebsocket.util import get_class_logger -from module.remote.json_converter import loads, dumps - - -class AbstractHandler: - """ - Abstract Handler providing common methods shared across WebSocket handlers - """ - PATH = "/" - - OK = 200 - BAD_REQUEST = 400 - UNAUTHORIZED = 401 - FORBIDDEN = 403 - NOT_FOUND = 404 - ERROR = 500 - - def __init__(self, api): - self.log = get_class_logger() - self.api = api - self.core = api.core - - def do_extra_handshake(self, req): - self.log.debug("WS Connected: %s" % req) - req.api = None #when api is set client is logged in - - # allow login via session when webinterface is active - if self.core.config['webinterface']['activated']: - cookie = req.headers_in.getheader('Cookie') - s = self.load_session(cookie) - if s: - uid = s.get('uid', None) - req.api = self.api.withUserContext(uid) - self.log.debug("WS authenticated user with cookie: %d" % uid) - - self.on_open(req) - - def on_open(self, req): - pass - - def load_session(self, cookies): - from Cookie import SimpleCookie - from beaker.session import Session - from module.web.webinterface import session - - cookies = SimpleCookie(cookies) - sid = cookies.get(session.options['key']) - if not sid: - return None - - s = Session({}, use_cookies=False, id=sid.value, **session.options) - if s.is_new: - return None - - return s - - def passive_closing_handshake(self, req): - self.log.debug("WS Closed: %s" % req) - self.on_close(req) - - def on_close(self, req): - pass - - def transfer_data(self, req): - raise NotImplemented - - def handle_call(self, msg, req): - """ Parses the msg for an argument call. If func is null an response was already sent. - - :return: func, args, kwargs - """ - try: - o = loads(msg) - except ValueError, e: #invalid json object - self.log.debug("Invalid Request: %s" % e) - self.send_result(req, self.ERROR, "No JSON request") - return None, None, None - - if not isinstance(o, basestring) and type(o) != list and len(o) not in range(1, 4): - self.log.debug("Invalid Api call: %s" % o) - self.send_result(req, self.ERROR, "Invalid Api call") - return None, None, None - - # called only with name, no args - if isinstance(o, basestring): - return o, [], {} - elif len(o) == 1: # arguments omitted - return o[0], [], {} - elif len(o) == 2: - func, args = o - if type(args) == list: - return func, args, {} - else: - return func, [], args - else: - return tuple(o) - - def do_login(self, req, args, kwargs): - user = self.api.checkAuth(*args, **kwargs) - if user: - req.api = self.api.withUserContext(user.uid) - return self.send_result(req, self.OK, True) - else: - return self.send_result(req, self.FORBIDDEN, "Forbidden") - - def do_logout(self, req): - req.api = None - return self.send_result(req, self.OK, True) - - def send_result(self, req, code, result): - return send_message(req, dumps([code, result])) - - def send(self, req, obj): - return send_message(req, dumps(obj))
\ No newline at end of file diff --git a/module/remote/wsbackend/ApiHandler.py b/module/remote/wsbackend/ApiHandler.py deleted file mode 100644 index e985e10be..000000000 --- a/module/remote/wsbackend/ApiHandler.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from mod_pywebsocket.msgutil import receive_message - -from module.Api import ExceptionObject - -from AbstractHandler import AbstractHandler - -class ApiHandler(AbstractHandler): - """Provides access to the API. - - Send your request as json encoded string in the following manner: - ["function", [*args]] or ["function", {**kwargs}] - - the result will be: - - [code, result] - - Don't forget to login first. - Non json request will be ignored. - """ - - PATH = "/api" - - def transfer_data(self, req): - while True: - try: - line = receive_message(req) - except TypeError, e: # connection closed - self.log.debug("WS Error: %s" % e) - return self.passive_closing_handshake(req) - - self.handle_message(line, req) - - def handle_message(self, msg, req): - - func, args, kwargs = self.handle_call(msg, req) - if not func: - return # handle_call already sent the result - - if func == 'login': - return self.do_login(req, args, kwargs) - elif func == 'logout': - return self.do_logout(req) - else: - if not req.api: - return self.send_result(req, self.FORBIDDEN, "Forbidden") - - if not self.api.isAuthorized(func, req.api.user): - return self.send_result(req, self.UNAUTHORIZED, "Unauthorized") - - try: - result = getattr(req.api, func)(*args, **kwargs) - except ExceptionObject, e: - return self.send_result(req, self.BAD_REQUEST, e) - except AttributeError: - return self.send_result(req, self.NOT_FOUND, "Not Found") - except Exception, e: - self.core.print_exc() - return self.send_result(req, self.ERROR, str(e)) - - # None is invalid json type - if result is None: result = True - - return self.send_result(req, self.OK, result)
\ No newline at end of file diff --git a/module/remote/wsbackend/AsyncHandler.py b/module/remote/wsbackend/AsyncHandler.py deleted file mode 100644 index 158033ee8..000000000 --- a/module/remote/wsbackend/AsyncHandler.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -import re -from Queue import Queue, Empty -from threading import Lock -from time import time - -from mod_pywebsocket.msgutil import receive_message - -from module.Api import EventInfo, Interaction -from module.utils import lock -from AbstractHandler import AbstractHandler - -class Mode: - STANDBY = 1 - RUNNING = 2 - -class AsyncHandler(AbstractHandler): - """ - Handler that provides asynchronous information about server status, running downloads, occurred events. - - Progress information are continuous and will be pushed in a fixed interval when available. - After connect you have to login and can set the interval by sending the json command ["setInterval", xy]. - To start receiving updates call "start", afterwards no more incoming messages will be accepted! - """ - - PATH = "/async" - COMMAND = "start" - - PROGRESS_INTERVAL = 2 - EVENT_PATTERN = re.compile(r"^(package|file|interaction)", re.I) - INTERACTION = Interaction.All - - def __init__(self, api): - AbstractHandler.__init__(self, api) - self.clients = [] - self.lock = Lock() - - self.core.evm.addEvent("event", self.add_event) - - @lock - def on_open(self, req): - req.queue = Queue() - req.interval = self.PROGRESS_INTERVAL - req.events = self.EVENT_PATTERN - req.interaction = self.INTERACTION - req.mode = Mode.STANDBY - req.t = time() # time when update should be pushed - self.clients.append(req) - - @lock - def on_close(self, req): - try: - del req.queue - self.clients.remove(req) - except ValueError: # ignore when not in list - pass - - @lock - def add_event(self, event, *args): - # Convert arguments to json suited instance - event = EventInfo(event, [x.toInfoData() if hasattr(x, 'toInfoData') else x for x in args]) - - for req in self.clients: - # Not logged in yet - if not req.api: continue - - # filter events that these user is no owner of - # TODO: events are security critical, this should be revised later - # TODO: permissions? interaction etc - if not req.api.user.isAdmin(): - skip = False - for arg in args: - if hasattr(arg, 'owner') and arg.owner != req.api.primaryUID: - skip = True - break - - # user should not get this event - if skip: break - - if req.events.search(event.eventname): - self.log.debug("Pushing event %s" % event) - req.queue.put(event) - - def transfer_data(self, req): - while True: - - if req.mode == Mode.STANDBY: - try: - line = receive_message(req) - except TypeError, e: # connection closed - self.log.debug("WS Error: %s" % e) - return self.passive_closing_handshake(req) - - self.mode_standby(line, req) - else: - if self.mode_running(req): - return self.passive_closing_handshake(req) - - def mode_standby(self, msg, req): - """ accepts calls before pushing updates """ - func, args, kwargs = self.handle_call(msg, req) - if not func: - return # Result was already sent - - if func == 'login': - return self.do_login(req, args, kwargs) - - elif func == 'logout': - return self.do_logout(req) - - else: - if not req.api: - return self.send_result(req, self.FORBIDDEN, "Forbidden") - - if func == "setInterval": - req.interval = args[0] - elif func == "setEvents": - req.events = re.compile(args[0], re.I) - elif func == "setInteraction": - req.interaction = args[0] - elif func == self.COMMAND: - req.mode = Mode.RUNNING - - - def mode_running(self, req): - """ Listen for events, closes socket when returning True """ - try: - # block length of update interval if necessary - ev = req.queue.get(True, req.interval) - try: - self.send(req, ev) - except TypeError: - self.log.debug("Event %s not converted" % ev) - ev.event_args = [] - # Resend the event without arguments - self.send(req, ev) - - except Empty: - pass - - if req.t <= time(): - # TODO: server status is not enough - # modify core api to include progress? think of other needed information to show - # eta is quite wrong currently - # notifications - self.send(req, self.api.getServerStatus()) - self.send(req, self.api.getProgressInfo()) - - # update time for next update - req.t = time() + req.interval
\ No newline at end of file diff --git a/module/remote/wsbackend/Dispatcher.py b/module/remote/wsbackend/Dispatcher.py deleted file mode 100644 index 44cc7555e..000000000 --- a/module/remote/wsbackend/Dispatcher.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from mod_pywebsocket import util -from mod_pywebsocket.dispatch import Dispatcher as BaseDispatcher - -class Dispatcher(BaseDispatcher): - - def __init__(self): - self._logger = util.get_class_logger(self) - - self._handler_suite_map = {} - self._source_warnings = [] - - def addHandler(self, path, handler): - self._handler_suite_map[path] = handler
\ No newline at end of file diff --git a/module/remote/wsbackend/Server.py b/module/remote/wsbackend/Server.py deleted file mode 100644 index af5e1cf19..000000000 --- a/module/remote/wsbackend/Server.py +++ /dev/null @@ -1,733 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -# A copy of standalone.py with uneeded stuff removed -# some logging methods removed -# Added api attribute to request - -import BaseHTTPServer -import CGIHTTPServer -import SocketServer -import httplib -import logging -import os -import re -import select -import socket -import sys -import threading - -_HAS_SSL = False -_HAS_OPEN_SSL = False - -from mod_pywebsocket import common -from mod_pywebsocket import dispatch -from mod_pywebsocket import handshake -from mod_pywebsocket import http_header_util -from mod_pywebsocket import memorizingfile -from mod_pywebsocket import util - - -_DEFAULT_LOG_MAX_BYTES = 1024 * 256 -_DEFAULT_LOG_BACKUP_COUNT = 5 - -_DEFAULT_REQUEST_QUEUE_SIZE = 128 - -# 1024 is practically large enough to contain WebSocket handshake lines. -_MAX_MEMORIZED_LINES = 1024 - -def import_ssl(): - global _HAS_SSL, _HAS_OPEN_SSL - try: - import ssl - _HAS_SSL = True - except ImportError: - try: - import OpenSSL.SSL - _HAS_OPEN_SSL = True - except ImportError: - pass - - -class _StandaloneConnection(object): - """Mimic mod_python mp_conn.""" - - def __init__(self, request_handler): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._request_handler = request_handler - - def get_local_addr(self): - """Getter to mimic mp_conn.local_addr.""" - - return (self._request_handler.server.server_name, - self._request_handler.server.server_port) - local_addr = property(get_local_addr) - - def get_remote_addr(self): - """Getter to mimic mp_conn.remote_addr. - - Setting the property in __init__ won't work because the request - handler is not initialized yet there.""" - - return self._request_handler.client_address - remote_addr = property(get_remote_addr) - - def write(self, data): - """Mimic mp_conn.write().""" - - return self._request_handler.wfile.write(data) - - def read(self, length): - """Mimic mp_conn.read().""" - - return self._request_handler.rfile.read(length) - - def get_memorized_lines(self): - """Get memorized lines.""" - - return self._request_handler.rfile.get_memorized_lines() - - -class _StandaloneRequest(object): - """Mimic mod_python request.""" - - def __init__(self, request_handler, use_tls): - """Construct an instance. - - Args: - request_handler: A WebSocketRequestHandler instance. - """ - - self._logger = util.get_class_logger(self) - - self._request_handler = request_handler - self.connection = _StandaloneConnection(request_handler) - self._use_tls = use_tls - self.headers_in = request_handler.headers - - def get_uri(self): - """Getter to mimic request.uri.""" - - return self._request_handler.path - uri = property(get_uri) - - def get_method(self): - """Getter to mimic request.method.""" - - return self._request_handler.command - method = property(get_method) - - def get_protocol(self): - """Getter to mimic request.protocol.""" - - return self._request_handler.request_version - protocol = property(get_protocol) - - def is_https(self): - """Mimic request.is_https().""" - - return self._use_tls - - def _drain_received_data(self): - """Don't use this method from WebSocket handler. Drains unread data - in the receive buffer. - """ - - raw_socket = self._request_handler.connection - drained_data = util.drain_received_data(raw_socket) - - if drained_data: - self._logger.debug( - 'Drained data following close frame: %r', drained_data) - - -class _StandaloneSSLConnection(object): - """A wrapper class for OpenSSL.SSL.Connection to provide makefile method - which is not supported by the class. - """ - - def __init__(self, connection): - self._connection = connection - - def __getattribute__(self, name): - if name in ('_connection', 'makefile'): - return object.__getattribute__(self, name) - return self._connection.__getattribute__(name) - - def __setattr__(self, name, value): - if name in ('_connection', 'makefile'): - return object.__setattr__(self, name, value) - return self._connection.__setattr__(name, value) - - def makefile(self, mode='r', bufsize=-1): - return socket._fileobject(self._connection, mode, bufsize) - - -def _alias_handlers(dispatcher, websock_handlers_map_file): - """Set aliases specified in websock_handler_map_file in dispatcher. - - Args: - dispatcher: dispatch.Dispatcher instance - websock_handler_map_file: alias map file - """ - - fp = open(websock_handlers_map_file) - try: - for line in fp: - if line[0] == '#' or line.isspace(): - continue - m = re.match('(\S+)\s+(\S+)', line) - if not m: - logging.warning('Wrong format in map file:' + line) - continue - try: - dispatcher.add_resource_path_alias( - m.group(1), m.group(2)) - except dispatch.DispatchException, e: - logging.error(str(e)) - finally: - fp.close() - - -class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): - """HTTPServer specialized for WebSocket.""" - - # Overrides SocketServer.ThreadingMixIn.daemon_threads - daemon_threads = True - # Overrides BaseHTTPServer.HTTPServer.allow_reuse_address - allow_reuse_address = True - - def __init__(self, options): - """Override SocketServer.TCPServer.__init__ to set SSL enabled - socket object to self.socket before server_bind and server_activate, - if necessary. - """ - # Removed dispatcher init here - self._logger = logging.getLogger("log") - - self.request_queue_size = options.request_queue_size - self.__ws_is_shut_down = threading.Event() - self.__ws_serving = False - - SocketServer.BaseServer.__init__( - self, (options.server_host, options.port), WebSocketRequestHandler) - - # Expose the options object to allow handler objects access it. We name - # it with websocket_ prefix to avoid conflict. - self.websocket_server_options = options - - self._create_sockets() - self.server_bind() - self.server_activate() - - def _create_sockets(self): - self.server_name, self.server_port = self.server_address - self._sockets = [] - if not self.server_name: - # On platforms that doesn't support IPv6, the first bind fails. - # On platforms that supports IPv6 - # - If it binds both IPv4 and IPv6 on call with AF_INET6, the - # first bind succeeds and the second fails (we'll see 'Address - # already in use' error). - # - If it binds only IPv6 on call with AF_INET6, both call are - # expected to succeed to listen both protocol. - addrinfo_array = [ - (socket.AF_INET6, socket.SOCK_STREAM, '', '', ''), - (socket.AF_INET, socket.SOCK_STREAM, '', '', '')] - else: - addrinfo_array = socket.getaddrinfo(self.server_name, - self.server_port, - socket.AF_UNSPEC, - socket.SOCK_STREAM, - socket.IPPROTO_TCP) - for addrinfo in addrinfo_array: - family, socktype, proto, canonname, sockaddr = addrinfo - try: - socket_ = socket.socket(family, socktype) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - continue - if self.websocket_server_options.use_tls: - if _HAS_SSL: - if self.websocket_server_options.tls_client_auth: - client_cert_ = ssl.CERT_REQUIRED - else: - client_cert_ = ssl.CERT_NONE - socket_ = ssl.wrap_socket(socket_, - keyfile=self.websocket_server_options.private_key, - certfile=self.websocket_server_options.certificate, - ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.websocket_server_options.tls_client_ca, - cert_reqs=client_cert_) - if _HAS_OPEN_SSL: - ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - ctx.use_privatekey_file( - self.websocket_server_options.private_key) - ctx.use_certificate_file( - self.websocket_server_options.certificate) - socket_ = OpenSSL.SSL.Connection(ctx, socket_) - self._sockets.append((socket_, addrinfo)) - - def server_bind(self): - """Override SocketServer.TCPServer.server_bind to enable multiple - sockets bind. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - if self.allow_reuse_address: - socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - socket_.bind(self.server_address) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - if self.server_address[1] == 0: - # The operating system assigns the actual port number for port - # number 0. This case, the second and later sockets should use - # the same port number. Also self.server_port is rewritten - # because it is exported, and will be used by external code. - self.server_address = ( - self.server_name, socket_.getsockname()[1]) - self.server_port = self.server_address[1] - self._logger.info('Port %r is assigned', self.server_port) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - def server_activate(self): - """Override SocketServer.TCPServer.server_activate to enable multiple - sockets listen. - """ - - failed_sockets = [] - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.debug('Listen on: %r', addrinfo) - try: - socket_.listen(self.request_queue_size) - except Exception, e: - self._logger.info('Skip by failure: %r', e) - socket_.close() - failed_sockets.append(socketinfo) - - for socketinfo in failed_sockets: - self._sockets.remove(socketinfo) - - if len(self._sockets) == 0: - self._logger.critical( - 'No sockets activated. Use info log level to see the reason.') - - def server_close(self): - """Override SocketServer.TCPServer.server_close to enable multiple - sockets close. - """ - - for socketinfo in self._sockets: - socket_, addrinfo = socketinfo - self._logger.info('Close on: %r', addrinfo) - socket_.close() - - def fileno(self): - """Override SocketServer.TCPServer.fileno.""" - - self._logger.critical('Not supported: fileno') - return self._sockets[0][0].fileno() - - def handle_error(self, rquest, client_address): - """Override SocketServer.handle_error.""" - - self._logger.error( - 'Exception in processing request from: %r\n%s', - client_address, - util.get_stack_trace()) - # Note: client_address is a tuple. - - def get_request(self): - """Override TCPServer.get_request to wrap OpenSSL.SSL.Connection - object with _StandaloneSSLConnection to provide makefile method. We - cannot substitute OpenSSL.SSL.Connection.makefile since it's readonly - attribute. - """ - - accepted_socket, client_address = self.socket.accept() - if self.websocket_server_options.use_tls and _HAS_OPEN_SSL: - accepted_socket = _StandaloneSSLConnection(accepted_socket) - return accepted_socket, client_address - - def serve_forever(self, poll_interval=0.5): - """Override SocketServer.BaseServer.serve_forever.""" - - self.__ws_serving = True - self.__ws_is_shut_down.clear() - handle_request = self.handle_request - if hasattr(self, '_handle_request_noblock'): - handle_request = self._handle_request_noblock - else: - self._logger.warning('Fallback to blocking request handler') - try: - while self.__ws_serving: - r, w, e = select.select( - [socket_[0] for socket_ in self._sockets], - [], [], poll_interval) - for socket_ in r: - self.socket = socket_ - handle_request() - self.socket = None - finally: - self.__ws_is_shut_down.set() - - def shutdown(self): - """Override SocketServer.BaseServer.shutdown.""" - - self.__ws_serving = False - self.__ws_is_shut_down.wait() - - -class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): - """CGIHTTPRequestHandler specialized for WebSocket.""" - - # Use httplib.HTTPMessage instead of mimetools.Message. - MessageClass = httplib.HTTPMessage - - def setup(self): - """Override SocketServer.StreamRequestHandler.setup to wrap rfile - with MemorizingFile. - - This method will be called by BaseRequestHandler's constructor - before calling BaseHTTPRequestHandler.handle. - BaseHTTPRequestHandler.handle will call - BaseHTTPRequestHandler.handle_one_request and it will call - WebSocketRequestHandler.parse_request. - """ - - # Call superclass's setup to prepare rfile, wfile, etc. See setup - # definition on the root class SocketServer.StreamRequestHandler to - # understand what this does. - CGIHTTPServer.CGIHTTPRequestHandler.setup(self) - - self.rfile = memorizingfile.MemorizingFile( - self.rfile, - max_memorized_lines=_MAX_MEMORIZED_LINES) - - def __init__(self, request, client_address, server): - self._logger = util.get_class_logger(self) - - self._options = server.websocket_server_options - - # Overrides CGIHTTPServerRequestHandler.cgi_directories. - self.cgi_directories = self._options.cgi_directories - # Replace CGIHTTPRequestHandler.is_executable method. - if self._options.is_executable_method is not None: - self.is_executable = self._options.is_executable_method - - # OWN MODIFICATION - # This actually calls BaseRequestHandler.__init__. - try: - CGIHTTPServer.CGIHTTPRequestHandler.__init__( - self, request, client_address, server) - except socket.error, e: - # Broken pipe, let it pass - if e.errno != 32: - raise - self._logger.debug("WS: Broken pipe") - - - - def parse_request(self): - """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. - - Return True to continue processing for HTTP(S), False otherwise. - - See BaseHTTPRequestHandler.handle_one_request method which calls - this method to understand how the return value will be handled. - """ - - # We hook parse_request method, but also call the original - # CGIHTTPRequestHandler.parse_request since when we return False, - # CGIHTTPRequestHandler.handle_one_request continues processing and - # it needs variables set by CGIHTTPRequestHandler.parse_request. - # - # Variables set by this method will be also used by WebSocket request - # handling (self.path, self.command, self.requestline, etc. See also - # how _StandaloneRequest's members are implemented using these - # attributes). - if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): - return False - - if self._options.use_basic_auth: - auth = self.headers.getheader('Authorization') - if auth != self._options.basic_auth_credential: - self.send_response(401) - self.send_header('WWW-Authenticate', - 'Basic realm="Pywebsocket"') - self.end_headers() - self._logger.info('Request basic authentication') - return True - - host, port, resource = http_header_util.parse_uri(self.path) - if resource is None: - self._logger.info('Invalid URI: %r', self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - server_options = self.server.websocket_server_options - if host is not None: - validation_host = server_options.validation_host - if validation_host is not None and host != validation_host: - self._logger.info('Invalid host: %r (expected: %r)', - host, - validation_host) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - if port is not None: - validation_port = server_options.validation_port - if validation_port is not None and port != validation_port: - self._logger.info('Invalid port: %r (expected: %r)', - port, - validation_port) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - self.path = resource - - request = _StandaloneRequest(self, self._options.use_tls) - - try: - # Fallback to default http handler for request paths for which - # we don't have request handlers. - if not self._options.dispatcher.get_handler_suite(self.path): - self._logger.info('No handler for resource: %r', - self.path) - self._logger.info('Fallback to CGIHTTPRequestHandler') - return True - except dispatch.DispatchException, e: - self._logger.info('%s', e) - self.send_error(e.status) - return False - - # If any Exceptions without except clause setup (including - # DispatchException) is raised below this point, it will be caught - # and logged by WebSocketServer. - - try: - try: - handshake.do_handshake( - request, - self._options.dispatcher, - allowDraft75=self._options.allow_draft75, - strict=self._options.strict) - except handshake.VersionException, e: - self._logger.info('%s', e) - self.send_response(common.HTTP_STATUS_BAD_REQUEST) - self.send_header(common.SEC_WEBSOCKET_VERSION_HEADER, - e.supported_versions) - self.end_headers() - return False - except handshake.HandshakeException, e: - # Handshake for ws(s) failed. - self._logger.info('%s', e) - self.send_error(e.status) - return False - - request._dispatcher = self._options.dispatcher - self._options.dispatcher.transfer_data(request) - except handshake.AbortedByUserException, e: - self._logger.info('%s', e) - return False - - def log_request(self, code='-', size='-'): - """Override BaseHTTPServer.log_request.""" - - self._logger.info('"%s" %s %s', - self.requestline, str(code), str(size)) - - def log_error(self, *args): - """Override BaseHTTPServer.log_error.""" - - # Despite the name, this method is for warnings than for errors. - # For example, HTTP status code is logged by this method. - self._logger.warning('%s - %s', - self.address_string(), - args[0] % args[1:]) - - def is_cgi(self): - """Test whether self.path corresponds to a CGI script. - - Add extra check that self.path doesn't contains .. - Also check if the file is a executable file or not. - If the file is not executable, it is handled as static file or dir - rather than a CGI script. - """ - - if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): - if '..' in self.path: - return False - # strip query parameter from request path - resource_name = self.path.split('?', 2)[0] - # convert resource_name into real path name in filesystem. - scriptfile = self.translate_path(resource_name) - if not os.path.isfile(scriptfile): - return False - if not self.is_executable(scriptfile): - return False - return True - return False - - -def _get_logger_from_class(c): - return logging.getLogger('%s.%s' % (c.__module__, c.__name__)) - - -def _configure_logging(options): - logging.addLevelName(common.LOGLEVEL_FINE, 'FINE') - - logger = logging.getLogger() - logger.setLevel(logging.getLevelName(options.log_level.upper())) - if options.log_file: - handler = logging.handlers.RotatingFileHandler( - options.log_file, 'a', options.log_max, options.log_count) - else: - handler = logging.StreamHandler() - formatter = logging.Formatter( - '[%(asctime)s] [%(levelname)s] %(name)s: %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - - deflate_log_level_name = logging.getLevelName( - options.deflate_log_level.upper()) - _get_logger_from_class(util._Deflater).setLevel( - deflate_log_level_name) - _get_logger_from_class(util._Inflater).setLevel( - deflate_log_level_name) - -class DefaultOptions: - server_host = '' - port = common.DEFAULT_WEB_SOCKET_PORT - use_tls = False - private_key = '' - certificate = '' - ca_certificate = '' - dispatcher = None - request_queue_size = _DEFAULT_REQUEST_QUEUE_SIZE - use_basic_auth = False - - allow_draft75 = False - strict = False - validation_host = None - validation_port = None - cgi_directories = '' - is_executable_method = False - -def _main(args=None): - """You can call this function from your own program, but please note that - this function has some side-effects that might affect your program. For - example, util.wrap_popen3_for_win use in this method replaces implementation - of os.popen3. - """ - - options, args = _parse_args_and_config(args=args) - - os.chdir(options.document_root) - - _configure_logging(options) - - # TODO(tyoshino): Clean up initialization of CGI related values. Move some - # of code here to WebSocketRequestHandler class if it's better. - options.cgi_directories = [] - options.is_executable_method = None - if options.cgi_paths: - options.cgi_directories = options.cgi_paths.split(',') - if sys.platform in ('cygwin', 'win32'): - cygwin_path = None - # For Win32 Python, it is expected that CYGWIN_PATH - # is set to a directory of cygwin binaries. - # For example, websocket_server.py in Chromium sets CYGWIN_PATH to - # full path of third_party/cygwin/bin. - if 'CYGWIN_PATH' in os.environ: - cygwin_path = os.environ['CYGWIN_PATH'] - util.wrap_popen3_for_win(cygwin_path) - - def __check_script(scriptpath): - return util.get_script_interp(scriptpath, cygwin_path) - - options.is_executable_method = __check_script - - if options.use_tls: - if not (_HAS_SSL or _HAS_OPEN_SSL): - logging.critical('TLS support requires ssl or pyOpenSSL module.') - sys.exit(1) - if not options.private_key or not options.certificate: - logging.critical( - 'To use TLS, specify private_key and certificate.') - sys.exit(1) - - if options.tls_client_auth: - if not options.use_tls: - logging.critical('TLS must be enabled for client authentication.') - sys.exit(1) - if not _HAS_SSL: - logging.critical('Client authentication requires ssl module.') - - if not options.scan_dir: - options.scan_dir = options.websock_handlers - - if options.use_basic_auth: - options.basic_auth_credential = 'Basic ' + base64.b64encode( - options.basic_auth_credential) - - try: - if options.thread_monitor_interval_in_sec > 0: - # Run a thread monitor to show the status of server threads for - # debugging. - ThreadMonitor(options.thread_monitor_interval_in_sec).start() - - server = WebSocketServer(options) - server.serve_forever() - except Exception, e: - logging.critical('mod_pywebsocket: %s' % e) - logging.critical('mod_pywebsocket: %s' % util.get_stack_trace()) - sys.exit(1) - - -if __name__ == '__main__': - _main(sys.argv[1:]) - - -# vi:sts=4 sw=4 et diff --git a/module/remote/wsbackend/__init__.py b/module/remote/wsbackend/__init__.py deleted file mode 100644 index de6d13128..000000000 --- a/module/remote/wsbackend/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'christian' -
\ No newline at end of file diff --git a/module/setup/System_Checks.py b/module/setup/System_Checks.py deleted file mode 100644 index cef46956b..000000000 --- a/module/setup/System_Checks.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -from getpass import getpass -import module.common.pylgettext as gettext -import os -from os import makedirs -from os.path import abspath, dirname, exists, join -from subprocess import PIPE, call -import sys -from sys import exit -from module.utils import get_console_encoding - -class System_Checks(): - def __init__(self): - self.result = "" - - def print_str(self, text, translate = True): - if translate: - self.result += _(text) + "\n" - else: - self.result += text + "\n" - - def print_dep(self, name, value): - """Print Status of dependency""" - if value: - self.print_str(name + ": OK", False) - else: - self.print_str(name + ": missing", False) - - def check_basic(self): - self.result = "" #clear result - python = False - if sys.version_info[:2] > (2, 7): - self.print_str("Your python version is to new, Please use Python 2.6/2.7") - elif sys.version_info[:2] < (2, 5): - self.print_str("Your python version is to old, Please use at least Python 2.5") - else: - self.print_str("Python Version: OK") - python = True - - curl = self.check_module("pycurl") - self.print_dep("pycurl", curl) - - sqlite = self.check_module("sqlite3") - self.print_dep("sqlite3", sqlite) - - beaker = self.check_module("beaker") - self.print_dep("beaker", beaker) - - jinja = True - try: - import jinja2 - v = jinja2.__version__ - if v and "unknown" not in v: - if not v.startswith("2.5") and not v.startswith("2.6"): - self.print_str("Your installed jinja2 version %s seems too old.") % jinja2.__version__ - self.print_str("You can safely continue but if the webinterface is not working,") - self.print_str("please upgrade or deinstall it, pyLoad includes a sufficient jinja2 library.") - jinja = False - except: - pass - self.print_dep("jinja2", jinja) - - return self.result, (python and curl and sqlite and (beaker or jinja)) - - def check_ssl(self): - self.result = "" #clear result - ssl = self.check_module("OpenSSL") - self.print_dep("py-OpenSSL", ssl) - return self.result, ssl - - def check_crypto(self): - self.result = "" #clear result - crypto = self.check_module("Crypto") - self.print_dep("pycrypto", crypto) - return self.result, crypto - - def check_captcha(self): - self.result = "" #clear result - pil = self.check_module("Image") - self.print_dep("py-imaging", pil) - if os.name == "nt": - tesser = self.check_prog([join(pypath, "tesseract", "tesseract.exe"), "-v"]) - else: - tesser = self.check_prog(["tesseract", "-v"]) - self.print_dep("tesseract", tesser) - return self.result, pil and tesser - - def check_js(self): - self.result = "" #clear result - from module.common import JsEngine - js = True if JsEngine.ENGINE else False - self.print_dep(_("JS engine"), js) - return self.result, pil and tesser - - def check_module(self, module): - try: - __import__(module) - return True - except: - return False - - def check_prog(self, command): - pipe = PIPE - try: - call(command, stdout=pipe, stderr=pipe) - return True - except: - return False - diff --git a/module/threads/AddonThread.py b/module/threads/AddonThread.py deleted file mode 100644 index afb56f66b..000000000 --- a/module/threads/AddonThread.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from copy import copy -from traceback import print_exc - -from BaseThread import BaseThread - -class AddonThread(BaseThread): - """thread for addons""" - - def __init__(self, m, function, args, kwargs): - """Constructor""" - BaseThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - def getActiveFiles(self): - return self.active - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - def run(self): #TODO: approach via func_code - try: - try: - self.kwargs["thread"] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs["thread"] - self.f(*self.args, **self.kwargs) - except Exception, e: - if hasattr(self.f, "im_self"): - addon = self.f.im_self - addon.logError(_("An Error occurred"), e) - if self.m.core.debug: - print_exc() - self.writeDebugReport(addon.__name__, plugin=addon) - - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self)
\ No newline at end of file diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py deleted file mode 100644 index c64678a72..000000000 --- a/module/threads/BaseThread.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from threading import Thread -from time import strftime, gmtime -from sys import exc_info -from types import MethodType -from pprint import pformat -from traceback import format_exc - -from module.utils import primary_uid -from module.utils.fs import listdir, join, save_join, stat, exists - -class BaseThread(Thread): - """abstract base class for thread types""" - - def __init__(self, manager): - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager - self.core = manager.core - self.log = manager.core.log - - #: Owner of the thread, every type should set it - self.owner = None - - @property - def user(self): - return primary_uid(self.owner) - - def getProgress(self): - """ retrieves progress information about the current running task - - :return: :class:`ProgressInfo` - """ - - # Debug Stuff - def writeDebugReport(self, name, pyfile=None, plugin=None): - """ writes a debug report to disk """ - - dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) - if pyfile: - dump = self.getFileDump(pyfile) - else: - dump = self.getPluginDump(plugin) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - if exists(join("tmp", name)): - for f in listdir(join("tmp", name)): - try: - # avoid encoding errors - zip.write(join("tmp", name, f), save_join(name, f)) - except: - pass - - info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L # change permissions - zip.writestr(info, dump) - - info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L - zip.writestr(info, self.getSystemDump()) - - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - f = open(dump_name, "wb") - f.write(dump) - f.close() - - self.log.info("Debug Report written to %s" % dump_name) - return dump_name - - def getFileDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - del frame - - del stack #delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.values) + "\n" - - return dump - - #TODO - def getPluginDump(self, plugin): - return "" - - def getSystemDump(self): - return "" diff --git a/module/threads/DecrypterThread.py b/module/threads/DecrypterThread.py deleted file mode 100644 index 39448a620..000000000 --- a/module/threads/DecrypterThread.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from time import sleep -from traceback import print_exc - -from module.utils import uniqify -from module.plugins.Base import Retry -from module.plugins.Crypter import Package - -from BaseThread import BaseThread - -class DecrypterThread(BaseThread): - """thread for decrypting""" - - def __init__(self, manager, data, pid): - """constructor""" - BaseThread.__init__(self, manager) - self.data = data - self.pid = pid - - self.start() - - def run(self): - plugin_map = {} - for url, plugin in self.data: - if plugin in plugin_map: - plugin_map[plugin].append(url) - else: - plugin_map[plugin] = [url] - - self.decrypt(plugin_map) - - def decrypt(self, plugin_map): - pack = self.m.core.files.getPackage(self.pid) - result = [] - - for name, urls in plugin_map.iteritems(): - klass = self.m.core.pluginManager.loadClass("crypter", name) - plugin = klass(self.m.core, pack, pack.password) - plugin_result = [] - - try: - try: - plugin_result = plugin._decrypt(urls) - except Retry: - sleep(1) - plugin_result = plugin._decrypt(urls) - except Exception, e: - plugin.logError(_("Decrypting failed"), e) - if self.m.core.debug: - print_exc() - self.writeDebugReport(plugin.__name__, plugin=plugin) - - plugin.logDebug("Decrypted", plugin_result) - result.extend(plugin_result) - - #TODO - result = uniqify(result) - pack_names = {} - urls = [] - - for p in result: - if isinstance(p, Package): - if p.name in pack_names: - pack_names[p.name].urls.extend(p.urls) - else: - pack_names[p.name] = p - else: - urls.append(p) - - if urls: - self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(urls), "name": pack.name}) - self.m.core.api.addFiles(self.pid, urls) - - for p in pack_names.itervalues(): - self.m.core.api.addPackage(p.name, p.urls, pack.password) - - if not result: - self.log.info(_("No links decrypted")) - diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py deleted file mode 100644 index cf59c5639..000000000 --- a/module/threads/DownloadThread.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from Queue import Queue -from time import sleep, time -from traceback import print_exc -from sys import exc_clear -from pycurl import error - -from module.plugins.Base import Fail, Retry, Abort -from module.plugins.Hoster import Reconnect, SkipDownload -from module.network.HTTPRequest import BadHeader - -from BaseThread import BaseThread - -class DownloadThread(BaseThread): - """thread for downloading files from 'real' hoster plugins""" - - def __init__(self, manager): - """Constructor""" - BaseThread.__init__(self, manager) - - self.queue = Queue() # job queue - self.active = None - - self.start() - - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = None - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): continue - #this pyfile was deleted while queuing - - pyfile.plugin.checkForSameFiles(starting=True) - self.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.core.addonManager.downloadPreparing(pyfile) - pyfile.plugin.preprocessing(self) - - self.log.info(_("Download finished: %s") % pyfile.name) - self.core.addonManager.downloadFinished(pyfile) - self.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.log.info(_("Download aborted: %s") % pyfile.name) - except: - pass - - pyfile.setStatus("aborted") - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - #pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - except Fail, e: - msg = e.args[0] - - # TODO: activate former skipped downloads - - if msg == "offline": - pyfile.setStatus("offline") - self.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - self.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.log.error("pycurl error %s: %s" % (code, msg)) - if self.core.debug: - print_exc() - self.writeDebugReport(pyfile.plugin.__name__, pyfile) - - self.core.addonManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.log.info(_("Download skipped: %(name)s due to %(plugin)s") - % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.core.files.checkPackageFinished(pyfile) - - self.active = False - self.core.files.save() - - continue - - - except Exception, e: - if isinstance(e, BadHeader) and e.code == 500: - pyfile.setStatus("temp. offline") - self.log.warning(_("Download is temporary offline: %s") % pyfile.name) - pyfile.error = _("Internal Server Error") - - else: - pyfile.setStatus("failed") - self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.core.debug: - print_exc() - self.writeDebugReport(pyfile.plugin.__name__, pyfile) - - self.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - - #pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.core.files.save() - - def getProgress(self): - if self.active: - return self.active.getProgressInfo() - - - def put(self, job): - """assign a job to the thread""" - self.queue.put(job) - - def clean(self, pyfile): - """ set thread inactive and release pyfile """ - self.active = False - pyfile.release() - - def stop(self): - """stops the thread""" - self.put("quit") diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py deleted file mode 100644 index bf5bb5777..000000000 --- a/module/threads/InfoThread.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from time import time -from traceback import print_exc - -from module.Api import LinkStatus -from module.utils.packagetools import parseNames -from module.utils import has_method, accumulate - -from BaseThread import BaseThread - -class InfoThread(BaseThread): - def __init__(self, manager, data, pid=-1, rid=-1): - """Constructor""" - BaseThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id - - self.cache = [] #accumulated data - - self.start() - - def run(self): - """run method""" - - plugins = accumulate(self.data) - crypter = {} - - # filter out crypter plugins - for name in self.m.core.pluginManager.getPlugins("crypter"): - if name in plugins: - crypter[name] = plugins[name] - del plugins[name] - - #directly write to database - if self.pid > -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) - klass = self.m.core.pluginManager.getPluginClass(pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, klass, urls, self.updateDB) - self.m.core.files.save() - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - else: #post the results - for name, urls in crypter: - #attach container content - try: - data = self.decrypt(name, urls) - except: - print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - accumulate(data, plugins) - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - klass = getattr(plugin, pluginname) - if has_method(klass, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - elif has_method(plugin, "getInfo"): - self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, (url, LinkStatus(name, plugin, "unknown", status, int(size)))) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #result loaded from cache - process = [] #urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] - - for res in result: - self.m.infoCache[res[3]] = res - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decrypt(self, plugin, urls): - self.m.log.debug("Pre decrypting %s" % plugin) - klass = self.m.core.pluginManager.loadClass("crypter", plugin) - - # only decrypt files - if has_method(klass, "decryptFile"): - urls = klass.decrypt(urls) - data, crypter = self.m.core.pluginManager.parseUrls(urls) - return data - - return [] diff --git a/module/threads/ThreadManager.py b/module/threads/ThreadManager.py deleted file mode 100644 index f67179d08..000000000 --- a/module/threads/ThreadManager.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from os.path import exists, join -import re -from subprocess import Popen -from threading import Event, Lock -from time import sleep, time -from traceback import print_exc -from random import choice - -import pycurl - -from module.datatypes.PyFile import PyFile -from module.network.RequestFactory import getURL -from module.utils import lock, uniqify -from module.utils.fs import free_space - -from DecrypterThread import DecrypterThread -from DownloadThread import DownloadThread -from InfoThread import InfoThread - -class ThreadManager: - """manages the download threads, assign jobs, reconnect etc""" - - - def __init__(self, core): - """Constructor""" - self.core = core - self.log = core.log - - self.threads = [] # thread list - self.localThreads = [] #addon+decrypter threads - - self.pause = True - - self.reconnecting = Event() - self.reconnecting.clear() - self.downloaded = 0 #number of files downloaded since last cleanup - - self.lock = Lock() - - # some operations require to fetch url info from hoster, so we caching them so it wont be done twice - # contains a timestamp and will be purged after timeout - self.infoCache = {} - - # pool of ids for online check - self.resultIDs = 0 - - # threads which are fetching hoster results - self.infoResults = {} - # timeout for cache purge - self.timestamp = 0 - - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - - for i in range(self.core.config.get("download", "max_downloads")): - self.createThread() - - - def createThread(self): - """create a download thread""" - - thread = DownloadThread(self) - self.threads.append(thread) - - def createInfoThread(self, data, pid): - """ start a thread which fetches online status and other info's """ - self.timestamp = time() + 5 * 60 - if data: InfoThread(self, data, pid) - - @lock - def createResultThread(self, data): - """ creates a thread to fetch online status, returns result id """ - self.timestamp = time() + 5 * 60 - - rid = self.resultIDs - self.resultIDs += 1 - - InfoThread(self, data, rid=rid) - - return rid - - @lock - def createDecryptThread(self, data, pid): - """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" - if data: DecrypterThread(self, data, pid) - - - @lock - def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} - - @lock - def setInfoResults(self, rid, result): - self.infoResults[rid].update(result) - - def getActiveDownloads(self, user=None): - # TODO: user context - return [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] - - def getProgressList(self, user=None): - info = [] - - # TODO: local threads can create multiple progresses - for thread in self.threads + self.localThreads: - # skip if not belong to current user - if user and thread.user != user: continue - - progress = thread.getProgress() - if progress: info.append(progress) - - return info - - def getActiveFiles(self): - active = self.getActiveDownloads() - - for t in self.localThreads: - active.extend(t.getActiveFiles()) - - return active - - def processingIds(self): - """get a id list of all pyfiles processed""" - return [x.id for x in self.getActiveFiles()] - - def work(self): - """run all task which have to be done (this is for repetetive call by core)""" - try: - self.tryReconnect() - except Exception, e: - self.log.error(_("Reconnect Failed: %s") % str(e) ) - self.reconnecting.clear() - self.core.print_exc() - - self.checkThreadCount() - - try: - self.assignJob() - except Exception, e: - self.log.warning("Assign job error", e) - self.core.print_exc() - - sleep(0.5) - self.assignJob() - #it may be failed non critical so we try it again - - if (self.infoCache or self.infoResults) and self.timestamp < time(): - self.infoCache.clear() - self.infoResults.clear() - self.log.debug("Cleared Result cache") - - def tryReconnect(self): - """checks if reconnect needed""" - - if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): - return False - - active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] - - if not (0 < active.count(True) == len(active)): - return False - - if not exists(self.core.config['reconnect']['method']): - if exists(join(pypath, self.core.config['reconnect']['method'])): - self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) - else: - self.core.config["reconnect"]["activated"] = False - self.log.warning(_("Reconnect script not found!")) - return - - self.reconnecting.set() - - #Do reconnect - self.log.info(_("Starting reconnect")) - - while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: - sleep(0.25) - - ip = self.getIP() - - self.core.addonManager.beforeReconnecting(ip) - - self.log.debug("Old IP: %s" % ip) - - try: - reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) - except: - self.log.warning(_("Failed executing reconnect script!")) - self.core.config["reconnect"]["activated"] = False - self.reconnecting.clear() - if self.core.debug: - print_exc() - return - - reconn.wait() - sleep(1) - ip = self.getIP() - self.core.addonManager.afterReconnecting(ip) - - self.log.info(_("Reconnected, new IP: %s") % ip) - - self.reconnecting.clear() - - def getIP(self): - """retrieve current ip""" - services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), - ("http://checkip.dyndns.org/",".*Current IP Address: (\S+)</body>.*")] - - ip = "" - for i in range(10): - try: - sv = choice(services) - ip = getURL(sv[0]) - ip = re.match(sv[1], ip).group(1) - break - except: - ip = "" - sleep(1) - - return ip - - def checkThreadCount(self): - """checks if there is a need for increasing or reducing thread count""" - - if len(self.threads) == self.core.config.get("download", "max_downloads"): - return True - elif len(self.threads) < self.core.config.get("download", "max_downloads"): - self.createThread() - else: - free = [x for x in self.threads if not x.active] - if free: - free[0].put("quit") - - - def cleanPycurl(self): - """ make a global curl cleanup (currently unused) """ - if self.processingIds(): - return False - pycurl.global_cleanup() - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - self.downloaded = 0 - self.log.debug("Cleaned up pycurl") - return True - - - def assignJob(self): - """assign a job to a thread if possible""" - - if self.pause or not self.core.api.isTimeDownload(): return - - #if self.downloaded > 20: - # if not self.cleanPyCurl(): return - - free = [x for x in self.threads if not x.active] - - inuse = [(x.active.pluginname, x.active.plugin.getDownloadLimit()) for x in self.threads if x.active and x.active.hasPlugin()] - inuse = [(x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) for x in inuse] - occ = tuple(sorted(uniqify([x[0] for x in inuse if 0 < x[1] <= x[2]]))) - - job = self.core.files.getJob(occ) - if job: - try: - job.initPlugin() - except Exception, e: - self.log.critical(str(e)) - print_exc() - job.setStatus("failed") - job.error = str(e) - job.release() - return - - spaceLeft = free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/module/threads/__init__.py b/module/threads/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/threads/__init__.py +++ /dev/null diff --git a/module/utils/ImportDebugger.py b/module/utils/ImportDebugger.py deleted file mode 100644 index a997f7b0c..000000000 --- a/module/utils/ImportDebugger.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -class ImportDebugger(object): - - def __init__(self): - self.imported = {} - - def find_module(self, name, path=None): - - if name not in self.imported: - self.imported[name] = 0 - - self.imported[name] += 1 - - print name, path - -sys.meta_path.append(ImportDebugger())
\ No newline at end of file diff --git a/module/utils/JsEngine.py b/module/utils/JsEngine.py deleted file mode 100644 index ef7494d16..000000000 --- a/module/utils/JsEngine.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from imp import find_module -from os.path import join, exists -from urllib import quote - - -ENGINE = "" - -DEBUG = False -JS = False -PYV8 = False -NODE = False -RHINO = False - -# TODO: Refactor + clean up this class - -if not ENGINE: - try: - import subprocess - - subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "js" - JS = True - except: - pass - -if not ENGINE or DEBUG: - try: - find_module("PyV8") - ENGINE = "pyv8" - PYV8 = True - except: - pass - -if not ENGINE or DEBUG: - try: - import subprocess - subprocess.Popen(["node", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["node", "-e", "console.log(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "node" - NODE = True - except: - pass - -if not ENGINE or DEBUG: - try: - path = "" #path where to find rhino - - if exists("/usr/share/java/js.jar"): - path = "/usr/share/java/js.jar" - elif exists("js.jar"): - path = "js.jar" - elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway - path = join(pypath, "js.jar") - - if not path: - raise Exception - - import subprocess - - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "rhino" - RHINO = True - except: - pass - -class JsEngine(): - def __init__(self): - self.engine = ENGINE - self.init = False - - def __nonzero__(self): - return False if not ENGINE else True - - def set_debug(self, value): - global DEBUG - DEBUG = value - - def eval(self, script): - if not self.init: - if ENGINE == "pyv8" or (DEBUG and PYV8): - import PyV8 - global PyV8 - - self.init = True - - if type(script) == unicode: - script = script.encode("utf8") - - if not ENGINE: - raise Exception("No JS Engine") - - if not DEBUG: - if ENGINE == "pyv8": - return self.eval_pyv8(script) - elif ENGINE == "js": - return self.eval_js(script) - elif ENGINE == "node": - return self.eval_node(script) - elif ENGINE == "rhino": - return self.eval_rhino(script) - else: - results = [] - if PYV8: - res = self.eval_pyv8(script) - print "PyV8:", res - results.append(res) - if JS: - res = self.eval_js(script) - print "JS:", res - results.append(res) - if NODE: - res = self.eval_node(script) - print "NODE:", res - results.append(res) - if RHINO: - res = self.eval_rhino(script) - print "Rhino:", res - results.append(res) - - warning = False - for x in results: - for y in results: - if x != y: - warning = True - - if warning: print "### WARNING ###: Different results" - - return results[0] - - def eval_pyv8(self, script): - rt = PyV8.JSContext() - rt.enter() - return rt.eval(script) - - def eval_js(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_node(self, script): - script = "console.log(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["node", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_rhino(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res.decode("utf8").encode("ISO-8859-1") - - def error(self): - return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8, nodejs or rhino") - -if __name__ == "__main__": - js = JsEngine() - js.set_debug(True) - - test = u'"Ì"+"À"' - js.eval(test)
\ No newline at end of file diff --git a/module/utils/__init__.py b/module/utils/__init__.py deleted file mode 100644 index ca00f9abe..000000000 --- a/module/utils/__init__.py +++ /dev/null @@ -1,253 +0,0 @@ -# -*- coding: utf-8 -*- - -""" Store all usefull functions here """ - -import os -import time -import re -from string import maketrans -from itertools import islice -from htmlentitydefs import name2codepoint - -# abstraction layer for json operations -try: # since python 2.6 - import json -except ImportError: #use system simplejson if available - import simplejson as json - -json_loads = json.loads -json_dumps = json.dumps - -def decode(string): - """ decode string with utf if possible """ - try: - if type(string) == str: - return string.decode("utf8", "replace") - else: - return string - except: - return string - -def encode(string): - """ decode string to utf if possible """ - try: - if type(string) == unicode: - return string.encode("utf8", "replace") - else: - return string - except: - return string - -def remove_chars(string, repl): - """ removes all chars in repl from string""" - if type(string) == str: - return string.translate(maketrans("", ""), repl) - elif type(string) == unicode: - return string.translate(dict([(ord(s), None) for s in repl])) - - -def get_console_encoding(enc): - if os.name == "nt": - if enc == "cp65001": # aka UTF-8 - print "WARNING: Windows codepage 65001 is not supported." - enc = "cp850" - else: - enc = "utf8" - - return enc - -def compare_time(start, end): - start = map(int, start) - end = map(int, end) - - if start == end: return True - - now = list(time.localtime()[3:5]) - if start < now < end: return True - elif start > end and (now > start or now < end): return True - elif start < now > end < start: return True - else: return False - -def to_list(value): - return value if type(value) == list else ([value] if value is not None else []) - -def formatSize(size): - print "Deprecated formatSize, use format_size" - return format_size(size) - -def format_size(bytes): - bytes = int(bytes) - steps = 0 - sizes = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") - while bytes > 1000: - bytes /= 1024.0 - steps += 1 - return "%.2f %s" % (bytes, sizes[steps]) - -def formatSpeed(speed): - print "Deprecated formatSpeed, use format_speed" - return format_speed(speed) - -def format_speed(speed): - return format_size(speed) + "/s" - -def format_time(seconds): - if seconds < 0: return "00:00:00" - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - -def uniqify(seq): #by Dave Kirby - """ removes duplicates from list, preserve order """ - seen = set() - return [x for x in seq if x not in seen and not seen.add(x)] - -def bits_set(bits, compare): - """ checks if all bits are set in compare, or bits is 0 """ - return bits == (bits & compare) - -def parseFileSize(string, unit=None): #returns bytes - if not unit: - m = re.match(r"([\d.,]+) *([a-zA-Z]*)", string.strip().lower()) - if m: - traffic = float(m.group(1).replace(",", ".")) - unit = m.group(2) - else: - return 0 - else: - if isinstance(string, basestring): - traffic = float(string.replace(",", ".")) - else: - traffic = string - - #ignore case - unit = unit.lower().strip() - - if unit in ("gb", "gig", "gbyte", "gigabyte", "gib", "g"): - traffic *= 1 << 30 - elif unit in ("mb", "mbyte", "megabyte", "mib", "m"): - traffic *= 1 << 20 - elif unit in ("kb", "kib", "kilobyte", "kbyte", "k"): - traffic *= 1 << 10 - - return traffic - - -def lock(func): - def new(*args, **kwargs): - #print "Handler: %s args: %s" % (func,args[1:]) - args[0].lock.acquire() - try: - return func(*args, **kwargs) - finally: - args[0].lock.release() - - return new - -def read_lock(func): - def new(*args, **kwargs): - args[0].lock.acquire(shared=True) - try: - return func(*args, **kwargs) - finally: - args[0].lock.release() - - return new - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) - - -def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - name = text[1:-1] - text = unichr(name2codepoint[name]) - except KeyError: - pass - - return text # leave as is - - -def has_method(obj, name): - """ checks if 'name' was defined in obj, (false if it was inhereted) """ - return name in obj.__dict__ - -def accumulate(it, inv_map=None): - """ accumulate (key, value) data to {value : [keylist]} dictionary """ - if not inv_map: - inv_map = {} - - for key, value in it: - if value in inv_map: - inv_map[value].append(key) - else: - inv_map[value] = [key] - - return inv_map - -def to_string(value): - return str(value) if not isinstance(value, basestring) else value - -def to_int(string, default=0): - """ return int from string or default """ - try: - return int(string) - except ValueError: - return default - -def from_string(value, typ=None): - """ cast value to given type, unicode for strings """ - - # value is no string - if not isinstance(value, basestring): - return value - - value = decode(value) - - if typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1", "true", "on", "an", "yes") else False - elif typ == "time": - if not value: value = "0:00" - if not ":" in value: value += ":00" - return value - else: - return value - -def get_index(l, value): - """ .index method that also works on tuple and python 2.5 """ - for pos, t in enumerate(l): - if t == value: - return pos - - # Matches behavior of list.index - raise ValueError("list.index(x): x not in list") - -def primary_uid(user): - """ Gets primary user id for user instances or ints """ - if type(user) == int: return user - return user.primary if user else None - -def html_unescape(text): - """Removes HTML or XML character references and entities from a text string""" - return re.sub("&#?\w+;", fixup, text) - -if __name__ == "__main__": - print remove_chars("ab'cdgdsf''ds'", "'ghd") diff --git a/module/utils/fs.py b/module/utils/fs.py deleted file mode 100644 index 92cc605e7..000000000 --- a/module/utils/fs.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import sys -from os.path import join -from . import decode, remove_chars - -# File System Encoding functions: -# Use fs_encode before accessing files on disk, it will encode the string properly - -if sys.getfilesystemencoding().startswith('ANSI'): - def fs_encode(string): - if type(string) == unicode: - return string.encode('utf8') - return string - - fs_decode = decode #decode utf8 - -else: - fs_encode = fs_decode = lambda x: x # do nothing - -# FS utilities -def chmod(path, mode): - try: - return os.chmod(fs_encode(path), mode) - except : - pass - -def dirname(path): - return fs_decode(os.path.dirname(fs_encode(path))) - -def abspath(path): - return fs_decode(os.path.abspath(fs_encode(path))) - -def chown(path, uid, gid): - return os.chown(fs_encode(path), uid, gid) - -def remove(path): - return os.remove(fs_encode(path)) - -def exists(path): - return os.path.exists(fs_encode(path)) - -def makedirs(path, mode=0755): - return os.makedirs(fs_encode(path), mode) - -# fs_decode? -def listdir(path): - return [fs_decode(x) for x in os.listdir(fs_encode(path))] - -def save_filename(name): - #remove some chars - if os.name == 'nt': - return remove_chars(name, '/\\?%*:|"<>,') - else: - return remove_chars(name, '/\\"') - -def stat(name): - return os.stat(fs_encode(name)) - -def save_join(*args): - """ joins a path, encoding aware """ - return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) - -def free_space(folder): - folder = fs_encode(folder) - - if os.name == "nt": - import ctypes - - free_bytes = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) - return free_bytes.value - else: - from os import statvfs - - s = statvfs(folder) - return s.f_frsize * s.f_bavail diff --git a/module/utils/json_layer.py b/module/utils/json_layer.py deleted file mode 100644 index cf9743603..000000000 --- a/module/utils/json_layer.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# abstraction layer for json operations - -print ".json_layer is deprecated, use .json instead" - -try: # since python 2.6 - import json - from json import loads as json_loads - from json import dumps as json_dumps -except ImportError: #use system simplejson if available - import simplejson as json - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps diff --git a/module/utils/packagetools.py b/module/utils/packagetools.py deleted file mode 100644 index 791a46d51..000000000 --- a/module/utils/packagetools.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python - -# JDownloader/src/jd/controlling/LinkGrabberPackager.java - -import re -from urlparse import urlparse - -def matchFirst(string, *args): - """ matches against list of regexp and returns first match""" - for patternlist in args: - for pattern in patternlist: - r = pattern.search(string) - if r is not None: - name = r.group(1) - return name - - return string - - -def parseNames(files): - """ Generates packages names from name, data lists - - :param files: list of (name, data) - :return: packagenames mapped to data lists (eg. urls) - """ - packs = {} - - endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" - - rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), - re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), - re.compile("(.*)\\.rar$", re.I), - re.compile("(.*)\\.r\\d+$", re.I), - re.compile("(.*)(\\.|_|-)\\d+$", re.I)] - - zipPats = [re.compile("(.*)\\.zip$", re.I), - re.compile("(.*)\\.z\\d+$", re.I), - re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), - re.compile("(.*)\\.a.$", re.I)] - - ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), - re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] - - iszPats = [re.compile("(.*)\\.isz$", re.I), - re.compile("(.*)\\.i\\d{2}$", re.I)] - - pat1 = re.compile("(\\.?CD\\d+)", re.I) - pat2 = re.compile("(\\.?part\\d+)", re.I) - - pat3 = re.compile("(.+)[\\.\\-_]+$") - pat4 = re.compile("(.+)\\.\\d+\\.xtm$") - - for file, url in files: - patternMatch = False - - if file is None: - continue - - # remove trailing / - name = file.rstrip('/') - - # extract last path part .. if there is a path - split = name.rsplit("/", 1) - if len(split) > 1: - name = split.pop(1) - - #check if an already existing package may be ok for this file - # found = False - # for pack in packs: - # if pack in file: - # packs[pack].append(url) - # found = True - # break - # - # if found: continue - - # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern - before = name - name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) - if before != name: - patternMatch = True - - # xtremsplit pattern - r = pat4.search(name) - if r is not None: - name = r.group(1) - - # remove part and cd pattern - r = pat1.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - r = pat2.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - # additional checks if extension pattern matched - if patternMatch: - # remove extension - index = name.rfind(".") - if index <= 0: - index = name.rfind("_") - if index > 0: - length = len(name) - index - if length <= 4: - name = name[:-length] - - # remove endings like . _ - - r = pat3.search(name) - if r is not None: - name = r.group(1) - - # replace . and _ with space - name = name.replace(".", " ") - name = name.replace("_", " ") - - name = name.strip() - else: - name = "" - - # fallback: package by hoster - if not name: - name = urlparse(file).hostname - if name: name = name.replace("www.", "") - - # fallback : default name - if not name: - name = "unknown" - - # build mapping - if name in packs: - packs[name].append(url) - else: - packs[name] = [url] - - return packs - - -if __name__ == "__main__": - from os.path import join - from pprint import pprint - - f = open(join("..", "..", "testlinks2.txt"), "rb") - urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()] - f.close() - - print "Having %d urls." % len(urls) - - packs = parseNames(urls) - - pprint(packs) - - print "Got %d urls." % sum([len(x) for x in packs.itervalues()]) diff --git a/module/utils/pylgettext.py b/module/utils/pylgettext.py deleted file mode 100644 index fb36fecee..000000000 --- a/module/utils/pylgettext.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from gettext import * - -_searchdirs = None - -origfind = find - -def setpaths(pathlist): - global _searchdirs - if isinstance(pathlist, list): - _searchdirs = pathlist - else: - _searchdirs = list(pathlist) - - -def addpath(path): - global _searchdirs - if _searchdirs is None: - _searchdirs = list(path) - else: - if path not in _searchdirs: - _searchdirs.append(path) - - -def delpath(path): - global _searchdirs - if _searchdirs is not None: - if path in _searchdirs: - _searchdirs.remove(path) - - -def clearpath(): - global _searchdirs - if _searchdirs is not None: - _searchdirs = None - - -def find(domain, localedir=None, languages=None, all=False): - if _searchdirs is None: - return origfind(domain, localedir, languages, all) - searches = [localedir] + _searchdirs - results = list() - for dir in searches: - res = origfind(domain, dir, languages, all) - if all is False: - results.append(res) - else: - results.extend(res) - if all is False: - results = filter(lambda x: x is not None, results) - if len(results) == 0: - return None - else: - return results[0] - else: - return results - -#Is there a smarter/cleaner pythonic way for this? -translation.func_globals['find'] = find diff --git a/module/web/.bowerrc b/module/web/.bowerrc deleted file mode 100644 index f594df7a7..000000000 --- a/module/web/.bowerrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "directory": "app/components" -} diff --git a/module/web/.jshintrc b/module/web/.jshintrc deleted file mode 100644 index 0cff430d9..000000000 --- a/module/web/.jshintrc +++ /dev/null @@ -1,29 +0,0 @@ -{ - "node": true, - "browser": true, - "esnext": true, - "bitwise": true, - "curly": true, - "eqeqeq": true, - "immed": true, - "indent": 4, - "latedef": true, - "newcap": true, - "noarg": true, - "quotmark": "single", - "regexp": true, - "undef": true, - "strict": true, - "trailing": true, - "smarttabs": true, - "unused": false, - "camelcase": false, - "-W030": false, - "-W015": false, - "-W116": false, - "predef": [ - "require", - "define", - "alert" - ] -} diff --git a/module/web/Gruntfile.js b/module/web/Gruntfile.js deleted file mode 100644 index 4799afb01..000000000 --- a/module/web/Gruntfile.js +++ /dev/null @@ -1,361 +0,0 @@ -// Generated on 2013-06-06 using generator-webapp 0.2.2 -'use strict'; -var LIVERELOAD_PORT = 35729; -var lrSnippet = require('connect-livereload')({port: LIVERELOAD_PORT}); -var mountFolder = function(connect, dir) { - return connect.static(require('path').resolve(dir)); -}; - -// # Globbing -// for performance reasons we're only matching one level down: -// 'test/spec/{,*/}*.js' -// use this if you want to recursively match all subfolders: -// 'test/spec/**/*.js' - -module.exports = function(grunt) { - // load all grunt tasks - require('matchdep').filterDev('grunt-*').forEach(grunt.loadNpmTasks); - - // configurable paths - var yeomanConfig = { - app: 'app', - dist: 'dist' - }; - - grunt.initConfig({ - yeoman: yeomanConfig, - watch: { - options: { - nospawn: true - }, - less: { - files: ['<%= yeoman.app %>/styles/**/*.less'], - tasks: ['less'] - }, - livereload: { - options: { - livereload: LIVERELOAD_PORT - }, - files: [ - '<%= yeoman.app %>/**/*.html', - '{.tmp,<%= yeoman.app %>}/styles/{,*/}*.css', - '{.tmp,<%= yeoman.app %>}/scripts/**/*.js', - '<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}' - ] - } - }, - connect: { - options: { - port: 9000, - // change this to '0.0.0.0' to access the server from outside - hostname: 'localhost' - }, - livereload: { - options: { - middleware: function(connect) { - return [ - mountFolder(connect, '.tmp'), - mountFolder(connect, yeomanConfig.app), - lrSnippet - ]; - } - } - }, - test: { - options: { - middleware: function(connect) { - return [ - mountFolder(connect, '.tmp'), - mountFolder(connect, 'test') - ]; - } - } - }, - dist: { - options: { - middleware: function(connect) { - return [ - mountFolder(connect, yeomanConfig.dist) - ]; - } - } - } - }, - open: { - server: { - path: 'http://localhost:<%= connect.options.port %>' - } - }, - clean: { - dist: { - files: [ - { - dot: true, - src: [ - '.tmp', - '<%= yeoman.dist %>/*', - '!<%= yeoman.dist %>/.git*' - ] - } - ] - }, - server: '.tmp' - }, - jshint: { - options: { - jshintrc: '.jshintrc' - }, - all: [ - 'Gruntfile.js', - '<%= yeoman.app %>/scripts/**/*.js', - '!<%= yeoman.app %>/scripts/vendor/*', - 'test/spec/{,*/}*.js' - ] - }, - mocha: { - all: { - options: { - run: true, - urls: ['http://localhost:<%= connect.options.port %>/index.html'] - } - } - }, - less: { - options: { - paths: [yeomanConfig.app + '/components', yeomanConfig.app + '/styles', yeomanConfig.app + '/styles/default'] - //dumpLineNumbers: true - }, - dist: { - files: [ - { - expand: true, // Enable dynamic expansion. - cwd: '<%= yeoman.app %>/styles/', // Src matches are relative to this path. - src: ['**/main.less'], // Actual pattern(s) to match. - dest: '.tmp/styles/', // Destination path prefix. - ext: '.css' // Dest filepaths will have this extension. - } - ] - } - }, - // not used since Uglify task does concat, - // but still available if needed - /*concat: { - dist: {} - },*/ - requirejs: { - dist: { - // Options: https://github.com/jrburke/r.js/blob/master/build/example.build.js - options: { - // `name` and `out` is set by grunt-usemin - baseUrl: yeomanConfig.app + '/scripts', - optimize: 'none', - // TODO: Figure out how to make sourcemaps work with grunt-usemin - // https://github.com/yeoman/grunt-usemin/issues/30 - //generateSourceMaps: true, - // required to support SourceMaps - // http://requirejs.org/docs/errors.html#sourcemapcomments - preserveLicenseComments: false, - useStrict: true, - wrap: true - //uglify2: {} // https://github.com/mishoo/UglifyJS2 - } - } - }, - rev: { - dist: { - files: { - src: [ - '<%= yeoman.dist %>/scripts/{,*/}*.js', - '<%= yeoman.dist %>/styles/{,*/}*.css', - '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp}', - '<%= yeoman.dist %>/fonts/*' - ] - } - } - }, - useminPrepare: { - options: { - dest: '<%= yeoman.dist %>' - }, - html: '<%= yeoman.app %>/index.html' - }, - usemin: { - options: { - dirs: ['<%= yeoman.dist %>'] - }, - html: ['<%= yeoman.dist %>/{,*/}*.html'], - css: ['<%= yeoman.dist %>/styles/{,*/}*.css'] - }, - imagemin: { - dist: { - files: [ - { - expand: true, - cwd: '<%= yeoman.app %>/images', - src: '{,*/}*.{png,jpg,jpeg}', - dest: '<%= yeoman.dist %>/images' - } - ] - } - }, - svgmin: { - dist: { - files: [ - { - expand: true, - cwd: '<%= yeoman.app %>/images', - src: '{,*/}*.svg', - dest: '<%= yeoman.dist %>/images' - } - ] - } - }, - cssmin: { - dist: { - expand: true, - cwd: '<%= yeoman.app %>/styles', - src: ['{,*/}*.css', '!*.min.css'], - dest: '<%= yeoman.dist %>/styles', - ext: '.min.css' - } - }, - htmlmin: { - dist: { - options: { - /*removeCommentsFromCDATA: true, - // https://github.com/yeoman/grunt-usemin/issues/44 - //collapseWhitespace: true, - collapseBooleanAttributes: true, - removeAttributeQuotes: true, - removeRedundantAttributes: true, - useShortDoctype: true, - removeEmptyAttributes: true, - removeOptionalTags: true*/ - }, - files: [ - { - expand: true, - cwd: '<%= yeoman.app %>', - src: '*.html', - dest: '<%= yeoman.dist %>' - } - ] - } - }, - // Put files not handled in other tasks here - copy: { - // Copy files from third party libraries - // TODO: copy also to dist folder - libs: { - files: [ - { - expand: true, - flatten: true, - cwd: '<% yeoman.app %>', - dest: '<% yeoman.app %>/images,', - src: [ - ] - }, - { - expand: true, - flatten: true, - cwd: '<% yeoman.app %>', - dest: '.tmp/fonts', - src: [ - '**/font-awesome/font/*' - ] - } - ] - }, - - dist: { - files: [ - { - expand: true, - dot: true, - cwd: '<%= yeoman.app %>', - dest: '<%= yeoman.dist %>', - src: [ - '*.{ico,txt}', - '.htaccess', - 'images/{,*/}*.{webp,gif}', - 'fonts/*' - ] - }, - { - expand: true, - cwd: '.tmp/images', - dest: '<%= yeoman.dist %>/images', - src: [ - 'generated/*' - ] - } - ] - } - }, - concurrent: { - server: [ - 'copy:libs', - 'less:dist' - ], - test: [ - 'less' - ], - dist: [ - 'less', - 'imagemin', - 'svgmin', - 'htmlmin' - ] - }, - bower: { - options: { - exclude: ['modernizr'] - }, - all: { - rjsConfig: '<%= yeoman.app %>/scripts/config.js' - } - } - }); - - grunt.registerTask('server', function(target) { - if (target === 'dist') { - return grunt.task.run(['build', 'open', 'connect:dist:keepalive']); - } - - grunt.task.run([ - 'clean:server', - 'concurrent:server', - 'connect:livereload', - 'open', - 'watch' - ]); - }); - - grunt.registerTask('test', [ - 'clean:server', - 'concurrent:test', - 'connect:test', - 'mocha' - ]); - - grunt.registerTask('build', [ - 'clean:dist', - 'copy:libs', - 'useminPrepare', - 'concurrent:dist', - 'requirejs', - 'cssmin', - 'concat', - 'uglify', - 'copy', - 'rev', - 'usemin' - ]); - - grunt.registerTask('default', [ - 'jshint', - 'test', - 'build' - ]); -}; diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py deleted file mode 100644 index a86ecca70..000000000 --- a/module/web/ServerThread.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python -from __future__ import with_statement -from time import time, sleep - -import threading -import logging - -from module.utils.fs import exists - -core = None -setup = None -log = logging.getLogger("log") - -class WebServer(threading.Thread): - def __init__(self, pycore=None, pysetup=None): - global core, setup - threading.Thread.__init__(self) - - if pycore: - core = pycore - config = pycore.config - elif pysetup: - setup = pysetup - config = pysetup.config - else: - raise Exception("No config context provided") - - self.server = config['webinterface']['server'] - self.https = config['webinterface']['https'] - self.cert = config["ssl"]["cert"] - self.key = config["ssl"]["key"] - self.host = config['webinterface']['host'] - self.port = config['webinterface']['port'] - self.debug = config['general']['debug_mode'] - self.force_server = config['webinterface']['force_server'] - self.error = None - - self.setDaemon(True) - - def run(self): - self.running = True - import webinterface - global webinterface - - if self.https: - if not exists(self.cert) or not exists(self.key): - log.warning(_("SSL certificates not found.")) - self.https = False - - prefer = None - - # These cases covers all settings - if self.server == "threaded": - prefer = "threaded" - elif self.server == "fastcgi": - prefer = "flup" - elif self.server == "fallback": - prefer = "wsgiref" - - server = self.select_server(prefer) - - try: - self.start_server(server) - - except Exception, e: - log.error(_("Failed starting webserver: " + e.message)) - self.error = e - if core: - core.print_exc() - - def select_server(self, prefer=None): - """ find a working server """ - from servers import all_server - - unavailable = [] - server = None - for server in all_server: - - if self.force_server and self.force_server == server.NAME: - break # Found server - # When force_server is set, no further checks have to be made - elif self.force_server: - continue - - if prefer and prefer == server.NAME: - break # found prefered server - elif prefer: # prefer is similar to force, but force has precedence - continue - - # Filter for server that offer ssl if needed - if self.https and not server.SSL: - continue - - try: - if server.find(): - break # Found a server - else: - unavailable.append(server.NAME) - except Exception, e: - log.error(_("Failed importing webserver: " + e.message)) - - if unavailable: # Just log whats not available to have some debug information - log.debug("Unavailable webserver: " + ",".join(unavailable)) - - if not server and self.force_server: - server = self.force_server # just return the name - - return server - - - def start_server(self, server): - - from servers import ServerAdapter - - if issubclass(server, ServerAdapter): - - if self.https and not server.SSL: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - elif not self.https: - self.cert = self.key = None # This implicitly disables SSL - # there is no extra argument for the server adapter - # TODO: check for openSSL ? - - # Now instantiate the serverAdapter - server = server(self.host, self.port, self.key, self.cert, 6, self.debug) # todo, num_connections - name = server.NAME - - else: # server is just a string - name = server - - log.info(_("Starting %(name)s webserver: %(host)s:%(port)d") % {"name": name, "host": self.host, "port": self.port}) - webinterface.run_server(host=self.host, port=self.port, server=server) - - - - # check if an error was raised for n seconds - def check_error(self, n=1): - t = time() + n - while time() < t: - if self.error: - return self.error - sleep(0.1) - diff --git a/module/web/__init__.py b/module/web/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/web/__init__.py +++ /dev/null diff --git a/module/web/api_app.py b/module/web/api_app.py deleted file mode 100644 index 3874b38a3..000000000 --- a/module/web/api_app.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from urllib import unquote -from itertools import chain -from traceback import format_exc, print_exc - -from bottle import route, request, response, HTTPError, parse_auth - -from utils import set_session, get_user_api -from webinterface import PYLOAD - -from module.Api import ExceptionObject -from module.remote.json_converter import loads, dumps -from module.utils import remove_chars - -def add_header(r): - r.headers.replace("Content-type", "application/json") - r.headers.append("Cache-Control", "no-cache, must-revalidate") - r.headers.append("Access-Control-Allow-Origin", request.get_header('Origin', '*')) - r.headers.append("Access-Control-Allow-Credentials", "true") - -# accepting positional arguments, as well as kwargs via post and get -# only forbidden path symbol are "?", which is used to separate GET data and # -@route("/api/<func><args:re:[^#?]*>") -@route("/api/<func><args:re:[^#?]*>", method="POST") -def call_api(func, args=""): - add_header(response) - - s = request.environ.get('beaker.session') - # Accepts standard http auth - auth = parse_auth(request.get_header('Authorization', '')) - if 'session' in request.POST or 'session' in request.GET: - # removes "' so it works on json strings - s = s.get_by_id(remove_chars(request.params.get('session'), "'\"")) - elif auth: - user = PYLOAD.checkAuth(auth[0], auth[1], request.environ.get('REMOTE_ADDR', None)) - # if auth is correct create a pseudo session - if user: s = {'uid': user.uid} - - api = get_user_api(s) - if not api: - return HTTPError(401, dumps("Unauthorized"), **response.headers) - - if not PYLOAD.isAuthorized(func, api.user): - return HTTPError(403, dumps("Forbidden"), **response.headers) - - if not hasattr(PYLOAD.EXTERNAL, func) or func.startswith("_"): - print "Invalid API call", func - return HTTPError(404, dumps("Not Found"), **response.headers) - - # TODO: possible encoding - # TODO Better error codes on invalid input - - args = [loads(unquote(arg)) for arg in args.split("/")[1:]] - kwargs = {} - - # accepts body as json dict - if request.json: - kwargs = request.json - - # convert arguments from json to obj separately - for x, y in chain(request.GET.iteritems(), request.POST.iteritems()): - if not x or not y or x == "session": continue - kwargs[x] = loads(unquote(y)) - - try: - result = getattr(api, func)(*args, **kwargs) - # null is invalid json response - if result is None: result = True - return dumps(result) - - except ExceptionObject, e: - return HTTPError(400, dumps(e), **response.headers) - except Exception, e: - print_exc() - return HTTPError(500, dumps({"error": e.message, "traceback": format_exc()}), **response.headers) - - -@route("/api/login") -@route("/api/login", method="POST") -def login(): - add_header(response) - - username = request.params.get("username") - password = request.params.get("password") - - user = PYLOAD.checkAuth(username, password, request.environ.get('REMOTE_ADDR', None)) - - if not user: - return dumps(False) - - s = set_session(request, user) - - # get the session id by dirty way, documentations seems wrong - try: - sid = s._headers["cookie_out"].split("=")[1].split(";")[0] - return dumps(sid) - except: - print "Could not get session" - return dumps(True) - - -@route("/api/logout") -@route("/api/logout", method="POST") -def logout(): - add_header(response) - - s = request.environ.get('beaker.session') - s.delete() - - return dumps(True) diff --git a/module/web/app/favicon.ico b/module/web/app/favicon.ico Binary files differdeleted file mode 100644 index d7f9f1857..000000000 --- a/module/web/app/favicon.ico +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Bold-webfont.eot b/module/web/app/fonts/Sansation_Bold-webfont.eot Binary files differdeleted file mode 100644 index 43ed2ee31..000000000 --- a/module/web/app/fonts/Sansation_Bold-webfont.eot +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Bold-webfont.ttf b/module/web/app/fonts/Sansation_Bold-webfont.ttf Binary files differdeleted file mode 100644 index d2e7c4c2a..000000000 --- a/module/web/app/fonts/Sansation_Bold-webfont.ttf +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Bold-webfont.woff b/module/web/app/fonts/Sansation_Bold-webfont.woff Binary files differdeleted file mode 100644 index 9ee938d55..000000000 --- a/module/web/app/fonts/Sansation_Bold-webfont.woff +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Light-webfont.eot b/module/web/app/fonts/Sansation_Light-webfont.eot Binary files differdeleted file mode 100644 index d83fa9cf6..000000000 --- a/module/web/app/fonts/Sansation_Light-webfont.eot +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Light-webfont.ttf b/module/web/app/fonts/Sansation_Light-webfont.ttf Binary files differdeleted file mode 100644 index 64d734bec..000000000 --- a/module/web/app/fonts/Sansation_Light-webfont.ttf +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Light-webfont.woff b/module/web/app/fonts/Sansation_Light-webfont.woff Binary files differdeleted file mode 100644 index 5f3dce493..000000000 --- a/module/web/app/fonts/Sansation_Light-webfont.woff +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Regular-webfont.eot b/module/web/app/fonts/Sansation_Regular-webfont.eot Binary files differdeleted file mode 100644 index 46219c9ff..000000000 --- a/module/web/app/fonts/Sansation_Regular-webfont.eot +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Regular-webfont.ttf b/module/web/app/fonts/Sansation_Regular-webfont.ttf Binary files differdeleted file mode 100644 index 92f686359..000000000 --- a/module/web/app/fonts/Sansation_Regular-webfont.ttf +++ /dev/null diff --git a/module/web/app/fonts/Sansation_Regular-webfont.woff b/module/web/app/fonts/Sansation_Regular-webfont.woff Binary files differdeleted file mode 100644 index 524b67992..000000000 --- a/module/web/app/fonts/Sansation_Regular-webfont.woff +++ /dev/null diff --git a/module/web/app/images/default/bgpattern.png b/module/web/app/images/default/bgpattern.png Binary files differdeleted file mode 100644 index 5111e6bdf..000000000 --- a/module/web/app/images/default/bgpattern.png +++ /dev/null diff --git a/module/web/app/images/default/checks_sheet.png b/module/web/app/images/default/checks_sheet.png Binary files differdeleted file mode 100644 index 9662b8070..000000000 --- a/module/web/app/images/default/checks_sheet.png +++ /dev/null diff --git a/module/web/app/images/default/fancy_deboss.png b/module/web/app/images/default/fancy_deboss.png Binary files differdeleted file mode 100644 index 926a762db..000000000 --- a/module/web/app/images/default/fancy_deboss.png +++ /dev/null diff --git a/module/web/app/images/default/logo.png b/module/web/app/images/default/logo.png Binary files differdeleted file mode 100644 index bb9c13679..000000000 --- a/module/web/app/images/default/logo.png +++ /dev/null diff --git a/module/web/app/images/default/logo_grey.png b/module/web/app/images/default/logo_grey.png Binary files differdeleted file mode 100644 index a4114d832..000000000 --- a/module/web/app/images/default/logo_grey.png +++ /dev/null diff --git a/module/web/app/images/icon.png b/module/web/app/images/icon.png Binary files differdeleted file mode 100644 index 1ab4ca081..000000000 --- a/module/web/app/images/icon.png +++ /dev/null diff --git a/module/web/app/index.html b/module/web/app/index.html deleted file mode 100644 index 87fd6c612..000000000 --- a/module/web/app/index.html +++ /dev/null @@ -1,107 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> - <!-- TODO: dynamic title --> - <title>pyLoad WebUI</title> - <meta name="description" content="pyLoad WebUI"> - <meta name="viewport" content="width=device-width"> - - <!-- TODO Include this font --> - <link href="http://fonts.googleapis.com/css?family=Abel" rel="stylesheet" type="text/css"/> - - <!-- TODO: basepath and templates --> - <link href="/styles/font.css" rel="stylesheet" type="text/css"/> - <link href="/styles/default/main.css" rel="stylesheet" type="text/css"> - - <!-- build:css styles/vendor.css --> - <link href="components/select2/select2.css" rel="stylesheet" type="text/css"/> - <!-- endbuild --> - - <!-- build:js scripts/default.js --> - <script data-main="scripts/config" src="components/requirejs/require.js"></script> - <!-- endbuild --> - - <script type="text/javascript"> - window.dates = { - weeks: ['week', 'weeks'], - days: ['day', 'days'], - hours: ['hour', 'hours'], - minutes: ['minute', 'minutes'], - seconds: ['second', 'seconds'] - }; // TODO carefully when translating - - window.hostProtocol = 'http://'; - window.hostAddress = 'localhost'; - window.hostPort = '8001'; - window.pathPrefix = "/"; - - window.wsAddress = "ws://%s:7227"; // TODO - - require(['config'], function(Config) { - require(['default'], function(App) { - }); - }) - </script> - -</head> -<body> -<div id="wrap"> - <header> - <div class="container-fluid"> - <div class="row-fluid" id="header"> - <div class="span3"> - <div class="logo"></div> - <span class="title visible-large-screen">pyLoad</span> - </div> - </div> - </div> - <div id="notification-area"></div> - <div id="selection-area"></div> - </header> - <div id="content-container" class="container-fluid"> - <div class="row-fluid" id="actionbar"> - </div> - <div class="row-fluid" id="content"> - </div> - </div> -</div> -<footer> - <div class="container-fluid"> - <div class="row-fluid"> - <div class="span2 offset1"> - <div class="copyright"> - © 2008-2013<br> - <a href="http://pyload.org/" target="_blank">The pyLoad Team</a><br> - </div> - </div> - <div class="span2"> - <h2 class="block-title">Powered by</h2> - <hr> - Bootstrap <br> - </div> - - <div class="span2"> - <h2 class="block-title">pyLoad</h2> - <hr> - dsfdsf <br> - </div> - - <div class="span2"> - <h2 class="block-title">Community</h2> - <hr> - asd <br> - </div> - - <div class="span2"> - <h2 class="block-title">Development</h2> - <hr> - asd <br> - </div> - </div> - </div> -</footer> -<div id="modal-overlay" class="hide"></div> -</body> -</html> diff --git a/module/web/app/scripts/app.js b/module/web/app/scripts/app.js deleted file mode 100644 index 427cb1bc8..000000000 --- a/module/web/app/scripts/app.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Global Application Object - * Contains all necessary logic shared across views - */ -define([ - - // Libraries. - 'jquery', - 'underscore', - 'backbone', - 'utils/initHB', - 'utils/animations', - 'utils/lazyRequire', - 'utils/dialogs', - 'marionette', - 'bootstrap', - 'animate' - -], function($, _, Backbone, Handlebars) { - 'use strict'; - - Backbone.Marionette.TemplateCache.prototype.compileTemplate = function(rawTemplate) { - return Handlebars.compile(rawTemplate); - }; - - // TODO: configurable root - var App = new Backbone.Marionette.Application({ - root: '/' - }); - - App.addRegions({ - header: '#header', - notification: '#notification-area', - selection: '#selection-area', - content: '#content', - actionbar: '#actionbar' - }); - - App.navigate = function(url) { - return Backbone.history.navigate(url, true); - }; - - App.apiUrl = function(path) { - var url = window.hostProtocol + window.hostAddress + ':' + window.hostPort + window.pathPrefix + path; - return url; - }; - - // Add Global Helper functions - // Generates options dict that can be used for xhr requests - App.apiRequest = function(method, data, options) { - options || (options = {}); - options.url = App.apiUrl('api/' + method); - options.dataType = 'json'; - - if (data) { - options.type = 'POST'; - options.data = {}; - // Convert arguments to json - _.keys(data).map(function(key) { - options.data[key] = JSON.stringify(data[key]); - }); - } - - return options; - }; - - App.setTitle = function(name) { - var title = window.document.title; - var newTitle; - // page name separator - var index = title.indexOf('-'); - if (index >= 0) - newTitle = name + ' - ' + title.substr(index + 2, title.length); - else - newTitle = name + ' - ' + title; - - window.document.title = newTitle; - }; - - App.openWebSocket = function(path) { - // TODO - return new WebSocket(window.wsAddress.replace('%s', window.hostAddress) + path); - }; - - App.on('initialize:after', function() { -// TODO pushState variable - Backbone.history.start({ - pushState: false, - root: App.root - }); - - // All links should be handled by backbone - $(document).on('click', 'a[data-nav]', function(evt) { - var href = { prop: $(this).prop('href'), attr: $(this).attr('href') }; - var root = location.protocol + '//' + location.host + App.root; - if (href.prop.slice(0, root.length) === root) { - evt.preventDefault(); - Backbone.history.navigate(href.attr, true); - } - }); - }); - - // Returns the app object to be available to other modules through require.js. - return App; -});
\ No newline at end of file diff --git a/module/web/app/scripts/collections/AccountList.js b/module/web/app/scripts/collections/AccountList.js deleted file mode 100644 index bfc2af5a3..000000000 --- a/module/web/app/scripts/collections/AccountList.js +++ /dev/null @@ -1,24 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/Account'], function($, Backbone, _, App, Account) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Account, - - comparator: function(account) { - return account.get('plugin'); - }, - - initialize: function() { - - }, - - fetch: function(options) { - // TODO: refresh options? - options = App.apiRequest('getAccounts/false', null, options); - return Backbone.Collection.prototype.fetch.call(this, options); - } - - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/collections/FileList.js b/module/web/app/scripts/collections/FileList.js deleted file mode 100644 index 873f4c0e3..000000000 --- a/module/web/app/scripts/collections/FileList.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/File'], function($, Backbone, _, File) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: File, - - comparator: function(file) { - return file.get('fileorder'); - }, - - initialize: function() { - - } - - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/collections/InteractionList.js b/module/web/app/scripts/collections/InteractionList.js deleted file mode 100644 index 24f8b9248..000000000 --- a/module/web/app/scripts/collections/InteractionList.js +++ /dev/null @@ -1,49 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/InteractionTask'], - function($, Backbone, _, App, InteractionTask) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: InteractionTask, - - comparator: function(task) { - return task.get('iid'); - }, - - fetch: function(options) { - options = App.apiRequest('getInteractionTasks/0', null, options); - var self = this; - options.success = function(data) { - self.set(data); - }; - - return $.ajax(options); - }, - - toJSON: function() { - var data = {queries: 0, notifications: 0}; - - this.map(function(task) { - if (task.isNotification()) - data.notifications++; - else - data.queries++; - }); - - return data; - }, - - // a task is waiting for attention (no notification) - hasTaskWaiting: function() { - var tasks = 0; - this.map(function(task) { - if (!task.isNotification()) - tasks++; - }); - - return tasks > 0; - } - - }); - - });
\ No newline at end of file diff --git a/module/web/app/scripts/collections/PackageList.js b/module/web/app/scripts/collections/PackageList.js deleted file mode 100644 index 7bee861a4..000000000 --- a/module/web/app/scripts/collections/PackageList.js +++ /dev/null @@ -1,16 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/Package'], function($, Backbone, _, Package) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Package, - - comparator: function(pack) { - return pack.get('packageorder'); - }, - - initialize: function() { - } - - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/collections/ProgressList.js b/module/web/app/scripts/collections/ProgressList.js deleted file mode 100644 index 51849d8de..000000000 --- a/module/web/app/scripts/collections/ProgressList.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'models/Progress'], function($, Backbone, _, Progress) { - 'use strict'; - - return Backbone.Collection.extend({ - - model: Progress, - - comparator: function(progress) { - return progress.get('eta'); - }, - - initialize: function() { - - } - - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/config.js b/module/web/app/scripts/config.js deleted file mode 100644 index 398d97e11..000000000 --- a/module/web/app/scripts/config.js +++ /dev/null @@ -1,73 +0,0 @@ -// Sets the require.js configuration for your application. -'use strict'; -require.config({ - - deps: ['default'], - - paths: { - - jquery: '../components/jquery/jquery', - flot: '../components/flot/jquery.flot', - transit: '../components/jquery.transit/jquery.transit', - animate: '../components/jquery.animate-enhanced/scripts/src/jquery.animate-enhanced', - cookie: '../components/jquery.cookie/jquery.cookie', - omniwindow: 'vendor/jquery.omniwindow', - select2: '../components/select2/select2', - bootstrap: 'vendor/bootstrap-2.3.2', - underscore: '../components/underscore/underscore', - backbone: '../components/backbone/backbone', - marionette: '../components/backbone.marionette/lib/backbone.marionette', -// handlebars: '../components/handlebars.js/dist/handlebars', - handlebars: 'vendor/Handlebars-1.0rc1', - jed: '../components/jed/jed', - - // TODO: Two hbs dependencies could be replaced - i18nprecompile: '../components/require-handlebars-plugin/hbs/i18nprecompile', - json2: '../components/require-handlebars-plugin/hbs/json2', - - // Plugins - text: '../components/requirejs-text/text', - hbs: '../components/require-handlebars-plugin/hbs', - - // Shortcut - tpl: '../templates/default' - }, - - hbs: { - disableI18n: true, - helperPathCallback: // Callback to determine the path to look for helpers - function(name) { - // Some helpers are accumulated into one file - if (name.indexOf('file') === 0) - name = 'fileHelper'; - - return 'helpers/' + name; - }, - templateExtension: 'html' - }, - - // Sets the configuration for your third party scripts that are not AMD compatible - shim: { - underscore: { - exports: '_' - }, - - backbone: { - deps: ['underscore', 'jquery'], - exports: 'Backbone' - }, - - marionette: ['backbone'], -// handlebars: { -// exports: 'Handlebars' -// }, - - flot: ['jquery'], - transit: ['jquery'], - cookie: ['jquery'], - omniwindow: ['jquery'], - select2: ['jquery'], - bootstrap: ['jquery'], - animate: ['jquery'] - } -});
\ No newline at end of file diff --git a/module/web/app/scripts/controller.js b/module/web/app/scripts/controller.js deleted file mode 100644 index 05237914d..000000000 --- a/module/web/app/scripts/controller.js +++ /dev/null @@ -1,67 +0,0 @@ -define([ - 'app', - 'backbone', - - // Views - 'views/headerView', - 'views/notificationView', - 'views/dashboard/dashboardView', - 'views/dashboard/selectionView', - 'views/dashboard/filterView', - 'views/loginView', - 'views/settings/settingsView', - 'views/accounts/accountListView' -], function( - App, Backbone, HeaderView, NotificationView, DashboardView, SelectionView, FilterView, LoginView, SettingsView, AccountListView) { - 'use strict'; - // TODO some views does not need to be loaded instantly - - return { - - header: function() { - if (!App.header.currentView) { - App.header.show(new HeaderView()); - App.header.currentView.init(); - App.notification.attachView(new NotificationView()); - } - }, - - dashboard: function() { - this.header(); - - App.actionbar.show(new FilterView()); - // TODO: not completly visible after reattaching - App.selection.attachView(new SelectionView()); - App.content.show(new DashboardView()); - }, - - login: function() { - App.content.show(new LoginView()); - }, - - logout: function() { - alert('Not implemented'); - }, - - settings: function() { - this.header(); - - var view = new SettingsView(); - App.actionbar.show(new view.actionbar()); - App.content.show(view); - }, - - accounts: function() { - this.header(); - - var view = new AccountListView(); - App.actionbar.show(new view.actionbar()); - App.content.show(view); - }, - - admin: function() { - alert('Not implemented'); - } - }; - -}); diff --git a/module/web/app/scripts/default.js b/module/web/app/scripts/default.js deleted file mode 100644 index a337cee21..000000000 --- a/module/web/app/scripts/default.js +++ /dev/null @@ -1,30 +0,0 @@ -define('default', ['backbone', 'jquery', 'app', 'router', 'models/userSession'], - function(Backbone, $, App, Router, UserSession) { - 'use strict'; - - // Global ajax options - var options = { - statusCode: { - 401: function() { - console.log('Not logged in.'); - App.navigate('login'); - } - }, - xhrFields: {withCredentials: true} - }; - - $.ajaxSetup(options); - - Backbone.ajax = function() { - Backbone.$.ajaxSetup.call(Backbone.$, options); - return Backbone.$.ajax.apply(Backbone.$, arguments); - }; - - $(function() { - App.session = new UserSession(); - App.router = new Router(); - App.start(); - }); - - return App; - });
\ No newline at end of file diff --git a/module/web/app/scripts/helpers/fileHelper.js b/module/web/app/scripts/helpers/fileHelper.js deleted file mode 100644 index 156be58f0..000000000 --- a/module/web/app/scripts/helpers/fileHelper.js +++ /dev/null @@ -1,55 +0,0 @@ -// Helpers to render the file view -define('helpers/fileHelper', ['handlebars', 'utils/apitypes', 'helpers/formatTime'], - function(Handlebars, Api, formatTime) { - 'use strict'; - - function fileClass(file, options) { - if (file.finished) - return 'finished'; - else if (file.failed) - return 'failed'; - else if (file.offline) - return 'offline'; - else if (file.online) - return 'online'; - else if (file.waiting) - return 'waiting'; - else if (file.downloading) - return 'downloading'; - - return ''; - } - - // TODO - function fileIcon(media, options) { - return 'icon-music'; - } - - // TODO rest of the states - function fileStatus(file, options) { - var s; - var msg = file.download.statusmsg; - - if (file.failed) { - s = '<i class="icon-remove"></i> '; - if (file.download.error) - s += file.download.error; - else s += msg; - } else if (file.finished) - s = '<i class="icon-ok"></i> ' + msg; - else if (file.downloading) - s = '<div class="progress"><div class="bar" style="width: ' + file.progress + '%"> ' + - formatTime(file.eta) + '</div></div>'; - else if (file.waiting) - s = '<i class="icon-time"></i> ' + formatTime(file.eta); - else - s = msg; - - return new Handlebars.SafeString(s); - } - - Handlebars.registerHelper('fileClass', fileClass); - Handlebars.registerHelper('fileIcon', fileIcon); - Handlebars.registerHelper('fileStatus', fileStatus); - return fileClass; - });
\ No newline at end of file diff --git a/module/web/app/scripts/helpers/formatSize.js b/module/web/app/scripts/helpers/formatSize.js deleted file mode 100644 index 3b62e74c7..000000000 --- a/module/web/app/scripts/helpers/formatSize.js +++ /dev/null @@ -1,15 +0,0 @@ -// Format bytes in human readable format -define('helpers/formatSize', ['handlebars'], function(Handlebars) { - 'use strict'; - - var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB']; - function formatSize(bytes, options) { - if (!bytes || bytes === 0) return '0 B'; - var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10); - // round to two digits - return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + sizes[i]; - } - - Handlebars.registerHelper('formatSize', formatSize); - return formatSize; -});
\ No newline at end of file diff --git a/module/web/app/scripts/helpers/formatTime.js b/module/web/app/scripts/helpers/formatTime.js deleted file mode 100644 index 757ff73ad..000000000 --- a/module/web/app/scripts/helpers/formatTime.js +++ /dev/null @@ -1,17 +0,0 @@ -// Format bytes in human readable format -define('helpers/formatTime', ['handlebars', 'vendor/remaining'], function(Handlebars, Remaining) { - 'use strict'; - - function formatTime(seconds, options) { - if (seconds === Infinity) - return 'â'; - else if (!seconds || seconds <= 0) - return '-'; - - // TODO: digital or written string - return Remaining.getStringDigital(seconds, window.dates); - } - - Handlebars.registerHelper('formatTime', formatTime); - return formatTime; -});
\ No newline at end of file diff --git a/module/web/app/scripts/helpers/pluginIcon.js b/module/web/app/scripts/helpers/pluginIcon.js deleted file mode 100644 index 6b2fdc67f..000000000 --- a/module/web/app/scripts/helpers/pluginIcon.js +++ /dev/null @@ -1,14 +0,0 @@ -// Resolves name of plugin to icon path -define('helpers/pluginIcon', ['handlebars', 'app'], function(Handlebars, App) { - 'use strict'; - - function pluginIcon(name) { - if (typeof name === 'object' && typeof name.get === 'function') - name = name.get('plugin'); - - return App.apiUrl('icons/' + name); - } - - Handlebars.registerHelper('pluginIcon', pluginIcon); - return pluginIcon; -});
\ No newline at end of file diff --git a/module/web/app/scripts/models/Account.js b/module/web/app/scripts/models/Account.js deleted file mode 100644 index a2e24b056..000000000 --- a/module/web/app/scripts/models/Account.js +++ /dev/null @@ -1,51 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - // TODO - // generated, not submitted - idAttribute: 'user', - - defaults: { - plugin: null, - loginname: null, - owner: -1, - valid: false, - validuntil: -1, - trafficleft: -1, - maxtraffic: -1, - premium: false, - activated: false, - shared: false, - options: null - }, - - // Model Constructor - initialize: function() { - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - save: function(options) { - options = App.apiRequest('updateAccountInfo', {account: this.toJSON()}, options); - return $.ajax(options); - }, - - destroy: function(options) { - options = App.apiRequest('removeAccount', {account: this.toJSON()}, options); - var self = this; - options.success = function() { - self.trigger('destroy', self, self.collection, options); - }; - - // TODO request is not dispatched -// return Backbone.Model.prototype.destroy.call(this, options); - return $.ajax(options); - } - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/models/ConfigHolder.js b/module/web/app/scripts/models/ConfigHolder.js deleted file mode 100644 index 40efbc7c0..000000000 --- a/module/web/app/scripts/models/ConfigHolder.js +++ /dev/null @@ -1,68 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', './ConfigItem'], - function($, Backbone, _, App, ConfigItem) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - name: '', - label: '', - description: '', - long_description: null, - // simple list but no collection - items: null, - info: null - }, - - // Model Constructor - initialize: function() { - - }, - - // Loads it from server by name - fetch: function(options) { - options = App.apiRequest('loadConfig/"' + this.get('name') + '"', null, options); - return Backbone.Model.prototype.fetch.call(this, options); - }, - - save: function(options) { - var config = this.toJSON(); - var items = []; - // Convert changed items to json - _.each(config.items, function(item) { - if (item.isChanged()) { - items.push(item.prepareSave()); - } - }); - config.items = items; - // TODO: only set new values on success - - options = App.apiRequest('saveConfig', {config: config}, options); - - return $.ajax(options); - }, - - parse: function(resp) { - // Create item models - resp.items = _.map(resp.items, function(item) { - return new ConfigItem(item); - }); - - return Backbone.Model.prototype.parse.call(this, resp); - }, - - isLoaded: function() { - return this.has('items') || this.has('long_description'); - }, - - // check if any of the items has changes - hasChanges: function() { - var items = this.get('items'); - if (!items) return false; - return _.reduce(items, function(a, b) { - return a || b.isChanged(); - }, false); - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/ConfigItem.js b/module/web/app/scripts/models/ConfigItem.js deleted file mode 100644 index 2d325c2a2..000000000 --- a/module/web/app/scripts/models/ConfigItem.js +++ /dev/null @@ -1,40 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], - function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - name: '', - label: '', - description: '', - input: null, - default_value: null, - value: null, - // additional attributes - inputView: null - }, - - // Model Constructor - initialize: function() { - - }, - - isChanged: function() { - return this.get('inputView') && this.get('inputView').getVal() !== this.get('value'); - }, - - // set new value and return json - prepareSave: function() { - // set the new value - if (this.get('inputView')) - this.set('value', this.get('inputView').getVal()); - - var data = this.toJSON(); - delete data.inputView; - delete data.description; - - return data; - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/File.js b/module/web/app/scripts/models/File.js deleted file mode 100644 index 3beb7f270..000000000 --- a/module/web/app/scripts/models/File.js +++ /dev/null @@ -1,92 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], function($, Backbone, _, App, Api) { - 'use strict'; - - var Finished = [Api.DownloadStatus.Finished, Api.DownloadStatus.Skipped]; - var Failed = [Api.DownloadStatus.Failed, Api.DownloadStatus.Aborted, Api.DownloadStatus.TempOffline, Api.DownloadStatus.Offline]; - // Unfinished - Other - - return Backbone.Model.extend({ - - idAttribute: 'fid', - - defaults: { - fid: -1, - name: null, - package: -1, - owner: -1, - size: -1, - status: -1, - media: -1, - added: -1, - fileorder: -1, - download: null, - - // UI attributes - selected: false, - visible: true, - progress: 0, - eta: 0 - }, - - // Model Constructor - initialize: function() { - - }, - - fetch: function(options) { - options = App.apiRequest( - 'getFileInfo', - {fid: this.get('fid')}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - destroy: function(options) { - // also not working when using data - options = App.apiRequest( - 'deleteFiles/[' + this.get('fid') + ']', - null, options); - options.method = 'post'; - - return Backbone.Model.prototype.destroy.call(this, options); - }, - - // Does not send a request to the server - destroyLocal: function(options) { - this.trigger('destroy', this, this.collection, options); - }, - - restart: function(options) { - options = App.apiRequest( - 'restartFile', - {fid: this.get('fid')}, - options); - - return $.ajax(options); - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - isDownload: function() { - return this.has('download'); - }, - - isFinished: function() { - return _.indexOf(Finished, this.get('download').status) > -1; - }, - - isUnfinished: function() { - return _.indexOf(Finished, this.get('download').status) === -1 && _.indexOf(Failed, this.get('download').status) === -1; - }, - - isFailed: function() { - return _.indexOf(Failed, this.get('download').status) > -1; - } - - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/models/InteractionTask.js b/module/web/app/scripts/models/InteractionTask.js deleted file mode 100644 index 54c739d4b..000000000 --- a/module/web/app/scripts/models/InteractionTask.js +++ /dev/null @@ -1,41 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes'], - function($, Backbone, _, App, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'iid', - - defaults: { - iid: -1, - type: null, - input: null, - default_value: null, - title: '', - description: '', - plugin: '', - // additional attributes - result: '' - }, - - // Model Constructor - initialize: function() { - - }, - - save: function(options) { - options = App.apiRequest('setInteractionResult/' + this.get('iid'), - {result: this.get('result')}, options); - - return $.ajax(options); - }, - - isNotification: function() { - return this.get('type') === Api.Interaction.Notification; - }, - - isCaptcha: function() { - return this.get('type') === Api.Interaction.Captcha; - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/Package.js b/module/web/app/scripts/models/Package.js deleted file mode 100644 index a34ec1c69..000000000 --- a/module/web/app/scripts/models/Package.js +++ /dev/null @@ -1,119 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'collections/FileList', 'require'], - function($, Backbone, _, App, FileList, require) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'pid', - - defaults: { - pid: -1, - name: null, - folder: '', - root: -1, - owner: -1, - site: '', - comment: '', - password: '', - added: -1, - tags: null, - status: -1, - shared: false, - packageorder: -1, - stats: null, - fids: null, - pids: null, - files: null, // Collection - packs: null, // Collection - - selected: false // For Checkbox - }, - - // Model Constructor - initialize: function() { - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - obj.percent = Math.round(obj.stats.linksdone * 100 / obj.stats.linkstotal); - - return obj; - }, - - // Changes url + method and delegates call to super class - fetch: function(options) { - options = App.apiRequest( - 'getFileTree/' + this.get('pid'), - {full: false}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - // Create a pseudo package und use search to populate data - search: function(qry, options) { - options = App.apiRequest( - 'findFiles', - {pattern: qry}, - options); - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - save: function(options) { - // TODO - }, - - destroy: function(options) { - // TODO: Not working when using data?, array seems to break it - options = App.apiRequest( - 'deletePackages/[' + this.get('pid') + ']', - null, options); - options.method = 'post'; - - console.log(options); - - return Backbone.Model.prototype.destroy.call(this, options); - }, - - restart: function(options) { - options = App.apiRequest( - 'restartPackage', - {pid: this.get('pid')}, - options); - - var self = this; - options.success = function() { - self.fetch(); - }; - return $.ajax(options); - }, - - parse: function(resp) { - // Package is loaded from tree collection - if (_.has(resp, 'root')) { - if (!this.has('files')) - resp.root.files = new FileList(_.values(resp.files)); - else - this.get('files').set(_.values(resp.files)); - - // circular dependencies needs to be avoided - var PackageList = require('collections/PackageList'); - - if (!this.has('packs')) - resp.root.packs = new PackageList(_.values(resp.packages)); - else - this.get('packs').set(_.values(resp.packages)); - - return resp.root; - } - return Backbone.model.prototype.parse.call(this, resp); - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/Progress.js b/module/web/app/scripts/models/Progress.js deleted file mode 100644 index b0bbb684d..000000000 --- a/module/web/app/scripts/models/Progress.js +++ /dev/null @@ -1,50 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'utils/apitypes'], function($, Backbone, _, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - // generated, not submitted - idAttribute: 'pid', - - defaults: { - pid: -1, - plugin: null, - name: null, - statusmsg: -1, - eta: -1, - done: -1, - total: -1, - download: null - }, - - getPercent: function() { - if (this.get('total') > 0) - return Math.round(this.get('done') * 100 / this.get('total')); - return 0; - }, - - // Model Constructor - initialize: function() { - - }, - - // Any time a model attribute is set, this method is called - validate: function(attrs) { - - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - obj.percent = this.getPercent(); - obj.downloading = this.isDownload() && this.get('download').status === Api.DownloadStatus.Downloading; - - return obj; - }, - - isDownload : function() { - return this.has('download'); - } - - }); - -});
\ No newline at end of file diff --git a/module/web/app/scripts/models/ServerStatus.js b/module/web/app/scripts/models/ServerStatus.js deleted file mode 100644 index 59739b41e..000000000 --- a/module/web/app/scripts/models/ServerStatus.js +++ /dev/null @@ -1,47 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], - function($, Backbone, _) { - 'use strict'; - - return Backbone.Model.extend({ - - defaults: { - speed: 0, - linkstotal: 0, - linksqueue: 0, - sizetotal: 0, - sizequeue: 0, - notifications: -1, - paused: false, - download: false, - reconnect: false - }, - - // Model Constructor - initialize: function() { - - }, - - fetch: function(options) { - options || (options = {}); - options.url = 'api/getServerStatus'; - - return Backbone.Model.prototype.fetch.call(this, options); - }, - - toJSON: function(options) { - var obj = Backbone.Model.prototype.toJSON.call(this, options); - - obj.linksdone = obj.linkstotal - obj.linksqueue; - obj.sizedone = obj.sizetotal - obj.sizequeue; - if (obj.speed && obj.speed > 0) - obj.eta = Math.round(obj.sizequeue / obj.speed); - else if (obj.sizequeue > 0) - obj.eta = Infinity; - else - obj.eta = 0; - - return obj; - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/TreeCollection.js b/module/web/app/scripts/models/TreeCollection.js deleted file mode 100644 index 2f761e6cc..000000000 --- a/module/web/app/scripts/models/TreeCollection.js +++ /dev/null @@ -1,50 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/Package', 'collections/FileList', 'collections/PackageList'], - function($, Backbone, _, App, Package, FileList, PackageList) { - 'use strict'; - - // TreeCollection - // A Model and not a collection, aggregates other collections - return Backbone.Model.extend({ - - defaults: { - root: null, - packages: null, - files: null - }, - - initialize: function() { - - }, - - fetch: function(options) { - options || (options = {}); - var pid = options.pid || -1; - - options = App.apiRequest( - 'getFileTree/' + pid, - {full: false}, - options); - - console.log('Fetching package tree ' + pid); - return Backbone.Model.prototype.fetch.call(this, options); - }, - - // Parse the response and updates the collections - parse: function(resp) { - var ret = {}; - if (!this.has('packages')) - ret.packages = new PackageList(_.values(resp.packages)); - else - this.get('packages').set(_.values(resp.packages)); - - if (!this.has('files')) - ret.files = new FileList(_.values(resp.files)); - else - this.get('files').set(_.values(resp.files)); - - ret.root = new Package(resp.root); - return ret; - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/models/UserSession.js b/module/web/app/scripts/models/UserSession.js deleted file mode 100644 index a7e9aa848..000000000 --- a/module/web/app/scripts/models/UserSession.js +++ /dev/null @@ -1,20 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'utils/apitypes', 'cookie'], - function($, Backbone, _, Api) { - 'use strict'; - - return Backbone.Model.extend({ - - idAttribute: 'username', - - defaults: { - username: null, - permissions: null, - session: null - }, - - // Model Constructor - initialize: function() { - this.set('session', $.cookie('beaker.session.id')); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/router.js b/module/web/app/scripts/router.js deleted file mode 100644 index 68ea5575d..000000000 --- a/module/web/app/scripts/router.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Router defines routes that are handled by registered controller - */ -define([ - // Libraries - 'backbone', - 'marionette', - - // Modules - 'controller' -], - function(Backbone, Marionette, Controller) { - 'use strict'; - - return Backbone.Marionette.AppRouter.extend({ - - appRoutes: { - '': 'dashboard', - 'login': 'login', - 'logout': 'logout', - 'settings': 'settings', - 'accounts': 'accounts', - 'admin': 'admin' - }, - - // Our controller to handle the routes - controller: Controller - }); - }); diff --git a/module/web/app/scripts/routers/defaultRouter.js b/module/web/app/scripts/routers/defaultRouter.js deleted file mode 100644 index 4b00d160c..000000000 --- a/module/web/app/scripts/routers/defaultRouter.js +++ /dev/null @@ -1,30 +0,0 @@ -define(['jquery', 'backbone', 'views/headerView'], function($, Backbone, HeaderView) { - 'use strict'; - - var Router = Backbone.Router.extend({ - - initialize: function() { - Backbone.history.start(); - }, - - // All of your Backbone Routes (add more) - routes: { - - // When there is no hash bang on the url, the home method is called - '': 'home' - - }, - - 'home': function() { - // Instantiating mainView and anotherView instances - var headerView = new HeaderView(); - - // Renders the mainView template - headerView.render(); - - } - }); - - // Returns the Router class - return Router; -});
\ No newline at end of file diff --git a/module/web/app/scripts/routers/mobileRouter.js b/module/web/app/scripts/routers/mobileRouter.js deleted file mode 100644 index e24cb7a34..000000000 --- a/module/web/app/scripts/routers/mobileRouter.js +++ /dev/null @@ -1,56 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - return Backbone.Router.extend({ - - initialize: function() { - _.bindAll(this, 'changePage'); - - this.$el = $('#content'); - - // Tells Backbone to start watching for hashchange events - Backbone.history.start(); - - }, - - // All of your Backbone Routes (add more) - routes: { - - // When there is no hash bang on the url, the home method is called - '': 'home' - - }, - - 'home': function() { - - var self = this; - - $('#p1').fastClick(function() { - self.changePage($('<div class=\'page\' style=\'background-color: #9acd32;\'><h1>Page 1</h1><br>some content<br>sdfdsf<br>sdffg<h3>oiuzz</h3></div>')); - }); - - $('#p2').bind('click', function() { - self.changePage($('<div class=\'page\' style=\'background-color: blue;\'><h1>Page 2</h1><br>some content<br>sdfdsf<br><h2>sdfsdf</h2>sdffg</div>')); - }); - - }, - - changePage: function(content) { - - var oldpage = this.$el.find('.page'); - content.css({x: '100%'}); - this.$el.append(content); - content.transition({x: 0}, function() { - window.setTimeout(function() { - oldpage.remove(); - }, 400); - }); - -// $("#viewport").transition({x: "100%"}, function(){ -// $("#viewport").html(content); -// $("#viewport").transition({x: 0}); -// }); - } - - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/utils/animations.js b/module/web/app/scripts/utils/animations.js deleted file mode 100644 index 7f89afef1..000000000 --- a/module/web/app/scripts/utils/animations.js +++ /dev/null @@ -1,129 +0,0 @@ -define(['jquery', 'underscore', 'transit'], function(jQuery, _) { - 'use strict'; - - // Adds an element and computes its height, which is saved as data attribute - // Important function to have slide animations - jQuery.fn.appendWithHeight = function(element, hide) { - var o = jQuery(this[0]); - element = jQuery(element); - - // TODO: additionally it could be placed out of viewport first - // The real height can only be retrieved when element is on DOM and display:true - element.css('visibility', 'hidden'); - o.append(element); - - var height = element.height(); - - // Hide the element - if (hide === true) { - element.hide(); - element.height(0); - } - - element.css('visibility', ''); - element.data('height', height); - - return this; - }; - - // Shortcut to have a animation when element is added - jQuery.fn.appendWithAnimation = function(element, animation) { - var o = jQuery(this[0]); - element = jQuery(element); - - if (animation === true) - element.hide(); - - o.append(element); - - if (animation === true) - element.fadeIn(); - -// element.calculateHeight(); - - return this; - }; - - // calculate the height and write it to data, should be used on invisible elements - jQuery.fn.calculateHeight = function(setHeight) { - var o = jQuery(this[0]); - var height = o.height(); - if (!height) { - var display = o.css('display'); - o.css('visibility', 'hidden'); - o.show(); - height = o.height(); - - o.css('display', display); - o.css('visibility', ''); - } - - if (setHeight) - o.css('height', height); - - o.data('height', height); - return this; - }; - - // TODO: carry arguments, optional height argument - - // reset arguments, sets overflow hidden - jQuery.fn.slideOut = function(reset) { - var o = jQuery(this[0]); - o.animate({height: o.data('height'), opacity: 'show'}, function() { - // reset css attributes; - if (reset) { - this.css('overflow', ''); - this.css('height', ''); - } - }); - return this; - }; - - jQuery.fn.slideIn = function(reset) { - var o = jQuery(this[0]); - if (reset) { - o.css('overflow', 'hidden'); - } - o.animate({height: 0, opacity: 'hide'}); - return this; - }; - - jQuery.fn.initTooltips = function(placement) { - placement || (placement = 'top'); - - var o = jQuery(this[0]); - o.find('[data-toggle="tooltip"]').tooltip( - { - delay: {show: 800, hide: 100}, - placement: placement - }); - - return this; - }; - - jQuery.fn._transit = jQuery.fn.transit; - - // Overriding transit plugin to support hide and show - jQuery.fn.transit = jQuery.fn.transition = function(props, duration, easing, callback) { - var self = this; - var cb = callback; - var newprops = _.extend({}, props); - - if (newprops && (newprops.opacity === 'hide')) { - newprops.opacity = 0; - - callback = function() { - self.css({display: 'none'}); - if (typeof cb === 'function') { - cb.apply(self); - } - }; - } else if (newprops && (newprops.opacity === 'show')) { - newprops.opacity = 1; - this.css({display: 'block'}); - } - - return this._transit(newprops, duration, easing, callback); - }; -});
\ No newline at end of file diff --git a/module/web/app/scripts/utils/apitypes.js b/module/web/app/scripts/utils/apitypes.js deleted file mode 100644 index cbbc9064f..000000000 --- a/module/web/app/scripts/utils/apitypes.js +++ /dev/null @@ -1,16 +0,0 @@ -// Autogenerated, do not edit! -/*jslint -W070: false*/ -define([], function() { - 'use strict'; - return { - DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2}, - DownloadStatus: {'Downloading': 10, 'NA': 0, 'Processing': 14, 'Waiting': 9, 'Decrypting': 13, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 16, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 15, 'Starting': 8, 'Queued': 3}, - FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1}, - InputType: {'Multiple': 10, 'Int': 2, 'NA': 0, 'List': 11, 'Bool': 7, 'File': 3, 'Text': 1, 'Table': 12, 'Folder': 4, 'Password': 6, 'Click': 8, 'Select': 9, 'Textbox': 5}, - Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1}, - MediaType: {'All': 0, 'Audio': 2, 'Image': 4, 'Other': 1, 'Video': 8, 'Document': 16, 'Archive': 32}, - PackageStatus: {'Paused': 1, 'Remote': 3, 'Folder': 2, 'Ok': 0}, - Permission: {'All': 0, 'Interaction': 32, 'Modify': 4, 'Add': 1, 'Accounts': 16, 'Plugins': 64, 'Download': 8, 'Delete': 2}, - Role: {'Admin': 0, 'User': 1}, - }; -});
\ No newline at end of file diff --git a/module/web/app/scripts/utils/dialogs.js b/module/web/app/scripts/utils/dialogs.js deleted file mode 100644 index 4933b7ed2..000000000 --- a/module/web/app/scripts/utils/dialogs.js +++ /dev/null @@ -1,16 +0,0 @@ -// Loads all helper and set own handlebars rules -define(['jquery', 'underscore', 'views/abstract/modalView'], function($, _, Modal) { - 'use strict'; - - // Shows the confirm dialog for given context - // on success executes func with context - _.confirm = function(template, func, context) { - template = 'text!tpl/' + template; - _.requireOnce([template], function(html) { - var template = _.compile(html); - var dialog = new Modal(template, _.bind(func, context)); - dialog.show(); - }); - - }; -});
\ No newline at end of file diff --git a/module/web/app/scripts/utils/initHB.js b/module/web/app/scripts/utils/initHB.js deleted file mode 100644 index d7f582521..000000000 --- a/module/web/app/scripts/utils/initHB.js +++ /dev/null @@ -1,11 +0,0 @@ -// Loads all helper and set own handlebars rules -define(['underscore', 'handlebars', - 'helpers/formatSize', 'helpers/fileHelper', 'helpers/formatTime'], - function(_, Handlebars) { - 'use strict'; - // Replace with own lexer rules compiled from handlebars.l - Handlebars.Parser.lexer.rules = [/^(?:[^\x00]*?(?=(<%)))/, /^(?:[^\x00]+)/, /^(?:[^\x00]{2,}?(?=(\{\{|$)))/, /^(?:\{\{>)/, /^(?:<%=)/, /^(?:<%\/)/, /^(?:\{\{\^)/, /^(?:<%\s*else\b)/, /^(?:\{<%%)/, /^(?:\{\{&)/, /^(?:<%![\s\S]*?%>)/, /^(?:<%)/, /^(?:=)/, /^(?:\.(?=[%} ]))/, /^(?:\.\.)/, /^(?:[\/.])/, /^(?:\s+)/, /^(?:%%>)/, /^(?:%>)/, /^(?:"(\\["]|[^"])*")/, /^(?:'(\\[']|[^'])*')/, /^(?:@[a-zA-Z]+)/, /^(?:true(?=[%}\s]))/, /^(?:false(?=[%}\s]))/, /^(?:[0-9]+(?=[%}\s]))/, /^(?:[a-zA-Z0-9_$-]+(?=[=%}\s\/.]))/, /^(?:\[[^\]]*\])/, /^(?:.)/, /^(?:$)/]; - _.compile = Handlebars.compile; - - return Handlebars; - });
\ No newline at end of file diff --git a/module/web/app/scripts/utils/lazyRequire.js b/module/web/app/scripts/utils/lazyRequire.js deleted file mode 100644 index 96c07aa24..000000000 --- a/module/web/app/scripts/utils/lazyRequire.js +++ /dev/null @@ -1,97 +0,0 @@ -// Define the module. -define( - [ - 'require', 'underscore' - ], - function( require, _ ){ - 'use strict'; - - - // Define the states of loading for a given set of modules - // within a require() statement. - var states = { - unloaded: 'UNLOADED', - loading: 'LOADING', - loaded: 'LOADED' - }; - - - // Define the top-level module container. Mostly, we're making - // the top-level container a non-Function so that users won't - // try to invoke this without calling the once() method below. - var lazyRequire = {}; - - - // I will return a new, unique instance of the requrieOnce() - // method. Each instance will only call the require() method - // once internally. - lazyRequire.once = function(){ - - // The modules start in an unloaded state before - // requireOnce() is invoked by the calling code. - var state = states.unloaded; - var args; - - var requireOnce = function(dependencies, loadCallback ){ - - // Use the module state to determine which method to - // invoke (or just to ignore the invocation). - if (state === states.loaded){ - loadCallback.apply(null, args); - - // The modules have not yet been requested - let's - // lazy load them. - } else if (state !== states.loading){ - - // We're about to load the modules asynchronously; - // flag the interim state. - state = states.loading; - - // Load the modules. - require( - dependencies, - function(){ - - args = arguments; - loadCallback.apply( null, args ); - state = states.loaded; - - - } - ); - - // RequireJS is currently loading the modules - // asynchronously, but they have not finished - // loading yet. - } else { - - // Simply ignore this call. - return; - - } - - }; - - // Return the new lazy loader. - return( requireOnce ); - - }; - - - // -------------------------------------------------- // - // -------------------------------------------------- // - - // Set up holder for underscore - var instances = {}; - _.requireOnce = function(dependencies, loadCallback) { - if (!_.has(instances, dependencies)) - instances[dependencies] = lazyRequire.once(); - - return instances[dependencies](dependencies, loadCallback); - }; - - - // Return the module definition. - return( lazyRequire ); - } -);
\ No newline at end of file diff --git a/module/web/app/scripts/vendor/Handlebars-1.0rc1.js b/module/web/app/scripts/vendor/Handlebars-1.0rc1.js deleted file mode 100644 index 991242461..000000000 --- a/module/web/app/scripts/vendor/Handlebars-1.0rc1.js +++ /dev/null @@ -1,1927 +0,0 @@ -// lib/handlebars/base.js -(function () { -/*jshint eqnull:true*/ -this.Handlebars = {}; - -(function(Handlebars) { - -Handlebars.VERSION = "1.0.rc.1"; - -Handlebars.helpers = {}; -Handlebars.partials = {}; - -Handlebars.registerHelper = function(name, fn, inverse) { - if(inverse) { fn.not = inverse; } - this.helpers[name] = fn; -}; - -Handlebars.registerPartial = function(name, str) { - this.partials[name] = str; -}; - -Handlebars.registerHelper('helperMissing', function(arg) { - if(arguments.length === 2) { - return undefined; - } else { - throw new Error("Could not find property '" + arg + "'"); - } -}); - -var toString = Object.prototype.toString, functionType = "[object Function]"; - -Handlebars.registerHelper('blockHelperMissing', function(context, options) { - var inverse = options.inverse || function() {}, fn = options.fn; - - - var ret = ""; - var type = toString.call(context); - - if(type === functionType) { context = context.call(this); } - - if(context === true) { - return fn(this); - } else if(context === false || context == null) { - return inverse(this); - } else if(type === "[object Array]") { - if(context.length > 0) { - return Handlebars.helpers.each(context, options); - } else { - return inverse(this); - } - } else { - return fn(context); - } -}); - -Handlebars.K = function() {}; - -Handlebars.createFrame = Object.create || function(object) { - Handlebars.K.prototype = object; - var obj = new Handlebars.K(); - Handlebars.K.prototype = null; - return obj; -}; - -Handlebars.registerHelper('each', function(context, options) { - var fn = options.fn, inverse = options.inverse; - var ret = "", data; - - if (options.data) { - data = Handlebars.createFrame(options.data); - } - - if(context && context.length > 0) { - for(var i=0, j=context.length; i<j; i++) { - if (data) { data.index = i; } - ret = ret + fn(context[i], { data: data }); - } - } else { - ret = inverse(this); - } - return ret; -}); - -Handlebars.registerHelper('if', function(context, options) { - var type = toString.call(context); - if(type === functionType) { context = context.call(this); } - - if(!context || Handlebars.Utils.isEmpty(context)) { - return options.inverse(this); - } else { - return options.fn(this); - } -}); - -Handlebars.registerHelper('unless', function(context, options) { - var fn = options.fn, inverse = options.inverse; - options.fn = inverse; - options.inverse = fn; - - return Handlebars.helpers['if'].call(this, context, options); -}); - -Handlebars.registerHelper('with', function(context, options) { - return options.fn(context); -}); - -Handlebars.registerHelper('log', function(context) { - Handlebars.log(context); -}); - -}(this.Handlebars)); -; -// lib/handlebars/compiler/parser.js -/* Jison generated parser */ -var handlebars = (function(){ -var parser = {trace: function trace() { }, -yy: {}, -symbols_: {"error":2,"root":3,"program":4,"EOF":5,"statements":6,"simpleInverse":7,"statement":8,"openInverse":9,"closeBlock":10,"openBlock":11,"mustache":12,"partial":13,"CONTENT":14,"COMMENT":15,"OPEN_BLOCK":16,"inMustache":17,"CLOSE":18,"OPEN_INVERSE":19,"OPEN_ENDBLOCK":20,"path":21,"OPEN":22,"OPEN_UNESCAPED":23,"OPEN_PARTIAL":24,"params":25,"hash":26,"DATA":27,"param":28,"STRING":29,"INTEGER":30,"BOOLEAN":31,"hashSegments":32,"hashSegment":33,"ID":34,"EQUALS":35,"pathSegments":36,"SEP":37,"$accept":0,"$end":1}, -terminals_: {2:"error",5:"EOF",14:"CONTENT",15:"COMMENT",16:"OPEN_BLOCK",18:"CLOSE",19:"OPEN_INVERSE",20:"OPEN_ENDBLOCK",22:"OPEN",23:"OPEN_UNESCAPED",24:"OPEN_PARTIAL",27:"DATA",29:"STRING",30:"INTEGER",31:"BOOLEAN",34:"ID",35:"EQUALS",37:"SEP"}, -productions_: [0,[3,2],[4,3],[4,1],[4,0],[6,1],[6,2],[8,3],[8,3],[8,1],[8,1],[8,1],[8,1],[11,3],[9,3],[10,3],[12,3],[12,3],[13,3],[13,4],[7,2],[17,3],[17,2],[17,2],[17,1],[17,1],[25,2],[25,1],[28,1],[28,1],[28,1],[28,1],[28,1],[26,1],[32,2],[32,1],[33,3],[33,3],[33,3],[33,3],[33,3],[21,1],[36,3],[36,1]], -performAction: function anonymous(yytext,yyleng,yylineno,yy,yystate,$$,_$) { - -var $0 = $$.length - 1; -switch (yystate) { -case 1: return $$[$0-1]; -break; -case 2: this.$ = new yy.ProgramNode($$[$0-2], $$[$0]); -break; -case 3: this.$ = new yy.ProgramNode($$[$0]); -break; -case 4: this.$ = new yy.ProgramNode([]); -break; -case 5: this.$ = [$$[$0]]; -break; -case 6: $$[$0-1].push($$[$0]); this.$ = $$[$0-1]; -break; -case 7: this.$ = new yy.BlockNode($$[$0-2], $$[$0-1].inverse, $$[$0-1], $$[$0]); -break; -case 8: this.$ = new yy.BlockNode($$[$0-2], $$[$0-1], $$[$0-1].inverse, $$[$0]); -break; -case 9: this.$ = $$[$0]; -break; -case 10: this.$ = $$[$0]; -break; -case 11: this.$ = new yy.ContentNode($$[$0]); -break; -case 12: this.$ = new yy.CommentNode($$[$0]); -break; -case 13: this.$ = new yy.MustacheNode($$[$0-1][0], $$[$0-1][1]); -break; -case 14: this.$ = new yy.MustacheNode($$[$0-1][0], $$[$0-1][1]); -break; -case 15: this.$ = $$[$0-1]; -break; -case 16: this.$ = new yy.MustacheNode($$[$0-1][0], $$[$0-1][1]); -break; -case 17: this.$ = new yy.MustacheNode($$[$0-1][0], $$[$0-1][1], true); -break; -case 18: this.$ = new yy.PartialNode($$[$0-1]); -break; -case 19: this.$ = new yy.PartialNode($$[$0-2], $$[$0-1]); -break; -case 20: -break; -case 21: this.$ = [[$$[$0-2]].concat($$[$0-1]), $$[$0]]; -break; -case 22: this.$ = [[$$[$0-1]].concat($$[$0]), null]; -break; -case 23: this.$ = [[$$[$0-1]], $$[$0]]; -break; -case 24: this.$ = [[$$[$0]], null]; -break; -case 25: this.$ = [[new yy.DataNode($$[$0])], null]; -break; -case 26: $$[$0-1].push($$[$0]); this.$ = $$[$0-1]; -break; -case 27: this.$ = [$$[$0]]; -break; -case 28: this.$ = $$[$0]; -break; -case 29: this.$ = new yy.StringNode($$[$0]); -break; -case 30: this.$ = new yy.IntegerNode($$[$0]); -break; -case 31: this.$ = new yy.BooleanNode($$[$0]); -break; -case 32: this.$ = new yy.DataNode($$[$0]); -break; -case 33: this.$ = new yy.HashNode($$[$0]); -break; -case 34: $$[$0-1].push($$[$0]); this.$ = $$[$0-1]; -break; -case 35: this.$ = [$$[$0]]; -break; -case 36: this.$ = [$$[$0-2], $$[$0]]; -break; -case 37: this.$ = [$$[$0-2], new yy.StringNode($$[$0])]; -break; -case 38: this.$ = [$$[$0-2], new yy.IntegerNode($$[$0])]; -break; -case 39: this.$ = [$$[$0-2], new yy.BooleanNode($$[$0])]; -break; -case 40: this.$ = [$$[$0-2], new yy.DataNode($$[$0])]; -break; -case 41: this.$ = new yy.IdNode($$[$0]); -break; -case 42: $$[$0-2].push($$[$0]); this.$ = $$[$0-2]; -break; -case 43: this.$ = [$$[$0]]; -break; -} -}, -table: [{3:1,4:2,5:[2,4],6:3,8:4,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,11],22:[1,13],23:[1,14],24:[1,15]},{1:[3]},{5:[1,16]},{5:[2,3],7:17,8:18,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,19],20:[2,3],22:[1,13],23:[1,14],24:[1,15]},{5:[2,5],14:[2,5],15:[2,5],16:[2,5],19:[2,5],20:[2,5],22:[2,5],23:[2,5],24:[2,5]},{4:20,6:3,8:4,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,11],20:[2,4],22:[1,13],23:[1,14],24:[1,15]},{4:21,6:3,8:4,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,11],20:[2,4],22:[1,13],23:[1,14],24:[1,15]},{5:[2,9],14:[2,9],15:[2,9],16:[2,9],19:[2,9],20:[2,9],22:[2,9],23:[2,9],24:[2,9]},{5:[2,10],14:[2,10],15:[2,10],16:[2,10],19:[2,10],20:[2,10],22:[2,10],23:[2,10],24:[2,10]},{5:[2,11],14:[2,11],15:[2,11],16:[2,11],19:[2,11],20:[2,11],22:[2,11],23:[2,11],24:[2,11]},{5:[2,12],14:[2,12],15:[2,12],16:[2,12],19:[2,12],20:[2,12],22:[2,12],23:[2,12],24:[2,12]},{17:22,21:23,27:[1,24],34:[1,26],36:25},{17:27,21:23,27:[1,24],34:[1,26],36:25},{17:28,21:23,27:[1,24],34:[1,26],36:25},{17:29,21:23,27:[1,24],34:[1,26],36:25},{21:30,34:[1,26],36:25},{1:[2,1]},{6:31,8:4,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,11],22:[1,13],23:[1,14],24:[1,15]},{5:[2,6],14:[2,6],15:[2,6],16:[2,6],19:[2,6],20:[2,6],22:[2,6],23:[2,6],24:[2,6]},{17:22,18:[1,32],21:23,27:[1,24],34:[1,26],36:25},{10:33,20:[1,34]},{10:35,20:[1,34]},{18:[1,36]},{18:[2,24],21:41,25:37,26:38,27:[1,45],28:39,29:[1,42],30:[1,43],31:[1,44],32:40,33:46,34:[1,47],36:25},{18:[2,25]},{18:[2,41],27:[2,41],29:[2,41],30:[2,41],31:[2,41],34:[2,41],37:[1,48]},{18:[2,43],27:[2,43],29:[2,43],30:[2,43],31:[2,43],34:[2,43],37:[2,43]},{18:[1,49]},{18:[1,50]},{18:[1,51]},{18:[1,52],21:53,34:[1,26],36:25},{5:[2,2],8:18,9:5,11:6,12:7,13:8,14:[1,9],15:[1,10],16:[1,12],19:[1,11],20:[2,2],22:[1,13],23:[1,14],24:[1,15]},{14:[2,20],15:[2,20],16:[2,20],19:[2,20],22:[2,20],23:[2,20],24:[2,20]},{5:[2,7],14:[2,7],15:[2,7],16:[2,7],19:[2,7],20:[2,7],22:[2,7],23:[2,7],24:[2,7]},{21:54,34:[1,26],36:25},{5:[2,8],14:[2,8],15:[2,8],16:[2,8],19:[2,8],20:[2,8],22:[2,8],23:[2,8],24:[2,8]},{14:[2,14],15:[2,14],16:[2,14],19:[2,14],20:[2,14],22:[2,14],23:[2,14],24:[2,14]},{18:[2,22],21:41,26:55,27:[1,45],28:56,29:[1,42],30:[1,43],31:[1,44],32:40,33:46,34:[1,47],36:25},{18:[2,23]},{18:[2,27],27:[2,27],29:[2,27],30:[2,27],31:[2,27],34:[2,27]},{18:[2,33],33:57,34:[1,58]},{18:[2,28],27:[2,28],29:[2,28],30:[2,28],31:[2,28],34:[2,28]},{18:[2,29],27:[2,29],29:[2,29],30:[2,29],31:[2,29],34:[2,29]},{18:[2,30],27:[2,30],29:[2,30],30:[2,30],31:[2,30],34:[2,30]},{18:[2,31],27:[2,31],29:[2,31],30:[2,31],31:[2,31],34:[2,31]},{18:[2,32],27:[2,32],29:[2,32],30:[2,32],31:[2,32],34:[2,32]},{18:[2,35],34:[2,35]},{18:[2,43],27:[2,43],29:[2,43],30:[2,43],31:[2,43],34:[2,43],35:[1,59],37:[2,43]},{34:[1,60]},{14:[2,13],15:[2,13],16:[2,13],19:[2,13],20:[2,13],22:[2,13],23:[2,13],24:[2,13]},{5:[2,16],14:[2,16],15:[2,16],16:[2,16],19:[2,16],20:[2,16],22:[2,16],23:[2,16],24:[2,16]},{5:[2,17],14:[2,17],15:[2,17],16:[2,17],19:[2,17],20:[2,17],22:[2,17],23:[2,17],24:[2,17]},{5:[2,18],14:[2,18],15:[2,18],16:[2,18],19:[2,18],20:[2,18],22:[2,18],23:[2,18],24:[2,18]},{18:[1,61]},{18:[1,62]},{18:[2,21]},{18:[2,26],27:[2,26],29:[2,26],30:[2,26],31:[2,26],34:[2,26]},{18:[2,34],34:[2,34]},{35:[1,59]},{21:63,27:[1,67],29:[1,64],30:[1,65],31:[1,66],34:[1,26],36:25},{18:[2,42],27:[2,42],29:[2,42],30:[2,42],31:[2,42],34:[2,42],37:[2,42]},{5:[2,19],14:[2,19],15:[2,19],16:[2,19],19:[2,19],20:[2,19],22:[2,19],23:[2,19],24:[2,19]},{5:[2,15],14:[2,15],15:[2,15],16:[2,15],19:[2,15],20:[2,15],22:[2,15],23:[2,15],24:[2,15]},{18:[2,36],34:[2,36]},{18:[2,37],34:[2,37]},{18:[2,38],34:[2,38]},{18:[2,39],34:[2,39]},{18:[2,40],34:[2,40]}], -defaultActions: {16:[2,1],24:[2,25],38:[2,23],55:[2,21]}, -parseError: function parseError(str, hash) { - throw new Error(str); -}, -parse: function parse(input) { - var self = this, stack = [0], vstack = [null], lstack = [], table = this.table, yytext = "", yylineno = 0, yyleng = 0, recovering = 0, TERROR = 2, EOF = 1; - this.lexer.setInput(input); - this.lexer.yy = this.yy; - this.yy.lexer = this.lexer; - this.yy.parser = this; - if (typeof this.lexer.yylloc == "undefined") - this.lexer.yylloc = {}; - var yyloc = this.lexer.yylloc; - lstack.push(yyloc); - var ranges = this.lexer.options && this.lexer.options.ranges; - if (typeof this.yy.parseError === "function") - this.parseError = this.yy.parseError; - function popStack(n) { - stack.length = stack.length - 2 * n; - vstack.length = vstack.length - n; - lstack.length = lstack.length - n; - } - function lex() { - var token; - token = self.lexer.lex() || 1; - if (typeof token !== "number") { - token = self.symbols_[token] || token; - } - return token; - } - var symbol, preErrorSymbol, state, action, a, r, yyval = {}, p, len, newState, expected; - while (true) { - state = stack[stack.length - 1]; - if (this.defaultActions[state]) { - action = this.defaultActions[state]; - } else { - if (symbol === null || typeof symbol == "undefined") { - symbol = lex(); - } - action = table[state] && table[state][symbol]; - } - if (typeof action === "undefined" || !action.length || !action[0]) { - var errStr = ""; - if (!recovering) { - expected = []; - for (p in table[state]) - if (this.terminals_[p] && p > 2) { - expected.push("'" + this.terminals_[p] + "'"); - } - if (this.lexer.showPosition) { - errStr = "Parse error on line " + (yylineno + 1) + ":\n" + this.lexer.showPosition() + "\nExpecting " + expected.join(", ") + ", got '" + (this.terminals_[symbol] || symbol) + "'"; - } else { - errStr = "Parse error on line " + (yylineno + 1) + ": Unexpected " + (symbol == 1?"end of input":"'" + (this.terminals_[symbol] || symbol) + "'"); - } - this.parseError(errStr, {text: this.lexer.match, token: this.terminals_[symbol] || symbol, line: this.lexer.yylineno, loc: yyloc, expected: expected}); - } - } - if (action[0] instanceof Array && action.length > 1) { - throw new Error("Parse Error: multiple actions possible at state: " + state + ", token: " + symbol); - } - switch (action[0]) { - case 1: - stack.push(symbol); - vstack.push(this.lexer.yytext); - lstack.push(this.lexer.yylloc); - stack.push(action[1]); - symbol = null; - if (!preErrorSymbol) { - yyleng = this.lexer.yyleng; - yytext = this.lexer.yytext; - yylineno = this.lexer.yylineno; - yyloc = this.lexer.yylloc; - if (recovering > 0) - recovering--; - } else { - symbol = preErrorSymbol; - preErrorSymbol = null; - } - break; - case 2: - len = this.productions_[action[1]][1]; - yyval.$ = vstack[vstack.length - len]; - yyval._$ = {first_line: lstack[lstack.length - (len || 1)].first_line, last_line: lstack[lstack.length - 1].last_line, first_column: lstack[lstack.length - (len || 1)].first_column, last_column: lstack[lstack.length - 1].last_column}; - if (ranges) { - yyval._$.range = [lstack[lstack.length - (len || 1)].range[0], lstack[lstack.length - 1].range[1]]; - } - r = this.performAction.call(yyval, yytext, yyleng, yylineno, this.yy, action[1], vstack, lstack); - if (typeof r !== "undefined") { - return r; - } - if (len) { - stack = stack.slice(0, -1 * len * 2); - vstack = vstack.slice(0, -1 * len); - lstack = lstack.slice(0, -1 * len); - } - stack.push(this.productions_[action[1]][0]); - vstack.push(yyval.$); - lstack.push(yyval._$); - newState = table[stack[stack.length - 2]][stack[stack.length - 1]]; - stack.push(newState); - break; - case 3: - return true; - } - } - return true; -} -}; -/* Jison generated lexer */ -var lexer = (function(){ -var lexer = ({EOF:1, -parseError:function parseError(str, hash) { - if (this.yy.parser) { - this.yy.parser.parseError(str, hash); - } else { - throw new Error(str); - } - }, -setInput:function (input) { - this._input = input; - this._more = this._less = this.done = false; - this.yylineno = this.yyleng = 0; - this.yytext = this.matched = this.match = ''; - this.conditionStack = ['INITIAL']; - this.yylloc = {first_line:1,first_column:0,last_line:1,last_column:0}; - if (this.options.ranges) this.yylloc.range = [0,0]; - this.offset = 0; - return this; - }, -input:function () { - var ch = this._input[0]; - this.yytext += ch; - this.yyleng++; - this.offset++; - this.match += ch; - this.matched += ch; - var lines = ch.match(/(?:\r\n?|\n).*/g); - if (lines) { - this.yylineno++; - this.yylloc.last_line++; - } else { - this.yylloc.last_column++; - } - if (this.options.ranges) this.yylloc.range[1]++; - - this._input = this._input.slice(1); - return ch; - }, -unput:function (ch) { - var len = ch.length; - var lines = ch.split(/(?:\r\n?|\n)/g); - - this._input = ch + this._input; - this.yytext = this.yytext.substr(0, this.yytext.length-len-1); - //this.yyleng -= len; - this.offset -= len; - var oldLines = this.match.split(/(?:\r\n?|\n)/g); - this.match = this.match.substr(0, this.match.length-1); - this.matched = this.matched.substr(0, this.matched.length-1); - - if (lines.length-1) this.yylineno -= lines.length-1; - var r = this.yylloc.range; - - this.yylloc = {first_line: this.yylloc.first_line, - last_line: this.yylineno+1, - first_column: this.yylloc.first_column, - last_column: lines ? - (lines.length === oldLines.length ? this.yylloc.first_column : 0) + oldLines[oldLines.length - lines.length].length - lines[0].length: - this.yylloc.first_column - len - }; - - if (this.options.ranges) { - this.yylloc.range = [r[0], r[0] + this.yyleng - len]; - } - return this; - }, -more:function () { - this._more = true; - return this; - }, -less:function (n) { - this.unput(this.match.slice(n)); - }, -pastInput:function () { - var past = this.matched.substr(0, this.matched.length - this.match.length); - return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, ""); - }, -upcomingInput:function () { - var next = this.match; - if (next.length < 20) { - next += this._input.substr(0, 20-next.length); - } - return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, ""); - }, -showPosition:function () { - var pre = this.pastInput(); - var c = new Array(pre.length + 1).join("-"); - return pre + this.upcomingInput() + "\n" + c+"^"; - }, -next:function () { - if (this.done) { - return this.EOF; - } - if (!this._input) this.done = true; - - var token, - match, - tempMatch, - index, - col, - lines; - if (!this._more) { - this.yytext = ''; - this.match = ''; - } - var rules = this._currentRules(); - for (var i=0;i < rules.length; i++) { - tempMatch = this._input.match(this.rules[rules[i]]); - if (tempMatch && (!match || tempMatch[0].length > match[0].length)) { - match = tempMatch; - index = i; - if (!this.options.flex) break; - } - } - if (match) { - lines = match[0].match(/(?:\r\n?|\n).*/g); - if (lines) this.yylineno += lines.length; - this.yylloc = {first_line: this.yylloc.last_line, - last_line: this.yylineno+1, - first_column: this.yylloc.last_column, - last_column: lines ? lines[lines.length-1].length-lines[lines.length-1].match(/\r?\n?/)[0].length : this.yylloc.last_column + match[0].length}; - this.yytext += match[0]; - this.match += match[0]; - this.matches = match; - this.yyleng = this.yytext.length; - if (this.options.ranges) { - this.yylloc.range = [this.offset, this.offset += this.yyleng]; - } - this._more = false; - this._input = this._input.slice(match[0].length); - this.matched += match[0]; - token = this.performAction.call(this, this.yy, this, rules[index],this.conditionStack[this.conditionStack.length-1]); - if (this.done && this._input) this.done = false; - if (token) return token; - else return; - } - if (this._input === "") { - return this.EOF; - } else { - return this.parseError('Lexical error on line '+(this.yylineno+1)+'. Unrecognized text.\n'+this.showPosition(), - {text: "", token: null, line: this.yylineno}); - } - }, -lex:function lex() { - var r = this.next(); - if (typeof r !== 'undefined') { - return r; - } else { - return this.lex(); - } - }, -begin:function begin(condition) { - this.conditionStack.push(condition); - }, -popState:function popState() { - return this.conditionStack.pop(); - }, -_currentRules:function _currentRules() { - return this.conditions[this.conditionStack[this.conditionStack.length-1]].rules; - }, -topState:function () { - return this.conditionStack[this.conditionStack.length-2]; - }, -pushState:function begin(condition) { - this.begin(condition); - }}); -lexer.options = {}; -lexer.performAction = function anonymous(yy,yy_,$avoiding_name_collisions,YY_START) { - -var YYSTATE=YY_START -switch($avoiding_name_collisions) { -case 0: - if(yy_.yytext.slice(-1) !== "\\") this.begin("mu"); - if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1), this.begin("emu"); - if(yy_.yytext) return 14; - -break; -case 1: return 14; -break; -case 2: - if(yy_.yytext.slice(-1) !== "\\") this.popState(); - if(yy_.yytext.slice(-1) === "\\") yy_.yytext = yy_.yytext.substr(0,yy_.yyleng-1); - return 14; - -break; -case 3: return 24; -break; -case 4: return 16; -break; -case 5: return 20; -break; -case 6: return 19; -break; -case 7: return 19; -break; -case 8: return 23; -break; -case 9: return 23; -break; -case 10: yy_.yytext = yy_.yytext.substr(3,yy_.yyleng-5); this.popState(); return 15; -break; -case 11: return 22; -break; -case 12: return 35; -break; -case 13: return 34; -break; -case 14: return 34; -break; -case 15: return 37; -break; -case 16: /*ignore whitespace*/ -break; -case 17: this.popState(); return 18; -break; -case 18: this.popState(); return 18; -break; -case 19: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; -break; -case 20: yy_.yytext = yy_.yytext.substr(1,yy_.yyleng-2).replace(/\\"/g,'"'); return 29; -break; -case 21: yy_.yytext = yy_.yytext.substr(1); return 27; -break; -case 22: return 31; -break; -case 23: return 31; -break; -case 24: return 30; -break; -case 25: return 34; -break; -case 26: yy_.yytext = yy_.yytext.substr(1, yy_.yyleng-2); return 34; -break; -case 27: return 'INVALID'; -break; -case 28: return 5; -break; -} -}; -lexer.rules = [/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[} ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:$)/]; -lexer.conditions = {"mu":{"rules":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],"inclusive":false},"emu":{"rules":[2],"inclusive":false},"INITIAL":{"rules":[0,1,28],"inclusive":true}}; -return lexer;})() -parser.lexer = lexer; -function Parser () { this.yy = {}; }Parser.prototype = parser;parser.Parser = Parser; -return new Parser; -})(); -if (typeof require !== 'undefined' && typeof exports !== 'undefined') { -exports.parser = handlebars; -exports.Parser = handlebars.Parser; -exports.parse = function () { return handlebars.parse.apply(handlebars, arguments); } -exports.main = function commonjsMain(args) { - if (!args[1]) - throw new Error('Usage: '+args[0]+' FILE'); - var source, cwd; - if (typeof process !== 'undefined') { - source = require('fs').readFileSync(require('path').resolve(args[1]), "utf8"); - } else { - source = require("file").path(require("file").cwd()).join(args[1]).read({charset: "utf-8"}); - } - return exports.parser.parse(source); -} -if (typeof module !== 'undefined' && require.main === module) { - exports.main(typeof process !== 'undefined' ? process.argv.slice(1) : require("system").args); -} -}; -; -// lib/handlebars/compiler/base.js -Handlebars.Parser = handlebars; - -Handlebars.parse = function(string) { - Handlebars.Parser.yy = Handlebars.AST; - return Handlebars.Parser.parse(string); -}; - -Handlebars.print = function(ast) { - return new Handlebars.PrintVisitor().accept(ast); -}; - -Handlebars.logger = { - DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3, level: 3, - - // override in the host environment - log: function(level, str) {} -}; - -Handlebars.log = function(level, str) { Handlebars.logger.log(level, str); }; -; -// lib/handlebars/compiler/ast.js -(function() { - - Handlebars.AST = {}; - - Handlebars.AST.ProgramNode = function(statements, inverse) { - this.type = "program"; - this.statements = statements; - if(inverse) { this.inverse = new Handlebars.AST.ProgramNode(inverse); } - }; - - Handlebars.AST.MustacheNode = function(rawParams, hash, unescaped) { - this.type = "mustache"; - this.escaped = !unescaped; - this.hash = hash; - - var id = this.id = rawParams[0]; - var params = this.params = rawParams.slice(1); - - // a mustache is an eligible helper if: - // * its id is simple (a single part, not `this` or `..`) - var eligibleHelper = this.eligibleHelper = id.isSimple; - - // a mustache is definitely a helper if: - // * it is an eligible helper, and - // * it has at least one parameter or hash segment - this.isHelper = eligibleHelper && (params.length || hash); - - // if a mustache is an eligible helper but not a definite - // helper, it is ambiguous, and will be resolved in a later - // pass or at runtime. - }; - - Handlebars.AST.PartialNode = function(id, context) { - this.type = "partial"; - - // TODO: disallow complex IDs - - this.id = id; - this.context = context; - }; - - var verifyMatch = function(open, close) { - if(open.original !== close.original) { - throw new Handlebars.Exception(open.original + " doesn't match " + close.original); - } - }; - - Handlebars.AST.BlockNode = function(mustache, program, inverse, close) { - verifyMatch(mustache.id, close); - this.type = "block"; - this.mustache = mustache; - this.program = program; - this.inverse = inverse; - - if (this.inverse && !this.program) { - this.isInverse = true; - } - }; - - Handlebars.AST.ContentNode = function(string) { - this.type = "content"; - this.string = string; - }; - - Handlebars.AST.HashNode = function(pairs) { - this.type = "hash"; - this.pairs = pairs; - }; - - Handlebars.AST.IdNode = function(parts) { - this.type = "ID"; - this.original = parts.join("."); - - var dig = [], depth = 0; - - for(var i=0,l=parts.length; i<l; i++) { - var part = parts[i]; - - if(part === "..") { depth++; } - else if(part === "." || part === "this") { this.isScoped = true; } - else { dig.push(part); } - } - - this.parts = dig; - this.string = dig.join('.'); - this.depth = depth; - - // an ID is simple if it only has one part, and that part is not - // `..` or `this`. - this.isSimple = parts.length === 1 && !this.isScoped && depth === 0; - }; - - Handlebars.AST.DataNode = function(id) { - this.type = "DATA"; - this.id = id; - }; - - Handlebars.AST.StringNode = function(string) { - this.type = "STRING"; - this.string = string; - }; - - Handlebars.AST.IntegerNode = function(integer) { - this.type = "INTEGER"; - this.integer = integer; - }; - - Handlebars.AST.BooleanNode = function(bool) { - this.type = "BOOLEAN"; - this.bool = bool; - }; - - Handlebars.AST.CommentNode = function(comment) { - this.type = "comment"; - this.comment = comment; - }; - -})();; -// lib/handlebars/utils.js -Handlebars.Exception = function(message) { - var tmp = Error.prototype.constructor.apply(this, arguments); - - for (var p in tmp) { - if (tmp.hasOwnProperty(p)) { this[p] = tmp[p]; } - } - - this.message = tmp.message; -}; -Handlebars.Exception.prototype = new Error(); - -// Build out our basic SafeString type -Handlebars.SafeString = function(string) { - this.string = string; -}; -Handlebars.SafeString.prototype.toString = function() { - return this.string.toString(); -}; - -(function() { - var escape = { - "&": "&", - "<": "<", - ">": ">", - '"': """, - "'": "'", - "`": "`" - }; - - var badChars = /[&<>"'`]/g; - var possible = /[&<>"'`]/; - - var escapeChar = function(chr) { - return escape[chr] || "&"; - }; - - Handlebars.Utils = { - escapeExpression: function(string) { - // don't escape SafeStrings, since they're already safe - if (string instanceof Handlebars.SafeString) { - return string.toString(); - } else if (string == null || string === false) { - return ""; - } - - if(!possible.test(string)) { return string; } - return string.replace(badChars, escapeChar); - }, - - isEmpty: function(value) { - if (typeof value === "undefined") { - return true; - } else if (value === null) { - return true; - } else if (value === false) { - return true; - } else if(Object.prototype.toString.call(value) === "[object Array]" && value.length === 0) { - return true; - } else { - return false; - } - } - }; -})();; -// lib/handlebars/compiler/compiler.js - -/*jshint eqnull:true*/ -Handlebars.Compiler = function() {}; -Handlebars.JavaScriptCompiler = function() {}; - -(function(Compiler, JavaScriptCompiler) { - // the foundHelper register will disambiguate helper lookup from finding a - // function in a context. This is necessary for mustache compatibility, which - // requires that context functions in blocks are evaluated by blockHelperMissing, - // and then proceed as if the resulting value was provided to blockHelperMissing. - - Compiler.prototype = { - compiler: Compiler, - - disassemble: function() { - var opcodes = this.opcodes, opcode, out = [], params, param; - - for (var i=0, l=opcodes.length; i<l; i++) { - opcode = opcodes[i]; - - if (opcode.opcode === 'DECLARE') { - out.push("DECLARE " + opcode.name + "=" + opcode.value); - } else { - params = []; - for (var j=0; j<opcode.args.length; j++) { - param = opcode.args[j]; - if (typeof param === "string") { - param = "\"" + param.replace("\n", "\\n") + "\""; - } - params.push(param); - } - out.push(opcode.opcode + " " + params.join(" ")); - } - } - - return out.join("\n"); - }, - - guid: 0, - - compile: function(program, options) { - this.children = []; - this.depths = {list: []}; - this.options = options; - - // These changes will propagate to the other compiler components - var knownHelpers = this.options.knownHelpers; - this.options.knownHelpers = { - 'helperMissing': true, - 'blockHelperMissing': true, - 'each': true, - 'if': true, - 'unless': true, - 'with': true, - 'log': true - }; - if (knownHelpers) { - for (var name in knownHelpers) { - this.options.knownHelpers[name] = knownHelpers[name]; - } - } - - return this.program(program); - }, - - accept: function(node) { - return this[node.type](node); - }, - - program: function(program) { - var statements = program.statements, statement; - this.opcodes = []; - - for(var i=0, l=statements.length; i<l; i++) { - statement = statements[i]; - this[statement.type](statement); - } - this.isSimple = l === 1; - - this.depths.list = this.depths.list.sort(function(a, b) { - return a - b; - }); - - return this; - }, - - compileProgram: function(program) { - var result = new this.compiler().compile(program, this.options); - var guid = this.guid++, depth; - - this.usePartial = this.usePartial || result.usePartial; - - this.children[guid] = result; - - for(var i=0, l=result.depths.list.length; i<l; i++) { - depth = result.depths.list[i]; - - if(depth < 2) { continue; } - else { this.addDepth(depth - 1); } - } - - return guid; - }, - - block: function(block) { - var mustache = block.mustache, - program = block.program, - inverse = block.inverse; - - if (program) { - program = this.compileProgram(program); - } - - if (inverse) { - inverse = this.compileProgram(inverse); - } - - var type = this.classifyMustache(mustache); - - if (type === "helper") { - this.helperMustache(mustache, program, inverse); - } else if (type === "simple") { - this.simpleMustache(mustache); - - // now that the simple mustache is resolved, we need to - // evaluate it by executing `blockHelperMissing` - this.opcode('pushProgram', program); - this.opcode('pushProgram', inverse); - this.opcode('pushLiteral', '{}'); - this.opcode('blockValue'); - } else { - this.ambiguousMustache(mustache, program, inverse); - - // now that the simple mustache is resolved, we need to - // evaluate it by executing `blockHelperMissing` - this.opcode('pushProgram', program); - this.opcode('pushProgram', inverse); - this.opcode('pushLiteral', '{}'); - this.opcode('ambiguousBlockValue'); - } - - this.opcode('append'); - }, - - hash: function(hash) { - var pairs = hash.pairs, pair, val; - - this.opcode('push', '{}'); - - for(var i=0, l=pairs.length; i<l; i++) { - pair = pairs[i]; - val = pair[1]; - - this.accept(val); - this.opcode('assignToHash', pair[0]); - } - }, - - partial: function(partial) { - var id = partial.id; - this.usePartial = true; - - if(partial.context) { - this.ID(partial.context); - } else { - this.opcode('push', 'depth0'); - } - - this.opcode('invokePartial', id.original); - this.opcode('append'); - }, - - content: function(content) { - this.opcode('appendContent', content.string); - }, - - mustache: function(mustache) { - var options = this.options; - var type = this.classifyMustache(mustache); - - if (type === "simple") { - this.simpleMustache(mustache); - } else if (type === "helper") { - this.helperMustache(mustache); - } else { - this.ambiguousMustache(mustache); - } - - if(mustache.escaped && !options.noEscape) { - this.opcode('appendEscaped'); - } else { - this.opcode('append'); - } - }, - - ambiguousMustache: function(mustache, program, inverse) { - var id = mustache.id, name = id.parts[0]; - - this.opcode('getContext', id.depth); - - this.opcode('pushProgram', program); - this.opcode('pushProgram', inverse); - - this.opcode('invokeAmbiguous', name); - }, - - simpleMustache: function(mustache, program, inverse) { - var id = mustache.id; - - if (id.type === 'DATA') { - this.DATA(id); - } else if (id.parts.length) { - this.ID(id); - } else { - // Simplified ID for `this` - this.addDepth(id.depth); - this.opcode('getContext', id.depth); - this.opcode('pushContext'); - } - - this.opcode('resolvePossibleLambda'); - }, - - helperMustache: function(mustache, program, inverse) { - var params = this.setupFullMustacheParams(mustache, program, inverse), - name = mustache.id.parts[0]; - - if (this.options.knownHelpers[name]) { - this.opcode('invokeKnownHelper', params.length, name); - } else if (this.knownHelpersOnly) { - throw new Error("You specified knownHelpersOnly, but used the unknown helper " + name); - } else { - this.opcode('invokeHelper', params.length, name); - } - }, - - ID: function(id) { - this.addDepth(id.depth); - this.opcode('getContext', id.depth); - - var name = id.parts[0]; - if (!name) { - this.opcode('pushContext'); - } else { - this.opcode('lookupOnContext', id.parts[0]); - } - - for(var i=1, l=id.parts.length; i<l; i++) { - this.opcode('lookup', id.parts[i]); - } - }, - - DATA: function(data) { - this.options.data = true; - this.opcode('lookupData', data.id); - }, - - STRING: function(string) { - this.opcode('pushString', string.string); - }, - - INTEGER: function(integer) { - this.opcode('pushLiteral', integer.integer); - }, - - BOOLEAN: function(bool) { - this.opcode('pushLiteral', bool.bool); - }, - - comment: function() {}, - - // HELPERS - opcode: function(name) { - this.opcodes.push({ opcode: name, args: [].slice.call(arguments, 1) }); - }, - - declare: function(name, value) { - this.opcodes.push({ opcode: 'DECLARE', name: name, value: value }); - }, - - addDepth: function(depth) { - if(isNaN(depth)) { throw new Error("EWOT"); } - if(depth === 0) { return; } - - if(!this.depths[depth]) { - this.depths[depth] = true; - this.depths.list.push(depth); - } - }, - - classifyMustache: function(mustache) { - var isHelper = mustache.isHelper; - var isEligible = mustache.eligibleHelper; - var options = this.options; - - // if ambiguous, we can possibly resolve the ambiguity now - if (isEligible && !isHelper) { - var name = mustache.id.parts[0]; - - if (options.knownHelpers[name]) { - isHelper = true; - } else if (options.knownHelpersOnly) { - isEligible = false; - } - } - - if (isHelper) { return "helper"; } - else if (isEligible) { return "ambiguous"; } - else { return "simple"; } - }, - - pushParams: function(params) { - var i = params.length, param; - - while(i--) { - param = params[i]; - - if(this.options.stringParams) { - if(param.depth) { - this.addDepth(param.depth); - } - - this.opcode('getContext', param.depth || 0); - this.opcode('pushStringParam', param.string); - } else { - this[param.type](param); - } - } - }, - - setupMustacheParams: function(mustache) { - var params = mustache.params; - this.pushParams(params); - - if(mustache.hash) { - this.hash(mustache.hash); - } else { - this.opcode('pushLiteral', '{}'); - } - - return params; - }, - - // this will replace setupMustacheParams when we're done - setupFullMustacheParams: function(mustache, program, inverse) { - var params = mustache.params; - this.pushParams(params); - - this.opcode('pushProgram', program); - this.opcode('pushProgram', inverse); - - if(mustache.hash) { - this.hash(mustache.hash); - } else { - this.opcode('pushLiteral', '{}'); - } - - return params; - } - }; - - var Literal = function(value) { - this.value = value; - }; - - JavaScriptCompiler.prototype = { - // PUBLIC API: You can override these methods in a subclass to provide - // alternative compiled forms for name lookup and buffering semantics - nameLookup: function(parent, name, type) { - if (/^[0-9]+$/.test(name)) { - return parent + "[" + name + "]"; - } else if (JavaScriptCompiler.isValidJavaScriptVariableName(name)) { - return parent + "." + name; - } - else { - return parent + "['" + name + "']"; - } - }, - - appendToBuffer: function(string) { - if (this.environment.isSimple) { - return "return " + string + ";"; - } else { - return "buffer += " + string + ";"; - } - }, - - initializeBuffer: function() { - return this.quotedString(""); - }, - - namespace: "Handlebars", - // END PUBLIC API - - compile: function(environment, options, context, asObject) { - this.environment = environment; - this.options = options || {}; - - Handlebars.log(Handlebars.logger.DEBUG, this.environment.disassemble() + "\n\n"); - - this.name = this.environment.name; - this.isChild = !!context; - this.context = context || { - programs: [], - aliases: { } - }; - - this.preamble(); - - this.stackSlot = 0; - this.stackVars = []; - this.registers = { list: [] }; - this.compileStack = []; - - this.compileChildren(environment, options); - - var opcodes = environment.opcodes, opcode; - - this.i = 0; - - for(l=opcodes.length; this.i<l; this.i++) { - opcode = opcodes[this.i]; - - if(opcode.opcode === 'DECLARE') { - this[opcode.name] = opcode.value; - } else { - this[opcode.opcode].apply(this, opcode.args); - } - } - - return this.createFunctionContext(asObject); - }, - - nextOpcode: function() { - var opcodes = this.environment.opcodes, opcode = opcodes[this.i + 1]; - return opcodes[this.i + 1]; - }, - - eat: function(opcode) { - this.i = this.i + 1; - }, - - preamble: function() { - var out = []; - - if (!this.isChild) { - var namespace = this.namespace; - var copies = "helpers = helpers || " + namespace + ".helpers;"; - if (this.environment.usePartial) { copies = copies + " partials = partials || " + namespace + ".partials;"; } - if (this.options.data) { copies = copies + " data = data || {};"; } - out.push(copies); - } else { - out.push(''); - } - - if (!this.environment.isSimple) { - out.push(", buffer = " + this.initializeBuffer()); - } else { - out.push(""); - } - - // track the last context pushed into place to allow skipping the - // getContext opcode when it would be a noop - this.lastContext = 0; - this.source = out; - }, - - createFunctionContext: function(asObject) { - var locals = this.stackVars.concat(this.registers.list); - - if(locals.length > 0) { - this.source[1] = this.source[1] + ", " + locals.join(", "); - } - - // Generate minimizer alias mappings - if (!this.isChild) { - var aliases = []; - for (var alias in this.context.aliases) { - this.source[1] = this.source[1] + ', ' + alias + '=' + this.context.aliases[alias]; - } - } - - if (this.source[1]) { - this.source[1] = "var " + this.source[1].substring(2) + ";"; - } - - // Merge children - if (!this.isChild) { - this.source[1] += '\n' + this.context.programs.join('\n') + '\n'; - } - - if (!this.environment.isSimple) { - this.source.push("return buffer;"); - } - - var params = this.isChild ? ["depth0", "data"] : ["Handlebars", "depth0", "helpers", "partials", "data"]; - - for(var i=0, l=this.environment.depths.list.length; i<l; i++) { - params.push("depth" + this.environment.depths.list[i]); - } - - if (asObject) { - params.push(this.source.join("\n ")); - - return Function.apply(this, params); - } else { - var functionSource = 'function ' + (this.name || '') + '(' + params.join(',') + ') {\n ' + this.source.join("\n ") + '}'; - Handlebars.log(Handlebars.logger.DEBUG, functionSource + "\n\n"); - return functionSource; - } - }, - - // [blockValue] - // - // On stack, before: hash, inverse, program, value - // On stack, after: return value of blockHelperMissing - // - // The purpose of this opcode is to take a block of the form - // `{{#foo}}...{{/foo}}`, resolve the value of `foo`, and - // replace it on the stack with the result of properly - // invoking blockHelperMissing. - blockValue: function() { - this.context.aliases.blockHelperMissing = 'helpers.blockHelperMissing'; - - var params = ["depth0"]; - this.setupParams(0, params); - - this.replaceStack(function(current) { - params.splice(1, 0, current); - return current + " = blockHelperMissing.call(" + params.join(", ") + ")"; - }); - }, - - // [ambiguousBlockValue] - // - // On stack, before: hash, inverse, program, value - // Compiler value, before: lastHelper=value of last found helper, if any - // On stack, after, if no lastHelper: same as [blockValue] - // On stack, after, if lastHelper: value - ambiguousBlockValue: function() { - this.context.aliases.blockHelperMissing = 'helpers.blockHelperMissing'; - - var params = ["depth0"]; - this.setupParams(0, params); - - var current = this.topStack(); - params.splice(1, 0, current); - - this.source.push("if (!" + this.lastHelper + ") { " + current + " = blockHelperMissing.call(" + params.join(", ") + "); }"); - }, - - // [appendContent] - // - // On stack, before: ... - // On stack, after: ... - // - // Appends the string value of `content` to the current buffer - appendContent: function(content) { - this.source.push(this.appendToBuffer(this.quotedString(content))); - }, - - // [append] - // - // On stack, before: value, ... - // On stack, after: ... - // - // Coerces `value` to a String and appends it to the current buffer. - // - // If `value` is truthy, or 0, it is coerced into a string and appended - // Otherwise, the empty string is appended - append: function() { - var local = this.popStack(); - this.source.push("if(" + local + " || " + local + " === 0) { " + this.appendToBuffer(local) + " }"); - if (this.environment.isSimple) { - this.source.push("else { " + this.appendToBuffer("''") + " }"); - } - }, - - // [appendEscaped] - // - // On stack, before: value, ... - // On stack, after: ... - // - // Escape `value` and append it to the buffer - appendEscaped: function() { - var opcode = this.nextOpcode(), extra = ""; - this.context.aliases.escapeExpression = 'this.escapeExpression'; - - if(opcode && opcode.opcode === 'appendContent') { - extra = " + " + this.quotedString(opcode.args[0]); - this.eat(opcode); - } - - this.source.push(this.appendToBuffer("escapeExpression(" + this.popStack() + ")" + extra)); - }, - - // [getContext] - // - // On stack, before: ... - // On stack, after: ... - // Compiler value, after: lastContext=depth - // - // Set the value of the `lastContext` compiler value to the depth - getContext: function(depth) { - if(this.lastContext !== depth) { - this.lastContext = depth; - } - }, - - // [lookupOnContext] - // - // On stack, before: ... - // On stack, after: currentContext[name], ... - // - // Looks up the value of `name` on the current context and pushes - // it onto the stack. - lookupOnContext: function(name) { - this.pushStack(this.nameLookup('depth' + this.lastContext, name, 'context')); - }, - - // [pushContext] - // - // On stack, before: ... - // On stack, after: currentContext, ... - // - // Pushes the value of the current context onto the stack. - pushContext: function() { - this.pushStackLiteral('depth' + this.lastContext); - }, - - // [resolvePossibleLambda] - // - // On stack, before: value, ... - // On stack, after: resolved value, ... - // - // If the `value` is a lambda, replace it on the stack by - // the return value of the lambda - resolvePossibleLambda: function() { - this.context.aliases.functionType = '"function"'; - - this.replaceStack(function(current) { - return "typeof " + current + " === functionType ? " + current + "() : " + current; - }); - }, - - // [lookup] - // - // On stack, before: value, ... - // On stack, after: value[name], ... - // - // Replace the value on the stack with the result of looking - // up `name` on `value` - lookup: function(name) { - this.replaceStack(function(current) { - return current + " == null || " + current + " === false ? " + current + " : " + this.nameLookup(current, name, 'context'); - }); - }, - - // [lookupData] - // - // On stack, before: ... - // On stack, after: data[id], ... - // - // Push the result of looking up `id` on the current data - lookupData: function(id) { - this.pushStack(this.nameLookup('data', id, 'data')); - }, - - // [pushStringParam] - // - // On stack, before: ... - // On stack, after: string, currentContext, ... - // - // This opcode is designed for use in string mode, which - // provides the string value of a parameter along with its - // depth rather than resolving it immediately. - pushStringParam: function(string) { - this.pushStackLiteral('depth' + this.lastContext); - this.pushString(string); - }, - - // [pushString] - // - // On stack, before: ... - // On stack, after: quotedString(string), ... - // - // Push a quoted version of `string` onto the stack - pushString: function(string) { - this.pushStackLiteral(this.quotedString(string)); - }, - - // [push] - // - // On stack, before: ... - // On stack, after: expr, ... - // - // Push an expression onto the stack - push: function(expr) { - this.pushStack(expr); - }, - - // [pushLiteral] - // - // On stack, before: ... - // On stack, after: value, ... - // - // Pushes a value onto the stack. This operation prevents - // the compiler from creating a temporary variable to hold - // it. - pushLiteral: function(value) { - this.pushStackLiteral(value); - }, - - // [pushProgram] - // - // On stack, before: ... - // On stack, after: program(guid), ... - // - // Push a program expression onto the stack. This takes - // a compile-time guid and converts it into a runtime-accessible - // expression. - pushProgram: function(guid) { - if (guid != null) { - this.pushStackLiteral(this.programExpression(guid)); - } else { - this.pushStackLiteral(null); - } - }, - - // [invokeHelper] - // - // On stack, before: hash, inverse, program, params..., ... - // On stack, after: result of helper invocation - // - // Pops off the helper's parameters, invokes the helper, - // and pushes the helper's return value onto the stack. - // - // If the helper is not found, `helperMissing` is called. - invokeHelper: function(paramSize, name) { - this.context.aliases.helperMissing = 'helpers.helperMissing'; - - var helper = this.lastHelper = this.setupHelper(paramSize, name); - this.register('foundHelper', helper.name); - - this.pushStack("foundHelper ? foundHelper.call(" + - helper.callParams + ") " + ": helperMissing.call(" + - helper.helperMissingParams + ")"); - }, - - // [invokeKnownHelper] - // - // On stack, before: hash, inverse, program, params..., ... - // On stack, after: result of helper invocation - // - // This operation is used when the helper is known to exist, - // so a `helperMissing` fallback is not required. - invokeKnownHelper: function(paramSize, name) { - var helper = this.setupHelper(paramSize, name); - this.pushStack(helper.name + ".call(" + helper.callParams + ")"); - }, - - // [invokeAmbiguous] - // - // On stack, before: hash, inverse, program, params..., ... - // On stack, after: result of disambiguation - // - // This operation is used when an expression like `{{foo}}` - // is provided, but we don't know at compile-time whether it - // is a helper or a path. - // - // This operation emits more code than the other options, - // and can be avoided by passing the `knownHelpers` and - // `knownHelpersOnly` flags at compile-time. - invokeAmbiguous: function(name) { - this.context.aliases.functionType = '"function"'; - - this.pushStackLiteral('{}'); - var helper = this.setupHelper(0, name); - - var helperName = this.lastHelper = this.nameLookup('helpers', name, 'helper'); - this.register('foundHelper', helperName); - - var nonHelper = this.nameLookup('depth' + this.lastContext, name, 'context'); - var nextStack = this.nextStack(); - - this.source.push('if (foundHelper) { ' + nextStack + ' = foundHelper.call(' + helper.callParams + '); }'); - this.source.push('else { ' + nextStack + ' = ' + nonHelper + '; ' + nextStack + ' = typeof ' + nextStack + ' === functionType ? ' + nextStack + '() : ' + nextStack + '; }'); - }, - - // [invokePartial] - // - // On stack, before: context, ... - // On stack after: result of partial invocation - // - // This operation pops off a context, invokes a partial with that context, - // and pushes the result of the invocation back. - invokePartial: function(name) { - var params = [this.nameLookup('partials', name, 'partial'), "'" + name + "'", this.popStack(), "helpers", "partials"]; - - if (this.options.data) { - params.push("data"); - } - - this.context.aliases.self = "this"; - this.pushStack("self.invokePartial(" + params.join(", ") + ");"); - }, - - // [assignToHash] - // - // On stack, before: value, hash, ... - // On stack, after: hash, ... - // - // Pops a value and hash off the stack, assigns `hash[key] = value` - // and pushes the hash back onto the stack. - assignToHash: function(key) { - var value = this.popStack(); - var hash = this.topStack(); - - this.source.push(hash + "['" + key + "'] = " + value + ";"); - }, - - // HELPERS - - compiler: JavaScriptCompiler, - - compileChildren: function(environment, options) { - var children = environment.children, child, compiler; - - for(var i=0, l=children.length; i<l; i++) { - child = children[i]; - compiler = new this.compiler(); - - this.context.programs.push(''); // Placeholder to prevent name conflicts for nested children - var index = this.context.programs.length; - child.index = index; - child.name = 'program' + index; - this.context.programs[index] = compiler.compile(child, options, this.context); - } - }, - - programExpression: function(guid) { - this.context.aliases.self = "this"; - - if(guid == null) { - return "self.noop"; - } - - var child = this.environment.children[guid], - depths = child.depths.list, depth; - - var programParams = [child.index, child.name, "data"]; - - for(var i=0, l = depths.length; i<l; i++) { - depth = depths[i]; - - if(depth === 1) { programParams.push("depth0"); } - else { programParams.push("depth" + (depth - 1)); } - } - - if(depths.length === 0) { - return "self.program(" + programParams.join(", ") + ")"; - } else { - programParams.shift(); - return "self.programWithDepth(" + programParams.join(", ") + ")"; - } - }, - - register: function(name, val) { - this.useRegister(name); - this.source.push(name + " = " + val + ";"); - }, - - useRegister: function(name) { - if(!this.registers[name]) { - this.registers[name] = true; - this.registers.list.push(name); - } - }, - - pushStackLiteral: function(item) { - this.compileStack.push(new Literal(item)); - return item; - }, - - pushStack: function(item) { - this.source.push(this.incrStack() + " = " + item + ";"); - this.compileStack.push("stack" + this.stackSlot); - return "stack" + this.stackSlot; - }, - - replaceStack: function(callback) { - var item = callback.call(this, this.topStack()); - - this.source.push(this.topStack() + " = " + item + ";"); - return "stack" + this.stackSlot; - }, - - nextStack: function(skipCompileStack) { - var name = this.incrStack(); - this.compileStack.push("stack" + this.stackSlot); - return name; - }, - - incrStack: function() { - this.stackSlot++; - if(this.stackSlot > this.stackVars.length) { this.stackVars.push("stack" + this.stackSlot); } - return "stack" + this.stackSlot; - }, - - popStack: function() { - var item = this.compileStack.pop(); - - if (item instanceof Literal) { - return item.value; - } else { - this.stackSlot--; - return item; - } - }, - - topStack: function() { - var item = this.compileStack[this.compileStack.length - 1]; - - if (item instanceof Literal) { - return item.value; - } else { - return item; - } - }, - - quotedString: function(str) { - return '"' + str - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') + '"'; - }, - - setupHelper: function(paramSize, name) { - var params = []; - this.setupParams(paramSize, params); - var foundHelper = this.nameLookup('helpers', name, 'helper'); - - return { - params: params, - name: foundHelper, - callParams: ["depth0"].concat(params).join(", "), - helperMissingParams: ["depth0", this.quotedString(name)].concat(params).join(", ") - }; - }, - - // the params and contexts arguments are passed in arrays - // to fill in - setupParams: function(paramSize, params) { - var options = [], contexts = [], param, inverse, program; - - options.push("hash:" + this.popStack()); - - inverse = this.popStack(); - program = this.popStack(); - - // Avoid setting fn and inverse if neither are set. This allows - // helpers to do a check for `if (options.fn)` - if (program || inverse) { - if (!program) { - this.context.aliases.self = "this"; - program = "self.noop"; - } - - if (!inverse) { - this.context.aliases.self = "this"; - inverse = "self.noop"; - } - - options.push("inverse:" + inverse); - options.push("fn:" + program); - } - - for(var i=0; i<paramSize; i++) { - param = this.popStack(); - params.push(param); - - if(this.options.stringParams) { - contexts.push(this.popStack()); - } - } - - if (this.options.stringParams) { - options.push("contexts:[" + contexts.join(",") + "]"); - } - - if(this.options.data) { - options.push("data:data"); - } - - params.push("{" + options.join(",") + "}"); - return params.join(", "); - } - }; - - var reservedWords = ( - "break else new var" + - " case finally return void" + - " catch for switch while" + - " continue function this with" + - " default if throw" + - " delete in try" + - " do instanceof typeof" + - " abstract enum int short" + - " boolean export interface static" + - " byte extends long super" + - " char final native synchronized" + - " class float package throws" + - " const goto private transient" + - " debugger implements protected volatile" + - " double import public let yield" - ).split(" "); - - var compilerWords = JavaScriptCompiler.RESERVED_WORDS = {}; - - for(var i=0, l=reservedWords.length; i<l; i++) { - compilerWords[reservedWords[i]] = true; - } - - JavaScriptCompiler.isValidJavaScriptVariableName = function(name) { - if(!JavaScriptCompiler.RESERVED_WORDS[name] && /^[a-zA-Z_$][0-9a-zA-Z_$]+$/.test(name)) { - return true; - } - return false; - }; - -})(Handlebars.Compiler, Handlebars.JavaScriptCompiler); - -Handlebars.precompile = function(string, options) { - options = options || {}; - - var ast = Handlebars.parse(string); - var environment = new Handlebars.Compiler().compile(ast, options); - return new Handlebars.JavaScriptCompiler().compile(environment, options); -}; - -Handlebars.compile = function(string, options) { - options = options || {}; - - var compiled; - function compile() { - var ast = Handlebars.parse(string); - var environment = new Handlebars.Compiler().compile(ast, options); - var templateSpec = new Handlebars.JavaScriptCompiler().compile(environment, options, undefined, true); - return Handlebars.template(templateSpec); - } - - // Template is only compiled on first use and cached after that point. - return function(context, options) { - if (!compiled) { - compiled = compile(); - } - return compiled.call(this, context, options); - }; -}; -; -// lib/handlebars/runtime.js -Handlebars.VM = { - template: function(templateSpec) { - // Just add water - var container = { - escapeExpression: Handlebars.Utils.escapeExpression, - invokePartial: Handlebars.VM.invokePartial, - programs: [], - program: function(i, fn, data) { - var programWrapper = this.programs[i]; - if(data) { - return Handlebars.VM.program(fn, data); - } else if(programWrapper) { - return programWrapper; - } else { - programWrapper = this.programs[i] = Handlebars.VM.program(fn); - return programWrapper; - } - }, - programWithDepth: Handlebars.VM.programWithDepth, - noop: Handlebars.VM.noop - }; - - return function(context, options) { - options = options || {}; - return templateSpec.call(container, Handlebars, context, options.helpers, options.partials, options.data); - }; - }, - - programWithDepth: function(fn, data, $depth) { - var args = Array.prototype.slice.call(arguments, 2); - - return function(context, options) { - options = options || {}; - - return fn.apply(this, [context, options.data || data].concat(args)); - }; - }, - program: function(fn, data) { - return function(context, options) { - options = options || {}; - - return fn(context, options.data || data); - }; - }, - noop: function() { return ""; }, - invokePartial: function(partial, name, context, helpers, partials, data) { - var options = { helpers: helpers, partials: partials, data: data }; - - if(partial === undefined) { - throw new Handlebars.Exception("The partial " + name + " could not be found"); - } else if(partial instanceof Function) { - return partial(context, options); - } else if (!Handlebars.compile) { - throw new Handlebars.Exception("The partial " + name + " could not be compiled when running in runtime-only mode"); - } else { - partials[name] = Handlebars.compile(partial, {data: data !== undefined}); - return partials[name](context, options); - } - } -}; - -Handlebars.template = Handlebars.VM.template; -; - -// AMD Define -define(function(){ - return Handlebars; -}); - -})(); diff --git a/module/web/app/scripts/vendor/bootstrap-2.3.2.js b/module/web/app/scripts/vendor/bootstrap-2.3.2.js deleted file mode 100755 index 96fed1387..000000000 --- a/module/web/app/scripts/vendor/bootstrap-2.3.2.js +++ /dev/null @@ -1,2291 +0,0 @@ -/* =================================================== - * bootstrap-transition.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#transitions - * =================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* CSS TRANSITION SUPPORT (http://www.modernizr.com/) - * ======================================================= */ - - $(function () { - - $.support.transition = (function () { - - var transitionEnd = (function () { - - var el = document.createElement('bootstrap') - , transEndEventNames = { - 'WebkitTransition' : 'webkitTransitionEnd' - , 'MozTransition' : 'transitionend' - , 'OTransition' : 'oTransitionEnd otransitionend' - , 'transition' : 'transitionend' - } - , name - - for (name in transEndEventNames){ - if (el.style[name] !== undefined) { - return transEndEventNames[name] - } - } - - }()) - - return transitionEnd && { - end: transitionEnd - } - - })() - - }) - -}(window.jQuery); -/* ========================================================= - * bootstrap-modal.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#modals - * ========================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================= */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* MODAL CLASS DEFINITION - * ====================== */ - - var Modal = function (element, options) { - this.options = options - this.$element = $(element) - .delegate('[data-dismiss="modal"]', 'click.dismiss.modal', $.proxy(this.hide, this)) - this.options.remote && this.$element.find('.modal-body').load(this.options.remote) - } - - Modal.prototype = { - - constructor: Modal - - , toggle: function () { - return this[!this.isShown ? 'show' : 'hide']() - } - - , show: function () { - var that = this - , e = $.Event('show') - - this.$element.trigger(e) - - if (this.isShown || e.isDefaultPrevented()) return - - this.isShown = true - - this.escape() - - this.backdrop(function () { - var transition = $.support.transition && that.$element.hasClass('fade') - - if (!that.$element.parent().length) { - that.$element.appendTo(document.body) //don't move modals dom position - } - - that.$element.show() - - if (transition) { - that.$element[0].offsetWidth // force reflow - } - - that.$element - .addClass('in') - .attr('aria-hidden', false) - - that.enforceFocus() - - transition ? - that.$element.one($.support.transition.end, function () { that.$element.focus().trigger('shown') }) : - that.$element.focus().trigger('shown') - - }) - } - - , hide: function (e) { - e && e.preventDefault() - - var that = this - - e = $.Event('hide') - - this.$element.trigger(e) - - if (!this.isShown || e.isDefaultPrevented()) return - - this.isShown = false - - this.escape() - - $(document).off('focusin.modal') - - this.$element - .removeClass('in') - .attr('aria-hidden', true) - - $.support.transition && this.$element.hasClass('fade') ? - this.hideWithTransition() : - this.hideModal() - } - - , enforceFocus: function () { - var that = this - $(document).on('focusin.modal', function (e) { - if (that.$element[0] !== e.target && !that.$element.has(e.target).length) { - that.$element.focus() - } - }) - } - - , escape: function () { - var that = this - if (this.isShown && this.options.keyboard) { - this.$element.on('keyup.dismiss.modal', function ( e ) { - e.which == 27 && that.hide() - }) - } else if (!this.isShown) { - this.$element.off('keyup.dismiss.modal') - } - } - - , hideWithTransition: function () { - var that = this - , timeout = setTimeout(function () { - that.$element.off($.support.transition.end) - that.hideModal() - }, 500) - - this.$element.one($.support.transition.end, function () { - clearTimeout(timeout) - that.hideModal() - }) - } - - , hideModal: function () { - var that = this - this.$element.hide() - this.backdrop(function () { - that.removeBackdrop() - that.$element.trigger('hidden') - }) - } - - , removeBackdrop: function () { - this.$backdrop && this.$backdrop.remove() - this.$backdrop = null - } - - , backdrop: function (callback) { - var that = this - , animate = this.$element.hasClass('fade') ? 'fade' : '' - - if (this.isShown && this.options.backdrop) { - var doAnimate = $.support.transition && animate - - this.$backdrop = $('<div class="modal-backdrop ' + animate + '" />') - .appendTo(document.body) - - this.$backdrop.click( - this.options.backdrop == 'static' ? - $.proxy(this.$element[0].focus, this.$element[0]) - : $.proxy(this.hide, this) - ) - - if (doAnimate) this.$backdrop[0].offsetWidth // force reflow - - this.$backdrop.addClass('in') - - if (!callback) return - - doAnimate ? - this.$backdrop.one($.support.transition.end, callback) : - callback() - - } else if (!this.isShown && this.$backdrop) { - this.$backdrop.removeClass('in') - - $.support.transition && this.$element.hasClass('fade')? - this.$backdrop.one($.support.transition.end, callback) : - callback() - - } else if (callback) { - callback() - } - } - } - - - /* MODAL PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.modal - - $.fn.modal = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('modal') - , options = $.extend({}, $.fn.modal.defaults, $this.data(), typeof option == 'object' && option) - if (!data) $this.data('modal', (data = new Modal(this, options))) - if (typeof option == 'string') data[option]() - else if (options.show) data.show() - }) - } - - $.fn.modal.defaults = { - backdrop: true - , keyboard: true - , show: true - } - - $.fn.modal.Constructor = Modal - - - /* MODAL NO CONFLICT - * ================= */ - - $.fn.modal.noConflict = function () { - $.fn.modal = old - return this - } - - - /* MODAL DATA-API - * ============== */ - - $(document).on('click.modal.data-api', '[data-toggle="modal"]', function (e) { - var $this = $(this) - , href = $this.attr('href') - , $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) //strip for ie7 - , option = $target.data('modal') ? 'toggle' : $.extend({ remote:!/#/.test(href) && href }, $target.data(), $this.data()) - - e.preventDefault() - - $target - .modal(option) - .one('hide', function () { - $this.focus() - }) - }) - -}(window.jQuery); - -/* ============================================================ - * bootstrap-dropdown.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#dropdowns - * ============================================================ - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* DROPDOWN CLASS DEFINITION - * ========================= */ - - var toggle = '[data-toggle=dropdown]' - , Dropdown = function (element) { - var $el = $(element).on('click.dropdown.data-api', this.toggle) - $('html').on('click.dropdown.data-api', function () { - $el.parent().removeClass('open') - }) - } - - Dropdown.prototype = { - - constructor: Dropdown - - , toggle: function (e) { - var $this = $(this) - , $parent - , isActive - - if ($this.is('.disabled, :disabled')) return - - $parent = getParent($this) - - isActive = $parent.hasClass('open') - - clearMenus() - - if (!isActive) { - if ('ontouchstart' in document.documentElement) { - // if mobile we we use a backdrop because click events don't delegate - $('<div class="dropdown-backdrop"/>').insertBefore($(this)).on('click', clearMenus) - } - $parent.toggleClass('open') - } - - $this.focus() - - return false - } - - , keydown: function (e) { - var $this - , $items - , $active - , $parent - , isActive - , index - - if (!/(38|40|27)/.test(e.keyCode)) return - - $this = $(this) - - e.preventDefault() - e.stopPropagation() - - if ($this.is('.disabled, :disabled')) return - - $parent = getParent($this) - - isActive = $parent.hasClass('open') - - if (!isActive || (isActive && e.keyCode == 27)) { - if (e.which == 27) $parent.find(toggle).focus() - return $this.click() - } - - $items = $('[role=menu] li:not(.divider):visible a', $parent) - - if (!$items.length) return - - index = $items.index($items.filter(':focus')) - - if (e.keyCode == 38 && index > 0) index-- // up - if (e.keyCode == 40 && index < $items.length - 1) index++ // down - if (!~index) index = 0 - - $items - .eq(index) - .focus() - } - - } - - function clearMenus() { - $('.dropdown-backdrop').remove() - $(toggle).each(function () { - getParent($(this)).removeClass('open') - }) - } - - function getParent($this) { - var selector = $this.attr('data-target') - , $parent - - if (!selector) { - selector = $this.attr('href') - selector = selector && /#/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - $parent = selector && $(selector) - - if (!$parent || !$parent.length) $parent = $this.parent() - - return $parent - } - - - /* DROPDOWN PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.dropdown - - $.fn.dropdown = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('dropdown') - if (!data) $this.data('dropdown', (data = new Dropdown(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - $.fn.dropdown.Constructor = Dropdown - - - /* DROPDOWN NO CONFLICT - * ==================== */ - - $.fn.dropdown.noConflict = function () { - $.fn.dropdown = old - return this - } - - - /* APPLY TO STANDARD DROPDOWN ELEMENTS - * =================================== */ - - $(document) - .on('click.dropdown.data-api', clearMenus) - .on('click.dropdown.data-api', '.dropdown form', function (e) { e.stopPropagation() }) - .on('click.dropdown.data-api' , toggle, Dropdown.prototype.toggle) - .on('keydown.dropdown.data-api', toggle + ', [role=menu]' , Dropdown.prototype.keydown) - -}(window.jQuery); - -/* ============================================================= - * bootstrap-scrollspy.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#scrollspy - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* SCROLLSPY CLASS DEFINITION - * ========================== */ - - function ScrollSpy(element, options) { - var process = $.proxy(this.process, this) - , $element = $(element).is('body') ? $(window) : $(element) - , href - this.options = $.extend({}, $.fn.scrollspy.defaults, options) - this.$scrollElement = $element.on('scroll.scroll-spy.data-api', process) - this.selector = (this.options.target - || ((href = $(element).attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 - || '') + ' .nav li > a' - this.$body = $('body') - this.refresh() - this.process() - } - - ScrollSpy.prototype = { - - constructor: ScrollSpy - - , refresh: function () { - var self = this - , $targets - - this.offsets = $([]) - this.targets = $([]) - - $targets = this.$body - .find(this.selector) - .map(function () { - var $el = $(this) - , href = $el.data('target') || $el.attr('href') - , $href = /^#\w/.test(href) && $(href) - return ( $href - && $href.length - && [[ $href.position().top + (!$.isWindow(self.$scrollElement.get(0)) && self.$scrollElement.scrollTop()), href ]] ) || null - }) - .sort(function (a, b) { return a[0] - b[0] }) - .each(function () { - self.offsets.push(this[0]) - self.targets.push(this[1]) - }) - } - - , process: function () { - var scrollTop = this.$scrollElement.scrollTop() + this.options.offset - , scrollHeight = this.$scrollElement[0].scrollHeight || this.$body[0].scrollHeight - , maxScroll = scrollHeight - this.$scrollElement.height() - , offsets = this.offsets - , targets = this.targets - , activeTarget = this.activeTarget - , i - - if (scrollTop >= maxScroll) { - return activeTarget != (i = targets.last()[0]) - && this.activate ( i ) - } - - for (i = offsets.length; i--;) { - activeTarget != targets[i] - && scrollTop >= offsets[i] - && (!offsets[i + 1] || scrollTop <= offsets[i + 1]) - && this.activate( targets[i] ) - } - } - - , activate: function (target) { - var active - , selector - - this.activeTarget = target - - $(this.selector) - .parent('.active') - .removeClass('active') - - selector = this.selector - + '[data-target="' + target + '"],' - + this.selector + '[href="' + target + '"]' - - active = $(selector) - .parent('li') - .addClass('active') - - if (active.parent('.dropdown-menu').length) { - active = active.closest('li.dropdown').addClass('active') - } - - active.trigger('activate') - } - - } - - - /* SCROLLSPY PLUGIN DEFINITION - * =========================== */ - - var old = $.fn.scrollspy - - $.fn.scrollspy = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('scrollspy') - , options = typeof option == 'object' && option - if (!data) $this.data('scrollspy', (data = new ScrollSpy(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.scrollspy.Constructor = ScrollSpy - - $.fn.scrollspy.defaults = { - offset: 10 - } - - - /* SCROLLSPY NO CONFLICT - * ===================== */ - - $.fn.scrollspy.noConflict = function () { - $.fn.scrollspy = old - return this - } - - - /* SCROLLSPY DATA-API - * ================== */ - - $(window).on('load', function () { - $('[data-spy="scroll"]').each(function () { - var $spy = $(this) - $spy.scrollspy($spy.data()) - }) - }) - -}(window.jQuery); -/* ======================================================== - * bootstrap-tab.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#tabs - * ======================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ======================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* TAB CLASS DEFINITION - * ==================== */ - - var Tab = function (element) { - this.element = $(element) - } - - Tab.prototype = { - - constructor: Tab - - , show: function () { - var $this = this.element - , $ul = $this.closest('ul:not(.dropdown-menu)') - , selector = $this.attr('data-target') - , previous - , $target - , e - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - if ( $this.parent('li').hasClass('active') ) return - - previous = $ul.find('.active:last a')[0] - - e = $.Event('show', { - relatedTarget: previous - }) - - $this.trigger(e) - - if (e.isDefaultPrevented()) return - - $target = $(selector) - - this.activate($this.parent('li'), $ul) - this.activate($target, $target.parent(), function () { - $this.trigger({ - type: 'shown' - , relatedTarget: previous - }) - }) - } - - , activate: function ( element, container, callback) { - var $active = container.find('> .active') - , transition = callback - && $.support.transition - && $active.hasClass('fade') - - function next() { - $active - .removeClass('active') - .find('> .dropdown-menu > .active') - .removeClass('active') - - element.addClass('active') - - if (transition) { - element[0].offsetWidth // reflow for transition - element.addClass('in') - } else { - element.removeClass('fade') - } - - if ( element.parent('.dropdown-menu') ) { - element.closest('li.dropdown').addClass('active') - } - - callback && callback() - } - - transition ? - $active.one($.support.transition.end, next) : - next() - - $active.removeClass('in') - } - } - - - /* TAB PLUGIN DEFINITION - * ===================== */ - - var old = $.fn.tab - - $.fn.tab = function ( option ) { - return this.each(function () { - var $this = $(this) - , data = $this.data('tab') - if (!data) $this.data('tab', (data = new Tab(this))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.tab.Constructor = Tab - - - /* TAB NO CONFLICT - * =============== */ - - $.fn.tab.noConflict = function () { - $.fn.tab = old - return this - } - - - /* TAB DATA-API - * ============ */ - - $(document).on('click.tab.data-api', '[data-toggle="tab"], [data-toggle="pill"]', function (e) { - e.preventDefault() - $(this).tab('show') - }) - -}(window.jQuery); -/* =========================================================== - * bootstrap-tooltip.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#tooltips - * Inspired by the original jQuery.tipsy by Jason Frame - * =========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* TOOLTIP PUBLIC CLASS DEFINITION - * =============================== */ - - var Tooltip = function (element, options) { - this.init('tooltip', element, options) - } - - Tooltip.prototype = { - - constructor: Tooltip - - , init: function (type, element, options) { - var eventIn - , eventOut - , triggers - , trigger - , i - - this.type = type - this.$element = $(element) - this.options = this.getOptions(options) - this.enabled = true - - triggers = this.options.trigger.split(' ') - - for (i = triggers.length; i--;) { - trigger = triggers[i] - if (trigger == 'click') { - this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) - } else if (trigger != 'manual') { - eventIn = trigger == 'hover' ? 'mouseenter' : 'focus' - eventOut = trigger == 'hover' ? 'mouseleave' : 'blur' - this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) - this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) - } - } - - this.options.selector ? - (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) : - this.fixTitle() - } - - , getOptions: function (options) { - options = $.extend({}, $.fn[this.type].defaults, this.$element.data(), options) - - if (options.delay && typeof options.delay == 'number') { - options.delay = { - show: options.delay - , hide: options.delay - } - } - - return options - } - - , enter: function (e) { - var defaults = $.fn[this.type].defaults - , options = {} - , self - - this._options && $.each(this._options, function (key, value) { - if (defaults[key] != value) options[key] = value - }, this) - - self = $(e.currentTarget)[this.type](options).data(this.type) - - if (!self.options.delay || !self.options.delay.show) return self.show() - - clearTimeout(this.timeout) - self.hoverState = 'in' - this.timeout = setTimeout(function() { - if (self.hoverState == 'in') self.show() - }, self.options.delay.show) - } - - , leave: function (e) { - var self = $(e.currentTarget)[this.type](this._options).data(this.type) - - if (this.timeout) clearTimeout(this.timeout) - if (!self.options.delay || !self.options.delay.hide) return self.hide() - - self.hoverState = 'out' - this.timeout = setTimeout(function() { - if (self.hoverState == 'out') self.hide() - }, self.options.delay.hide) - } - - , show: function () { - var $tip - , pos - , actualWidth - , actualHeight - , placement - , tp - , e = $.Event('show') - - if (this.hasContent() && this.enabled) { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $tip = this.tip() - this.setContent() - - if (this.options.animation) { - $tip.addClass('fade') - } - - placement = typeof this.options.placement == 'function' ? - this.options.placement.call(this, $tip[0], this.$element[0]) : - this.options.placement - - $tip - .detach() - .css({ top: 0, left: 0, display: 'block' }) - - this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) - - pos = this.getPosition() - - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - - switch (placement) { - case 'bottom': - tp = {top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2} - break - case 'top': - tp = {top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2} - break - case 'left': - tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth} - break - case 'right': - tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width} - break - } - - this.applyPlacement(tp, placement) - this.$element.trigger('shown') - } - } - - , applyPlacement: function(offset, placement){ - var $tip = this.tip() - , width = $tip[0].offsetWidth - , height = $tip[0].offsetHeight - , actualWidth - , actualHeight - , delta - , replace - - $tip - .offset(offset) - .addClass(placement) - .addClass('in') - - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - - if (placement == 'top' && actualHeight != height) { - offset.top = offset.top + height - actualHeight - replace = true - } - - if (placement == 'bottom' || placement == 'top') { - delta = 0 - - if (offset.left < 0){ - delta = offset.left * -2 - offset.left = 0 - $tip.offset(offset) - actualWidth = $tip[0].offsetWidth - actualHeight = $tip[0].offsetHeight - } - - this.replaceArrow(delta - width + actualWidth, actualWidth, 'left') - } else { - this.replaceArrow(actualHeight - height, actualHeight, 'top') - } - - if (replace) $tip.offset(offset) - } - - , replaceArrow: function(delta, dimension, position){ - this - .arrow() - .css(position, delta ? (50 * (1 - delta / dimension) + "%") : '') - } - - , setContent: function () { - var $tip = this.tip() - , title = this.getTitle() - - $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title) - $tip.removeClass('fade in top bottom left right') - } - - , hide: function () { - var that = this - , $tip = this.tip() - , e = $.Event('hide') - - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - - $tip.removeClass('in') - - function removeWithAnimation() { - var timeout = setTimeout(function () { - $tip.off($.support.transition.end).detach() - }, 500) - - $tip.one($.support.transition.end, function () { - clearTimeout(timeout) - $tip.detach() - }) - } - - $.support.transition && this.$tip.hasClass('fade') ? - removeWithAnimation() : - $tip.detach() - - this.$element.trigger('hidden') - - return this - } - - , fixTitle: function () { - var $e = this.$element - if ($e.attr('title') || typeof($e.attr('data-original-title')) != 'string') { - $e.attr('data-original-title', $e.attr('title') || '').attr('title', '') - } - } - - , hasContent: function () { - return this.getTitle() - } - - , getPosition: function () { - var el = this.$element[0] - return $.extend({}, (typeof el.getBoundingClientRect == 'function') ? el.getBoundingClientRect() : { - width: el.offsetWidth - , height: el.offsetHeight - }, this.$element.offset()) - } - - , getTitle: function () { - var title - , $e = this.$element - , o = this.options - - title = $e.attr('data-original-title') - || (typeof o.title == 'function' ? o.title.call($e[0]) : o.title) - - return title - } - - , tip: function () { - return this.$tip = this.$tip || $(this.options.template) - } - - , arrow: function(){ - return this.$arrow = this.$arrow || this.tip().find(".tooltip-arrow") - } - - , validate: function () { - if (!this.$element[0].parentNode) { - this.hide() - this.$element = null - this.options = null - } - } - - , enable: function () { - this.enabled = true - } - - , disable: function () { - this.enabled = false - } - - , toggleEnabled: function () { - this.enabled = !this.enabled - } - - , toggle: function (e) { - var self = e ? $(e.currentTarget)[this.type](this._options).data(this.type) : this - self.tip().hasClass('in') ? self.hide() : self.show() - } - - , destroy: function () { - this.hide().$element.off('.' + this.type).removeData(this.type) - } - - } - - - /* TOOLTIP PLUGIN DEFINITION - * ========================= */ - - var old = $.fn.tooltip - - $.fn.tooltip = function ( option ) { - return this.each(function () { - var $this = $(this) - , data = $this.data('tooltip') - , options = typeof option == 'object' && option - if (!data) $this.data('tooltip', (data = new Tooltip(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.tooltip.Constructor = Tooltip - - $.fn.tooltip.defaults = { - animation: true - , placement: 'top' - , selector: false - , template: '<div class="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>' - , trigger: 'hover focus' - , title: '' - , delay: 0 - , html: false - , container: false - } - - - /* TOOLTIP NO CONFLICT - * =================== */ - - $.fn.tooltip.noConflict = function () { - $.fn.tooltip = old - return this - } - -}(window.jQuery); - -/* =========================================================== - * bootstrap-popover.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#popovers - * =========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* POPOVER PUBLIC CLASS DEFINITION - * =============================== */ - - var Popover = function (element, options) { - this.init('popover', element, options) - } - - - /* NOTE: POPOVER EXTENDS BOOTSTRAP-TOOLTIP.js - ========================================== */ - - Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype, { - - constructor: Popover - - , setContent: function () { - var $tip = this.tip() - , title = this.getTitle() - , content = this.getContent() - - $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) - $tip.find('.popover-content')[this.options.html ? 'html' : 'text'](content) - - $tip.removeClass('fade top bottom left right in') - } - - , hasContent: function () { - return this.getTitle() || this.getContent() - } - - , getContent: function () { - var content - , $e = this.$element - , o = this.options - - content = (typeof o.content == 'function' ? o.content.call($e[0]) : o.content) - || $e.attr('data-content') - - return content - } - - , tip: function () { - if (!this.$tip) { - this.$tip = $(this.options.template) - } - return this.$tip - } - - , destroy: function () { - this.hide().$element.off('.' + this.type).removeData(this.type) - } - - }) - - - /* POPOVER PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.popover - - $.fn.popover = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('popover') - , options = typeof option == 'object' && option - if (!data) $this.data('popover', (data = new Popover(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.popover.Constructor = Popover - - $.fn.popover.defaults = $.extend({} , $.fn.tooltip.defaults, { - placement: 'right' - , trigger: 'click' - , content: '' - , template: '<div class="popover"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>' - }) - - - /* POPOVER NO CONFLICT - * =================== */ - - $.fn.popover.noConflict = function () { - $.fn.popover = old - return this - } - -}(window.jQuery); - -/* ========================================================== - * bootstrap-affix.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#affix - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* AFFIX CLASS DEFINITION - * ====================== */ - - var Affix = function (element, options) { - this.options = $.extend({}, $.fn.affix.defaults, options) - this.$window = $(window) - .on('scroll.affix.data-api', $.proxy(this.checkPosition, this)) - .on('click.affix.data-api', $.proxy(function () { setTimeout($.proxy(this.checkPosition, this), 1) }, this)) - this.$element = $(element) - this.checkPosition() - } - - Affix.prototype.checkPosition = function () { - if (!this.$element.is(':visible')) return - - var scrollHeight = $(document).height() - , scrollTop = this.$window.scrollTop() - , position = this.$element.offset() - , offset = this.options.offset - , offsetBottom = offset.bottom - , offsetTop = offset.top - , reset = 'affix affix-top affix-bottom' - , affix - - if (typeof offset != 'object') offsetBottom = offsetTop = offset - if (typeof offsetTop == 'function') offsetTop = offset.top() - if (typeof offsetBottom == 'function') offsetBottom = offset.bottom() - - affix = this.unpin != null && (scrollTop + this.unpin <= position.top) ? - false : offsetBottom != null && (position.top + this.$element.height() >= scrollHeight - offsetBottom) ? - 'bottom' : offsetTop != null && scrollTop <= offsetTop ? - 'top' : false - - if (this.affixed === affix) return - - this.affixed = affix - this.unpin = affix == 'bottom' ? position.top - scrollTop : null - - this.$element.removeClass(reset).addClass('affix' + (affix ? '-' + affix : '')) - } - - - /* AFFIX PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.affix - - $.fn.affix = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('affix') - , options = typeof option == 'object' && option - if (!data) $this.data('affix', (data = new Affix(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.affix.Constructor = Affix - - $.fn.affix.defaults = { - offset: 0 - } - - - /* AFFIX NO CONFLICT - * ================= */ - - $.fn.affix.noConflict = function () { - $.fn.affix = old - return this - } - - - /* AFFIX DATA-API - * ============== */ - - $(window).on('load', function () { - $('[data-spy="affix"]').each(function () { - var $spy = $(this) - , data = $spy.data() - - data.offset = data.offset || {} - - data.offsetBottom && (data.offset.bottom = data.offsetBottom) - data.offsetTop && (data.offset.top = data.offsetTop) - - $spy.affix(data) - }) - }) - - -}(window.jQuery); -/* ========================================================== - * bootstrap-alert.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#alerts - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* ALERT CLASS DEFINITION - * ====================== */ - - var dismiss = '[data-dismiss="alert"]' - , Alert = function (el) { - $(el).on('click', dismiss, this.close) - } - - Alert.prototype.close = function (e) { - var $this = $(this) - , selector = $this.attr('data-target') - , $parent - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - $parent = $(selector) - - e && e.preventDefault() - - $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent()) - - $parent.trigger(e = $.Event('close')) - - if (e.isDefaultPrevented()) return - - $parent.removeClass('in') - - function removeElement() { - $parent - .trigger('closed') - .remove() - } - - $.support.transition && $parent.hasClass('fade') ? - $parent.on($.support.transition.end, removeElement) : - removeElement() - } - - - /* ALERT PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.alert - - $.fn.alert = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('alert') - if (!data) $this.data('alert', (data = new Alert(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - $.fn.alert.Constructor = Alert - - - /* ALERT NO CONFLICT - * ================= */ - - $.fn.alert.noConflict = function () { - $.fn.alert = old - return this - } - - - /* ALERT DATA-API - * ============== */ - - $(document).on('click.alert.data-api', dismiss, Alert.prototype.close) - -}(window.jQuery); -/* ============================================================ - * bootstrap-button.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#buttons - * ============================================================ - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* BUTTON PUBLIC CLASS DEFINITION - * ============================== */ - - var Button = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.button.defaults, options) - } - - Button.prototype.setState = function (state) { - var d = 'disabled' - , $el = this.$element - , data = $el.data() - , val = $el.is('input') ? 'val' : 'html' - - state = state + 'Text' - data.resetText || $el.data('resetText', $el[val]()) - - $el[val](data[state] || this.options[state]) - - // push to event loop to allow forms to submit - setTimeout(function () { - state == 'loadingText' ? - $el.addClass(d).attr(d, d) : - $el.removeClass(d).removeAttr(d) - }, 0) - } - - Button.prototype.toggle = function () { - var $parent = this.$element.closest('[data-toggle="buttons-radio"]') - - $parent && $parent - .find('.active') - .removeClass('active') - - this.$element.toggleClass('active') - } - - - /* BUTTON PLUGIN DEFINITION - * ======================== */ - - var old = $.fn.button - - $.fn.button = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('button') - , options = typeof option == 'object' && option - if (!data) $this.data('button', (data = new Button(this, options))) - if (option == 'toggle') data.toggle() - else if (option) data.setState(option) - }) - } - - $.fn.button.defaults = { - loadingText: 'loading...' - } - - $.fn.button.Constructor = Button - - - /* BUTTON NO CONFLICT - * ================== */ - - $.fn.button.noConflict = function () { - $.fn.button = old - return this - } - - - /* BUTTON DATA-API - * =============== */ - - $(document).on('click.button.data-api', '[data-toggle^=button]', function (e) { - var $btn = $(e.target) - if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') - $btn.button('toggle') - }) - -}(window.jQuery); -/* ============================================================= - * bootstrap-collapse.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#collapse - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* COLLAPSE PUBLIC CLASS DEFINITION - * ================================ */ - - var Collapse = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.collapse.defaults, options) - - if (this.options.parent) { - this.$parent = $(this.options.parent) - } - - this.options.toggle && this.toggle() - } - - Collapse.prototype = { - - constructor: Collapse - - , dimension: function () { - var hasWidth = this.$element.hasClass('width') - return hasWidth ? 'width' : 'height' - } - - , show: function () { - var dimension - , scroll - , actives - , hasData - - if (this.transitioning || this.$element.hasClass('in')) return - - dimension = this.dimension() - scroll = $.camelCase(['scroll', dimension].join('-')) - actives = this.$parent && this.$parent.find('> .accordion-group > .in') - - if (actives && actives.length) { - hasData = actives.data('collapse') - if (hasData && hasData.transitioning) return - actives.collapse('hide') - hasData || actives.data('collapse', null) - } - - this.$element[dimension](0) - this.transition('addClass', $.Event('show'), 'shown') - $.support.transition && this.$element[dimension](this.$element[0][scroll]) - } - - , hide: function () { - var dimension - if (this.transitioning || !this.$element.hasClass('in')) return - dimension = this.dimension() - this.reset(this.$element[dimension]()) - this.transition('removeClass', $.Event('hide'), 'hidden') - this.$element[dimension](0) - } - - , reset: function (size) { - var dimension = this.dimension() - - this.$element - .removeClass('collapse') - [dimension](size || 'auto') - [0].offsetWidth - - this.$element[size !== null ? 'addClass' : 'removeClass']('collapse') - - return this - } - - , transition: function (method, startEvent, completeEvent) { - var that = this - , complete = function () { - if (startEvent.type == 'show') that.reset() - that.transitioning = 0 - that.$element.trigger(completeEvent) - } - - this.$element.trigger(startEvent) - - if (startEvent.isDefaultPrevented()) return - - this.transitioning = 1 - - this.$element[method]('in') - - $.support.transition && this.$element.hasClass('collapse') ? - this.$element.one($.support.transition.end, complete) : - complete() - } - - , toggle: function () { - this[this.$element.hasClass('in') ? 'hide' : 'show']() - } - - } - - - /* COLLAPSE PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.collapse - - $.fn.collapse = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('collapse') - , options = $.extend({}, $.fn.collapse.defaults, $this.data(), typeof option == 'object' && option) - if (!data) $this.data('collapse', (data = new Collapse(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.collapse.defaults = { - toggle: true - } - - $.fn.collapse.Constructor = Collapse - - - /* COLLAPSE NO CONFLICT - * ==================== */ - - $.fn.collapse.noConflict = function () { - $.fn.collapse = old - return this - } - - - /* COLLAPSE DATA-API - * ================= */ - - $(document).on('click.collapse.data-api', '[data-toggle=collapse]', function (e) { - var $this = $(this), href - , target = $this.attr('data-target') - || e.preventDefault() - || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7 - , option = $(target).data('collapse') ? 'toggle' : $this.data() - $this[$(target).hasClass('in') ? 'addClass' : 'removeClass']('collapsed') - $(target).collapse(option) - }) - -}(window.jQuery); -/* ========================================================== - * bootstrap-carousel.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#carousel - * ========================================================== - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* CAROUSEL CLASS DEFINITION - * ========================= */ - - var Carousel = function (element, options) { - this.$element = $(element) - this.$indicators = this.$element.find('.carousel-indicators') - this.options = options - this.options.pause == 'hover' && this.$element - .on('mouseenter', $.proxy(this.pause, this)) - .on('mouseleave', $.proxy(this.cycle, this)) - } - - Carousel.prototype = { - - cycle: function (e) { - if (!e) this.paused = false - if (this.interval) clearInterval(this.interval); - this.options.interval - && !this.paused - && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) - return this - } - - , getActiveIndex: function () { - this.$active = this.$element.find('.item.active') - this.$items = this.$active.parent().children() - return this.$items.index(this.$active) - } - - , to: function (pos) { - var activeIndex = this.getActiveIndex() - , that = this - - if (pos > (this.$items.length - 1) || pos < 0) return - - if (this.sliding) { - return this.$element.one('slid', function () { - that.to(pos) - }) - } - - if (activeIndex == pos) { - return this.pause().cycle() - } - - return this.slide(pos > activeIndex ? 'next' : 'prev', $(this.$items[pos])) - } - - , pause: function (e) { - if (!e) this.paused = true - if (this.$element.find('.next, .prev').length && $.support.transition.end) { - this.$element.trigger($.support.transition.end) - this.cycle(true) - } - clearInterval(this.interval) - this.interval = null - return this - } - - , next: function () { - if (this.sliding) return - return this.slide('next') - } - - , prev: function () { - if (this.sliding) return - return this.slide('prev') - } - - , slide: function (type, next) { - var $active = this.$element.find('.item.active') - , $next = next || $active[type]() - , isCycling = this.interval - , direction = type == 'next' ? 'left' : 'right' - , fallback = type == 'next' ? 'first' : 'last' - , that = this - , e - - this.sliding = true - - isCycling && this.pause() - - $next = $next.length ? $next : this.$element.find('.item')[fallback]() - - e = $.Event('slide', { - relatedTarget: $next[0] - , direction: direction - }) - - if ($next.hasClass('active')) return - - if (this.$indicators.length) { - this.$indicators.find('.active').removeClass('active') - this.$element.one('slid', function () { - var $nextIndicator = $(that.$indicators.children()[that.getActiveIndex()]) - $nextIndicator && $nextIndicator.addClass('active') - }) - } - - if ($.support.transition && this.$element.hasClass('slide')) { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $next.addClass(type) - $next[0].offsetWidth // force reflow - $active.addClass(direction) - $next.addClass(direction) - this.$element.one($.support.transition.end, function () { - $next.removeClass([type, direction].join(' ')).addClass('active') - $active.removeClass(['active', direction].join(' ')) - that.sliding = false - setTimeout(function () { that.$element.trigger('slid') }, 0) - }) - } else { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $active.removeClass('active') - $next.addClass('active') - this.sliding = false - this.$element.trigger('slid') - } - - isCycling && this.cycle() - - return this - } - - } - - - /* CAROUSEL PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.carousel - - $.fn.carousel = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('carousel') - , options = $.extend({}, $.fn.carousel.defaults, typeof option == 'object' && option) - , action = typeof option == 'string' ? option : options.slide - if (!data) $this.data('carousel', (data = new Carousel(this, options))) - if (typeof option == 'number') data.to(option) - else if (action) data[action]() - else if (options.interval) data.pause().cycle() - }) - } - - $.fn.carousel.defaults = { - interval: 5000 - , pause: 'hover' - } - - $.fn.carousel.Constructor = Carousel - - - /* CAROUSEL NO CONFLICT - * ==================== */ - - $.fn.carousel.noConflict = function () { - $.fn.carousel = old - return this - } - - /* CAROUSEL DATA-API - * ================= */ - - $(document).on('click.carousel.data-api', '[data-slide], [data-slide-to]', function (e) { - var $this = $(this), href - , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 - , options = $.extend({}, $target.data(), $this.data()) - , slideIndex - - $target.carousel(options) - - if (slideIndex = $this.attr('data-slide-to')) { - $target.data('carousel').pause().to(slideIndex).cycle() - } - - e.preventDefault() - }) - -}(window.jQuery); -/* ============================================================= - * bootstrap-typeahead.js v2.3.2 - * http://twitter.github.com/bootstrap/javascript.html#typeahead - * ============================================================= - * Copyright 2012 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function($){ - - "use strict"; // jshint ;_; - - - /* TYPEAHEAD PUBLIC CLASS DEFINITION - * ================================= */ - - var Typeahead = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.typeahead.defaults, options) - this.matcher = this.options.matcher || this.matcher - this.sorter = this.options.sorter || this.sorter - this.highlighter = this.options.highlighter || this.highlighter - this.updater = this.options.updater || this.updater - this.source = this.options.source - this.$menu = $(this.options.menu) - this.shown = false - this.listen() - } - - Typeahead.prototype = { - - constructor: Typeahead - - , select: function () { - var val = this.$menu.find('.active').attr('data-value') - this.$element - .val(this.updater(val)) - .change() - return this.hide() - } - - , updater: function (item) { - return item - } - - , show: function () { - var pos = $.extend({}, this.$element.position(), { - height: this.$element[0].offsetHeight - }) - - this.$menu - .insertAfter(this.$element) - .css({ - top: pos.top + pos.height - , left: pos.left - }) - .show() - - this.shown = true - return this - } - - , hide: function () { - this.$menu.hide() - this.shown = false - return this - } - - , lookup: function (event) { - var items - - this.query = this.$element.val() - - if (!this.query || this.query.length < this.options.minLength) { - return this.shown ? this.hide() : this - } - - items = $.isFunction(this.source) ? this.source(this.query, $.proxy(this.process, this)) : this.source - - return items ? this.process(items) : this - } - - , process: function (items) { - var that = this - - items = $.grep(items, function (item) { - return that.matcher(item) - }) - - items = this.sorter(items) - - if (!items.length) { - return this.shown ? this.hide() : this - } - - return this.render(items.slice(0, this.options.items)).show() - } - - , matcher: function (item) { - return ~item.toLowerCase().indexOf(this.query.toLowerCase()) - } - - , sorter: function (items) { - var beginswith = [] - , caseSensitive = [] - , caseInsensitive = [] - , item - - while (item = items.shift()) { - if (!item.toLowerCase().indexOf(this.query.toLowerCase())) beginswith.push(item) - else if (~item.indexOf(this.query)) caseSensitive.push(item) - else caseInsensitive.push(item) - } - - return beginswith.concat(caseSensitive, caseInsensitive) - } - - , highlighter: function (item) { - var query = this.query.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g, '\\$&') - return item.replace(new RegExp('(' + query + ')', 'ig'), function ($1, match) { - return '<strong>' + match + '</strong>' - }) - } - - , render: function (items) { - var that = this - - items = $(items).map(function (i, item) { - i = $(that.options.item).attr('data-value', item) - i.find('a').html(that.highlighter(item)) - return i[0] - }) - - items.first().addClass('active') - this.$menu.html(items) - return this - } - - , next: function (event) { - var active = this.$menu.find('.active').removeClass('active') - , next = active.next() - - if (!next.length) { - next = $(this.$menu.find('li')[0]) - } - - next.addClass('active') - } - - , prev: function (event) { - var active = this.$menu.find('.active').removeClass('active') - , prev = active.prev() - - if (!prev.length) { - prev = this.$menu.find('li').last() - } - - prev.addClass('active') - } - - , listen: function () { - this.$element - .on('focus', $.proxy(this.focus, this)) - .on('blur', $.proxy(this.blur, this)) - .on('keypress', $.proxy(this.keypress, this)) - .on('keyup', $.proxy(this.keyup, this)) - - if (this.eventSupported('keydown')) { - this.$element.on('keydown', $.proxy(this.keydown, this)) - } - - this.$menu - .on('click', $.proxy(this.click, this)) - .on('mouseenter', 'li', $.proxy(this.mouseenter, this)) - .on('mouseleave', 'li', $.proxy(this.mouseleave, this)) - } - - , eventSupported: function(eventName) { - var isSupported = eventName in this.$element - if (!isSupported) { - this.$element.setAttribute(eventName, 'return;') - isSupported = typeof this.$element[eventName] === 'function' - } - return isSupported - } - - , move: function (e) { - if (!this.shown) return - - switch(e.keyCode) { - case 9: // tab - case 13: // enter - case 27: // escape - e.preventDefault() - break - - case 38: // up arrow - e.preventDefault() - this.prev() - break - - case 40: // down arrow - e.preventDefault() - this.next() - break - } - - e.stopPropagation() - } - - , keydown: function (e) { - this.suppressKeyPressRepeat = ~$.inArray(e.keyCode, [40,38,9,13,27]) - this.move(e) - } - - , keypress: function (e) { - if (this.suppressKeyPressRepeat) return - this.move(e) - } - - , keyup: function (e) { - switch(e.keyCode) { - case 40: // down arrow - case 38: // up arrow - case 16: // shift - case 17: // ctrl - case 18: // alt - break - - case 9: // tab - case 13: // enter - if (!this.shown) return - this.select() - break - - case 27: // escape - if (!this.shown) return - this.hide() - break - - default: - this.lookup() - } - - e.stopPropagation() - e.preventDefault() - } - - , focus: function (e) { - this.focused = true - } - - , blur: function (e) { - this.focused = false - if (!this.mousedover && this.shown) this.hide() - } - - , click: function (e) { - e.stopPropagation() - e.preventDefault() - this.select() - this.$element.focus() - } - - , mouseenter: function (e) { - this.mousedover = true - this.$menu.find('.active').removeClass('active') - $(e.currentTarget).addClass('active') - } - - , mouseleave: function (e) { - this.mousedover = false - if (!this.focused && this.shown) this.hide() - } - - } - - - /* TYPEAHEAD PLUGIN DEFINITION - * =========================== */ - - var old = $.fn.typeahead - - $.fn.typeahead = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('typeahead') - , options = typeof option == 'object' && option - if (!data) $this.data('typeahead', (data = new Typeahead(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.typeahead.defaults = { - source: [] - , items: 8 - , menu: '<ul class="typeahead dropdown-menu"></ul>' - , item: '<li><a href="#"></a></li>' - , minLength: 1 - } - - $.fn.typeahead.Constructor = Typeahead - - - /* TYPEAHEAD NO CONFLICT - * =================== */ - - $.fn.typeahead.noConflict = function () { - $.fn.typeahead = old - return this - } - - - /* TYPEAHEAD DATA-API - * ================== */ - - $(document).on('focus.typeahead.data-api', '[data-provide="typeahead"]', function (e) { - var $this = $(this) - if ($this.data('typeahead')) return - $this.typeahead($this.data()) - }) - -}(window.jQuery); diff --git a/module/web/app/scripts/vendor/jquery.omniwindow.js b/module/web/app/scripts/vendor/jquery.omniwindow.js deleted file mode 100644 index e1f0b8f77..000000000 --- a/module/web/app/scripts/vendor/jquery.omniwindow.js +++ /dev/null @@ -1,141 +0,0 @@ -// jQuery OmniWindow plugin -// @version: 0.7.0 -// @author: Rudenka Alexander (mur.mailbox@gmail.com) -// @license: MIT - -;(function($) { - "use strict"; - $.fn.extend({ - omniWindow: function(options) { - - options = $.extend(true, { - animationsPriority: { - show: ['overlay', 'modal'], - hide: ['modal', 'overlay'] - }, - overlay: { - selector: '.ow-overlay', - hideClass: 'ow-closed', - animations: { - show: function(subjects, internalCallback) { return internalCallback(subjects); }, - hide: function(subjects, internalCallback) { return internalCallback(subjects); }, - internal: { - show: function(subjects){ subjects.overlay.removeClass(options.overlay.hideClass); }, - hide: function(subjects){ subjects.overlay.addClass(options.overlay.hideClass); } - } - } - }, - modal: { - hideClass: 'ow-closed', - animations: { - show: function(subjects, internalCallback) { return internalCallback(subjects); }, - hide: function(subjects, internalCallback) { return internalCallback(subjects); }, - internal: { - show: function(subjects){ subjects.modal.removeClass(options.modal.hideClass); }, - hide: function(subjects){ subjects.modal.addClass(options.modal.hideClass); } - } - }, - internal: { - stateAttribute: 'ow-active' - } - }, - eventsNames: { - show: 'show.ow', - hide: 'hide.ow', - internal: { - overlayClick: 'click.ow', - keyboardKeyUp: 'keyup.ow' - } - }, - callbacks: { // Callbacks execution chain - beforeShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 1 (stop if retruns false) - positioning: function(subjects, internalCallback) { return internalCallback(subjects); }, // 2 - afterShow: function(subjects, internalCallback) { return internalCallback(subjects); }, // 3 - beforeHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 4 (stop if retruns false) - afterHide: function(subjects, internalCallback) { return internalCallback(subjects); }, // 5 - internal: { - beforeShow: function(subjects) { - if (subjects.modal.data(options.modal.internal.stateAttribute)) { - return false; - } else { - subjects.modal.data(options.modal.internal.stateAttribute, true); - return true; - } - }, - afterShow: function(subjects) { - $(document).on(options.eventsNames.internal.keyboardKeyUp, function(e) { - if (e.keyCode === 27) { // if the key pressed is the ESC key - subjects.modal.trigger(options.eventsNames.hide); - } - }); - - subjects.overlay.on(options.eventsNames.internal.overlayClick, function(){ - subjects.modal.trigger(options.eventsNames.hide); - }); - }, - positioning: function(subjects) { - subjects.modal.css('margin-left', Math.round(subjects.modal.outerWidth() / -2)); - }, - beforeHide: function(subjects) { - if (subjects.modal.data(options.modal.internal.stateAttribute)) { - subjects.modal.data(options.modal.internal.stateAttribute, false); - return true; - } else { - return false; - } - }, - afterHide: function(subjects) { - subjects.overlay.off(options.eventsNames.internal.overlayClick); - $(document).off(options.eventsNames.internal.keyboardKeyUp); - - subjects.overlay.css('display', ''); // clear inline styles after jQ animations - subjects.modal.css('display', ''); - } - } - } - }, options); - - var animate = function(process, subjects, callbackName) { - var first = options.animationsPriority[process][0], - second = options.animationsPriority[process][1]; - - options[first].animations[process](subjects, function(subjs) { // call USER's FIRST animation (depends on priority) - options[first].animations.internal[process](subjs); // call internal FIRST animation - - options[second].animations[process](subjects, function(subjs) { // call USER's SECOND animation - options[second].animations.internal[process](subjs); // call internal SECOND animation - - // then we need to call USER's - // afterShow of afterHide callback - options.callbacks[callbackName](subjects, options.callbacks.internal[callbackName]); - }); - }); - }; - - var showModal = function(subjects) { - if (!options.callbacks.beforeShow(subjects, options.callbacks.internal.beforeShow)) { return; } // cancel showing if beforeShow callback return false - - options.callbacks.positioning(subjects, options.callbacks.internal.positioning); - - animate('show', subjects, 'afterShow'); - }; - - var hideModal = function(subjects) { - if (!options.callbacks.beforeHide(subjects, options.callbacks.internal.beforeHide)) { return; } // cancel hiding if beforeHide callback return false - - animate('hide', subjects, 'afterHide'); - }; - - - var $overlay = $(options.overlay.selector); - - return this.each(function() { - var $modal = $(this); - var subjects = {modal: $modal, overlay: $overlay}; - - $modal.bind(options.eventsNames.show, function(){ showModal(subjects); }) - .bind(options.eventsNames.hide, function(){ hideModal(subjects); }); - }); - } - }); -})(jQuery);
\ No newline at end of file diff --git a/module/web/app/scripts/vendor/remaining.js b/module/web/app/scripts/vendor/remaining.js deleted file mode 100644 index d66a2931a..000000000 --- a/module/web/app/scripts/vendor/remaining.js +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Javascript Countdown - * Copyright (c) 2009 Markus Hedlund - * Version 1.1 - * Licensed under MIT license - * http://www.opensource.org/licenses/mit-license.php - * http://labs.mimmin.com/countdown - */ -define([], function() { - var remaining = { - /** - * Get the difference of the passed date, and now. The different formats of the taget parameter are: - * January 12, 2009 15:14:00 (Month dd, yyyy hh:mm:ss) - * January 12, 2009 (Month dd, yyyy) - * 09,00,12,15,14,00 (yy,mm,dd,hh,mm,ss) Months range from 0-11, not 1-12. - * 09,00,12 (yy,mm,dd) Months range from 0-11, not 1-12. - * 500 (milliseconds) - * 2009-01-12 15:14:00 (yyyy-mm-dd hh-mm-ss) - * 2009-01-12 15:14 (yyyy-mm-dd hh-mm) - * @param target Target date. Can be either a date object or a string (formated like '24 December, 2010 15:00:00') - * @return Difference in seconds - */ - getSeconds: function(target) { - var today = new Date(); - - if (typeof(target) == 'object') { - var targetDate = target; - } else { - var matches = target.match(/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2})(:(\d{2}))?/); // YYYY-MM-DD HH-MM-SS - if (matches != null) { - matches[7] = typeof(matches[7]) == 'undefined' ? '00' : matches[7]; - var targetDate = new Date(matches[1], matches[2] - 1, matches[3], matches[4], matches[5], matches[7]); - } else { - var targetDate = new Date(target); - } - } - - return Math.floor((targetDate.getTime() - today.getTime()) / 1000); - }, - - /** - * @param seconds Difference in seconds - * @param i18n A language object (see code) - * @param onlyLargestUnit Return only the largest unit (see documentation) - * @param hideEmpty Hide empty units (see documentation) - * @return String formated something like '1 week, 1 hours, 1 second' - */ - getString: function(seconds, i18n, onlyLargestUnit, hideEmpty) { - if (seconds < 1) { - return ''; - } - - if (typeof(hideEmpty) == 'undefined' || hideEmpty == null) { - hideEmpty = true; - } - if (typeof(onlyLargestUnit) == 'undefined' || onlyLargestUnit == null) { - onlyLargestUnit = false; - } - if (typeof(i18n) == 'undefined' || i18n == null) { - i18n = { - weeks: ['week', 'weeks'], - days: ['day', 'days'], - hours: ['hour', 'hours'], - minutes: ['minute', 'minutes'], - seconds: ['second', 'seconds'] - }; - } - - var units = { - weeks: 7 * 24 * 60 * 60, - days: 24 * 60 * 60, - hours: 60 * 60, - minutes: 60, - seconds: 1 - }; - - var returnArray = []; - var value; - for (unit in units) { - value = units[unit]; - if (seconds / value >= 1 || unit == 'seconds' || !hideEmpty) { - secondsConverted = Math.floor(seconds / value); - var i18nUnit = i18n[unit][secondsConverted == 1 ? 0 : 1]; - returnArray.push(secondsConverted + ' ' + i18nUnit); - seconds -= secondsConverted * value; - - if (onlyLargestUnit) { - break; - } - } - } - ; - - return returnArray.join(', '); - }, - - /** - * @param seconds Difference in seconds - * @return String formated something like '169:00:01' - */ - getStringDigital: function(seconds) { - if (seconds < 1) { - return ''; - } - - remainingTime = remaining.getArray(seconds); - - for (index in remainingTime) { - remainingTime[index] = remaining.padNumber(remainingTime[index]); - } - ; - - return remainingTime.join(':'); - }, - - /** - * @param seconds Difference in seconds - * @return Array with hours, minutes and seconds - */ - getArray: function(seconds) { - if (seconds < 1) { - return []; - } - - var units = [60 * 60, 60, 1]; - - var returnArray = []; - var value; - for (index in units) { - value = units[index]; - secondsConverted = Math.floor(seconds / value); - returnArray.push(secondsConverted); - seconds -= secondsConverted * value; - } - ; - - return returnArray; - }, - - /** - * @param number An integer - * @return Integer padded with a 0 if necessary - */ - padNumber: function(number) { - return (number >= 0 && number < 10) ? '0' + number : number; - } - }; - return remaining; -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/abstract/itemView.js b/module/web/app/scripts/views/abstract/itemView.js deleted file mode 100644 index c37118a4c..000000000 --- a/module/web/app/scripts/views/abstract/itemView.js +++ /dev/null @@ -1,47 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - // A view that is meant for temporary displaying - // All events must be unbound in onDestroy - return Backbone.View.extend({ - - tagName: 'li', - destroy: function() { - this.undelegateEvents(); - this.unbind(); - if (this.onDestroy) { - this.onDestroy(); - } - this.$el.removeData().unbind(); - this.remove(); - }, - - hide: function() { - this.$el.slideUp(); - }, - - show: function() { - this.$el.slideDown(); - }, - - unrender: function() { - var self = this; - this.$el.slideUp(function() { - self.destroy(); - }); - }, - - deleteItem: function(e) { - if (e) - e.stopPropagation(); - this.model.destroy(); - }, - - restart: function(e) { - if(e) - e.stopPropagation(); - this.model.restart(); - } - - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/abstract/modalView.js b/module/web/app/scripts/views/abstract/modalView.js deleted file mode 100644 index 9d1d72869..000000000 --- a/module/web/app/scripts/views/abstract/modalView.js +++ /dev/null @@ -1,125 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'omniwindow'], function($, Backbone, _) { - 'use strict'; - - return Backbone.View.extend({ - - events: { - 'click .btn-confirm': 'confirm', - 'click .btn-close': 'hide', - 'click .close': 'hide' - }, - - template: null, - dialog: null, - - onHideDestroy: false, - confirmCallback: null, - - initialize: function(template, confirm) { - this.confirmCallback = confirm; - var self = this; - if (this.template === null) { - if (template) { - this.template = template; - // When template was provided this is a temporary dialog - this.onHideDestroy = true; - } - else - require(['text!tpl/default/modal.html'], function(template) { - self.template = template; - }); - } - - }, - - // TODO: whole modal stuff is not very elegant - render: function() { - this.$el.html(this.template(this.renderContent())); - this.onRender(); - - if (this.dialog === null) { - this.$el.addClass('modal hide'); - this.$el.css({opacity: 0, scale: 0.7}); - - var self = this; - $('body').append(this.el); - this.dialog = this.$el.omniWindow({ - overlay: { - selector: '#modal-overlay', - hideClass: 'hide', - animations: { - hide: function(subjects, internalCallback) { - subjects.overlay.transition({opacity: 'hide', delay: 100}, 300, function() { - internalCallback(subjects); - self.onHide(); - if (self.onHideDestroy) - self.destroy(); - }); - }, - show: function(subjects, internalCallback) { - subjects.overlay.fadeIn(300); - internalCallback(subjects); - }}}, - modal: { - hideClass: 'hide', - animations: { - hide: function(subjects, internalCallback) { - subjects.modal.transition({opacity: 'hide', scale: 0.7}, 300); - internalCallback(subjects); - }, - - show: function(subjects, internalCallback) { - subjects.modal.transition({opacity: 'show', scale: 1, delay: 100}, 300, function() { - internalCallback(subjects); - }); - }} - }}); - } - - return this; - }, - - onRender: function() { - - }, - - renderContent: function() { - return {}; - }, - - show: function() { - if (this.dialog === null) - this.render(); - - this.dialog.trigger('show'); - - this.onShow(); - }, - - onShow: function() { - - }, - - hide: function() { - this.dialog.trigger('hide'); - }, - - onHide: function() { - - }, - - confirm: function() { - if (this.confirmCallback) - this.confirmCallback.apply(); - - this.hide(); - }, - - destroy: function() { - this.$el.remove(); - this.dialog = null; - this.remove(); - } - - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountListView.js b/module/web/app/scripts/views/accounts/accountListView.js deleted file mode 100644 index 4eb5bfe7d..000000000 --- a/module/web/app/scripts/views/accounts/accountListView.js +++ /dev/null @@ -1,52 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'collections/AccountList', './accountView', - 'hbs!tpl/accounts/layout', 'hbs!tpl/accounts/actionbar'], - function($, _, Backbone, App, AccountList, accountView, template, templateBar) { - 'use strict'; - - // Renders settings over view page - return Backbone.Marionette.CollectionView.extend({ - - itemView: accountView, - template: template, - - collection: null, - modal: null, - - initialize: function() { - this.actionbar = Backbone.Marionette.ItemView.extend({ - template: templateBar, - events: { - 'click .btn': 'addAccount' - }, - addAccount: _.bind(this.addAccount, this) - }); - - this.collection = new AccountList(); - this.update(); - - this.listenTo(App.vent, 'accounts:updated', this.update); - }, - - update: function() { - this.collection.fetch(); - }, - - onBeforeRender: function() { - this.$el.html(template()); - }, - - appendHtml: function(collectionView, itemView, index) { - this.$('.account-list').append(itemView.el); - }, - - addAccount: function() { - var self = this; - _.requireOnce(['views/accounts/accountModal'], function(Modal) { - if (self.modal === null) - self.modal = new Modal(); - - self.modal.show(); - }); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountModal.js b/module/web/app/scripts/views/accounts/accountModal.js deleted file mode 100644 index 6c2b226df..000000000 --- a/module/web/app/scripts/views/accounts/accountModal.js +++ /dev/null @@ -1,72 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addAccount', 'helpers/pluginIcon', 'select2'], - function($, _, App, modalView, template, pluginIcon) { - 'use strict'; - return modalView.extend({ - - events: { - 'submit form': 'add', - 'click .btn-add': 'add' - }, - template: template, - plugins: null, - select: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - var self = this; - $.ajax(App.apiRequest('getAccountTypes', null, {success: function(data) { - self.plugins = _.sortBy(data, function(item) { - return item; - }); - self.render(); - }})); - }, - - onRender: function() { - // TODO: could be a separate input type if needed on multiple pages - if (this.plugins) - this.select = this.$('#pluginSelect').select2({ - escapeMarkup: function(m) { - return m; - }, - formatResult: this.format, - formatSelection: this.format, - data: {results: this.plugins, text: function(item) { - return item; - }}, - id: function(item) { - return item; - } - }); - }, - - onShow: function() { - }, - - onHide: function() { - }, - - format: function(data) { - return '<img class="logo-select" src="' + pluginIcon(data) + '"> ' + data; - }, - - add: function(e) { - e.stopPropagation(); - if (this.select) { - var plugin = this.select.val(), - login = this.$('#login').val(), - password = this.$('#password').val(), - self = this; - - $.ajax(App.apiRequest('updateAccount', { - plugin: plugin, login: login, password: password - }, { success: function() { - App.vent.trigger('accounts:updated'); - self.hide(); - }})); - } - return false; - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/accounts/accountView.js b/module/web/app/scripts/views/accounts/accountView.js deleted file mode 100644 index 89f69d7e7..000000000 --- a/module/web/app/scripts/views/accounts/accountView.js +++ /dev/null @@ -1,18 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'hbs!tpl/accounts/account'], - function($, _, Backbone, App, template) { - 'use strict'; - - return Backbone.Marionette.ItemView.extend({ - - tagName: 'tr', - template: template, - - events: { - 'click .btn-danger': 'deleteAccount' - }, - - deleteAccount: function() { - this.model.destroy(); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/dashboardView.js b/module/web/app/scripts/views/dashboard/dashboardView.js deleted file mode 100644 index e7adba475..000000000 --- a/module/web/app/scripts/views/dashboard/dashboardView.js +++ /dev/null @@ -1,168 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'models/TreeCollection', - './packageView', './fileView', 'hbs!tpl/dashboard/layout', 'select2'], - function($, Backbone, _, App, TreeCollection, PackageView, FileView, template) { - 'use strict'; - // Renders whole dashboard - return Backbone.Marionette.ItemView.extend({ - - // TODO: refactor - active: $('.breadcrumb .active'), - - template: template, - - events: { - }, - - ui: { - 'packages': '.package-list', - 'files': '.file-list' - }, - - // Package tree - tree: null, - // Current open files - files: null, - // True when loading animation is running - isLoading: false, - - initialize: function() { - App.dashboard = this; - this.tree = new TreeCollection(); - - var self = this; - // When package is added we reload the data - App.vent.on('package:added', function() { - console.log('Package tree caught, package:added event'); - self.tree.fetch(); - }); - - App.vent.on('file:updated', _.bind(this.fileUpdated, this)); - - // TODO: merge? - this.init(); - // TODO: file:added - // TODO: package:deleted - // TODO: package:updated - }, - - init: function() { - var self = this; - // TODO: put in separated function - // TODO: order of elements? - // Init the tree and callback for package added - this.tree.fetch({success: function() { - self.update(); - self.tree.get('packages').on('add', function(pack) { - console.log('Package ' + pack.get('pid') + ' added to tree'); - self.appendPackage(pack, 0, true); - self.openPackage(pack); - }); - }}); - - this.$('.input').select2({tags: ['a', 'b', 'sdf']}); - }, - - update: function() { - console.log('Update package list'); - - // TODO: Both null - var packs = this.tree.get('packages'); - this.files = this.tree.get('files'); - - if (packs) - packs.each(_.bind(this.appendPackage, this)); - - if (!this.files || this.files.length === 0) { - // no files are displayed - this.files = null; - // Open the first package - if (packs && packs.length >= 1) - this.openPackage(packs.at(0)); - } - else - this.files.each(_.bind(this.appendFile, this)); - - return this; - }, - - // TODO sorting ?! - // Append a package to the list, index, animate it - appendPackage: function(pack, i, animation) { - var el = new PackageView({model: pack}).render().el; - $(this.ui.packages).appendWithAnimation(el, animation); - }, - - appendFile: function(file, i, animation) { - var el = new FileView({model: file}).render().el; - $(this.ui.files).appendWithAnimation(el, animation); - }, - - // Show content of the packages on main view - openPackage: function(pack) { - var self = this; - - // load animation only when something is shown and its different from current package - if (this.files && this.files !== pack.get('files')) - self.loading(); - - pack.fetch({silent: true, success: function() { - console.log('Package ' + pack.get('pid') + ' loaded'); - self.active.text(pack.get('name')); - self.contentReady(pack.get('files')); - }, failure: function() { - self.failure(); - }}); - - }, - - contentReady: function(files) { - var old_files = this.files; - this.files = files; - App.vent.trigger('dashboard:contentReady'); - - // show the files when no loading animation is running and not already open - if (!this.isLoading && old_files !== files) - this.show(); - }, - - // Do load animation, remove the old stuff - loading: function() { - this.isLoading = true; - this.files = null; - var self = this; - $(this.ui.files).fadeOut({complete: function() { - // All file views should vanish - App.vent.trigger('dashboard:destroyContent'); - - // Loading was faster than animation - if (self.files) - self.show(); - - self.isLoading = false; - }}); - }, - - failure: function() { - // TODO - }, - - show: function() { - // fileUL has to be resetted before - this.files.each(_.bind(this.appendFile, this)); - //TODO: show placeholder when nothing is displayed (filtered content empty) - $(this.ui.files).fadeIn(); - App.vent.trigger('dashboard:updated'); - }, - - // Refresh the file if it is currently shown - fileUpdated: function(data) { - // this works with ids and object - var file = this.files.get(data); - if (file) - if (_.isObject(data)) // update directly - file.set(data); - else // fetch from server - file.fetch(); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/fileView.js b/module/web/app/scripts/views/dashboard/fileView.js deleted file mode 100644 index 4e5884ed8..000000000 --- a/module/web/app/scripts/views/dashboard/fileView.js +++ /dev/null @@ -1,102 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', 'helpers/formatTime', 'hbs!tpl/dashboard/file'], - function($, Backbone, _, App, Api, ItemView, formatTime, template) { - 'use strict'; - - // Renders single file item - return ItemView.extend({ - - tagName: 'li', - className: 'file-view row-fluid', - template: template, - events: { - 'click .checkbox': 'select', - 'click .btn-delete': 'deleteItem', - 'click .btn-restart': 'restart' - }, - - initialize: function() { - this.listenTo(this.model, 'change', this.render); - // This will be triggered manually and changed before with silent=true - this.listenTo(this.model, 'change:visible', this.visibility_changed); - this.listenTo(this.model, 'change:progress', this.progress_changed); - this.listenTo(this.model, 'remove', this.unrender); - this.listenTo(App.vent, 'dashboard:destroyContent', this.destroy); - }, - - onDestroy: function() { - }, - - render: function() { - var data = this.model.toJSON(); - if (data.download) { - var status = data.download.status; - if (status === Api.DownloadStatus.Offline || status === Api.DownloadStatus.TempOffline) - data.offline = true; - else if (status === Api.DownloadStatus.Online) - data.online = true; - else if (status === Api.DownloadStatus.Waiting) - data.waiting = true; - else if (status === Api.DownloadStatus.Downloading) - data.downloading = true; - else if (this.model.isFailed()) - data.failed = true; - else if (this.model.isFinished()) - data.finished = true; - } - - this.$el.html(this.template(data)); - if (this.model.get('selected')) - this.$el.addClass('ui-selected'); - else - this.$el.removeClass('ui-selected'); - - if (this.model.get('visible')) - this.$el.show(); - else - this.$el.hide(); - - return this; - }, - - select: function(e) { - e.preventDefault(); - var checked = this.$el.hasClass('ui-selected'); - // toggle class immediately, so no re-render needed - this.model.set('selected', !checked, {silent: true}); - this.$el.toggleClass('ui-selected'); - App.vent.trigger('file:selection'); - }, - - visibility_changed: function(visible) { - // TODO: improve animation, height is not available when element was not visible - if (visible) - this.$el.slideOut(true); - else { - this.$el.calculateHeight(true); - this.$el.slideIn(true); - } - }, - - progress_changed: function() { - if (!this.model.isDownload()) - return; - - if (this.model.get('download').status === Api.DownloadStatus.Downloading) { - var bar = this.$('.progress .bar'); - if (!bar) { // ensure that the dl bar is rendered - this.render(); - bar = this.$('.progress .bar'); - } - - bar.width(this.model.get('progress') + '%'); - bar.html(' ' + formatTime(this.model.get('eta'))); - } else if (this.model.get('download').status === Api.DownloadStatus.Waiting) { - this.$('.second').html( - '<i class="icon-time"></i> ' + formatTime(this.model.get('eta'))); - - } else // Every else state can be renderred normally - this.render(); - - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/filterView.js b/module/web/app/scripts/views/dashboard/filterView.js deleted file mode 100644 index 64bc56724..000000000 --- a/module/web/app/scripts/views/dashboard/filterView.js +++ /dev/null @@ -1,133 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'models/Package', 'hbs!tpl/dashboard/actionbar'], - /*jslint -W040: false*/ - function($, Backbone, _, App, Api, Package, template) { - 'use strict'; - - // Modified version of type ahead show, nearly the same without absolute positioning - function show() { - this.$menu - .insertAfter(this.$element) - .show(); - - this.shown = true; - return this; - } - - // Renders the actionbar for the dashboard, handles everything related to filtering displayed files - return Backbone.Marionette.ItemView.extend({ - - events: { - 'click .filter-type': 'filter_type', - 'click .filter-state': 'switch_filter', - 'submit .form-search': 'search' - }, - - ui: { - 'search': '.search-query', - 'stateMenu': '.dropdown-toggle .state' - }, - - template: template, - state: null, - - initialize: function() { - this.state = Api.DownloadState.All; - - // Apply the filter before the content is shown - App.vent.on('dashboard:contentReady', _.bind(this.apply_filter, this)); - }, - - onRender: function() { - // use our modified method - $.fn.typeahead.Constructor.prototype.show = show; - this.ui.search.typeahead({ - minLength: 2, - source: this.getSuggestions - }); - - }, - - // TODO: app level api request - search: function(e) { - e.stopPropagation(); - var query = this.ui.search.val(); - this.ui.search.val(''); - - var pack = new Package(); - // Overwrite fetch method to use a search - // TODO: quite hackish, could be improved to filter packages - // or show performed search - pack.fetch = function(options) { - pack.search(query, options); - }; - - App.dashboard.openPackage(pack); - }, - - getSuggestions: function(query, callback) { - $.ajax(App.apiRequest('searchSuggestions', {pattern: query}, { - method: 'POST', - success: function(data) { - callback(data); - } - })); - }, - - switch_filter: function(e) { - e.stopPropagation(); - var element = $(e.target); - var state = parseInt(element.data('state'), 10); - var menu = this.ui.stateMenu.parent().parent(); - menu.removeClass('open'); - - if (state === Api.DownloadState.Finished) { - menu.removeClass().addClass('dropdown finished'); - } else if (state === Api.DownloadState.Unfinished) { - menu.removeClass().addClass('dropdown active'); - } else if (state === Api.DownloadState.Failed) { - menu.removeClass().addClass('dropdown failed'); - } else { - menu.removeClass().addClass('dropdown'); - } - - this.state = state; - this.ui.stateMenu.text(element.text()); - this.apply_filter(); - }, - - // Applies the filtering to current open files - apply_filter: function() { - if (!App.dashboard.files) - return; - - var self = this; - App.dashboard.files.map(function(file) { - var visible = file.get('visible'); - if (visible !== self.is_visible(file)) { - file.set('visible', !visible, {silent: true}); - file.trigger('change:visible', !visible); - } - }); - - App.vent.trigger('dashboard:filtered'); - }, - - // determine if a file should be visible - // TODO: non download files - is_visible: function(file) { - if (this.state === Api.DownloadState.Finished) - return file.isFinished(); - else if (this.state === Api.DownloadState.Unfinished) - return file.isUnfinished(); - else if (this.state === Api.DownloadState.Failed) - return file.isFailed(); - - return true; - }, - - filter_type: function(e) { - - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/packageView.js b/module/web/app/scripts/views/dashboard/packageView.js deleted file mode 100644 index 2738fcbea..000000000 --- a/module/web/app/scripts/views/dashboard/packageView.js +++ /dev/null @@ -1,75 +0,0 @@ -define(['jquery', 'app', 'views/abstract/itemView', 'underscore', 'hbs!tpl/dashboard/package'], - function($, App, itemView, _, template) { - 'use strict'; - - // Renders a single package item - return itemView.extend({ - - tagName: 'li', - className: 'package-view', - template: template, - events: { - 'click .package-name, .btn-open': 'open', - 'click .icon-refresh': 'restart', - 'click .select': 'select', - 'click .btn-delete': 'deleteItem' - }, - - // Ul for child packages (unused) - ul: null, - // Currently unused - expanded: false, - - initialize: function() { - this.listenTo(this.model, 'filter:added', this.hide); - this.listenTo(this.model, 'filter:removed', this.show); - this.listenTo(this.model, 'change', this.render); - this.listenTo(this.model, 'remove', this.unrender); - - // Clear drop down menu - var self = this; - this.$el.on('mouseleave', function() { - self.$('.dropdown-menu').parent().removeClass('open'); - }); - }, - - onDestroy: function() { - }, - - // Render everything, optional only the fileViews - render: function() { - this.$el.html(this.template(this.model.toJSON())); - this.$el.initTooltips(); - - return this; - }, - - unrender: function() { - itemView.prototype.unrender.apply(this); - - // TODO: display other package - App.vent.trigger('dashboard:loading', null); - }, - - - // TODO - // Toggle expanding of packages - expand: function(e) { - e.preventDefault(); - }, - - open: function(e) { - e.preventDefault(); - App.dashboard.openPackage(this.model); - }, - - select: function(e) { - e.preventDefault(); - var checked = this.$('.select').hasClass('icon-check'); - // toggle class immediately, so no re-render needed - this.model.set('selected', !checked, {silent: true}); - this.$('.select').toggleClass('icon-check').toggleClass('icon-check-empty'); - App.vent.trigger('package:selection'); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/dashboard/selectionView.js b/module/web/app/scripts/views/dashboard/selectionView.js deleted file mode 100644 index 8685fd849..000000000 --- a/module/web/app/scripts/views/dashboard/selectionView.js +++ /dev/null @@ -1,155 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/dashboard/select'], - function($, Backbone, _, App, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.Marionette.ItemView.extend({ - - el: '#selection-area', - template: template, - - events: { - 'click .icon-check': 'deselect', - 'click .icon-pause': 'pause', - 'click .icon-trash': 'trash', - 'click .icon-refresh': 'restart' - }, - - // Element of the action bar - actionBar: null, - // number of currently selected elements - current: 0, - - initialize: function() { - this.$el.calculateHeight().height(0); - var render = _.bind(this.render, this); - - App.vent.on('dashboard:updated', render); - App.vent.on('dashboard:filtered', render); - App.vent.on('package:selection', render); - App.vent.on('file:selection', render); - - this.actionBar = $('.actionbar .btn-check'); - this.actionBar.parent().click(_.bind(this.select_toggle, this)); - - // API events, maybe better to rely on internal ones? - App.vent.on('package:deleted', render); - App.vent.on('file:deleted', render); - }, - - get_files: function(all) { - var files = []; - if (App.dashboard.files) - if (all) - files = App.dashboard.files.where({visible: true}); - else - files = App.dashboard.files.where({selected: true, visible: true}); - - return files; - }, - - get_packs: function() { - if (!App.dashboard.tree.get('packages')) - return []; // TODO - - return App.dashboard.tree.get('packages').where({selected: true}); - }, - - render: function() { - var files = this.get_files().length; - var packs = this.get_packs().length; - - if (files + packs > 0) { - this.$el.html(this.template({files: files, packs: packs})); - this.$el.initTooltips('bottom'); - } - - if (files + packs > 0 && this.current === 0) - this.$el.slideOut(); - else if (files + packs === 0 && this.current > 0) - this.$el.slideIn(); - - // TODO: accessing ui directly, should be events - if (files > 0) { - this.actionBar.addClass('icon-check').removeClass('icon-check-empty'); - App.dashboard.ui.packages.addClass('ui-files-selected'); - } - else { - this.actionBar.addClass('icon-check-empty').removeClass('icon-check'); - App.dashboard.ui.packages.removeClass('ui-files-selected'); - } - - this.current = files + packs; - }, - - // Deselects all items - deselect: function() { - this.get_files().map(function(file) { - file.set('selected', false); - }); - - this.get_packs().map(function(pack) { - pack.set('selected', false); - }); - - this.render(); - }, - - pause: function() { - alert('Not implemented yet'); - this.deselect(); - }, - - trash: function() { - _.confirm('default/confirmDialog.html', function() { - - var pids = []; - // TODO: delete many at once - this.get_packs().map(function(pack) { - pids.push(pack.get('pid')); - pack.destroy(); - }); - - // get only the fids of non deleted packages - var fids = _.filter(this.get_files(),function(file) { - return !_.contains(pids, file.get('package')); - }).map(function(file) { - file.destroyLocal(); - return file.get('fid'); - }); - - if (fids.length > 0) - $.ajax(App.apiRequest('deleteFiles', {fids: fids})); - - this.deselect(); - }, this); - }, - - restart: function() { - this.get_files().map(function(file) { - file.restart(); - }); - this.get_packs().map(function(pack) { - pack.restart(); - }); - - this.deselect(); - }, - - // Select or deselect all visible files - select_toggle: function() { - var files = this.get_files(); - if (files.length === 0) { - this.get_files(true).map(function(file) { - file.set('selected', true); - }); - - } else - files.map(function(file) { - file.set('selected', false); - }); - - this.render(); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/headerView.js b/module/web/app/scripts/views/headerView.js deleted file mode 100644 index 512c7259b..000000000 --- a/module/web/app/scripts/views/headerView.js +++ /dev/null @@ -1,240 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'collections/ProgressList', - 'views/progressView', 'views/notificationView', 'helpers/formatSize', 'hbs!tpl/header/layout', - 'hbs!tpl/header/status', 'hbs!tpl/header/progressbar' , 'flot'], - function($, _, Backbone, App, ServerStatus, ProgressList, ProgressView, NotificationView, formatSize, - template, templateStatus, templateHeader) { - 'use strict'; - // Renders the header with all information - return Backbone.Marionette.ItemView.extend({ - - events: { - 'click .icon-list': 'toggle_taskList', - 'click .popover .close': 'toggle_taskList', - 'click .btn-grabber': 'open_grabber' - }, - - ui: { - progress: '.progress-list', - speedgraph: '#speedgraph' - }, - - // todo: maybe combine these - template: template, - templateStatus: templateStatus, - templateHeader: templateHeader, - - // view - grabber: null, - speedgraph: null, - - // models and data - ws: null, - status: null, - progressList: null, - speeds: null, - - // sub view - notificationView: null, - - // save if last progress was empty - wasEmpty: false, - - initialize: function() { - var self = this; - this.notificationView = new NotificationView(); - - this.status = new ServerStatus(); - this.listenTo(this.status, 'change', this.update); - - this.progressList = new ProgressList(); - this.listenTo(this.progressList, 'add', function(model) { - self.ui.progress.appendWithAnimation(new ProgressView({model: model}).render().el); - }); - - // TODO: button to start stop refresh - var ws = App.openWebSocket('/async'); - ws.onopen = function() { - ws.send(JSON.stringify('start')); - }; - // TODO compare with polling - ws.onmessage = _.bind(this.onData, this); - ws.onerror = function(error) { - console.log(error); - alert('WebSocket error' + error); - }; - - this.ws = ws; - }, - - initGraph: function() { - var totalPoints = 120; - var data = []; - - // init with empty data - while (data.length < totalPoints) - data.push([data.length, 0]); - - this.speeds = data; - this.speedgraph = $.plot(this.ui.speedgraph, [this.speeds], { - series: { - lines: { show: true, lineWidth: 2 }, - shadowSize: 0, - color: '#fee247' - }, - xaxis: { ticks: [] }, - yaxis: { ticks: [], min: 1, autoscaleMargin: 0.1, tickFormatter: function(data) { - return formatSize(data * 1024); - }, position: 'right' }, - grid: { - show: true, -// borderColor: "#757575", - borderColor: 'white', - borderWidth: 1, - labelMargin: 0, - axisMargin: 0, - minBorderMargin: 0 - } - }); - - }, - - // Must be called after view was attached - init: function() { - this.initGraph(); - this.update(); - }, - - update: function() { - // TODO: what should be displayed in the header - // queue/processing size? - - var status = this.status.toJSON(); - status.maxspeed = _.max(this.speeds, function(speed) { - return speed[1]; - })[1] * 1024; - this.$('.status-block').html( - this.templateStatus(status) - ); - - var data = {tasks: 0, downloads: 0, speed: 0, single: false}; - this.progressList.each(function(progress) { - if (progress.isDownload()) { - data.downloads += 1; - data.speed += progress.get('download').speed; - } else - data.tasks++; - }); - - // Show progress of one task - if (data.tasks + data.downloads === 1) { - var progress = this.progressList.at(0); - data.single = true; - data.eta = progress.get('eta'); - data.percent = progress.getPercent(); - data.name = progress.get('name'); - data.statusmsg = progress.get('statusmsg'); - } - // TODO: better progressbar rendering - - data.etaqueue = status.eta; - data.linksqueue = status.linksqueue; - data.sizequeue = status.sizequeue; - - this.$('#progress-info').html( - this.templateHeader(data) - ); - return this; - }, - - toggle_taskList: function() { - this.$('.popover').animate({opacity: 'toggle'}); - }, - - open_grabber: function() { - var self = this; - _.requireOnce(['views/linkGrabberModal'], function(ModalView) { - if (self.grabber === null) - self.grabber = new ModalView(); - - self.grabber.show(); - }); - }, - - onData: function(evt) { - var data = JSON.parse(evt.data); - if (data === null) return; - - if (data['@class'] === 'ServerStatus') { - // TODO: load interaction when none available - this.status.set(data); - - // There tasks at the server, but not in queue: so fetch them - // or there are tasks in our queue but not on the server - if (this.status.get('notifications') && !this.notificationView.tasks.hasTaskWaiting() || - !this.status.get('notifications') && this.notificationView.tasks.hasTaskWaiting()) - this.notificationView.tasks.fetch(); - - this.speeds = this.speeds.slice(1); - this.speeds.push([this.speeds[this.speeds.length - 1][0] + 1, Math.floor(data.speed / 1024)]); - - // TODO: if everything is 0 re-render is not needed - this.speedgraph.setData([this.speeds]); - // adjust the axis - this.speedgraph.setupGrid(); - this.speedgraph.draw(); - - } - else if (_.isArray(data)) - this.onProgressUpdate(data); - else if (data['@class'] === 'EventInfo') - this.onEvent(data.eventname, data.event_args); - else - console.log('Unknown Async input', data); - - }, - - onProgressUpdate: function(progress) { - // generate a unique id - _.each(progress, function(prog) { - if (prog.download) - prog.pid = prog.download.fid; - else - prog.pid = prog.plugin + prog.name; - }); - - this.progressList.set(progress); - // update currently open files with progress - this.progressList.each(function(prog) { - if (prog.isDownload() && App.dashboard.files) { - var file = App.dashboard.files.get(prog.get('download').fid); - if (file) { - file.set({ - progress: prog.getPercent(), - eta: prog.get('eta') - }, {silent: true}); - - file.trigger('change:progress'); - } - } - }); - - if (progress.length === 0) { - // only render one time when last was not empty already - if (!this.wasEmpty) { - this.update(); - this.wasEmpty = true; - } - } else { - this.wasEmpty = false; - this.update(); - } - }, - - onEvent: function(event, args) { - args.unshift(event); - console.log('Core send event', args); - App.vent.trigger.apply(App.vent, args); - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/input/inputLoader.js b/module/web/app/scripts/views/input/inputLoader.js deleted file mode 100644 index 11665abb4..000000000 --- a/module/web/app/scripts/views/input/inputLoader.js +++ /dev/null @@ -1,8 +0,0 @@ -define(['./textInput'], function(textInput) { - 'use strict'; - - // selects appropriate input element - return function(input, value, default_value, description) { - return textInput; - }; -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/input/inputView.js b/module/web/app/scripts/views/input/inputView.js deleted file mode 100644 index 1fbe5042d..000000000 --- a/module/web/app/scripts/views/input/inputView.js +++ /dev/null @@ -1,86 +0,0 @@ -define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { - 'use strict'; - - // Renders input elements - return Backbone.View.extend({ - - tagName: 'input', - - input: null, - value: null, - default_value: null, - description: null, - - // enables tooltips - tooltip: true, - - initialize: function(options) { - this.input = options.input; - this.value = options.value; - this.default_value = options.default_value; - this.description = options.description; - }, - - render: function() { - this.renderInput(); - // data for tooltips - if (this.description && this.tooltip) { - this.$el.data('content', this.description); - // TODO: render default value in popup? -// this.$el.data('title', "TODO: title"); - this.$el.popover({ - placement: 'right', - trigger: 'hover' -// delay: { show: 500, hide: 100 } - }); - } - - return this; - }, - - renderInput: function() { - // Overwrite this - }, - - showTooltip: function() { - if (this.description && this.tooltip) - this.$el.popover('show'); - }, - - hideTooltip: function() { - if (this.description && this.tooltip) - this.$el.popover('hide'); - }, - - destroy: function() { - this.undelegateEvents(); - this.unbind(); - if (this.onDestroy) { - this.onDestroy(); - } - this.$el.removeData().unbind(); - this.remove(); - }, - - // focus the input element - focus: function() { - this.$el.focus(); - }, - - // Clear the input - clear: function() { - - }, - - // retrieve value of the input - getVal: function() { - return this.value; - }, - - // the child class must call this when the value changed - setVal: function(value) { - this.value = value; - this.trigger('change', value); - } - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/input/textInput.js b/module/web/app/scripts/views/input/textInput.js deleted file mode 100644 index 0eebbf91e..000000000 --- a/module/web/app/scripts/views/input/textInput.js +++ /dev/null @@ -1,36 +0,0 @@ -define(['jquery', 'backbone', 'underscore', './inputView'], function($, Backbone, _, inputView) { - 'use strict'; - - return inputView.extend({ - - // TODO - tagName: 'input', - events: { - 'keyup': 'onChange', - 'focus': 'showTooltip', - 'focusout': 'hideTooltip' - }, - - renderInput: function() { - this.$el.attr('type', 'text'); - this.$el.attr('name', 'textInput'); - - if (this.default_value) - this.$el.attr('placeholder', this.default_value); - - if (this.value) - this.$el.val(this.value); - - return this; - }, - - clear: function() { - this.$el.val(''); - }, - - onChange: function(e) { - this.setVal(this.$el.val()); - } - - }); -});
\ No newline at end of file diff --git a/module/web/app/scripts/views/linkGrabberModal.js b/module/web/app/scripts/views/linkGrabberModal.js deleted file mode 100644 index e6f59c134..000000000 --- a/module/web/app/scripts/views/linkGrabberModal.js +++ /dev/null @@ -1,49 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/linkgrabber'], - function($, _, App, modalView, template) { - 'use strict'; - // Modal dialog for package adding - triggers package:added when package was added - return modalView.extend({ - - events: { - 'click .btn-success': 'addPackage', - 'keypress #inputPackageName': 'addOnEnter' - }, - - template: template, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - }, - - addOnEnter: function(e) { - if (e.keyCode !== 13) return; - this.addPackage(e); - }, - - addPackage: function(e) { - var self = this; - var options = App.apiRequest('addPackage', - { - name: $('#inputPackageName').val(), - // TODO: better parsing / tokenization - links: $('#inputLinks').val().split('\n') - }, - { - success: function() { - App.vent.trigger('package:added'); - self.hide(); - } - }); - - $.ajax(options); - $('#inputPackageName').val(''); - $('#inputLinks').val(''); - }, - - onShow: function() { - this.$('#inputPackageName').focus(); - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/loginView.js b/module/web/app/scripts/views/loginView.js deleted file mode 100644 index 891b3ec99..000000000 --- a/module/web/app/scripts/views/loginView.js +++ /dev/null @@ -1,37 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'hbs!tpl/login'], - function($, Backbone, _, App, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.Marionette.ItemView.extend({ - template: template, - - events: { - 'submit form': 'login' - }, - - ui: { - 'form': 'form' - }, - - login: function(e) { - e.stopPropagation(); - - var options = App.apiRequest('login', null, { - data: this.ui.form.serialize(), - type : 'post', - success: function(data) { - // TODO: go to last page, better error - if (data) - App.navigate(''); - else - alert('Wrong login'); - } - }); - - $.ajax(options); - return false; - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/notificationView.js b/module/web/app/scripts/views/notificationView.js deleted file mode 100644 index abfcd8079..000000000 --- a/module/web/app/scripts/views/notificationView.js +++ /dev/null @@ -1,83 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'collections/InteractionList', 'hbs!tpl/notification'], - function($, Backbone, _, App, InteractionList, queryModal, template) { - 'use strict'; - - // Renders context actions for selection packages and files - return Backbone.View.extend({ - - // Only view for this area so it's hardcoded - el: '#notification-area', - template: template, - - events: { - 'click .btn-query': 'openQuery', - 'click .btn-notification': 'openNotifications' - }, - - tasks: null, - // area is slided out - visible: false, - // the dialog - modal: null, - - initialize: function() { - this.tasks = new InteractionList(); - - this.$el.calculateHeight().height(0); - - App.vent.on('interaction:added', _.bind(this.onAdd, this)); - App.vent.on('interaction:deleted', _.bind(this.onDelete, this)); - - var render = _.bind(this.render, this); - this.listenTo(this.tasks, 'add', render); - this.listenTo(this.tasks, 'remove', render); - - }, - - onAdd: function(task) { - this.tasks.add(task); - }, - - onDelete: function(task) { - this.tasks.remove(task); - }, - - render: function() { - - // only render when it will be visible - if (this.tasks.length > 0) - this.$el.html(this.template(this.tasks.toJSON())); - - if (this.tasks.length > 0 && !this.visible) { - this.$el.slideOut(); - this.visible = true; - } - else if (this.tasks.length === 0 && this.visible) { - this.$el.slideIn(); - this.visible = false; - } - - return this; - }, - - openQuery: function() { - var self = this; - - _.requireOnce(['views/queryModal'], function(ModalView) { - if (self.modal === null) { - self.modal = new ModalView(); - self.modal.parent = self; - } - - self.modal.model = self.tasks.at(0); - self.modal.render(); - self.modal.show(); - }); - - }, - - openNotifications: function() { - - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/progressView.js b/module/web/app/scripts/views/progressView.js deleted file mode 100644 index 3a4bb2825..000000000 --- a/module/web/app/scripts/views/progressView.js +++ /dev/null @@ -1,33 +0,0 @@ -define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abstract/itemView', - 'hbs!tpl/header/progress', 'helpers/pluginIcon'], - function($, Backbone, _, App, Api, ItemView, template, pluginIcon) { - 'use strict'; - - // Renders single file item - return ItemView.extend({ - - idAttribute: 'pid', - tagName: 'li', - template: template, - events: { - }, - - initialize: function() { - this.listenTo(this.model, 'change', this.render); - this.listenTo(this.model, 'remove', this.unrender); - }, - - onDestroy: function() { - }, - - render: function() { - // TODO: icon - // TODO: other states - // TODO: non download progress - // TODO: better progressbar rendering - this.$el.css('background-image', 'url('+ pluginIcon('todo') +')'); - this.$el.html(this.template(this.model.toJSON())); - return this; - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/queryModal.js b/module/web/app/scripts/views/queryModal.js deleted file mode 100644 index 7c6439b49..000000000 --- a/module/web/app/scripts/views/queryModal.js +++ /dev/null @@ -1,69 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', './input/inputLoader', 'text!tpl/default/queryDialog.html'], - function($, _, App, modalView, load_input, template) { - 'use strict'; - return modalView.extend({ - - // TODO: submit on enter reloads the page sometimes - events: { - 'click .btn-success': 'submit', - 'submit form': 'submit' - }, - template: _.compile(template), - - // the notificationView - parent: null, - - model: null, - input: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - }, - - renderContent: function() { - var data = { - title: this.model.get('title'), - plugin: this.model.get('plugin'), - description: this.model.get('description') - }; - - var input = this.model.get('input').data; - if (this.model.isCaptcha()) { - data.captcha = input[0]; - data.type = input[1]; - } - return data; - }, - - onRender: function() { - // instantiate the input - var input = this.model.get('input'); - var InputView = load_input(input); - this.input = new InputView(input); - // only renders after wards - this.$('#inputField').append(this.input.render().el); - }, - - submit: function(e) { - e.stopPropagation(); - // TODO: load next task - - this.model.set('result', this.input.getVal()); - var self = this; - this.model.save({success: function() { - self.hide(); - }}); - - this.input.clear(); - }, - - onShow: function() { - this.input.focus(); - }, - - onHide: function() { - this.input.destroy(); - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/settings/configSectionView.js b/module/web/app/scripts/views/settings/configSectionView.js deleted file mode 100644 index e05701b2a..000000000 --- a/module/web/app/scripts/views/settings/configSectionView.js +++ /dev/null @@ -1,99 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', '../abstract/itemView', '../input/inputLoader', - 'hbs!tpl/settings/config', 'hbs!tpl/settings/configItem'], - function($, _, Backbone, App, itemView, load_input, template, templateItem) { - 'use strict'; - - // Renders settings over view page - return itemView.extend({ - - tagName: 'div', - - template: template, - templateItem: templateItem, - - // Will only render one time with further attribute updates - rendered: false, - - events: { - 'click .btn-primary': 'submit', - 'click .btn-reset': 'reset' - }, - - initialize: function() { - this.listenTo(this.model, 'destroy', this.destroy); - }, - - render: function() { - if (!this.rendered) { - this.$el.html(this.template(this.model.toJSON())); - - // initialize the popover - this.$('.page-header a').popover({ - placement: 'left' -// trigger: 'hover' - }); - - var container = this.$('.control-content'); - var self = this; - _.each(this.model.get('items'), function(item) { - var json = item.toJSON(); - var el = $('<div>').html(self.templateItem(json)); - var InputView = load_input(item.get('input')); - var input = new InputView(json).render(); - item.set('inputView', input); - - self.listenTo(input, 'change', _.bind(self.render, self)); - el.find('.controls').append(input.el); - container.append(el); - }); - this.rendered = true; - } - // Enable button if something is changed - if (this.model.hasChanges()) - this.$('.btn-primary').removeClass('disabled'); - else - this.$('.btn-primary').addClass('disabled'); - - // Mark all inputs that are modified - _.each(this.model.get('items'), function(item) { - var input = item.get('inputView'); - var el = input.$el.parent().parent(); - if (item.isChanged()) - el.addClass('info'); - else - el.removeClass('info'); - }); - - return this; - }, - - onDestroy: function() { - // TODO: correct cleanup after building up so many views and models - }, - - submit: function(e) { - e.stopPropagation(); - // TODO: success / failure popups - var self = this; - this.model.save({success: function() { - self.render(); - App.settingsView.refresh(); - }}); - - }, - - reset: function(e) { - e.stopPropagation(); - // restore the original value - _.each(this.model.get('items'), function(item) { - if (item.has('inputView')) { - var input = item.get('inputView'); - input.setVal(item.get('value')); - input.render(); - } - }); - this.render(); - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/settings/pluginChooserModal.js b/module/web/app/scripts/views/settings/pluginChooserModal.js deleted file mode 100644 index 91e9f11b3..000000000 --- a/module/web/app/scripts/views/settings/pluginChooserModal.js +++ /dev/null @@ -1,69 +0,0 @@ -define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dialogs/addPluginConfig', - 'helpers/pluginIcon', 'select2'], - function($, _, App, modalView, template, pluginIcon) { - 'use strict'; - return modalView.extend({ - - events: { - 'click .btn-add': 'add' - }, - template: template, - plugins: null, - select: null, - - initialize: function() { - // Inherit parent events - this.events = _.extend({}, modalView.prototype.events, this.events); - var self = this; - $.ajax(App.apiRequest('getAvailablePlugins', null, {success: function(data) { - self.plugins = _.sortBy(data, function(item) { - return item.name; - }); - self.render(); - }})); - }, - - onRender: function() { - // TODO: could be a seperate input type if needed on multiple pages - if (this.plugins) - this.select = this.$('#pluginSelect').select2({ - escapeMarkup: function(m) { - return m; - }, - formatResult: this.format, - formatSelection: this.formatSelection, - data: {results: this.plugins, text: function(item) { - return item.label; - }}, - id: function(item) { - return item.name; - } - }); - }, - - onShow: function() { - }, - - onHide: function() { - }, - - format: function(data) { - var s = '<div class="plugin-select" style="background-image: url(' + pluginIcon(data.name) +')">' + data.label; - s += '<br><span>' + data.description + '<span></div>'; - return s; - }, - - formatSelection: function(data) { - return '<img class="logo-select" src="' + pluginIcon(data.name) + '"> ' + data.label; - }, - - add: function(e) { - e.stopPropagation(); - if (this.select) { - var plugin = this.select.val(); - App.vent.trigger('config:open', plugin); - this.hide(); - } - } - }); - });
\ No newline at end of file diff --git a/module/web/app/scripts/views/settings/settingsView.js b/module/web/app/scripts/views/settings/settingsView.js deleted file mode 100644 index cad5ab075..000000000 --- a/module/web/app/scripts/views/settings/settingsView.js +++ /dev/null @@ -1,184 +0,0 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'models/ConfigHolder', './configSectionView', - 'hbs!tpl/settings/layout', 'hbs!tpl/settings/menu', 'hbs!tpl/settings/actionbar'], - function($, _, Backbone, App, ConfigHolder, ConfigSectionView, template, templateMenu, templateBar) { - 'use strict'; - - // Renders settings over view page - return Backbone.Marionette.ItemView.extend({ - - template: template, - templateMenu: templateMenu, - - events: { - 'click .settings-menu li > a': 'change_section', - 'click .btn-add': 'choosePlugin', // TODO not in scope - 'click .icon-remove': 'deleteConfig' - }, - - ui: { - 'menu': '.settings-menu', - 'content': '.setting-box > form' - }, - - selected: null, - modal: null, - - coreConfig: null, // It seems collections are not needed - pluginConfig: null, - - // currently open configHolder - config: null, - lastConfig: null, - isLoading: false, - - initialize: function() { - this.actionbar = Backbone.Marionette.ItemView.extend({ - template: templateBar, - events: { - 'click .btn': 'choosePlugin' - }, - choosePlugin: _.bind(this.choosePlugin, this) - - }); - this.listenTo(App.vent, 'config:open', this.openConfig); - - this.refresh(); - }, - - refresh: function() { - var self = this; - $.ajax(App.apiRequest('getCoreConfig', null, {success: function(data) { - self.coreConfig = data; - self.renderMenu(); - }})); - $.ajax(App.apiRequest('getPluginConfig', null, {success: function(data) { - self.pluginConfig = data; - self.renderMenu(); - }})); - }, - - onRender: function() { - // set a height with css so animations will work - this.ui.content.height(this.ui.content.height()); - }, - - renderMenu: function() { - var plugins = [], - addons = []; - - // separate addons and default plugins - // addons have an activated state - _.each(this.pluginConfig, function(item) { - if (item.activated === null) - plugins.push(item); - else - addons.push(item); - }); - - this.ui.menu.html(this.templateMenu({ - core: this.coreConfig, - plugin: plugins, - addon: addons - })); - - // mark the selected element - this.$('li[data-name="' + this.selected + '"]').addClass('active'); - }, - - openConfig: function(name) { - // Do nothing when this config is already open - if (this.config && this.config.get('name') === name) - return; - - this.lastConfig = this.config; - this.config = new ConfigHolder({name: name}); - this.loading(); - - var self = this; - this.config.fetch({success: function() { - if (!self.isLoading) - self.show(); - - }, failure: _.bind(this.failure, this)}); - - }, - - loading: function() { - this.isLoading = true; - var self = this; - this.ui.content.fadeOut({complete: function() { - if (self.config.isLoaded()) - self.show(); - - self.isLoading = false; - }}); - - }, - - show: function() { - // TODO animations are bit sloppy - this.ui.content.css('display', 'block'); - var oldHeight = this.ui.content.height(); - - // this will destroy the old view - if (this.lastConfig) - this.lastConfig.trigger('destroy'); - else - this.ui.content.empty(); - - // reset the height - this.ui.content.css('height', ''); - // append the new element - this.ui.content.append(new ConfigSectionView({model: this.config}).render().el); - // get the new height - var height = this.ui.content.height(); - // set the old height again - this.ui.content.height(oldHeight); - this.ui.content.animate({ - opacity: 'show', - height: height - }); - }, - - failure: function() { - // TODO - this.config = null; - }, - - change_section: function(e) { - // TODO check for changes - // TODO move this into render? - - var el = $(e.target).closest('li'); - - this.selected = el.data('name'); - this.openConfig(this.selected); - - this.ui.menu.find('li.active').removeClass('active'); - el.addClass('active'); - e.preventDefault(); - }, - - choosePlugin: function(e) { - var self = this; - _.requireOnce(['views/settings/pluginChooserModal'], function(Modal) { - if (self.modal === null) - self.modal = new Modal(); - - self.modal.show(); - }); - }, - - deleteConfig: function(e) { - e.stopPropagation(); - var el = $(e.target).parent().parent(); - var name = el.data('name'); - var self = this; - $.ajax(App.apiRequest('deleteConfig', {plugin: name}, { success: function() { - self.refresh(); - }})); - - } - - }); - });
\ No newline at end of file diff --git a/module/web/app/styles/default/accounts.less b/module/web/app/styles/default/accounts.less deleted file mode 100644 index 9b45b64b3..000000000 --- a/module/web/app/styles/default/accounts.less +++ /dev/null @@ -1,6 +0,0 @@ -@import "common"; - -.logo-select { - width: 20px; - height: 20px; -}
\ No newline at end of file diff --git a/module/web/app/styles/default/admin.less b/module/web/app/styles/default/admin.less deleted file mode 100644 index 92524c153..000000000 --- a/module/web/app/styles/default/admin.less +++ /dev/null @@ -1,17 +0,0 @@ -@import "common"; - -/* - Admin -*/ - -#btn_newuser { - float: right; -} - -#user_permissions { - float: right; -} - -.userperm { - width: 115px; -}
\ No newline at end of file diff --git a/module/web/app/styles/default/base.less b/module/web/app/styles/default/base.less deleted file mode 100644 index bd318127e..000000000 --- a/module/web/app/styles/default/base.less +++ /dev/null @@ -1,163 +0,0 @@ -@import "common"; - -/* - General -*/ - -* { - margin: 0; -} - -html, body { - height: 100%; -} - -body { - margin: 0; - padding: 0; - font-family: 'Abel', sans-serif; - font-size: 16px; - background: url("../../images/default/bgpattern.png") repeat scroll 0 0 transparent; -} - -h1, h2, h3 { - margin: 0; - padding: 0; - font-weight: normal; -} - -a { - text-decoration: none; - color: @blue; -} - -a:hover { - text-decoration: none; - color: @emph; -} - -#wrap { - min-height: 100%; -} - -#content { - padding-bottom: @footer-height; -} - -#content-container:before { - display: block; - content: " "; - height: @header-height; -} - -/* - Additional Responsive Class for larger desktop -*/ - -.visible-large-screen { - display: inherit !important; -} - -@media (max-width: @large-screen) { - .visible-large-screen { - display: none !important; - } -} - -.btn-blue { - background-color: hsl(206, 49%, 35%) !important; - background-repeat: repeat-x; - background-image: -khtml-gradient(linear, left top, left bottom, from(#5493c4), to(#2d5f84)); - background-image: -moz-linear-gradient(top, #5493c4, #2d5f84); - background-image: -ms-linear-gradient(top, #5493c4, #2d5f84); - background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #5493c4), color-stop(100%, #2d5f84)); - background-image: -webkit-linear-gradient(top, #5493c4, #2d5f84); - background-image: -o-linear-gradient(top, #5493c4, #2d5f84); - background-image: linear-gradient(#5493c4, #2d5f84); - border-color: #2d5f84 #2d5f84 hsl(206, 49%, 30%); - color: #fff !important; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.33); - -webkit-font-smoothing: antialiased; - - .caret { - border-bottom-color: #FFFFFF; - border-top-color: #FFFFFF; - } - -} - -.btn-yellow { - background-color: hsl(46, 100%, 57%) !important; - background-repeat: repeat-x; - background-image: -khtml-gradient(linear, left top, left bottom, from(#ffe389), to(#fecb23)); - background-image: -moz-linear-gradient(top, #ffe389, #fecb23); - background-image: -ms-linear-gradient(top, #ffe389, #fecb23); - background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffe389), color-stop(100%, #fecb23)); - background-image: -webkit-linear-gradient(top, #ffe389, #fecb23); - background-image: -o-linear-gradient(top, #ffe389, #fecb23); - background-image: linear-gradient(#ffe389, #fecb23); - border-color: #fecb23 #fecb23 hsl(46, 100%, 52%); - color: #333 !important; - text-shadow: 0 1px 1px rgba(255, 255, 255, 0.33); - -webkit-font-smoothing: antialiased; - - .caret { - border-bottom-color: #FFFFFF; - border-top-color: #FFFFFF; - } -} - -.icon-8x { - font-size: 8em; -} - -// Will not work well in buttons/navs, would require more rules -.icon-larger { - vertical-align: -10%; - font-size: 1.5em; -} - -/* - Modal Overlay -*/ -#modal-overlay { - content: " "; - height: 100%; - width: 100%; - position: absolute; - left: 0; - top: 0; - background: -moz-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -webkit-gradient(radial, center center, 0px, center center, 100%, color-stop(0%, rgba(236, 208, 66, 0)), color-stop(100%, rgba(40, 119, 171, 0.9))); - background: -webkit-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -o-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: -ms-radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - background: radial-gradient(center, ellipse cover, rgba(236, 208, 66, 0) 0%, rgba(40, 119, 171, 0.9) 100%); - z-index: 100; - opacity: 0; -} - -div.modal-header { - border-top-left-radius: 4px; - border-top-right-radius: 4px; - background-color: @blueDark; - color: #FFFFFF; - - .close { - color: @light; - } - -} - -.select2-container { - min-width: 220px; // same as other input fields -} - -div.modal-body { - max-height: 100%; -} - -div.modal { - width: 600px; - -}
\ No newline at end of file diff --git a/module/web/app/styles/default/common.less b/module/web/app/styles/default/common.less deleted file mode 100644 index 338ed38e7..000000000 --- a/module/web/app/styles/default/common.less +++ /dev/null @@ -1,90 +0,0 @@ -/* - Definitions -*/ - - -/* - Threshold for slightly larger screen -*/ -@large-screen: 1150px; - -@header-height: 70px; -@actionbar-height: 40px; -@footer-height: 66px; - -@light: #ffffff; -@dark: #333333; - -@grey: #757575; -@greyLight: #E5E5E5; -@greyLighter: #F5F5F5; -@greyDark: #444444; -@greyDarker: #111113; - -@yellow: #ffd856; -@yellowLighter: lighten(spin(@yellow, 10), 20%); -@yellowLightest: lighten(spin(@yellow, 15), 30%); -@yellowDark: darken(@yellow, 10%); - -@blue: #3571a2; -@blueLight: lighten(spin(@blue, 5), 10%); -@blueLighter: lighten(spin(@blue, 10), 20%); -@blueLightest: lighten(spin(@blue, 20), 40%); -@blueDark: darken(spin(@blue, -5), 10%); -@blueDarker: darken(spin(@blue, -10), 20%); - -@green: #468847; -@greenLight: lighten(spin(@green, 5), 10%); -@greenDark: darken(spin(@green, -5), 10%); - -@red: #b94a48; -@redLight: lighten(spin(@red, 5), 10%); -@redDark: darken(spin(@red, -5), 10%); - -@emph: #FF7637; - - -/* - Mixins -*/ - -.gradient(@origin: left, @start: #ffffff, @stop: #000000) { - background-color: @start; - background-image: -webkit-linear-gradient(@origin, @start, @stop); - background-image: -moz-linear-gradient(@origin, @start, @stop); - background-image: -o-linear-gradient(@origin, @start, @stop); - background-image: -ms-linear-gradient(@origin, @start, @stop); - background-image: linear-gradient(@origin, @start, @stop); -} - -.transition(@prop: all, @time: 0.25s, @ease: linear) { - -webkit-transition: @prop @time @ease; - -moz-transition: @prop @time @ease; - -o-transition: @prop @time @ease; - -ms-transition: @prop @time @ease; - transition: @prop @time @ease; -} - -.stripes(@color, @color2: rgba(255, 255, 255, 0.15)) { - background-color: @color; - background-image: -webkit-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, @color2 25%, transparent 25%, transparent 50%, @color2 50%, @color2 75%, transparent 75%, transparent); - -webkit-background-size: 40px 40px; - -moz-background-size: 40px 40px; - -o-background-size: 40px 40px; - background-size: 40px 40px; -} -.stripes-animated { - -webkit-animation: progress-bar-stripes 2s linear infinite; - -moz-animation: progress-bar-stripes 2s linear infinite; - -ms-animation: progress-bar-stripes 2s linear infinite; - -o-animation: progress-bar-stripes 2s linear infinite; - animation: progress-bar-stripes 2s linear infinite; -} - - -.default-shadow { - box-shadow: 0 0 5px @dark; -}
\ No newline at end of file diff --git a/module/web/app/styles/default/dashboard.less b/module/web/app/styles/default/dashboard.less deleted file mode 100644 index ab61ba29b..000000000 --- a/module/web/app/styles/default/dashboard.less +++ /dev/null @@ -1,331 +0,0 @@ -@import "common"; - -/* - Dashboard -*/ - -#dashboard ul { - margin: 0; - list-style: none; -} - -.sidebar-header { - font-size: 25px; - line-height: 25px; - margin: 4px 0; - border-bottom: 1px dashed @grey; -} - -/* - Packages -*/ - -.package-list { - list-style: none; - margin-left: 0; -} - -@frame-top: 20px; -@frame-bottom: 18px; - -.package-frame { - position: absolute; - top: -@frame-top; - left: -@frame-top / 2; - right: -@frame-top / 2; - bottom: -@frame-bottom + 2px; // + size of visible bar - z-index: -1; // lies under package - border: 1px solid @grey; - border-radius: 5px; - box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); -} - -.package-view { - padding-bottom: 4px; - margin: 8px 0; - position: relative; - overflow: hidden; - - -webkit-hyphens: auto; - -moz-hyphens: auto; - hyphens: auto; - - i { - cursor: pointer; - } - - & > i { - vertical-align: middle; - } - - .progress { - position: absolute; - height: @frame-bottom; - line-height: @frame-bottom; - font-size: 12px; - text-align: center; - border-radius: 0; - border-bottom-left-radius: 5px; - border-bottom-right-radius: 5px; - bottom: 0; - left: 0; - right: 0; - margin-bottom: 0; - background-image: none; - color: @light; - background-color: @yellow; - } - - .bar-info { - background-image: none; - background-color: @blue; - } - - &:hover { - overflow: visible; - z-index: 10; - - .package-frame { - background-color: @light; - } - } - - &.ui-selected:hover { - color: @light; - - .package-frame { - background-color: @dark; - } - - } -} - -.package-name { - cursor: pointer; -} - -.package-indicator { - position: absolute; - top: 0; - right: 0; - float: right; - color: @blue; - text-shadow: @yellowDark 1px 1px; - height: @frame-top; - line-height: @frame-top; - - & > i:hover { - color: @green; - } - - .dropdown-menu { - text-shadow: none; - } - - .tooltip { - text-shadow: none; - width: 100%; - } - - .btn-move { - color: @green; - display: none; - } - -} - -.ui-files-selected .btn-move { - display: inline; -} - -// Tag area with different effect on hover -.tag-area { - position: absolute; - top: -2px; - left: 0; - - .badge { - font-size: 11px; - line-height: 11px; - } - - .badge i { - cursor: pointer; - &:hover:before { - content: "\f024"; // show Remove icon - } - } - - .badge-ghost { - visibility: hidden; - cursor: pointer; - opacity: 0.5; - } - - &:hover .badge-ghost { - visibility: visible; - } - -} - -/* - File View -*/ - -.file-list { - list-style: none; - margin: 0; -} - -@file-height: 22px; - -.file-view { - position: relative; - padding: 0 4px; - border-top: 1px solid #dddddd; - line-height: @file-height; - - &:first-child { - border-top: none; - } - - &:hover, &.ui-selected:hover { - border-radius: 5px; - .gradient(top, @blue, @blueLight); - color: @light; - } - - &.ui-selected { - .gradient(top, @yellow, @yellowDark); - color: @dark; - border-color: @greenDark; - - .file-row.downloading .bar { - .gradient(top, @green, @greenLight); - } - - } - - img { // plugin logo - margin-top: -2px; - padding: 0 2px; - height: @file-height; - width: @file-height; - } - - .icon-chevron-down:hover { - cursor: pointer; - color: @yellow; - } - -} - -.file-row { - min-height: 0 !important; -// padding-left: 5px; - padding-top: 4px; - padding-bottom: 4px; - - // TODO: better styling for filestatus - &.second { -// border-radius: 4px; -// background: @light; - font-size: small; - font-weight: bold; -// box-shadow: 3px 3px 6px rgba(0, 0, 0, 0.75); -// .default-shadow; - } - - &.third { - margin-left: 0; - position: relative; - font-size: small; - } - - .dropdown-menu { - font-size: medium; - } -} - -/* - TODO: more colorful states - better fileView design -*/ - -.file-row.finished { -// .gradient(top, @green, @greenLight); -// color: @light; - color: @green; -} - -.file-row.failed { -// .gradient(top, @red, @redLight); -// color: @light; - color: @red; -} - -.file-row.downloading { - - .progress { - height: @file-height; - background: @light; - margin: 0; - } - - .bar { - text-align: left; - .gradient(top, @yellow, @yellowDark); - color: @dark; - } - -} - -/* -FANCY CHECKBOXES -*/ -.file-view .checkbox { - width: 20px; - height: 21px; - background: url(../../images/default/checks_sheet.png) left top no-repeat; - cursor: pointer; -} - -.file-view.ui-selected .checkbox { - background: url(../../images/default/checks_sheet.png) -21px top no-repeat; -} - -/* - Actionbar -*/ - -.form-search { - position: relative; - - .dropdown-menu { - min-width: 100%; - position: absolute; - right: 0; - left: auto; - } - -} - -li.finished > a, li.finished:hover > a { - background-color: @green; - color: @light; - - .caret, .caret:hover { - border-bottom-color: @light !important; - border-top-color: @light !important; - } -} - -li.failed > a, li.failed:hover > a { - background-color: @red; - color: @light; - - .caret, .caret:hover { - border-bottom-color: @light !important; - border-top-color: @light !important; - } -}
\ No newline at end of file diff --git a/module/web/app/styles/default/main.less b/module/web/app/styles/default/main.less deleted file mode 100644 index 15d9943f8..000000000 --- a/module/web/app/styles/default/main.less +++ /dev/null @@ -1,11 +0,0 @@ -@import "bootstrap/less/bootstrap"; -@import "bootstrap/less/responsive"; -@import "font-awesome/build/assets/font-awesome/less/font-awesome"; - -@FontAwesomePath: "../../fonts"; - -@import "style"; -@import "dashboard"; -@import "settings"; -@import "accounts"; -@import "admin"; diff --git a/module/web/app/styles/default/settings.less b/module/web/app/styles/default/settings.less deleted file mode 100644 index 34bfcb92a..000000000 --- a/module/web/app/styles/default/settings.less +++ /dev/null @@ -1,121 +0,0 @@ -@import "common"; - -/* - Settings -*/ -.settings-menu { - background-color: #FFF; - box-shadow: 0 0 5px #000; // border: 10px solid #EEE; - - .nav-header { - background: @blueDark; - color: @light; - } - - li > a, .nav-header { - margin-left: -16px; - margin-right: -16px; - text-shadow: none; - } - - i { - margin-top: 0; - } - - .plugin, .addon { - a { - padding-left: 28px; - background-position: 4px 2px; - background-repeat: no-repeat; - background-size: 20px 20px; - } - - .icon-remove { - display: none; - } - - &:hover { - i { - display: block; - } - } - - } - - .addon { - div { - font-size: small; - } - .addon-on { - color: @green; - } - - .addon-off { - color: @red; - } - - } - - border-top-left-radius: 0; - border-top-right-radius: 0; - - .nav > li > a:hover { - color: @blueDark; - } -} - -.setting-box { - border: 10px solid @blueDark; - box-shadow: 0 0 5px @dark; // .gradient(bottom, @yellowLightest, @light); - overflow: hidden; - - .page-header { - margin: 0; - - .btn { - float: right; - margin-top: 5px; - } - - .popover { - font-size: medium; - } - - } - - // Bit wider control labels - .control-label { - width: 180px; - } - .controls { - margin-left: 200px; - } - .form-actions { - padding-left: 200px; - } - -} - -/* - Plugin select -*/ - -.plugin-select { - background-position: left 2px; - background-repeat: no-repeat; - background-size: 20px 20px; - padding-left: 24px; - - font-weight: bold; - span { - line-height: 14px; - font-size: small; - font-weight: normal; - } - -} - -.logo-select { - width: 20px; - height: 20px; -}
\ No newline at end of file diff --git a/module/web/app/styles/default/style.less b/module/web/app/styles/default/style.less deleted file mode 100644 index 6fb5a4857..000000000 --- a/module/web/app/styles/default/style.less +++ /dev/null @@ -1,366 +0,0 @@ -@import "default/base"; - - -/* - Header -*/ - -header { // background-color: @greyDark; - .gradient(to bottom, #222222, #111111); - height: @header-height; - position: fixed; - top: 0; - vertical-align: top; - width: 100%; - z-index: 10; - color: #ffffff; - - a { - color: #ffffff; - } - .container-fluid, .row-fluid { - height: @header-height; - } - -} - -@header-inner-height: @header-height - 16px; - -// centered header element -.centered { - height: @header-inner-height; - margin: 8px 0; -} - -header:before { - position: absolute; - content: ' '; - top: 0; - right: 0; - bottom: 0; - left: 0; - background-color: transparent; - z-index: -1; - .default-shadow; -} - -header span.title { - color: white; - float: left; - font-family: SansationRegular, sans-serif; - font-size: 40px; - line-height: @header-height; - cursor: default; -} - -header .logo { - float: left; - margin-right: 10px; - margin-top: 10px; - width: 105px; - height: 107px; - background: url("../../images/default/logo.png") no-repeat; -} - -.header-block { - .centered; - float: left; - line-height: @header-inner-height / 3; // 3 rows - font-size: small; -} - -.status-block { - min-width: 15%; -} - -.header-btn { - float: right; - position: relative; - .centered; - - .lower { - position: absolute; - bottom: 0; - left: 0; - right: 0; - margin-left: 0; - - button { - width: 100% / 3; // 3 buttons - } - - } -} - -#progress-area { - .centered; - position: relative; - margin-top: 8px; - line-height: 16px; - - #progress-info { - padding-left: 2px; - } - - .sub { - font-size: small; - padding: 0 2px; - } - - .popover { // display: block; - max-width: none; - width: 120%; - left: -60%; // Half of width - margin-left: 50%; - top: 100%; - } - - .popover-title, .popover-content { - color: @greyDark; - } - - .icon-list { - cursor: pointer; - margin-right: 2px; // same as globalprogress margin - - &:hover { - color: @yellow; - } - } - .close { - line-height: 14px; - } -} - -.progress-list { - list-style: none; - margin: 0; - font-size: small; - - li { - background-repeat: no-repeat; - background-size: 32px 32px; - background-position: 0px 8px; - padding-left: 40px; - - &:not(:last-child) { - margin-bottom: 5px; - padding-bottom: 5px; - border-bottom: 1px dashed @greyLight; - } - - .progress { - height: 8px; - margin-bottom: 0; - - .bar { - .gradient(bottom, @blue, @blueLight); - } - } - } -} - -#globalprogress { - background-color: @greyDark; - background-image: none; - height: 8px; - margin: 4px 0; - border-radius: 8px; - border: 2px solid @grey; - - .bar { - color: @dark; - background-image: none; - background-color: @yellow; - - &.running { - width: 100%; - .stripes(@yellowLighter, @yellowDark); - } - } -} - -.speedgraph-container { - // Allows speedgraph to take up remaining space - display: block; - overflow: hidden; - padding: 0 8px; - - #speedgraph { - float: right; - width: 100%; - .centered; -// height: @header-height - 16px; -// margin: 8px 0; - font-family: sans-serif; - } -} - -.header-area { - display: none; // hidden by default - position: absolute; - bottom: -28px; - line-height: 18px; - top: @header-height; - padding: 4px 10px 6px 10px; - text-align: center; - border-radius: 0 0 6px 6px; - color: @light; - background-color: @greyDark; - .default-shadow; -} - -#notification-area { - .header-area; - left: 140px; - - .badge { - vertical-align: top; - } - - .btn-query, .btn-notification { - cursor: pointer; - } -} - -#selection-area { - .header-area; - left: 50%; - min-width: 15%; - - i { - cursor: pointer; - - &:hover { - color: @yellow; - } - } - -} - -/* - Actionbar -*/ - -.nav > li > a:hover { - color: @blue; -} - -.actionbar { - padding-bottom: 3px; - margin-bottom: 0; - border-bottom: 1px dashed @grey; - - height: @actionbar-height; - - padding-top: 2px; - margin-bottom: 5px; - -} - -.actionbar > li > a { - margin-top: 4px; -} - -.actionbar .breadcrumb { - margin: 0; - padding-top: 10px; - padding-bottom: 0; - - .active { - color: @grey; - } - -} - -.actionbar form { - margin-top: 6px; - margin-bottom: 0; -} - -.actionbar input, .actionbar button { - padding-top: 2px; - padding-bottom: 2px; -} - -.actionbar .dropdown-menu i { - margin-top: 4px; - padding-right: 5px; -} - -/* - Login -*/ -.login { - vertical-align: middle; - border: 2px solid @dark; - padding: 15px 50px; - font-size: 17px; - border-radius: 15px; - -moz-border-radius: 15px; - -webkit-border-radius: 15px; -} - - -/* - Footer -*/ -footer { // Same gradient as navbar - .gradient(top, #222222, #111111); - color: @grey; - min-height: @footer-height; - margin-top: -@footer-height; - position: relative; - width: 100%; - line-height: 16px; - z-index: 10; -} - -footer:before { - position: absolute; - content: ' '; - top: 0; - right: 0; - bottom: 0; - left: 0; - background-color: transparent; - box-shadow: 0 0 5px black; - z-index: -1; -} - -footer hr { - margin: 4px 0; - border-top-color: @greyDarker; - border-bottom-color: @greyDark; -} - -footer .span2 { - font-size: 12px; - padding-top: 12px; - padding-bottom: 12px; -} - -// This is the copyright span -footer .offset1 { - padding-top: 8px; - padding-bottom: 0; -} - -footer .copyright { - background: url(../../images/default/logo_grey.png) no-repeat; - background-size: 40px 40px; - background-position: 12px center; - height: 40px; - padding-left: 40px; - padding-top: 10px; - text-align: center; -} - -footer h2 { - color: @light; - font-family: SansationLight, sans-serif; - font-size: 16px; - font-weight: normal; - line-height: 16px; - margin: 0; -}
\ No newline at end of file diff --git a/module/web/app/styles/font.css b/module/web/app/styles/font.css deleted file mode 100644 index ee117d43b..000000000 --- a/module/web/app/styles/font.css +++ /dev/null @@ -1,37 +0,0 @@ -/** - * @file - * Font styling - */ - -@font-face { - font-family: 'SansationRegular'; - src: url('../fonts/Sansation_Regular-webfont.eot'); - src: url('../fonts/Sansation_Regular-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Regular-webfont.woff') format('woff'), - url('../fonts/Sansation_Regular-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Regular-webfont.svg#SansationRegular') format('svg'); - font-weight: normal; - font-style: normal; -} - -@font-face { - font-family: 'SansationLight'; - src: url('../fonts/Sansation_Light-webfont.eot'); - src: url('../fonts/Sansation_Light-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Light-webfont.woff') format('woff'), - url('../fonts/Sansation_Light-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Light-webfont.svg#SansationLight') format('svg'); - font-weight: normal; - font-style: normal; -} - -@font-face { - font-family: 'SansationBold'; - src: url('../fonts/Sansation_Bold-webfont.eot'); - src: url('../fonts/Sansation_Bold-webfont.eot?#iefix') format('embedded-opentype'), - url('../fonts/Sansation_Bold-webfont.woff') format('woff'), - url('../fonts/Sansation_Bold-webfont.ttf') format('truetype'), - url('../fonts/Sansation_Bold-webfont.svg#SansationBold') format('svg'); - font-weight: normal; - font-style: normal; -}
\ No newline at end of file diff --git a/module/web/app/templates/default/accounts/account.html b/module/web/app/templates/default/accounts/account.html deleted file mode 100644 index c2ded16f6..000000000 --- a/module/web/app/templates/default/accounts/account.html +++ /dev/null @@ -1,10 +0,0 @@ -<td><% plugin %></td> -<td><% loginname %></td> -<td><% valid %></td> -<td><% premium %></td> -<td><% trafficleft %></td> -<td><% shared %></td> -<td><% activated %></td> -<td> - <button type="button" class="btn btn-danger">Delete</button> -</td>
\ No newline at end of file diff --git a/module/web/app/templates/default/accounts/actionbar.html b/module/web/app/templates/default/accounts/actionbar.html deleted file mode 100644 index f4652ec42..000000000 --- a/module/web/app/templates/default/accounts/actionbar.html +++ /dev/null @@ -1,5 +0,0 @@ -<div class="span2 offset1"> -</div> -<span class="span9"> - <button class="btn btn-small btn-blue btn-add">Add Account</button> -</span>
\ No newline at end of file diff --git a/module/web/app/templates/default/accounts/layout.html b/module/web/app/templates/default/accounts/layout.html deleted file mode 100644 index e6627500d..000000000 --- a/module/web/app/templates/default/accounts/layout.html +++ /dev/null @@ -1,19 +0,0 @@ -<!--{# TODO: responsive layout instead of table #}--> -<div class="span10 offset2"> - <table class="table table-striped"> - <thead> - <tr> - <th>Plugin</th> - <th>Name</th> - <th>Valid</th> - <th>Premium</th> - <th>Traffic</th> - <th>Shared</th> - <th>Activated</th> - <th>Delete</th> - </tr> - </thead> - <tbody class="account-list"> - </tbody> - </table> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/admin.html b/module/web/app/templates/default/admin.html deleted file mode 100644 index 2eb90d7e0..000000000 --- a/module/web/app/templates/default/admin.html +++ /dev/null @@ -1,223 +0,0 @@ -{% extends 'default/base.html' %} - -{% block title %}{{ _("Admin") }} - {{ super() }} {% endblock %} -{% block subtitle %}{{ _("Admin") }} -{% endblock %} - -{% block css %} - <link href="static/css/default/admin.less" rel="stylesheet/less" type="text/css" media="screen"/> - <link rel="stylesheet" type="text/css" href="static/css/fontawesome.css" /> -{% endblock %} - -{% block require %} -{% endblock %} - -{% block content %} - <div class="container-fluid"> - <div class="row-fluid"> - <div id="userlist" class="span10"> - <div class="page-header"> - <h1>Admin Bereich - <small>Userverwaltung, Systeminfos</small> - <a id="btn_newuser" class="btn btn-warning btn-large" type="button"><i class="iconf-plus-sign iconf-large "></i></a> - </h1> - - - - </div> - - <div class="dropdown"> - <span class="label name">User</span> - <a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="iconf-user iconf-8x"></i></a> - <ul class="dropdown-menu" role="menu" aria-labelledby="dropdownMenu"> - <li><a tabindex="-1" id="useredit" href="#" role="button" data-backdrop="true" data-controls-modal="event-modal" data-keyboard="true"><i class="icon-pencil"></i>Edit</a></li> - <li><a tabindex="-1" href="#"><i class="icon-tasks"></i>Statistik</a></li> - <li class="divider"></li> - <li><a tabindex="-1" href="#"><i class="icon-remove-sign"></i>Delete</a></li> - </ul> - </div> - - <div id="event-modal" class="modal hide fade"> - <div class="modal-header"> - <a class="close" id="useredit_close" href="#">x</a> - <h3>User Settings</h3> - </div> - <div class="modal-body"> - <p>Set password and permissions</p> - <table style="width:100%;" class="table "> - <td> - <div class="input-prepend"> - <span class="add-on"><i class="iconf-key"></i></span> - <input class="span2" style="min-width:120px;" id="prependedInput" type="text" placeholder="New Password"> - </div> - <div class="input-prepend"> - <span class="add-on"><i class="icon-repeat"></i></span> - <input class="span2" style="min-width:120px;" id="prependedInput" type="text" placeholder="Repeat"> - </div> - <br> - <br> - <br> - <form class="form-horizontal"> - <div class="control-group"> - <label class="control-label" for="onoff">Administrator</label> - - <div class="controls"> - <div class="btn-group" id="onoff" data-toggle="buttons-radio"> - <button type="button" class="btn btn-primary" >On</button> - <button type="button" class="btn btn-primary active">Off</button> - </div> - </div> - </div> - </form> - </td> - <td> - <div id="user_permissions"> - <h3>Permissions</h3> - <div class="btn-group btn-group-vertical" data-toggle="buttons-checkbox"> - <button type="button" class="btn btn-inverse userperm">Accounts</button> - <button type="button" class="btn btn-inverse userperm active">Add</button> - <button type="button" class="btn btn-inverse userperm">Delete</button> - <button type="button" class="btn btn-inverse userperm active">Download</button> - <button type="button" class="btn btn-inverse userperm active">List</button> - <button type="button" class="btn btn-inverse userperm">Logs</button> - <button type="button" class="btn btn-inverse userperm">Modify</button> - <button type="button" class="btn btn-inverse userperm">Settings</button> - <button type="button" class="btn btn-inverse userperm active">Status</button> - </div> - </div> - </td> - </table> - </div> - <div class="modal-footer"> - <a class="btn btn-primary" id="useredit_save"href="#">Save</a> - - </div> - </div> - - - - </div> - - <div class="span2"> - <br> - <h2>Support</h2> - <table> - <tr> - <td> - <i class="icon-globe"></i> - </td> - <td> - <a href="#">Wiki |</a> - <a href="#">Forum |</a> - <a href="#">Chat</a> - </td> - </tr> - <tr> - <td> - <i class="icon-book"></i> - </td> - <td> - <a href="#">Documentation</a> - </td> - </tr> - <tr> - <td> - <i class="icon-fire"></i> - </td> - <td> - <a href="#">Development</a> - </td> - </tr> - <tr> - <td> - <i class="icon-bullhorn"></i> - </td> - <td> - <a href="#">Issue Tracker</a> - </td> - </tr> - </table> - <br> - <a href="#" class="btn btn-inverse" id="info" rel="popover" data-content="<table class='table table-striped'> - <tbody> - <tr> - <td>Python:</td> - <td>2.6.4 </td> - </tr> - <tr> - <td>Betriebssystem:</td> - <td>nt win32</td> - </tr> - <tr> - <td>pyLoad Version:</td> - <td>0.4.9</td> - </tr> - <tr> - <td>Installationsordner:</td> - <td>C:\pyLoad</td> - </tr> - <tr> - <td>Konfigurationsordner:</td> - <td>C:\Users\Marvin\pyload</td> - </tr> - <tr> - <td>Downloadordner:</td> - <td>C:\Users\Marvin\new</td> - </tr> - <tr> - <td>HDD:</td> - <td>1.67 TiB <div class='progress progress-striped active'> - <div class='bar' style='width: 40%;'></div> -</div></td> - </tr> - <tr> - <td>Sprache:</td> - <td>de</td> - </tr> - <tr> - <td>Webinterface Port:</td> - <td>8000</td> - </tr> - <tr> - <td>Remote Interface Port:</td> - <td>7227</td> - </tr> - </tbody> - </table>" title="Systeminformationen">System</a> - - </div> - </div> - </div> - - <script src="static/js/libs/jquery-1.9.0.js"></script> - {##} - <script src="static/js/libs/bootstrap-2.2.2.js"></script> - <script type="text/javascript"> - $('#info').popover({ - placement: 'left', - trigger: 'click', - html:'true', - }); - - $('.dropdown-toggle').dropdown(); - - $("#btn_newuser").click(function() { - - str = "<div class='dropdown1'><span class='label name'>User</span><a class='dropdown-toggle' data-toggle='dropdown1' href='#'><i class='iconf-user iconf-8x'></i></a><ul class='dropdown-menu' role='menu' aria-labelledby='dropdownMenu'><li><a tabindex='-1' href='#'>Action</a></li><li><a tabindex='-1' href='#'>Another action</a></li><li><a tabindex='-1' href='#'>Something else here</a></li><li class='divider'></li><li><a tabindex='-1' href='#'>Separated link</a></li></ul></div>"; - - $("#userlist").append(str); - - }); - - $("#useredit").click(function() { - $('#event-modal').modal(); - }); - $("#useredit_close").click(function() { - $('#event-modal').modal('hide'); - }); - $("#useredit_save").click(function() { - $('#event-modal').modal('hide'); - }); - - </script> -{% endblock %}
\ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/actionbar.html b/module/web/app/templates/default/dashboard/actionbar.html deleted file mode 100644 index 815d4593c..000000000 --- a/module/web/app/templates/default/dashboard/actionbar.html +++ /dev/null @@ -1,54 +0,0 @@ -<div class="span2 offset1"> -</div> -<ul class="actionbar nav nav-pills span9"> - <li> - <ul class="breadcrumb"> - <li><a href="#">Local</a> <span class="divider">/</span></li> - <li class="active"></li> - </ul> - </li> - - <li style="float: right;"> - <form class="form-search" action="#"> - <div class="input-append"> - <input type="text" class="search-query" style="width: 120px"> - <button type="submit" class="btn">Search</button> - </div> - </form> - </li> - <li style="float: right"> - <a href="#"><i class="icon-check-empty btn-check"></i></a> - </li> - <li class="dropdown" style="float: right;"> - <a class="dropdown-toggle type" - data-toggle="dropdown" - href="#"> - Type - <b class="caret"></b> - </a> - <ul class="dropdown-menu"> - <li><a class="filter-type" data-type="2" href="#"><i class="icon-ok"></i> Audio</a></li> - <li><a class="filter-type" data-type="4" href="#"><i class="icon-ok"></i> Image</a></li> - <li><a class="filter-type" data-type="8" href="#"><i class="icon-ok"></i> Video</a></li> - <li><a class="filter-type" data-type="16" href="#"><i class="icon-ok"></i> Document</a></li> - <li><a class="filter-type" data-type="32" href="#"><i class="icon-remove"></i> Archive</a></li> - <li><a class="filter-type" data-type="1" href="#"><i class="icon-remove"></i> Other</a></li> - </ul> - </li> - <li class="dropdown" style="float: right;"> - <a class="dropdown-toggle" - data-toggle="dropdown" - href="#"> - <span class="state"> - All - </span> - <b class="caret"></b> - </a> - <ul class="dropdown-menu"> - <li><a class="filter-state" data-state="0" href="#">All</a></li> - <li><a class="filter-state" data-state="1" href="#">Finished</a></li> - <li><a class="filter-state" data-state="2" href="#">Unfinished</a></li> - <li><a class="filter-state" data-state="3" href="#">Failed</a></li> - </ul> - </li> -</ul>
\ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/file.html b/module/web/app/templates/default/dashboard/file.html deleted file mode 100644 index 6256254da..000000000 --- a/module/web/app/templates/default/dashboard/file.html +++ /dev/null @@ -1,34 +0,0 @@ -<div class="file-row first span6"> - <i class="checkbox"></i> - <span class="name"> - <% name %> - </span> -</div> -<div class="file-row second span3 <% fileClass this %>"> - <% fileStatus this %> -</div> - -<div class="file-row third span3 pull-right"> - <i class="<% fileIcon media %>"></i> - <% formatSize size %> - <span class="pull-right"> - <img src="<% pluginIcon download.plugin %>"/> - <% download.plugin %> - <i class="icon-chevron-down" data-toggle="dropdown"></i> - <ul class="dropdown-menu" role="menu"> - <li><a href="#" class="btn-delete"><i class="icon-trash"></i> Delete</a></li> - <li><a href="#" class="btn-restart"><i class="icon-refresh"></i> Restart</a></li> - <!--{# TODO: only show when finished #}--> - <li><a href="download/<% fid %>" target="_blank" class="btn-dowload"><i class="icon-download"></i> - Download</a></li> - <li><a href="#" class="btn-share"><i class="icon-share"></i> Share</a></li> - <li class="divider"></li> - <li class="dropdown-submenu pull-left"> - <a>Addons</a> - <ul class="dropdown-menu"> - <li><a>Test</a></li> - </ul> - </li> - </ul> - </span> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/layout.html b/module/web/app/templates/default/dashboard/layout.html deleted file mode 100644 index 945d11762..000000000 --- a/module/web/app/templates/default/dashboard/layout.html +++ /dev/null @@ -1,35 +0,0 @@ -<div class="span3"> - <div class="sidebar-header"> - <i class="icon-hdd"></i> Local - <div class="pull-right" style="font-size: medium; line-height: normal"> - <i class="icon-chevron-down" style="font-size: 20px"></i> - </div> - <div class="clearfix"></div> - </div> - <ul class="package-list"> - - </ul> - <div class="sidebar-header"> - <i class="icon-group"></i> Shared - </div> - <ul class="package-list"> - <li>Content from</li> - <li>Other user</li> - <li>which they shared</li> - </ul> - <div class="sidebar-header"> - <i class="icon-sitemap"></i> Remote - </div> - <ul> - <li>Content from</li> - <li>remote sites</li> - <li>mega</li> - <li>dropbox</li> - <li>other pyloads</li> - </ul> -</div> -<div class="span9"> - <ul class="file-list"> - - </ul> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/package.html b/module/web/app/templates/default/dashboard/package.html deleted file mode 100644 index c0690a9bf..000000000 --- a/module/web/app/templates/default/dashboard/package.html +++ /dev/null @@ -1,50 +0,0 @@ -<%= if selected %> - <i class="icon-check select"></i> - <% else %> - <i class="icon-check-empty select"></i> - <%/if%> - <span class="package-name"> - <% name %> - </span> - - <div class="package-frame"> - <div class="tag-area"> - <span class="badge badge-success"><i class="icon-tag"></i>video</span> - <span class="badge badge-success badge-ghost"><i class="icon-tag"></i> Add Tag</span> - </div> - <div class="package-indicator"> - <i class="icon-plus-sign btn-move" data-toggle="tooltip" title="Move files here"></i> - <i class="icon-pause" data-toggle="tooltip" title="Pause Package"></i> - <i class="icon-refresh" data-toggle="tooltip" title="Restart Package"></i> - <%= if shared %> - <i class="icon-eye-open" data-toggle="tooltip" title="Package is public"></i> - <% else %> - <i class="icon-eye-close" data-toggle="tooltip" title="Package is private"></i> - <%/if%> - <i class="icon-chevron-down" data-toggle="dropdown"> - </i> - <ul class="dropdown-menu" role="menu"> - <li><a href="#" class="btn-open"><i class="icon-folder-open-alt"></i> Open</a></li> - <li><a href="#"><i class="icon-plus-sign"></i> Add links</a></li> - <li><a href="#"><i class="icon-edit"></i> Details</a></li> - <li><a href="#" class="btn-delete"><i class="icon-trash"></i> Delete</a></li> - <li><a href="#" class="btn-recheck"><i class="icon-refresh"></i> Recheck</a></li> - <li class="divider"></li> - <li class="dropdown-submenu"> - <a>Addons</a> - <ul class="dropdown-menu"> - <li><a>Test</a></li> - </ul> - </li> - </ul> - </div> - <div class="progress"> - <span style="position: absolute; left: 5px"> - <% stats.linksdone %> / <% stats.linkstotal %> - </span> - <div class="bar bar-info" style="width: <% percent %>%"></div> - <span style="position: absolute; right: 5px"> - <% formatSize stats.sizedone %> / <% formatSize stats.sizetotal %> - </span> - </div> - </div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dashboard/select.html b/module/web/app/templates/default/dashboard/select.html deleted file mode 100644 index 73ea391cd..000000000 --- a/module/web/app/templates/default/dashboard/select.html +++ /dev/null @@ -1,11 +0,0 @@ -<i class="icon-check" data-toggle="tooltip" title="Deselect"></i> -<%= if packs %><% packs %> package(s)<%/if %> -<%= if files %> -<%= if packs %>, <%/if %> -<% files %> file(s) -<%/if %> -selected - | -<i class="icon-pause" data-toggle="tooltip" title="Pause"></i> -<i class="icon-trash" data-toggle="tooltip" title="Delete"></i> -<i class="icon-refresh" data-toggle="tooltip" title="Restart"></i>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/addAccount.html b/module/web/app/templates/default/dialogs/addAccount.html deleted file mode 100755 index bdc8a609a..000000000 --- a/module/web/app/templates/default/dialogs/addAccount.html +++ /dev/null @@ -1,42 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3>Add an account</h3> -</div> -<div class="modal-body"> - <form class="form-horizontal" autocomplete="off"> - <legend> - Please enter your account data - </legend> - <div class="control-group"> - <label class="control-label" for="pluginSelect"> - Plugin - </label> - - <div class="controls"> - <input type="hidden" id="pluginSelect"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="login"> - Loginname - </label> - - <div class="controls"> - <input type="text" id="login"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="password"> - Password - </label> - - <div class="controls"> - <input type="password" id="password"> - </div> - </div> - </form> -</div> -<div class="modal-footer"> - <a class="btn btn-success btn-add">Add</a> - <a class="btn btn-close">Close</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/addPluginConfig.html b/module/web/app/templates/default/dialogs/addPluginConfig.html deleted file mode 100755 index e7a42a208..000000000 --- a/module/web/app/templates/default/dialogs/addPluginConfig.html +++ /dev/null @@ -1,26 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3> - Choose a plugin - </h3> -</div> -<div class="modal-body"> - <form class="form-horizontal"> - <legend> - Please choose a plugin, which you want to configure - </legend> - <div class="control-group"> - <label class="control-label" for="pluginSelect"> - Plugin - </label> - - <div class="controls"> - <input type="hidden" id="pluginSelect"> - </div> - </div> - </form> -</div> -<div class="modal-footer"> - <a class="btn btn-success btn-add">Add</a> - <a class="btn btn-close">Close</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/confirmDelete.html b/module/web/app/templates/default/dialogs/confirmDelete.html deleted file mode 100644 index 65ae1cb21..000000000 --- a/module/web/app/templates/default/dialogs/confirmDelete.html +++ /dev/null @@ -1,11 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3>Please confirm</h3> -</div> -<div class="modal-body"> - Do you want to delete the selected items? -</div> -<div class="modal-footer"> - <a class="btn btn-danger btn-confirm"><i class="icon-trash icon-white"></i> Delete</a> - <a class="btn btn-close">Cancel</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/interactionTask.html b/module/web/app/templates/default/dialogs/interactionTask.html deleted file mode 100755 index ae325e83d..000000000 --- a/module/web/app/templates/default/dialogs/interactionTask.html +++ /dev/null @@ -1,37 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3> - <% title %> - <small style="background: url('<% pluginIcon plugin %>') no-repeat right 0; background-size: 20px; padding-right: 22px"> - <% plugin %> - </small> - </h3> -</div> -<div class="modal-body"> - <form class="form-horizontal" action="#"> - <legend><% description %></legend> - <%= if captcha %> - <div class="control-group"> - <label class="control-label" for="captchaImage"> - Captcha Image - </label> - - <div class="controls"> - <img id="captchaImage" src="data:image/<% type %>;base64,<% captcha %>"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="inputField">Captcha Text</label> - - <div class="controls" id="inputField"> - </div> - </div> - <% else %> - <% content %> - <%/if%> - </form> -</div> -<div class="modal-footer"> - <a class="btn btn-success">Submit</a> - <a class="btn btn-close">Close</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/linkgrabber.html b/module/web/app/templates/default/dialogs/linkgrabber.html deleted file mode 100755 index 08418cf03..000000000 --- a/module/web/app/templates/default/dialogs/linkgrabber.html +++ /dev/null @@ -1,49 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3> - AddPackage - <small>paste&add links to pyLoad</small> - </h3> -</div> - -<div class="modal-body"> - <div class="alert alert-error hidden"> - Upload files container failed. Please try again. - </div> - <form class="form-horizontal"> - <div class="control-group"> - <label class="control-label" for="inputPackageName">Package name</label> - - <div class="controls"> - <input type="text" class="span4" id="inputPackageName" placeholder="Name of your package"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="inputLinks">Links</label> - - <div class="controls"> - <textarea id="inputLinks" class="span4" rows="10" placeholder="Paste your links here..."></textarea> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="inputPassword">Password</label> - - <div class="controls"> - <input type="text" id="inputPassword" class="span4" placeholder="Password for .rar files"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="inputContainer">Upload links container</label> - - <div class="controls controls-row"> - <input type="text" id="inputContainer" class="span3" placeholder="Path to your container"> - <button id="inputContainer-btn" class="btn span1" type="button">Browse…</button> - </div> - </div> - </form> -</div> - -<div class="modal-footer"> - <a class="btn btn-success"><i class="icon-plus icon-white"></i> Add</a> - <a class="btn btn-close">Close</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/dialogs/modal.html b/module/web/app/templates/default/dialogs/modal.html deleted file mode 100755 index 1e44cc99c..000000000 --- a/module/web/app/templates/default/dialogs/modal.html +++ /dev/null @@ -1,10 +0,0 @@ -<div class="modal-header"> - <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> - <h3>Dialog</h3> -</div> -<div class="modal-body"> -</div> -<div class="modal-footer"> - <a class="btn btn-close">Close</a> - <a class="btn btn-primary">Save</a> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/header/layout.html b/module/web/app/templates/default/header/layout.html deleted file mode 100644 index 890a5b018..000000000 --- a/module/web/app/templates/default/header/layout.html +++ /dev/null @@ -1,62 +0,0 @@ -<div class="span3"> - <div class="logo"></div> - <span class="title visible-large-screen">pyLoad</span> -</div> -<div class="span4 offset1"> - <div id="progress-area"> - <span id="progress-info"> - </span> - - <div class="popover bottom"> - <div class="arrow"></div> - <div class="popover-inner"> - <h3 class="popover-title"> - Running... - <button type="button" class="close" aria-hidden="true">×</button> - </h3> - <div class="popover-content"> - <ul class="progress-list"></ul> - </div> - </div> - </div> - </div> -</div> -<div class="span4"> - <div class="header-block"> - <i class="icon-download-alt icon-white"></i> Max. Speed:<br> - <i class="icon-off icon-white"></i> Running:<br> - <i class="icon-refresh icon-white"></i> Reconnect:<br> - </div> - - <div class="header-block status-block"></div> - - <div class="header-btn"> - <div class="btn-group"> - <a class="btn btn-blue btn-small" href="#"><i class="icon-user icon-white"></i> User</a> - <a class="btn btn-blue btn-small dropdown-toggle" data-toggle="dropdown" href="#"><span - class="caret"></span></a> - <ul class="dropdown-menu" style="right: 0; left: -100%"> - <li><a data-nav href="/"><i class="icon-list-alt"></i> Dashboard</a></li> - <li><a data-nav href="/settings"><i class="icon-wrench"></i> Settings</a></li> - <li><a data-nav href="/accounts"><i class="icon-key"></i> Accounts</a></li> - <li><a data-nav href="/admin"><i class="icon-cogs"></i> Admin</a></li> - <li class="divider"></li> - <li><a data-nav href="/logout"><i class="icon-signout"></i> Logout</a></li> - </ul> - </div> - <div class="btn-group lower"> - <button class="btn btn-success btn-grabber btn-mini" href="#"> - <i class="icon-plus icon-white"></i> - </button> - <button class="btn btn-blue btn-play btn-mini" href="#"> - <i class="icon-play icon-white"></i> - </button> - <button class="btn btn-danger btn-delete btn-mini" href="#"> - <i class="icon-remove icon-white"></i> - </button> - </div> - </div> -<span class="visible-desktop speedgraph-container"> - <div id="speedgraph"></div> -</span> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/header/progress.html b/module/web/app/templates/default/header/progress.html deleted file mode 100644 index 65ae9a880..000000000 --- a/module/web/app/templates/default/header/progress.html +++ /dev/null @@ -1,14 +0,0 @@ -<% name %> -<span class="pull-right"><% plugin %></span> - -<div class="progress"> - <div class="bar" style="width: <% percent %>%"></div> -</div> -<%= if downloading %> -<% formatSize done %> of <% formatSize total %> (<% formatSize download.speed %>/s) -<% else %> -<% statusmsg %> -<%/if%> -<span class="pull-right"> - <% formatTime eta %> -</span>
\ No newline at end of file diff --git a/module/web/app/templates/default/header/progressbar.html b/module/web/app/templates/default/header/progressbar.html deleted file mode 100644 index 41645c92f..000000000 --- a/module/web/app/templates/default/header/progressbar.html +++ /dev/null @@ -1,27 +0,0 @@ - <%= if single %> - <% name %> (<% statusmsg %>) - <% else %> - <%= if downloads %> - <% downloads %> downloads running <%= if speed %>(<% formatSize speed %>/s)<%/if%> - <% else %> - No running tasks - <%/if%> - <%/if%> - <i class="icon-list pull-right"></i> - <!-- TODO active animation --> - <div class="progress" id="globalprogress"> - <%= if single %> - <div class="bar" style="width: <% percent %>%"> - <% else %> - <div class="bar <%= if downloads %>running<%/if%>"> - <%/if%> - </div> - </div> - <div class="sub"> - <%= if linksqueue %> - <% linksqueue %> downloads left (<% formatSize sizequeue %>) - <%/if%> - <span class="pull-right"> - <% formatTime etaqueue %> - </span> - </div>
\ No newline at end of file diff --git a/module/web/app/templates/default/header/status.html b/module/web/app/templates/default/header/status.html deleted file mode 100644 index 3a22bb75b..000000000 --- a/module/web/app/templates/default/header/status.html +++ /dev/null @@ -1,3 +0,0 @@ -<span class="pull-right maxspeed"><% formatSize maxspeed %>/s</span><br> -<span class="pull-right running"><% paused %></span><br> -<span class="pull-right reconnect"><%= if reconnect %>true<% else %>false<%/if%></span>
\ No newline at end of file diff --git a/module/web/app/templates/default/login.html b/module/web/app/templates/default/login.html deleted file mode 100644 index 9e8d9eeb6..000000000 --- a/module/web/app/templates/default/login.html +++ /dev/null @@ -1,28 +0,0 @@ -<br> -<div class="login"> - <form method="post" class="form-horizontal"> - <legend>Login</legend> - <div class="control-group"> - <label class="control-label" for="inputUser">Username</label> - <div class="controls"> - <input type="text" id="inputUser" placeholder="Username" name="username"> - </div> - </div> - <div class="control-group"> - <label class="control-label" for="inputPassword">Password</label> - <div class="controls"> - <input type="password" id="inputPassword" placeholder="Password" name="password"> - </div> - </div> - <div class="control-group"> - <div class="controls"> - <label class="checkbox"> - <input type="checkbox"> Remember me - </label> - <button type="submit" class="btn">Login</button> - </div> - </div> - </form> -</div> -<br> -<!-- TODO: Errors --> diff --git a/module/web/app/templates/default/notification.html b/module/web/app/templates/default/notification.html deleted file mode 100644 index 0f9b2c9d2..000000000 --- a/module/web/app/templates/default/notification.html +++ /dev/null @@ -1,11 +0,0 @@ -<%= if queries %> -<span class="btn-query"> -Queries <span class="badge badge-info"><% queries %></span> -</span> -<%/if%> -<%= if notifications %> -<span class="btn-notification"> -Notifications <span class="badge badge-success"><% notifications %></span> -</span> -<%/if%> -</%if%>
\ No newline at end of file diff --git a/module/web/app/templates/default/settings/actionbar.html b/module/web/app/templates/default/settings/actionbar.html deleted file mode 100644 index 25b10d463..000000000 --- a/module/web/app/templates/default/settings/actionbar.html +++ /dev/null @@ -1,5 +0,0 @@ -<div class="span2 offset1"> -</div> -<span class="span9"> - <button class="btn btn-small btn-blue btn-add">Add Plugin</button> -</span>
\ No newline at end of file diff --git a/module/web/app/templates/default/settings/config.html b/module/web/app/templates/default/settings/config.html deleted file mode 100644 index a9ca6214c..000000000 --- a/module/web/app/templates/default/settings/config.html +++ /dev/null @@ -1,17 +0,0 @@ -<legend> - <div class="page-header"> - <h1><% label %> - <small><% description %></small> - <%= if long_description %> - <a class="btn btn-small" data-title="Help" data-content="<% long_description %>"><i - class="icon-question-sign"></i></a> - <%/if%> - </h1> - </div> -</legend> -<div class="control-content"> -</div> -<div class="form-actions"> - <button type="button" class="btn btn-primary">Save changes</button> - <button type="button" class="btn btn-reset">Reset</button> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/settings/configItem.html b/module/web/app/templates/default/settings/configItem.html deleted file mode 100644 index 3ddf16c84..000000000 --- a/module/web/app/templates/default/settings/configItem.html +++ /dev/null @@ -1,7 +0,0 @@ - <div class="control-group"> - <label class="control-label"><% label %></label> - - <div class="controls"> - <!--{# <span class="help-inline"><% description %></span>#}--> - </div> - </div>
\ No newline at end of file diff --git a/module/web/app/templates/default/settings/layout.html b/module/web/app/templates/default/settings/layout.html deleted file mode 100644 index 39f1a2ec9..000000000 --- a/module/web/app/templates/default/settings/layout.html +++ /dev/null @@ -1,11 +0,0 @@ -<div class="span2"> - <ul class="nav nav-list well settings-menu"> - </ul> -</div> -<div class="span10"> - <div class="well setting-box"> - <form class="form-horizontal" action="#"> - <h1>Please choose a config section</h1> - </form> - </div> -</div>
\ No newline at end of file diff --git a/module/web/app/templates/default/settings/menu.html b/module/web/app/templates/default/settings/menu.html deleted file mode 100644 index ef814414a..000000000 --- a/module/web/app/templates/default/settings/menu.html +++ /dev/null @@ -1,40 +0,0 @@ -<%=if core%> -<li class="nav-header"><i class="icon-globe icon-white"></i> General</li> -<%= each core%> -<li data-name="<% name %>"><a href="#"><% label %></a></li> -<%/each%> -<%/if%> -<li class="divider"></li> -<li class="nav-header"><i class="icon-th-large icon-white"></i> Addons</li> -<%= each addon %> -<li class="addon" data-name="<% name %>"> - <a href="#" style="background-image: url(<% pluginIcon name %>);"> - <% label %> - <i class="icon-remove pull-right"></i> - <%= if activated %> - <div class="addon-on"> - active - <%else%> - <div class="addon-off"> - inactive - <%/if%> - <%= if user_context %> - <!--{# TODO: tooltip #}--> - <i class="icon-user pull-right"></i> - <%else%> - <i class="icon-globe pull-right"></i> - <%/if%> - </div> - </a> -</li> -<%/each%> -<li class="divider"></li> -<li class="nav-header"><i class="icon-th-list icon-white"></i> Plugin Configs</li> -<%= each plugin %> -<li class="plugin" data-name="<% name %>"> - <a href="#" style="background-image: url(<% pluginIcon name %>);"> - <% label %> - <i class="icon-remove pull-right"></i> - </a> -</li> -<%/each%>
\ No newline at end of file diff --git a/module/web/app/templates/default/setup.html b/module/web/app/templates/default/setup.html deleted file mode 100644 index e5c9f4b8c..000000000 --- a/module/web/app/templates/default/setup.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends 'default/base.html' %} -{% block title %} - {{_("Setup")}} - {{ super()}} -{% endblock %} - -{% block content %} - <div class="hero-unit"> - <h1>You did it!</h1> - <p>pyLoad is running and ready for configuration.</p> - <p> - <a class="btn btn-primary btn-large"> - Go on - </a> - </p> - </div> -{% endblock %}
\ No newline at end of file diff --git a/module/web/app/unavailable.html b/module/web/app/unavailable.html deleted file mode 100644 index 6706a693c..000000000 --- a/module/web/app/unavailable.html +++ /dev/null @@ -1,18 +0,0 @@ -<!DOCTYPE html> -<html> -<head> - <title>WebUI not available</title> -</head> -<body> - -<h1>WebUI not available</h1> -You are running a pyLoad version without prebuilt webUI. You can download a build from our website or deactivate the dev mode. -If desired you can build it yourself by running: -<ul> - <li>npm install</li> - <li>bower install</li> - <li>grunt build</li> -</ul> - -</body> -</html>
\ No newline at end of file diff --git a/module/web/bower.json b/module/web/bower.json deleted file mode 100644 index 6d8adb8a7..000000000 --- a/module/web/bower.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "pyload", - "version": "0.1.0", - "dependencies": { - "requirejs": "~2.1.6", - "requirejs-text": "*", - "require-handlebars-plugin": "*", - "jquery": "~1.9.1", - "jquery.transit": "~0.9.9", - "jquery.cookie": "~1.3.1", - "jquery.animate-enhanced": "*", - "flot": "~0.8.1", - "underscore": "~1.4.4", - "backbone": "~1.0.0", - "backbone.marionette": "~1.0.3", - "handlebars.js": "~1.0.0", - "jed": "~0.5.4", - "select2": "~3.4.0", - "bootstrap": "~2.3.2", - "font-awesome": "~3.1.1" - }, - "devDependencies": {} -} diff --git a/module/web/cnl_app.py b/module/web/cnl_app.py deleted file mode 100644 index b6a98a0a8..000000000 --- a/module/web/cnl_app.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from os.path import join -import re -from urllib import unquote -from base64 import standard_b64decode -from binascii import unhexlify - -from module.utils.fs import save_filename - -from bottle import route, request, HTTPError -from webinterface import PYLOAD, DL_ROOT, JS - -try: - from Crypto.Cipher import AES -except: - pass - - -def local_check(function): - def _view(*args, **kwargs): - if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ - or request.environ.get('HTTP_HOST','0') in ('127.0.0.1:9666', 'localhost:9666'): - return function(*args, **kwargs) - else: - return HTTPError(403, "Forbidden") - - return _view - - -@route("/flash") -@route("/flash/:id") -@route("/flash", method="POST") -@local_check -def flash(id="0"): - return "JDownloader\r\n" - -@route("/flash/add", method="POST") -@local_check -def add(request): - package = request.POST.get('referer', None) - urls = filter(lambda x: x != "", request.POST['urls'].split("\n")) - - if package: - PYLOAD.addPackage(package, urls, 0) - else: - PYLOAD.generateAndAddPackages(urls, 0) - - return "" - -@route("/flash/addcrypted", method="POST") -@local_check -def addcrypted(): - - package = request.forms.get('referer', 'ClickAndLoad Package') - dlc = request.forms['crypted'].replace(" ", "+") - - dlc_path = join(DL_ROOT, save_filename(package) + ".dlc") - dlc_file = open(dlc_path, "wb") - dlc_file.write(dlc) - dlc_file.close() - - try: - PYLOAD.addPackage(package, [dlc_path], 0) - except: - return HTTPError() - else: - return "success\r\n" - -@route("/flash/addcrypted2", method="POST") -@local_check -def addcrypted2(): - - package = request.forms.get("source", None) - crypted = request.forms["crypted"] - jk = request.forms["jk"] - - crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) - if JS: - jk = "%s f()" % jk - jk = JS.eval(jk) - - else: - try: - jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] - except: - ## Test for some known js functions to decode - if jk.find("dec") > -1 and jk.find("org") > -1: - org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] - jk = list(org) - jk.reverse() - jk = "".join(jk) - else: - print "Could not decrypt key, please install py-spidermonkey or ossp-js" - - try: - Key = unhexlify(jk) - except: - print "Could not decrypt key, please install py-spidermonkey or ossp-js" - return "failed" - - IV = Key - - obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") - - result = filter(lambda x: x != "", result) - - try: - if package: - PYLOAD.addPackage(package, result, 0) - else: - PYLOAD.generateAndAddPackages(result, 0) - except: - return "failed can't add" - else: - return "success\r\n" - -@route("/flashgot_pyload") -@route("/flashgot_pyload", method="POST") -@route("/flashgot") -@route("/flashgot", method="POST") -@local_check -def flashgot(): - if request.environ['HTTP_REFERER'] != "http://localhost:9666/flashgot" and request.environ['HTTP_REFERER'] != "http://127.0.0.1:9666/flashgot": - return HTTPError() - - autostart = int(request.forms.get('autostart', 0)) - package = request.forms.get('package', None) - urls = filter(lambda x: x != "", request.forms['urls'].split("\n")) - folder = request.forms.get('dir', None) - - if package: - PYLOAD.addPackage(package, urls, autostart) - else: - PYLOAD.generateAndAddPackages(urls, autostart) - - return "" - -@route("/crossdomain.xml") -@local_check -def crossdomain(): - rep = "<?xml version=\"1.0\"?>\n" - rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\n" - rep += "<cross-domain-policy>\n" - rep += "<allow-access-from domain=\"*\" />\n" - rep += "</cross-domain-policy>" - return rep - - -@route("/flash/checkSupportForUrl") -@local_check -def checksupport(): - - url = request.GET.get("url") - res = PYLOAD.checkURLs([url]) - supported = (not res[0][1] is None) - - return str(supported).lower() - -@route("/jdcheck.js") -@local_check -def jdcheck(): - rep = "jdownloader=true;\n" - rep += "var version='9.581;'" - return rep diff --git a/module/web/middlewares.py b/module/web/middlewares.py deleted file mode 100644 index ae0911cc3..000000000 --- a/module/web/middlewares.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# gzip is optional on some platform -try: - import gzip -except ImportError: - gzip = None - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -class StripPathMiddleware(object): - def __init__(self, app): - self.app = app - - def __call__(self, e, h): - e['PATH_INFO'] = e['PATH_INFO'].rstrip('/') - return self.app(e, h) - - -class PrefixMiddleware(object): - def __init__(self, app, prefix="/pyload"): - self.app = app - self.prefix = prefix - - def __call__(self, e, h): - path = e["PATH_INFO"] - if path.startswith(self.prefix): - e['PATH_INFO'] = path.replace(self.prefix, "", 1) - return self.app(e, h) - -# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php - -# WSGI middleware -# Gzip-encodes the response. - -class GZipMiddleWare(object): - - def __init__(self, application, compress_level=6): - self.application = application - self.compress_level = int(compress_level) - - def __call__(self, environ, start_response): - if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): - # nothing for us to do, so this middleware will - # be a no-op: - return self.application(environ, start_response) - response = GzipResponse(start_response, self.compress_level) - app_iter = self.application(environ, - response.gzip_start_response) - if app_iter is not None: - response.finish_response(app_iter) - - return response.write() - -def header_value(headers, key): - for header, value in headers: - if key.lower() == header.lower(): - return value - -def update_header(headers, key, value): - remove_header(headers, key) - headers.append((key, value)) - -def remove_header(headers, key): - for header, value in headers: - if key.lower() == header.lower(): - headers.remove((header, value)) - break - -class GzipResponse(object): - - def __init__(self, start_response, compress_level): - self.start_response = start_response - self.compress_level = compress_level - self.buffer = StringIO() - self.compressible = False - self.content_length = None - self.headers = () - - def gzip_start_response(self, status, headers, exc_info=None): - self.headers = headers - ct = header_value(headers,'content-type') - ce = header_value(headers,'content-encoding') - cl = header_value(headers, 'content-length') - - # don't compress on unknown size, it may be too huge - cl = int(cl) if cl else 0 - - if ce: - self.compressible = False - elif gzip is not None and ct and (ct.startswith('text/') or ct.startswith('application/')) \ - and 'zip' not in ct and 200 < cl < 1024*1024: - self.compressible = True - headers.append(('content-encoding', 'gzip')) - headers.append(('vary', 'Accept-Encoding')) - - remove_header(headers, 'content-length') - self.headers = headers - self.status = status - return self.buffer.write - - def write(self): - out = self.buffer - out.seek(0) - s = out.getvalue() - out.close() - return [s] - - def finish_response(self, app_iter): - if self.compressible: - output = gzip.GzipFile(mode='wb', compresslevel=self.compress_level, - fileobj=self.buffer) - else: - output = self.buffer - try: - for s in app_iter: - output.write(s) - if self.compressible: - output.close() - finally: - if hasattr(app_iter, 'close'): - try: - app_iter.close() - except : - pass - - content_length = self.buffer.tell() - update_header(self.headers, "Content-Length" , str(content_length)) - self.start_response(self.status, self.headers)
\ No newline at end of file diff --git a/module/web/package.json b/module/web/package.json deleted file mode 100644 index aaf1c9a8d..000000000 --- a/module/web/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "pyload", - "version": "0.1.0", - "dependencies": {}, - "devDependencies": { - "grunt": "~0.4.1", - "grunt-contrib-copy": "~0.4.1", - "grunt-contrib-concat": "~0.1.3", - "grunt-contrib-uglify": "~0.2.0", - "grunt-contrib-jshint": "~0.4.1", - "grunt-contrib-less": "~0.5.2", - "grunt-contrib-cssmin": "~0.6.0", - "grunt-contrib-connect": "~0.2.0", - "grunt-contrib-clean": "~0.4.0", - "grunt-contrib-htmlmin": "~0.1.3", - "grunt-bower-requirejs": "~0.4.3", - "grunt-contrib-requirejs": "~0.4.0", - "grunt-contrib-imagemin": "~0.1.3", - "grunt-contrib-watch": "~0.4.0", - "grunt-rev": "~0.1.0", - "grunt-usemin": "~0.1.10", - "grunt-mocha": "~0.3.0", - "grunt-open": "~0.2.0", - "grunt-svgmin": "~0.1.0", - "grunt-concurrent": "~0.1.0", - "matchdep": "~0.1.1", - "connect-livereload": "~0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } -} diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py deleted file mode 100644 index 724fddec2..000000000 --- a/module/web/pyload_app.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -import time -from os.path import join, exists - -from bottle import route, static_file, response, redirect - -from webinterface import PROJECT_DIR, SETUP, DEVELOP - -from utils import login_required - -########## -# Helper -########## - -app_path = "app" -UNAVAILALBE = False - -# webUI build is available -if exists(join(PROJECT_DIR, "dist", "index.html")) and not DEVELOP: - app_path = "dist" -elif not exists(join(PROJECT_DIR, "app", "components")) or not exists(join(PROJECT_DIR, ".tmp")): - UNAVAILALBE = True - - -@route('/icons/<path:path>') -def serve_icon(path): - # TODO - return redirect('/images/icon.png') - # return static_file(path, root=join("tmp", "icons")) - -@route("/download/:fid") -@login_required('Download') -def download(fid, api): - path, name = api.getFilePath(fid) - return static_file(name, path, download=True) - - -@route('/') -def index(): - if UNAVAILALBE: - return server_static("unavailable.html") - - if SETUP: - # TODO show different page - pass - - # TODO: render it as simple template with configuration - return server_static('index.html') - -# Very last route that is registered, could match all uris -@route('/<path:path>') -def server_static(path): - response.headers['Expires'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime(time.time() + 60 * 60 * 24 * 7)) - response.headers['Cache-control'] = "public" - resp = static_file(path, root=join(PROJECT_DIR, app_path)) - # Also serve from .tmp folder in dev mode - if resp.status_code == 404 and app_path == "app": - return static_file(path, root=join(PROJECT_DIR, '.tmp')) - - return resp
\ No newline at end of file diff --git a/module/web/servers.py b/module/web/servers.py deleted file mode 100644 index a3c51e36b..000000000 --- a/module/web/servers.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from bottle import ServerAdapter as BaseAdapter - -class ServerAdapter(BaseAdapter): - SSL = False - NAME = "" - - def __init__(self, host, port, key, cert, connections, debug, **kwargs): - BaseAdapter.__init__(self, host, port, **kwargs) - self.key = key - self.cert = cert - self.connection = connections - self.debug = debug - - @classmethod - def find(cls): - """ Check if server is available by trying to import it - - :raises Exception: importing C dependant library could also fail with other reasons - :return: True on success - """ - try: - __import__(cls.NAME) - return True - except ImportError: - return False - - def run(self, handler): - raise NotImplementedError - - -class CherryPyWSGI(ServerAdapter): - SSL = True - NAME = "threaded" - - @classmethod - def find(cls): - return True - - def run(self, handler): - from wsgiserver import CherryPyWSGIServer - - if self.cert and self.key: - CherryPyWSGIServer.ssl_certificate = self.cert - CherryPyWSGIServer.ssl_private_key = self.key - - server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.connection) - server.start() - - -class FapwsServer(ServerAdapter): - """ Does not work very good currently """ - - NAME = "fapws" - - def run(self, handler): # pragma: no cover - import fapws._evwsgi as evwsgi - from fapws import base, config - - port = self.port - if float(config.SERVER_IDENT[-2:]) > 0.4: - # fapws3 silently changed its API in 0.5 - port = str(port) - evwsgi.start(self.host, port) - evwsgi.set_base_module(base) - - def app(environ, start_response): - environ['wsgi.multiprocess'] = False - return handler(environ, start_response) - - evwsgi.wsgi_cb(('', app)) - evwsgi.run() - - -# TODO: ssl -class MeinheldServer(ServerAdapter): - SSL = True - NAME = "meinheld" - - def run(self, handler): - from meinheld import server - - if self.quiet: - server.set_access_logger(None) - server.set_error_logger(None) - - server.listen((self.host, self.port)) - server.run(handler) - -# todo:ssl -class TornadoServer(ServerAdapter): - """ The super hyped asynchronous server by facebook. Untested. """ - - SSL = True - NAME = "tornado" - - def run(self, handler): # pragma: no cover - import tornado.wsgi, tornado.httpserver, tornado.ioloop - - container = tornado.wsgi.WSGIContainer(handler) - server = tornado.httpserver.HTTPServer(container) - server.listen(port=self.port) - tornado.ioloop.IOLoop.instance().start() - - -class BjoernServer(ServerAdapter): - """ Fast server written in C: https://github.com/jonashaag/bjoern """ - - NAME = "bjoern" - - def run(self, handler): - from bjoern import run - - run(handler, self.host, self.port) - - -# todo: ssl -class EventletServer(ServerAdapter): - - SSL = True - NAME = "eventlet" - - def run(self, handler): - from eventlet import wsgi, listen - - try: - wsgi.server(listen((self.host, self.port)), handler, - log_output=(not self.quiet)) - except TypeError: - # Needed to ignore the log - class NoopLog: - def write(self, *args): - pass - - # Fallback, if we have old version of eventlet - wsgi.server(listen((self.host, self.port)), handler, log=NoopLog()) - - -class FlupFCGIServer(ServerAdapter): - - SSL = False - NAME = "flup" - - def run(self, handler): # pragma: no cover - import flup.server.fcgi - from flup.server.threadedserver import ThreadedServer - - def noop(*args, **kwargs): - pass - - # Monkey patch signal handler, it does not work from threads - ThreadedServer._installSignalHandlers = noop - - self.options.setdefault('bindAddress', (self.host, self.port)) - flup.server.fcgi.WSGIServer(handler, **self.options).run() - -# Order is important and gives every server precedence over others! -all_server = [BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] -# Some are deactivated because they have some flaws -##all_server = [FapwsServer, MeinheldServer, BjoernServer, TornadoServer, EventletServer, CherryPyWSGI]
\ No newline at end of file diff --git a/module/web/setup_app.py b/module/web/setup_app.py deleted file mode 100644 index cd44ad08e..000000000 --- a/module/web/setup_app.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from bottle import route, request, response, HTTPError, redirect - -from webinterface import PROJECT_DIR, SETUP - -def setup_required(func): - def _view(*args, **kwargs): - # setup needs to be running - if SETUP is None: - redirect("/nopermission") - - return func(*args, **kwargs) - return _view - - -@route("/setup") -@setup_required -def setup(): - pass # TODO diff --git a/module/web/utils.py b/module/web/utils.py deleted file mode 100644 index b5a933b26..000000000 --- a/module/web/utils.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from bottle import request, HTTPError, redirect - -from webinterface import PYLOAD, SETUP - - -def set_session(request, user): - s = request.environ.get('beaker.session') - s["uid"] = user.uid - s.save() - return s - -def get_user_api(s): - if s: - uid = s.get("uid", None) - if (uid is not None) and (PYLOAD is not None): - return PYLOAD.withUserContext(uid) - return None - -def is_mobile(): - if request.get_cookie("mobile"): - if request.get_cookie("mobile") == "True": - return True - else: - return False - mobile_ua = request.headers.get('User-Agent', '').lower() - if mobile_ua.find('opera mini') > 0: - return True - if mobile_ua.find('windows') > 0: - return False - if request.headers.get('Accept', '').lower().find('application/vnd.wap.xhtml+xml') > 0: - return True - if re.search('(up.browser|up.link|mmp|symbian|smartphone|midp|wap|phone|android)', mobile_ua) is not None: - return True - mobile_ua = mobile_ua[:4] - mobile_agents = ['w3c ','acs-','alav','alca','amoi','audi','avan','benq','bird','blac','blaz','brew','cell','cldc','cmd-', - 'dang','doco','eric','hipt','inno','ipaq','java','jigs','kddi','keji','leno','lg-c','lg-d','lg-g','lge-', - 'maui','maxo','midp','mits','mmef','mobi','mot-','moto','mwbp','nec-','newt','noki','palm','pana','pant', - 'phil','play','port','prox','qwap','sage','sams','sany','sch-','sec-','send','seri','sgh-','shar','sie-', - 'siem','smal','smar','sony','sph-','symb','t-mo','teli','tim-','tosh','tsm-','upg1','upsi','vk-v','voda', - 'wap-','wapa','wapi','wapp','wapr','webc','winw','winw','xda ','xda-'] - if mobile_ua in mobile_agents: - return True - return False - - -def login_required(perm=None): - def _dec(func): - def _view(*args, **kwargs): - - # In case of setup, no login methods can be accessed - if SETUP is not None: - redirect("/setup") - - s = request.environ.get('beaker.session') - api = get_user_api(s) - if api is not None: - if perm: - if api.user.hasPermission(perm): - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return HTTPError(403, "Forbidden") - else: - return redirect("/nopermission") - - kwargs["api"] = api - return func(*args, **kwargs) - else: - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return HTTPError(403, "Forbidden") - else: - return redirect("/login") - - return _view - - return _dec diff --git a/module/web/webinterface.py b/module/web/webinterface.py deleted file mode 100644 index 30c7d141f..000000000 --- a/module/web/webinterface.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -import sys - -from os.path import join, abspath, dirname - -PROJECT_DIR = abspath(dirname(__file__)) -PYLOAD_DIR = abspath(join(PROJECT_DIR, "..", "..")) - -sys.path.append(PYLOAD_DIR) - -from module import InitHomeDir - -import bottle -from bottle import run, app - -from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware - -SETUP = None -PYLOAD = None - -from module.web import ServerThread - -if not ServerThread.core: - if ServerThread.setup: - SETUP = ServerThread.setup - config = SETUP.config - else: - raise Exception("Could not access pyLoad Core") -else: - PYLOAD = ServerThread.core.api - config = ServerThread.core.config - -from module.utils.JsEngine import JsEngine - -JS = JsEngine() - -TEMPLATE = config.get('webinterface', 'template') -DL_ROOT = config.get('general', 'download_folder') -PREFIX = config.get('webinterface', 'prefix') -DEVELOP = config.get('webinterface', 'develop') - -if PREFIX: - PREFIX = PREFIX.rstrip("/") - if PREFIX and not PREFIX.startswith("/"): - PREFIX = "/" + PREFIX - -DEBUG = config.get("general", "debug_mode") or "-d" in sys.argv or "--debug" in sys.argv -bottle.debug(DEBUG) - - -# Middlewares -from beaker.middleware import SessionMiddleware - -session_opts = { - 'session.type': 'file', - 'session.cookie_expires': False, - 'session.data_dir': './tmp', - 'session.auto': False -} - -session = SessionMiddleware(app(), session_opts) -web = StripPathMiddleware(session) -web = GZipMiddleWare(web) - -if PREFIX: - web = PrefixMiddleware(web, prefix=PREFIX) - -import api_app -import cnl_app -import setup_app -# Last routes to register, -import pyload_app - -# Server Adapter -def run_server(host, port, server): - run(app=web, host=host, port=port, quiet=True, server=server) - - -if __name__ == "__main__": - run(app=web, port=8001) |
