diff options
author | RaNaN <Mast3rRaNaN@hotmail.de> | 2010-07-31 20:44:42 +0200 |
---|---|---|
committer | RaNaN <Mast3rRaNaN@hotmail.de> | 2010-07-31 20:44:42 +0200 |
commit | b06286d2d6693931e7956dba10c07073e69b5909 (patch) | |
tree | e195016b9b58b4c0c3e996ffb3a0cb234cafc217 /module | |
parent | daily commit (diff) | |
download | pyload-b06286d2d6693931e7956dba10c07073e69b5909.tar.xz |
some changes
Diffstat (limited to 'module')
-rw-r--r-- | module/FileDatabase.py | 73 | ||||
-rw-r--r-- | module/HookManager.py | 6 | ||||
-rw-r--r-- | module/PluginThread.py | 59 | ||||
-rw-r--r-- | module/ThreadManager.py | 20 | ||||
-rw-r--r-- | module/plugins/Crypter.py | 67 | ||||
-rw-r--r-- | module/plugins/Hook.py | 6 | ||||
-rw-r--r-- | module/plugins/Plugin.py | 20 | ||||
-rw-r--r-- | module/plugins/container/LinkList.py | 56 |
8 files changed, 238 insertions, 69 deletions
diff --git a/module/FileDatabase.py b/module/FileDatabase.py index 53e50bb72..e371754e6 100644 --- a/module/FileDatabase.py +++ b/module/FileDatabase.py @@ -93,12 +93,12 @@ class FileHandler: data = self.db.getAllLinks(queue) packs = self.db.getAllPackages(queue) - data.update( [ (x.id, x.toDbDict()[x.id]) for x in self.cache.itervalues() ] ) - packs.update( [ (x.id, x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue] ) + data.update( [ (str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues() ] ) + packs.update( [ (str(x.id), x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue] ) for key, value in data.iteritems(): - if packs.has_key(value["package"]): - packs[value["package"]]["links"][key] = value + if packs.has_key(str(value["package"])): + packs[str(value["package"])]["links"][key] = value return packs @@ -152,9 +152,11 @@ class FileHandler: self.lock.acquire() if self.cache.has_key(id): - self.cache[id].abortDownload() - del self.cache[id] - + if id in self.core.threadManager.processingIds(): + self.cache[id].abortDownload() + + #del self.cache[id] + self.lock.release() self.db.deleteLink(id) @@ -191,6 +193,20 @@ class FileHandler: return self.db.getPackage(id) #---------------------------------------------------------------------- + def getPackageData(self, id): + """returns dict with package information""" + pack = self.getPackage(id) + pack = pack.toDict()[id] + + data = self.db.getPackageData(id) + + data.update( [ (str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues() ] ) + + pack["links"] = data + + return pack + + #---------------------------------------------------------------------- def getFile(self, id): """returns pyfile instance""" if self.cache.has_key(id): @@ -253,13 +269,21 @@ class FileHandler: return self.filecount #---------------------------------------------------------------------- + def getQueueCount(self): + """number of files that have to be processed""" + pass + + #---------------------------------------------------------------------- def restartPackage(self, id): """restart package""" - if self.packageCache.has_key(id): - pass + for pyfile in self.cache.itervalues(): + if pyfile.packageid == id: + self.restartFile(pyfile.id) + + self.db.restartPackage(id) def restartFile(self, id): - """ restart link""" + """ restart file""" if self.cache.has_key(id): self.cache[id].abortDownload() self.cache[id].status = 3 @@ -386,7 +410,7 @@ class FileDatabaseBackend(Thread): self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.id', (q, )) data = {} for r in self.c: - data[int(r[0])] = { + data[str(r[0])] = { 'url': r[1], 'name': r[2], 'size': r[3], @@ -417,7 +441,7 @@ class FileDatabaseBackend(Thread): data = {} for r in self.c: - data[int(r[0])] = { + data[str(r[0])] = { 'name': r[1], 'folder': r[2], 'site': r[3], @@ -433,9 +457,25 @@ class FileDatabaseBackend(Thread): """get link information""" pass + @queue def getPackageData(self, id): - """get package data _with_ link data""" - pass + """get package data""" + self.c.execute('SELECT id,url,name,size,status,error,plugin,package FROM links WHERE package=? ORDER BY id', (str(id),)) + + data = {} + for r in self.c: + data[str(r[0])] = { + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7] + } + + return data @async @@ -452,7 +492,7 @@ class FileDatabaseBackend(Thread): @async def restartPackage(self, id): - pass + self.c.execute('UPDATE links SET status=3 WHERE package=?', ( str(id), ) ) @async def commit(self): @@ -575,8 +615,7 @@ class PyFile(): def abortDownload(self): """abort pyfile if possible""" - print "abort" - + self.m.core.log.info(_("Download aborted: %s" % self.name)) while self.id in self.m.core.threadManager.processingIds(): self.abort = True if self.plugin: self.plugin.req.abort = True diff --git a/module/HookManager.py b/module/HookManager.py index 9579c74a1..1fd8a5844 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -21,6 +21,7 @@ import traceback from threading import RLock from module.PluginThread import HookThread +from time import time class HookManager(): def __init__(self, core): @@ -60,7 +61,10 @@ class HookManager(): def periodical(self): - pass + for plugin in self.plugins: + if plugin.lastCall + plugin.interval < time(): + plugin.periodical() + plugin.lastCall = time() def coreReady(self): for plugin in self.plugins: diff --git a/module/PluginThread.py b/module/PluginThread.py index a073c9334..a2ac6e027 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -147,7 +147,6 @@ class DownloadThread(PluginThread): self.active = False pyfile.finishIfDone() self.m.core.files.save() - print "done with", pyfile #---------------------------------------------------------------------- def put(self, job): @@ -170,15 +169,68 @@ class DecrypterThread(PluginThread): """constructor""" PluginThread.__init__(self, manager) - self.pyfile = pyfile + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") self.start() #---------------------------------------------------------------------- def run(self): """run method""" - self.pyfile.plugin.preprocessing() + pyfile = self.active + + try: + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) + + except NotImplementedError: + + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) + return + + except Fail,e : + + msg = e.args[0] + + if msg == "offline": + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) + else: + self.active.setStatus("failed") + self.m.log.warning(_("Decrypting failed: %s | %s") % (self.active.name, msg)) + self.active.error = msg + + return + + + except Exception, e: + + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %s | %s") % (self.active.name, str(e))) + self.active.error = str(e) + + if self.m.core.debug: + print_exc() + + return + + + finally: + self.active.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + + + #self.m.core.hookManager.downloadFinished(pyfile) + + + #self.m.localThreads.remove(self) + #self.active.finishIfDone() + pyfile.delete() ######################################################################## class HookThread(PluginThread): @@ -201,6 +253,7 @@ class HookThread(PluginThread): def run(self): self.f(self.active) + self.m.localThreads.remove(self) self.active.finishIfDone() diff --git a/module/ThreadManager.py b/module/ThreadManager.py index 5186c189e..e6d4747b0 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -54,12 +54,12 @@ class ThreadManager: #---------------------------------------------------------------------- def downloadingIds(self): """get a list of the currently downloading pyfile's ids""" - return [x.active.id for x in self.threads if x.active] + return [x.active.id for x in self.threads if x.active and x.active != "quit"] #---------------------------------------------------------------------- def processingIds(self): """get a id list of all pyfiles processed""" - return [x.active.id for x in self.threads+self.localThreads if x.active] + return [x.active.id for x in self.threads+self.localThreads if x.active and x.active != "quit"] #---------------------------------------------------------------------- @@ -73,7 +73,15 @@ class ThreadManager: #---------------------------------------------------------------------- def checkReconnect(self): """checks if reconnect needed""" - pass + active = [x.active.plugin.wantReconnect for x in self.threads if x.active] + + if active.count(True) > 0 and len(active) == active.count(True): + self.reconnecting.set() + + #Do reconnect + self.log.info(_("Reconnecting")) + + self.reconnecting.clear() #---------------------------------------------------------------------- def checkThreadCount(self): @@ -100,8 +108,8 @@ class ThreadManager: occ = [x.active.pluginname for x in self.threads if x.active and not x.active.plugin.multiDL ] occ.sort() - occ = set(occ) - job = self.core.files.getJob(tuple(occ)) + occ = tuple(set(occ)) + job = self.core.files.getJob(occ) if job: try: job.initPlugin() @@ -117,7 +125,7 @@ class ThreadManager: self.core.files.jobCache[occ].append(job.id) else: - thread = PluginThread.DecrypterThread(job) + thread = PluginThread.DecrypterThread(self, job) diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index e0459c714..a14a51e79 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -19,6 +19,8 @@ from module.plugins.Plugin import Plugin +from os.path import join, exists, basename + class Crypter(Plugin): __name__ = "Crypter" __version__ = "0.1" @@ -27,13 +29,62 @@ class Crypter(Plugin): __description__ = """Base crypter plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - + + def __init__(self, pyfile): + Plugin.__init__(self, pyfile) + + self.packages = [] #put all packages here [ .. (name, folder, [urls]) ..] + #---------------------------------------------------------------------- - def createPackage(self, name, urls): - """ create a new package """ - pass + def preprocessing(self, thread): + """prepare""" + self.thread = thread + + self.decrypt(self.pyfile) + + self.createPackages() + - def fillCurrentPackage(self, name, urls): - """ rename current package and fill with urls""" - pass -
\ No newline at end of file + #---------------------------------------------------------------------- + def loadToDisk(self): + """loads container to disk if its stored remotely and overwrite url, + or check existent on several places at disk""" + + if self.pyfile.url.startswith("http://"): + self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] + content = self.load(self.pyfile.url) + self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) + f = open(self.pyfile.url, "wb" ) + f.write(content) + f.close() + + else: + self.pyfile.name = basename(self.pyfile.url) + if not exists(self.pyfile.url): + if exists(join(pypath, self.pyfile.url)): + self.pyfile.url = join(pypath, self.pyfile.url) + else: + self.fail(_("File not exists.")) + + + #---------------------------------------------------------------------- + def createPackages(self): + """ create new packages from self.packages """ + i = 0 + for pack in self.packages: + + self.log.info(_("Parsed package %s with %s links") % (pack[0], len(pack[1]) ) ) + + if i == 0: + # replace current package with new one + self.pyfile.package().name = pack[0] + self.pyfile.package().folder = pack[2] + + self.core.files.addLinks(pack[1], self.pyfile.package().id) + + self.pyfile.package().sync() + else: + self.core.server_methods.add_package(pack[0], pack[1]) + + i += 1 +
\ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index a928d7173..7adbe2bbd 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -35,6 +35,9 @@ class Hook(): self.log = core.log self.config = core.config + self.interval = 60 + self.lastCall = 0 + self.setup() def setup(self): @@ -60,3 +63,6 @@ class Hook(): def afterReconnecting(self, ip): pass + + def periodical(self): + pass diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index d8d8aae3c..1d3fb4309 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -61,14 +61,6 @@ class Plugin(object): __author_name__ = ("RaNaN", "spoob", "mkaay") __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - def __new__(cls, *args, **kws): - for f in dir(cls): - if not f.startswith("_") and f not in ("checksum"): - setattr(cls, f, dec(getattr(cls, f)) ) - - o = super(cls.__class__, cls).__new__(cls) - #wrap decorator around every method - return o def __init__(self, pyfile): self.config = pyfile.m.core.config @@ -151,7 +143,7 @@ class Plugin(object): def wait(): """ waits the time previously set """ - pass + if self.pyfile.abort: raise Abort def fail(self, reason): """ fail and give reason """ @@ -178,10 +170,10 @@ class Plugin(object): - ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__) - if ocr: - #@TODO decrypt - result = "" + Ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__) + if Ocr: + ocr = Ocr() + result = ocr.get_captcha(temp.name) else: captchaManager = self.core.captchaManager mime = guess_type(temp.name) @@ -203,6 +195,8 @@ class Plugin(object): def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False): """ returns the content loaded """ + if self.pyfile.abort: raise Abort + return self.req.load(url, get, post, ref, cookies, just_header) def download(self, url, get={}, post={}, ref=True, cookies=True): diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py index 3ffeeb193..9668b34ad 100644 --- a/module/plugins/container/LinkList.py +++ b/module/plugins/container/LinkList.py @@ -7,38 +7,52 @@ from module.plugins.Container import Container class LinkList(Container): __name__ = "LinkList" __version__ = "0.1" - __pattern__ = r"(?!http://).*\.txt" + __pattern__ = r".*\.txt$" __description__ = """Read Link Lists in txt format""" __author_name__ = ("spoob", "jeix") __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") - def proceed(self, linkList, location): - txt = open(linkList, 'r') + def decrypt(self, pyfile): + + self.loadToDisk() + + txt = open(pyfile.url, 'r') links = txt.readlines() - packages = {"Parsed links":[],} - curPack = "Parsed links" + curPack = "Parsed links %s" % pyfile.name + + packages = {curPack:[],} + for link in links: - if link != "\n": - link = link.strip() - if link.startswith(";"): - continue - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - packages[curPack].append(link.replace("\n", "")) + link = link.strip() + if not link: continue + + if link.startswith(";"): + continue + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + packages[curPack].append(link.replace("\n", "")) txt.close() - # empty Parsed links fix - if len(packages["Parsed links"]) < 1: - del packages["Parsed links"] + # empty packages fix - if not self.parent.core.config['general']['debug_mode']: + delete = [] + + for key,value in packages.iteritems(): + if not value: + delete.append(key) + + for key in delete: + del packages[key] + + if not self.core.debug: txt = open(linkList, 'w') txt.write("") txt.close() #@TODO: maybe delete read txt file? - - self.links = packages + + for name, links in packages.iteritems(): + self.packages.append((name, links, name)) |