summaryrefslogtreecommitdiffstats
path: root/module
diff options
context:
space:
mode:
authorGravatar RaNaN <Mast3rRaNaN@hotmail.de> 2010-07-31 20:44:42 +0200
committerGravatar RaNaN <Mast3rRaNaN@hotmail.de> 2010-07-31 20:44:42 +0200
commitb06286d2d6693931e7956dba10c07073e69b5909 (patch)
treee195016b9b58b4c0c3e996ffb3a0cb234cafc217 /module
parentdaily commit (diff)
downloadpyload-b06286d2d6693931e7956dba10c07073e69b5909.tar.xz
some changes
Diffstat (limited to 'module')
-rw-r--r--module/FileDatabase.py73
-rw-r--r--module/HookManager.py6
-rw-r--r--module/PluginThread.py59
-rw-r--r--module/ThreadManager.py20
-rw-r--r--module/plugins/Crypter.py67
-rw-r--r--module/plugins/Hook.py6
-rw-r--r--module/plugins/Plugin.py20
-rw-r--r--module/plugins/container/LinkList.py56
8 files changed, 238 insertions, 69 deletions
diff --git a/module/FileDatabase.py b/module/FileDatabase.py
index 53e50bb72..e371754e6 100644
--- a/module/FileDatabase.py
+++ b/module/FileDatabase.py
@@ -93,12 +93,12 @@ class FileHandler:
data = self.db.getAllLinks(queue)
packs = self.db.getAllPackages(queue)
- data.update( [ (x.id, x.toDbDict()[x.id]) for x in self.cache.itervalues() ] )
- packs.update( [ (x.id, x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue] )
+ data.update( [ (str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues() ] )
+ packs.update( [ (str(x.id), x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue] )
for key, value in data.iteritems():
- if packs.has_key(value["package"]):
- packs[value["package"]]["links"][key] = value
+ if packs.has_key(str(value["package"])):
+ packs[str(value["package"])]["links"][key] = value
return packs
@@ -152,9 +152,11 @@ class FileHandler:
self.lock.acquire()
if self.cache.has_key(id):
- self.cache[id].abortDownload()
- del self.cache[id]
-
+ if id in self.core.threadManager.processingIds():
+ self.cache[id].abortDownload()
+
+ #del self.cache[id]
+
self.lock.release()
self.db.deleteLink(id)
@@ -191,6 +193,20 @@ class FileHandler:
return self.db.getPackage(id)
#----------------------------------------------------------------------
+ def getPackageData(self, id):
+ """returns dict with package information"""
+ pack = self.getPackage(id)
+ pack = pack.toDict()[id]
+
+ data = self.db.getPackageData(id)
+
+ data.update( [ (str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues() ] )
+
+ pack["links"] = data
+
+ return pack
+
+ #----------------------------------------------------------------------
def getFile(self, id):
"""returns pyfile instance"""
if self.cache.has_key(id):
@@ -253,13 +269,21 @@ class FileHandler:
return self.filecount
#----------------------------------------------------------------------
+ def getQueueCount(self):
+ """number of files that have to be processed"""
+ pass
+
+ #----------------------------------------------------------------------
def restartPackage(self, id):
"""restart package"""
- if self.packageCache.has_key(id):
- pass
+ for pyfile in self.cache.itervalues():
+ if pyfile.packageid == id:
+ self.restartFile(pyfile.id)
+
+ self.db.restartPackage(id)
def restartFile(self, id):
- """ restart link"""
+ """ restart file"""
if self.cache.has_key(id):
self.cache[id].abortDownload()
self.cache[id].status = 3
@@ -386,7 +410,7 @@ class FileDatabaseBackend(Thread):
self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.id', (q, ))
data = {}
for r in self.c:
- data[int(r[0])] = {
+ data[str(r[0])] = {
'url': r[1],
'name': r[2],
'size': r[3],
@@ -417,7 +441,7 @@ class FileDatabaseBackend(Thread):
data = {}
for r in self.c:
- data[int(r[0])] = {
+ data[str(r[0])] = {
'name': r[1],
'folder': r[2],
'site': r[3],
@@ -433,9 +457,25 @@ class FileDatabaseBackend(Thread):
"""get link information"""
pass
+ @queue
def getPackageData(self, id):
- """get package data _with_ link data"""
- pass
+ """get package data"""
+ self.c.execute('SELECT id,url,name,size,status,error,plugin,package FROM links WHERE package=? ORDER BY id', (str(id),))
+
+ data = {}
+ for r in self.c:
+ data[str(r[0])] = {
+ 'url': r[1],
+ 'name': r[2],
+ 'size': r[3],
+ 'status': r[4],
+ 'statusmsg': self.manager.statusMsg[r[4]],
+ 'error': r[5],
+ 'plugin': r[6],
+ 'package': r[7]
+ }
+
+ return data
@async
@@ -452,7 +492,7 @@ class FileDatabaseBackend(Thread):
@async
def restartPackage(self, id):
- pass
+ self.c.execute('UPDATE links SET status=3 WHERE package=?', ( str(id), ) )
@async
def commit(self):
@@ -575,8 +615,7 @@ class PyFile():
def abortDownload(self):
"""abort pyfile if possible"""
- print "abort"
-
+ self.m.core.log.info(_("Download aborted: %s" % self.name))
while self.id in self.m.core.threadManager.processingIds():
self.abort = True
if self.plugin: self.plugin.req.abort = True
diff --git a/module/HookManager.py b/module/HookManager.py
index 9579c74a1..1fd8a5844 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -21,6 +21,7 @@
import traceback
from threading import RLock
from module.PluginThread import HookThread
+from time import time
class HookManager():
def __init__(self, core):
@@ -60,7 +61,10 @@ class HookManager():
def periodical(self):
- pass
+ for plugin in self.plugins:
+ if plugin.lastCall + plugin.interval < time():
+ plugin.periodical()
+ plugin.lastCall = time()
def coreReady(self):
for plugin in self.plugins:
diff --git a/module/PluginThread.py b/module/PluginThread.py
index a073c9334..a2ac6e027 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -147,7 +147,6 @@ class DownloadThread(PluginThread):
self.active = False
pyfile.finishIfDone()
self.m.core.files.save()
- print "done with", pyfile
#----------------------------------------------------------------------
def put(self, job):
@@ -170,15 +169,68 @@ class DecrypterThread(PluginThread):
"""constructor"""
PluginThread.__init__(self, manager)
- self.pyfile = pyfile
+ self.active = pyfile
+ manager.localThreads.append(self)
+
+ pyfile.setStatus("decrypting")
self.start()
#----------------------------------------------------------------------
def run(self):
"""run method"""
- self.pyfile.plugin.preprocessing()
+ pyfile = self.active
+
+ try:
+ self.m.log.info(_("Decrypting starts: %s") % self.active.name)
+ self.active.plugin.preprocessing(self)
+
+ except NotImplementedError:
+
+ self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname)
+ return
+
+ except Fail,e :
+
+ msg = e.args[0]
+
+ if msg == "offline":
+ self.active.setStatus("offline")
+ self.m.log.warning(_("Download is offline: %s") % self.active.name)
+ else:
+ self.active.setStatus("failed")
+ self.m.log.warning(_("Decrypting failed: %s | %s") % (self.active.name, msg))
+ self.active.error = msg
+
+ return
+
+
+ except Exception, e:
+
+ self.active.setStatus("failed")
+ self.m.log.error(_("Decrypting failed: %s | %s") % (self.active.name, str(e)))
+ self.active.error = str(e)
+
+ if self.m.core.debug:
+ print_exc()
+
+ return
+
+
+ finally:
+ self.active.release()
+ self.active = False
+ self.m.core.files.save()
+ self.m.localThreads.remove(self)
+
+
+ #self.m.core.hookManager.downloadFinished(pyfile)
+
+
+ #self.m.localThreads.remove(self)
+ #self.active.finishIfDone()
+ pyfile.delete()
########################################################################
class HookThread(PluginThread):
@@ -201,6 +253,7 @@ class HookThread(PluginThread):
def run(self):
self.f(self.active)
+
self.m.localThreads.remove(self)
self.active.finishIfDone()
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 5186c189e..e6d4747b0 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -54,12 +54,12 @@ class ThreadManager:
#----------------------------------------------------------------------
def downloadingIds(self):
"""get a list of the currently downloading pyfile's ids"""
- return [x.active.id for x in self.threads if x.active]
+ return [x.active.id for x in self.threads if x.active and x.active != "quit"]
#----------------------------------------------------------------------
def processingIds(self):
"""get a id list of all pyfiles processed"""
- return [x.active.id for x in self.threads+self.localThreads if x.active]
+ return [x.active.id for x in self.threads+self.localThreads if x.active and x.active != "quit"]
#----------------------------------------------------------------------
@@ -73,7 +73,15 @@ class ThreadManager:
#----------------------------------------------------------------------
def checkReconnect(self):
"""checks if reconnect needed"""
- pass
+ active = [x.active.plugin.wantReconnect for x in self.threads if x.active]
+
+ if active.count(True) > 0 and len(active) == active.count(True):
+ self.reconnecting.set()
+
+ #Do reconnect
+ self.log.info(_("Reconnecting"))
+
+ self.reconnecting.clear()
#----------------------------------------------------------------------
def checkThreadCount(self):
@@ -100,8 +108,8 @@ class ThreadManager:
occ = [x.active.pluginname for x in self.threads if x.active and not x.active.plugin.multiDL ]
occ.sort()
- occ = set(occ)
- job = self.core.files.getJob(tuple(occ))
+ occ = tuple(set(occ))
+ job = self.core.files.getJob(occ)
if job:
try:
job.initPlugin()
@@ -117,7 +125,7 @@ class ThreadManager:
self.core.files.jobCache[occ].append(job.id)
else:
- thread = PluginThread.DecrypterThread(job)
+ thread = PluginThread.DecrypterThread(self, job)
diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py
index e0459c714..a14a51e79 100644
--- a/module/plugins/Crypter.py
+++ b/module/plugins/Crypter.py
@@ -19,6 +19,8 @@
from module.plugins.Plugin import Plugin
+from os.path import join, exists, basename
+
class Crypter(Plugin):
__name__ = "Crypter"
__version__ = "0.1"
@@ -27,13 +29,62 @@ class Crypter(Plugin):
__description__ = """Base crypter plugin"""
__author_name__ = ("mkaay")
__author_mail__ = ("mkaay@mkaay.de")
-
+
+ def __init__(self, pyfile):
+ Plugin.__init__(self, pyfile)
+
+ self.packages = [] #put all packages here [ .. (name, folder, [urls]) ..]
+
#----------------------------------------------------------------------
- def createPackage(self, name, urls):
- """ create a new package """
- pass
+ def preprocessing(self, thread):
+ """prepare"""
+ self.thread = thread
+
+ self.decrypt(self.pyfile)
+
+ self.createPackages()
+
- def fillCurrentPackage(self, name, urls):
- """ rename current package and fill with urls"""
- pass
- \ No newline at end of file
+ #----------------------------------------------------------------------
+ def loadToDisk(self):
+ """loads container to disk if its stored remotely and overwrite url,
+ or check existent on several places at disk"""
+
+ if self.pyfile.url.startswith("http://"):
+ self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
+ content = self.load(self.pyfile.url)
+ self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name)
+ f = open(self.pyfile.url, "wb" )
+ f.write(content)
+ f.close()
+
+ else:
+ self.pyfile.name = basename(self.pyfile.url)
+ if not exists(self.pyfile.url):
+ if exists(join(pypath, self.pyfile.url)):
+ self.pyfile.url = join(pypath, self.pyfile.url)
+ else:
+ self.fail(_("File not exists."))
+
+
+ #----------------------------------------------------------------------
+ def createPackages(self):
+ """ create new packages from self.packages """
+ i = 0
+ for pack in self.packages:
+
+ self.log.info(_("Parsed package %s with %s links") % (pack[0], len(pack[1]) ) )
+
+ if i == 0:
+ # replace current package with new one
+ self.pyfile.package().name = pack[0]
+ self.pyfile.package().folder = pack[2]
+
+ self.core.files.addLinks(pack[1], self.pyfile.package().id)
+
+ self.pyfile.package().sync()
+ else:
+ self.core.server_methods.add_package(pack[0], pack[1])
+
+ i += 1
+ \ No newline at end of file
diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py
index a928d7173..7adbe2bbd 100644
--- a/module/plugins/Hook.py
+++ b/module/plugins/Hook.py
@@ -35,6 +35,9 @@ class Hook():
self.log = core.log
self.config = core.config
+ self.interval = 60
+ self.lastCall = 0
+
self.setup()
def setup(self):
@@ -60,3 +63,6 @@ class Hook():
def afterReconnecting(self, ip):
pass
+
+ def periodical(self):
+ pass
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index d8d8aae3c..1d3fb4309 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -61,14 +61,6 @@ class Plugin(object):
__author_name__ = ("RaNaN", "spoob", "mkaay")
__author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
- def __new__(cls, *args, **kws):
- for f in dir(cls):
- if not f.startswith("_") and f not in ("checksum"):
- setattr(cls, f, dec(getattr(cls, f)) )
-
- o = super(cls.__class__, cls).__new__(cls)
- #wrap decorator around every method
- return o
def __init__(self, pyfile):
self.config = pyfile.m.core.config
@@ -151,7 +143,7 @@ class Plugin(object):
def wait():
""" waits the time previously set """
- pass
+ if self.pyfile.abort: raise Abort
def fail(self, reason):
""" fail and give reason """
@@ -178,10 +170,10 @@ class Plugin(object):
- ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__)
- if ocr:
- #@TODO decrypt
- result = ""
+ Ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__)
+ if Ocr:
+ ocr = Ocr()
+ result = ocr.get_captcha(temp.name)
else:
captchaManager = self.core.captchaManager
mime = guess_type(temp.name)
@@ -203,6 +195,8 @@ class Plugin(object):
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False):
""" returns the content loaded """
+ if self.pyfile.abort: raise Abort
+
return self.req.load(url, get, post, ref, cookies, just_header)
def download(self, url, get={}, post={}, ref=True, cookies=True):
diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py
index 3ffeeb193..9668b34ad 100644
--- a/module/plugins/container/LinkList.py
+++ b/module/plugins/container/LinkList.py
@@ -7,38 +7,52 @@ from module.plugins.Container import Container
class LinkList(Container):
__name__ = "LinkList"
__version__ = "0.1"
- __pattern__ = r"(?!http://).*\.txt"
+ __pattern__ = r".*\.txt$"
__description__ = """Read Link Lists in txt format"""
__author_name__ = ("spoob", "jeix")
__author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com")
- def proceed(self, linkList, location):
- txt = open(linkList, 'r')
+ def decrypt(self, pyfile):
+
+ self.loadToDisk()
+
+ txt = open(pyfile.url, 'r')
links = txt.readlines()
- packages = {"Parsed links":[],}
- curPack = "Parsed links"
+ curPack = "Parsed links %s" % pyfile.name
+
+ packages = {curPack:[],}
+
for link in links:
- if link != "\n":
- link = link.strip()
- if link.startswith(";"):
- continue
- if link.startswith("[") and link.endswith("]"):
- # new package
- curPack = link[1:-1]
- packages[curPack] = []
- continue
- packages[curPack].append(link.replace("\n", ""))
+ link = link.strip()
+ if not link: continue
+
+ if link.startswith(";"):
+ continue
+ if link.startswith("[") and link.endswith("]"):
+ # new package
+ curPack = link[1:-1]
+ packages[curPack] = []
+ continue
+ packages[curPack].append(link.replace("\n", ""))
txt.close()
- # empty Parsed links fix
- if len(packages["Parsed links"]) < 1:
- del packages["Parsed links"]
+ # empty packages fix
- if not self.parent.core.config['general']['debug_mode']:
+ delete = []
+
+ for key,value in packages.iteritems():
+ if not value:
+ delete.append(key)
+
+ for key in delete:
+ del packages[key]
+
+ if not self.core.debug:
txt = open(linkList, 'w')
txt.write("")
txt.close()
#@TODO: maybe delete read txt file?
-
- self.links = packages
+
+ for name, links in packages.iteritems():
+ self.packages.append((name, links, name))