diff options
Diffstat (limited to 'module/plugins/hooks/ExtractArchive.py')
| -rw-r--r-- | module/plugins/hooks/ExtractArchive.py | 268 |
1 files changed, 114 insertions, 154 deletions
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 2e9efa2b0..5c4e9bc82 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -6,8 +6,6 @@ import os import sys from copy import copy -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir from traceback import print_exc # monkey patch bug in python 2.6 and lower @@ -47,18 +45,17 @@ if sys.version_info < (2, 7) and os.name != "nt": if os.name != "nt": from grp import getgrnam - from os import chown from pwd import getpwnam from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.utils import save_join, uniqify +from module.utils import fs_decode, save_join, uniqify class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.03" + __version__ = "1.10" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract full path" , True ), @@ -77,7 +74,9 @@ class ExtractArchive(Hook): __description__ = """Extract different kind of archives""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("AndroKev", ""), + ("Walter Purcaro", "vuolter@gmail.com")] event_list = ["allDownloadsProcessed"] @@ -93,17 +92,18 @@ class ExtractArchive(Hook): def setup(self): - self.plugins = [] - self.passwords = [] - names = [] + self.interval = 300 + self.extractors = [] + self.passwords = [] + names = [] for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.checkDeps(): names.append(p) - self.plugins.append(klass) + self.extractors.append(klass) except OSError, e: if e.errno == 2: @@ -123,66 +123,48 @@ class ExtractArchive(Hook): else: self.logInfo(_("No Extract plugins activated")) - # queue with package ids - self.queue = [] - def periodical(self): - if not self.queue or self.extracting: - return - - local = copy(self.queue) - self.queue[:] = [] - - self.extractPackages(*local) + if not self.extracting: + self.extractPackage(*self.getQueue()) @Expose - def extractPackage(self, id): - """ Extract package wrapper""" - self.extractPackages(id) - - - @Expose - def extractPackages(self, *ids): + def extractPackage(self, *ids): """ Extract packages with given id""" self.manager.startThread(self.extract, ids) def packageFinished(self, pypack): - if self.getConfig("queue") or self.extracting: + if self.extracting or self.getConfig("queue"): self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pypack.id) + self.addToQueue(pypack.id) else: self.extractPackage(pypack.id) @threaded - def allDownloadsProcessed(self, thread): - local = copy(self.queue) - self.queue[:] = [] - - if self.extract(local): #: check only if all gone fine, no failed reporting for now + def allDownloadsProcessed(self): + if self.extract(self.getQueue()): #@NOTE: check only if all gone fine, no failed reporting for now self.manager.dispatchEvent("all_archives_extracted") self.manager.dispatchEvent("all_archives_processed") def extract(self, ids): - self.extracting = True - processed = [] extracted = [] failed = [] - clearlist = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] + clearList = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] destination = self.getConfig("destination") subfolder = self.getConfig("subfolder") fullpath = self.getConfig("fullpath") overwrite = self.getConfig("overwrite") - extensions = clearlist(self.getConfig("extensions")) - excludefiles = clearlist(self.getConfig("excludefiles")) + extensions = clearList(self.getConfig("extensions")) + excludefiles = clearList(self.getConfig("excludefiles")) + excludefiles = self.getConfig("excludefiles") renice = self.getConfig("renice") recursive = self.getConfig("recursive") delete = self.getConfig("delete") @@ -197,25 +179,27 @@ class ExtractArchive(Hook): # dl folder dl = self.config['general']['download_folder'] - #iterate packages -> plugins -> targets + #iterate packages -> extractors -> targets for pid in ids: - p = self.core.files.getPackage(pid) - self.logInfo(_("Check package: %s") % p.name) - if not p: + pypack = self.core.files.getPackage(pid) + + self.logInfo(_("Check package: %s") % pypack.name) + + if not pypack: continue # determine output folder - out = save_join(dl, p.folder, destination, "") #: force trailing slash + out = save_join(dl, pypack.folder, destination, "") #: force trailing slash if subfolder: - out = save_join(out, p.folder) + out = save_join(out, pypack.folder) - if not exists(out): - makedirs(out) + if not os.path.exists(out): + os.makedirs(out) - files_ids = [(save_join(dl, p.folder, x['name']), x['id']) for x in p.getChildren().itervalues()] matched = False success = True + files_ids = [(save_join(dl, pypack.folder, x['name']), x['id']) for x in pypack.getChildren().itervalues()] # check as long there are unseen files while files_ids: @@ -224,42 +208,41 @@ class ExtractArchive(Hook): if extensions: files_ids = [(file, id) for file, id in files_ids if filter(lambda ext: file.endswith(ext), extensions)] - for plugin in self.plugins: - targets = plugin.getTargets(files_ids) - + for Extractor in self.extractors: + targets = Extractor.getTargets(files_ids) if targets: - self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) + self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True - for target, fid in targets: - if target in processed: - self.logDebug(basename(target), "skipped") + for filename, fid in targets: + fname = os.path.basename(filename) + + if filename in processed: + self.logDebug(fname, "Skipped") continue - processed.append(target) # prevent extracting same file twice + processed.append(filename) # prevent extracting same file twice - self.logInfo(basename(target), _("Extract to: %s") % out) + self.logInfo(fname, _("Extract to: %s") % out) try: - klass = plugin(self, - target, - out, - p.password, - fullpath, - overwrite, - excludefiles, - renice, - delete, - keepbroken) + self.extracting = True + + klass = Extractor(self, + filename, + out, + fullpath, + overwrite, + excludefiles, + renice, + delete, + keepbroken, + fid) klass.init() - new_files = self._extract(klass, fid) + new_files = self._extract(klass, fid, pypack.password) except Exception, e: - self.logError(basename(target), e) - new_files = None - - if new_files is None: - self.logWarning(basename(target), _("No files extracted")) + self.logError(fname, e) success = False continue @@ -267,10 +250,10 @@ class ExtractArchive(Hook): self.setPermissions(new_files) for file in new_files: - if not exists(file): + if not os.path.exists(file): self.logDebug("New file %s does not exists" % file) continue - if recursive and isfile(file): + if recursive and os.path.isfile(file): new_files_ids.append((file, fid)) # append as new target files_ids = new_files_ids # also check extracted files @@ -278,10 +261,10 @@ class ExtractArchive(Hook): if matched: if success: extracted.append(pid) - self.manager.dispatchEvent("package_extracted", p) + self.manager.dispatchEvent("package_extracted", pypack) else: failed.append(pid) - self.manager.dispatchEvent("package_extract_failed", p) + self.manager.dispatchEvent("package_extract_failed", pypack) else: self.logInfo(_("No files found to extract")) @@ -295,124 +278,103 @@ class ExtractArchive(Hook): return True if not failed else False - def _extract(self, plugin, fid): + def _extract(self, archive, fid, password): pyfile = self.core.files.getFile(fid) + fname = os.path.basename(fs_decode(archive.target)) pyfile.setCustomStatus(_("extracting")) + pyfile.setProgress(0) try: - progress = lambda x: pyfile.setProgress(x) - encrypted = False - passwords = self.getPasswords() - - try: - self.logInfo(basename(plugin.file), "Verifying...") - - tmp_password = plugin.password - plugin.password = "" #: Force verifying without password - - plugin.verify() - - except PasswordError: - encrypted = True - - except CRCError: - self.logWarning(basename(plugin.file), _("Archive damaged")) - - if not self.getConfig("repair"): - raise CRCError - - elif plugin.repair(): - self.logInfo(basename(plugin.file), _("Successfully repaired")) - - elif not self.getConfig("keepbroken"): - raise ArchiveError(_("Broken archive")) - - else: - self.logInfo(basename(plugin.file), _("All OK")) - - plugin.password = tmp_password - - if not encrypted: - plugin.extract(progress) + success = False + if not archive.checkArchive(): + archive.extract(password) + success = True else: - self.logInfo(basename(plugin.file), _("Password protected")) + self.logInfo(fname, _("Password protected")) + self.logDebug("Password: %s" % (password or "No provided")) - if plugin.password: - passwords.insert(0, plugin.password) - passwords = uniqify(self.passwords) - self.logDebug("Password: %s" % plugin.password) - else: - self.logDebug("No package password provided") - - for pw in passwords: + for pw in set(self.getPasswords(False) + [password]): try: self.logDebug("Try password: %s" % pw) - - if plugin.setPassword(pw): - plugin.extract(progress) + if archive.checkPassword(pw): + archive.extract(pw) self.addPassword(pw) + success = True break - else: - raise PasswordError except PasswordError: self.logDebug("Password was wrong") else: raise PasswordError + pyfile.setProgress(100) + pyfile.setStatus("processing") + if self.core.debug: self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) if self.getConfig("delete"): - files = plugin.getDeleteFiles() + files = archive.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: - if exists(f): - remove(f) + if os.path.exists(f): + os.remove(f) else: self.logDebug("%s does not exists" % f) - self.logInfo(basename(plugin.file), _("Extracting finished")) + self.logInfo(fname, _("Extracting finished")) - extracted_files = plugin.getExtractedFiles() - self.manager.dispatchEvent("archive_extracted", pyfile, plugin.out, plugin.file, extracted_files) + extracted_files = archive.getExtractedFiles() + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.file, extracted_files) return extracted_files except PasswordError: - self.logError(basename(plugin.file), _("Wrong password" if passwords else "No password found")) - plugin.password = "" + self.logError(fname, _("Wrong password" if password else "No password found")) except CRCError: - self.logError(basename(plugin.file), _("CRC Mismatch")) + self.logError(fname, _("CRC Mismatch")) except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), e) + self.logError(fname, _("Archive Error"), e) except Exception, e: if self.core.debug: print_exc() - self.logError(basename(plugin.file), _("Unknown Error"), e) + self.logError(fname, _("Unknown Error"), e) + + finally: + pyfile.finishIfDone() self.manager.dispatchEvent("archive_extract_failed", pyfile) - self.logError(basename(plugin.file), _("Extract failed")) + raise Exception(_("Extract failed")) + + + def getQueue(self): + return self.getStorage("ExtractArchive", []) + + + def addToQueue(self, item): + queue = self.getQueue() + return self.setStorage("ExtractArchive", queue + [item] if item not in queue else queue) @Expose - def getPasswords(self): + def getPasswords(self, reload=True): """ List of saved passwords """ + if reload: + self.reloadPasswords() + return self.passwords def reloadPasswords(self): - passwordfile = self.getConfig("passwordfile") - try: passwords = [] - with open(passwordfile, "a+") as f: + with open(self.getConfig("passwordfile")) as f: for pw in f.read().splitlines(): passwords.append(pw) @@ -426,13 +388,10 @@ class ExtractArchive(Hook): @Expose def addPassword(self, pw): """ Adds a password to saved list""" - passwordfile = self.getConfig("passwordfile") - - self.passwords.insert(0, pw) - self.passwords = uniqify(self.passwords) - try: - with open(passwordfile, "wb") as f: + self.passwords = uniqify([pw] + self.passwords) + + with open(self.getConfig("passwordfile"), "wb") as f: for pw in self.passwords: f.write(pw + '\n') @@ -442,20 +401,21 @@ class ExtractArchive(Hook): def setPermissions(self, files): for f in files: - if not exists(f): + if not os.path.exists(f): continue try: if self.config['permission']['change_file']: - if isfile(f): - chmod(f, int(self.config['permission']['file'], 8)) - elif isdir(f): - chmod(f, int(self.config['permission']['folder'], 8)) + if os.path.isfile(f): + os.chmod(f, int(self.config['permission']['file'], 8)) + + elif os.path.isdir(f): + os.chmod(f, int(self.config['permission']['folder'], 8)) if self.config['permission']['change_dl'] and os.name != "nt": uid = getpwnam(self.config['permission']['user'])[2] gid = getgrnam(self.config['permission']['group'])[2] - chown(f, uid, gid) + os.chown(f, uid, gid) except Exception, e: self.logWarning(_("Setting User and Group failed"), e) |