summaryrefslogtreecommitdiffstats
path: root/module/plugins/internal
diff options
context:
space:
mode:
Diffstat (limited to 'module/plugins/internal')
-rw-r--r--module/plugins/internal/Extractor.py76
-rw-r--r--module/plugins/internal/SevenZip.py163
-rw-r--r--module/plugins/internal/UnRar.py175
-rw-r--r--module/plugins/internal/UnZip.py47
4 files changed, 320 insertions, 141 deletions
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 55d9b2e83..ddf0f8a85 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -1,5 +1,11 @@
# -*- coding: utf-8 -*-
+import os
+
+from module.PyFile import PyFile
+from module.utils import fs_encode
+
+
class ArchiveError(Exception):
pass
@@ -8,29 +14,39 @@ class CRCError(Exception):
pass
-class WrongPassword(Exception):
+class PasswordError(Exception):
pass
class Extractor:
__name__ = "Extractor"
- __version__ = "0.14"
+ __version__ = "0.15"
__description__ = """Base extractor plugin"""
__license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org")]
+ __authors__ = [("RaNaN", "ranan@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com")]
+
+
+ EXTENSIONS = []
- @staticmethod
- def checkDeps():
+ @classmethod
+ def isArchive(cls, filename):
+ name = os.path.basename(filename).lower()
+ return any(name.endswith(ext) for ext in cls.EXTENSIONS)
+
+
+ @classmethod
+ def checkDeps(cls):
""" Check if system statisfy dependencies
:return: boolean
"""
return True
- @staticmethod
- def getTargets(files_ids):
+ @classmethod
+ def getTargets(cls, files_ids):
""" Filter suited targets from list of filename id tuple list
:param files_ids: List of filepathes
:return: List of targets, id tuple list
@@ -38,24 +54,28 @@ class Extractor:
raise NotImplementedError
- def __init__(self, m, file, out, fullpath, overwrite, excludefiles, renice):
- """Initialize extractor for specific file
-
- :param m: ExtractArchive Hook plugin
- :param file: Absolute filepath
- :param out: Absolute path to destination directory
- :param fullpath: extract to fullpath
- :param overwrite: Overwrite existing archives
- :param renice: Renice value
- """
- self.m = m
- self.file = file
- self.out = out
- self.fullpath = fullpath
- self.overwrite = overwrite
- self.excludefiles = excludefiles
- self.renice = renice
- self.files = [] #: Store extracted files here
+ def __init__(self, manager, filename, out,
+ fullpath=True,
+ overwrite=False,
+ excludefiles=[],
+ renice=0,
+ delete=False,
+ keepbroken=False,
+ fid=None):
+ """ Initialize extractor for specific file """
+ self.manager = manager
+ self.target = fs_encode(filename)
+ self.out = out
+ self.fullpath = fullpath
+ self.overwrite = overwrite
+ self.excludefiles = excludefiles
+ self.renice = renice
+ self.delete = delete
+ self.keepbroken = keepbroken
+ self.files = [] #: Store extracted files here
+
+ pyfile = self.manager.core.files.getFile(fid) if fid else None
+ self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None
def init(self):
@@ -83,12 +103,12 @@ class Extractor:
return True
- def extract(self, progress, password=None):
+ def extract(self, password=None):
"""Extract the archive. Raise specific errors in case of failure.
:param progress: Progress function, call this to update status
:param password password to use
- :raises WrongPassword
+ :raises PasswordError
:raises CRCError
:raises ArchiveError
:return:
@@ -101,7 +121,7 @@ class Extractor:
:return: List with paths of files to delete
"""
- raise NotImplementedError
+ return [self.target]
def getExtractedFiles(self):
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
new file mode 100644
index 000000000..508cf9c8d
--- /dev/null
+++ b/module/plugins/internal/SevenZip.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+
+from subprocess import Popen, PIPE
+
+from module.plugins.internal.UnRar import UnRar, renice
+from module.utils import save_join
+
+
+class SevenZip(UnRar):
+ __name__ = "SevenZip"
+ __version__ = "0.02"
+
+ __description__ = """7-Zip extractor plugin"""
+ __license__ = "GPLv3"
+ __authors__ = [("Michael Nowak", ""),
+ ("Walter Purcaro", "vuolter@gmail.com")]
+
+
+ CMD = "7z"
+
+ EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2",
+ ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab",
+ ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha",
+ ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt",
+ ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr",
+ ".squashfs", ".cramfs", ".scap"]
+
+
+ #@NOTE: there are some more uncovered 7z formats
+ re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
+ re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I)
+ re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I)
+
+
+ @classmethod
+ def checkDeps(cls):
+ if os.name == "nt":
+ cls.CMD = os.path.join(pypath, "7z.exe")
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
+ p.communicate()
+ else:
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
+ p.communicate()
+
+ return True
+
+
+ def checkArchive(self):
+ p = self.call_cmd("l", "-slt", self.target)
+ out, err = p.communicate()
+
+ if p.returncode > 1:
+ raise ArchiveError("Process terminated")
+
+ # check if output or error macthes the 'wrong password'-Regexp
+ if self.re_wrongpwd.search(out):
+ return True
+
+ # check if output matches 'Encrypted = +'
+ if self.re_wrongcrc.search(out):
+ return True
+
+ # check if archive is empty
+ self.files = self.list()
+ if not self.files:
+ raise ArchiveError("Empty Archive")
+
+ return False
+
+
+ def checkPassword(self, password):
+ p = self.call_cmd("l", self.target, password=password)
+ p.communicate()
+ return p.returncode == 0
+
+
+ def extract(self, password=None):
+ command = "x" if self.fullpath else "e"
+
+ p = self.call_cmd(command, '-o' + self.out, self.target, password=password)
+
+ renice(p.pid, self.renice)
+
+ progressstring = ""
+ while True:
+ c = p.stdout.read(1)
+ # quit loop on eof
+ if not c:
+ break
+ # reading a percentage sign -> set progress and restart
+ if c == '%':
+ self.notifyProgress(int(progressstring))
+ progressstring = ""
+ # not reading a digit -> therefore restart
+ elif c not in digits:
+ progressstring = ""
+ # add digit to progressstring
+ else:
+ progressstring += c
+
+ # retrieve stderr
+ err = p.stderr.read()
+
+ if self.re_wrongpwd.search(err):
+ raise PasswordError
+
+ elif self.re_wrongcrc.search(err):
+ raise CRCError
+
+ elif err.strip(): #: raise error if anything is on stderr
+ raise ArchiveError(err.strip())
+
+ if p.returncode > 1:
+ raise ArchiveError("Process terminated")
+
+ if not self.files:
+ self.files = self.list(password)
+
+
+ def list(self, password=None):
+ command = "l" if self.fullpath else "l"
+
+ p = self.call_cmd(command, self.target, password=password)
+ out, err = p.communicate()
+ code = p.returncode
+
+ if "Can not open" in err:
+ raise ArchiveError("Cannot open file")
+
+ if code != 0:
+ raise ArchiveError("Process terminated unsuccessful")
+
+ result = set()
+ for groups in self.re_filelist.findall(out):
+ f = groups[-1].strip()
+ result.add(save_join(self.out, f))
+
+ return list(result)
+
+
+ def call_cmd(self, command, *xargs, **kwargs):
+ args = []
+
+ #overwrite flag
+ if self.overwrite:
+ args.append("-y")
+
+ #set a password
+ if "password" in kwargs and kwargs["password"]:
+ args.append("-p%s" % kwargs["password"])
+ else:
+ args.append("-p-")
+
+ #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
+ call = [self.cmd, command] + args + list(xargs)
+
+ self.manager.logDebug(" ".join([decode(arg) for arg in call]))
+
+ p = Popen(call, stdout=PIPE, stderr=PIPE)
+ return p
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 43592c3de..7f1b08caf 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -4,110 +4,92 @@ import os
import re
from glob import glob
-from os.path import basename, join
from string import digits
from subprocess import Popen, PIPE
-from module.plugins.internal.Extractor import Extractor, WrongPassword, ArchiveError, CRCError
+from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
from module.utils import save_join, decode
def renice(pid, value):
- if os.name != "nt" and value:
+ if value and os.name != "nt":
try:
Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
except Exception:
- print "Renice failed"
+ pass
class UnRar(Extractor):
__name__ = "UnRar"
- __version__ = "1.03"
+ __version__ = "1.04"
__description__ = """Rar extractor plugin"""
__license__ = "GPLv3"
- __authors__ = [("RaNaN", "RaNaN@pyload.org")]
+ __authors__ = [("RaNaN", "RaNaN@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com")]
CMD = "unrar"
- # there are some more uncovered rar formats
- re_version = re.compile(r'UNRAR 5[\d.]+')
- re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I)
- re_partfiles = re.compile(r'.*\.(rar|r\d+)$', re.I)
- re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+')
- re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)')
- re_wrongpwd = re.compile(r'Corrupt file or wrong password|password incorrect', re.I)
+ EXTENSIONS = [".rar", ".zip", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2",
+ ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"]
+
+ #@NOTE: there are some more uncovered rar formats
+ re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I)
+ re_rarpart2 = re.compile(r'\.r(\d+)$', re.I)
+
+ re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)')
+ re_wrongpwd = re.compile(r'password', re.I)
re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I)
- @staticmethod
- def checkDeps():
+ @classmethod
+ def checkDeps(cls):
if os.name == "nt":
- UnRar.CMD = join(pypath, "UnRAR.exe")
- p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE)
+ cls.CMD = os.path.join(pypath, "UnRAR.exe")
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
else:
try:
- p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE)
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
- except OSError:
- # fallback to rar
- UnRar.CMD = "rar"
- p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE)
+ except OSError: #: fallback to rar
+ cls.CMD = "rar"
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
return True
- @staticmethod
- def getTargets(files_ids):
- result = []
+ @classmethod
+ def getTargets(cls, files_ids):
+ targets = []
- for file, id in files_ids:
- if not file.endswith(".rar"):
+ for filename, id in files_ids:
+ if not cls.isArchive(filename):
continue
- match = UnRar.re_splitfile.findall(file)
- if match:
- # only add first parts
- if int(match[0][1]) == 1:
- result.append((file, id))
- else:
- result.append((file, id))
-
- return result
-
+ m = cls.re_rarpart1.match(filename)
+ if not m or int(m.group(1)) is 1: #@NOTE: only add first part file
+ targets.append((filename, id))
- def init(self):
- self.passwordProtected = False
- self.headerProtected = False #: list files will not work without password
- self.smallestFile = None #: small file to test passwords
- self.password = "" #: save the correct password
+ return targets
def checkArchive(self):
- p = self.call_unrar("l", "-v", self.file)
+ p = self.call_cmd("l", "-v", self.target)
out, err = p.communicate()
+
if self.re_wrongpwd.search(err):
- self.passwordProtected = True
- self.headerProtected = True
return True
# output only used to check if passworded files are present
- if self.re_version.search(out):
- for attr, size, name in self.re_filelist5.findall(out):
- if attr.startswith("*"):
- self.passwordProtected = True
- return True
- else:
- for name, size, packed in self.re_filelist.findall(out):
- if name.startswith("*"):
- self.passwordProtected = True
- return True
+ for attr in self.re_filelist.findall(out):
+ if attr[0].startswith("*"):
+ return True
- self.listContent()
+ self.files = self.list()
if not self.files:
raise ArchiveError("Empty Archive")
@@ -116,22 +98,18 @@ class UnRar(Extractor):
def checkPassword(self, password):
# at this point we can only verify header protected files
- if self.headerProtected:
- p = self.call_unrar("l", "-v", self.file, password=password)
- out, err = p.communicate()
- if self.re_wrongpwd.search(err):
- return False
-
- return True
+ p = self.call_cmd("l", "-v", self.target, password=password)
+ out, err = p.communicate()
+ return False if self.re_wrongpwd.search(err) else True
- def extract(self, progress, password=None):
+ def extract(self, password=None):
command = "x" if self.fullpath else "e"
- p = self.call_unrar(command, self.file, self.out, password=password)
+ p = self.call_cmd(command, self.target, self.out, password=password)
+
renice(p.pid, self.renice)
- progress(0)
progressstring = ""
while True:
c = p.stdout.read(1)
@@ -140,21 +118,20 @@ class UnRar(Extractor):
break
# reading a percentage sign -> set progress and restart
if c == '%':
- progress(int(progressstring))
+ self.notifyProgress(int(progressstring))
progressstring = ""
# not reading a digit -> therefore restart
elif c not in digits:
progressstring = ""
# add digit to progressstring
else:
- progressstring = progressstring + c
- progress(100)
+ progressstring += c
# retrieve stderr
err = p.stderr.read()
if self.re_wrongpwd.search(err):
- raise WrongPassword
+ raise PasswordError
elif self.re_wrongcrc.search(err):
raise CRCError
@@ -162,50 +139,65 @@ class UnRar(Extractor):
elif err.strip(): #: raise error if anything is on stderr
raise ArchiveError(err.strip())
- if p.returncode:
+ if p.returncode != 0:
raise ArchiveError("Process terminated")
if not self.files:
- self.password = password
- self.listContent()
+ self.files = self.list(password)
def getDeleteFiles(self):
- if ".part" in basename(self.file):
- return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I))
- # get files which matches .r* and filter unsuited files out
- parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I))
- return filter(lambda x: self.re_partfiles.match(x), parts)
+ files = []
+
+ for i in [1, 2]:
+ try:
+ dir, name = os.path.split(self.target)
+ part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1)
+ filename = os.path.join(dir, name.replace(part, '*', 1))
+ files.extend(glob(filename))
+
+ except Exception:
+ continue
+ if self.target not in files:
+ files.insert(0, self.target)
- def listContent(self):
+ return files
+
+
+ def list(self, password=None):
command = "vb" if self.fullpath else "lb"
- p = self.call_unrar(command, "-v", self.file, password=self.password)
+
+ p = self.call_cmd(command, "-v", self.target, password=password)
out, err = p.communicate()
if "Cannot open" in err:
raise ArchiveError("Cannot open file")
if err.strip(): #: only log error at this point
- self.m.logError(err.strip())
+ self.manager.logError(err.strip())
result = set()
-
for f in decode(out).splitlines():
f = f.strip()
result.add(save_join(self.out, f))
- self.files = result
+ return list(result)
- def call_unrar(self, command, *xargs, **kwargs):
+ def call_cmd(self, command, *xargs, **kwargs):
args = []
+
# overwrite flag
- args.append("-o+") if self.overwrite else args.append("-o-")
+ if self.overwrite:
+ args.append("-o+")
+ else:
+ args.append("-o-")
+ if self.delete:
+ args.append("-or")
- if self.excludefiles:
- for word in self.excludefiles.split(';'):
- args.append("-x%s" % word)
+ for word in self.excludefiles:
+ args.append("-x%s" % word.strip())
# assume yes on all queries
args.append("-y")
@@ -216,10 +208,13 @@ class UnRar(Extractor):
else:
args.append("-p-")
+ if self.keepbroken:
+ args.append("-kb")
+
# NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
call = [self.CMD, command] + args + list(xargs)
- self.m.logDebug(" ".join(call))
- p = Popen(call, stdout=PIPE, stderr=PIPE)
+ self.manager.logDebug(" ".join(call))
+ p = Popen(call, stdout=PIPE, stderr=PIPE)
return p
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index 888ae7ebe..026503be5 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -1,51 +1,52 @@
# -*- coding: utf-8 -*-
+from __future__ import with_statement
+
+import os
import sys
import zipfile
-from module.plugins.internal.Extractor import Extractor, WrongPassword, ArchiveError
+from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
class UnZip(Extractor):
__name__ = "UnZip"
- __version__ = "1.02"
+ __version__ = "1.03"
__description__ = """Zip extractor plugin"""
__license__ = "GPLv3"
- __authors__ = [("RaNaN", "RaNaN@pyload.org")]
+ __authors__ = [("RaNaN", "RaNaN@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com")]
- @staticmethod
- def checkDeps():
- return sys.version_info[:2] >= (2, 6)
+ EXTENSIONS = [".zip", ".zip64"]
- @staticmethod
- def getTargets(files_ids):
- result = []
+ @classmethod
+ def checkDeps(cls):
+ return sys.version_info[:2] >= (2, 6)
- for file, id in files_ids:
- if file.endswith(".zip"):
- result.append((file, id))
- return result
+ @classmethod
+ def getTargets(cls, files_ids):
+ return [(filename, id) for filename, id in files_ids if cls.isArchive(filename)]
- def extract(self, progress, password=None):
+ def extract(self, password=None):
try:
- z = zipfile.ZipFile(self.file)
- self.files = z.namelist()
- z.extractall(self.out, pwd=password)
+ with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
+ z.setpassword(self.password)
+ if not z.testzip():
+ z.extractall(self.out)
+ self.files = z.namelist()
+ else:
+ raise CRCError
except (BadZipfile, LargeZipFile), e:
raise ArchiveError(e)
except RuntimeError, e:
- if e is "Bad password for file":
- raise WrongPassword
+ if "encrypted" in e:
+ raise PasswordError
else:
raise ArchiveError(e)
-
-
- def getDeleteFiles(self):
- return [self.file]
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-21 19:34:35 +0100