From 1f2ddda833a0d4d98c08ca69213262f4de6113e6 Mon Sep 17 00:00:00 2001 From: stickell Date: Wed, 18 Feb 2015 11:17:15 +0100 Subject: [180Upload] new offline pattern Should fix #1189 --- module/plugins/hoster/HundredEightyUploadCom.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/module/plugins/hoster/HundredEightyUploadCom.py b/module/plugins/hoster/HundredEightyUploadCom.py index 32d36ddb9..2f226721c 100644 --- a/module/plugins/hoster/HundredEightyUploadCom.py +++ b/module/plugins/hoster/HundredEightyUploadCom.py @@ -6,14 +6,15 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class HundredEightyUploadCom(XFSHoster): __name__ = "HundredEightyUploadCom" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'http://(?:www\.)?180upload\.com/\w{12}' __description__ = """180upload.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - + + OFFLINE_PATTERN = r'>File Not Found' getInfo = create_getInfo(HundredEightyUploadCom) -- cgit v1.2.3 From e98807ebce1220afdac21e7dfabe04f7ee2acdcf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 18 Feb 2015 22:09:19 +0100 Subject: [ClickAndLoad] Fix https://github.com/pyload/pyload/issues/1186 --- module/plugins/hooks/ClickAndLoad.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index 8b53872e4..a0eacd10f 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -21,7 +21,7 @@ def forward(source, destination): #: create_connection wrapper for python 2.5 socket module def create_connection(address, timeout=object(), source_address=None): if hasattr(socket, 'create_connection'): - if type(timeout) == object: + if type(timeout) is object: timeout = socket._GLOBAL_DEFAULT_TIMEOUT return socket.create_connection(address, timeout, source_address) @@ -29,11 +29,11 @@ def create_connection(address, timeout=object(), source_address=None): else: host, port = address err = None - for res in getaddrinfo(host, port, 0, SOCK_STREAM): + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None try: - sock = socket(af, socktype, proto) + sock = socket.socket(af, socktype, proto) if type(timeout) != object: sock.settimeout(timeout) if source_address: @@ -41,21 +41,18 @@ def create_connection(address, timeout=object(), source_address=None): sock.connect(sa) return sock - except socket.error, _: - err = _ + except socket.error, e: + err = e if sock is not None: sock.close() - - if err is not None: - raise err else: - raise socket.error("getaddrinfo returns an empty list") + raise socket.error(err or "getaddrinfo returns an empty list") class ClickAndLoad(Hook): __name__ = "ClickAndLoad" __type__ = "hook" - __version__ = "0.35" + __version__ = "0.36" __config__ = [("activated", "bool", "Activated" , True), ("port" , "int" , "Port" , 9666), @@ -76,7 +73,7 @@ class ClickAndLoad(Hook): if not self.config['webinterface']['activated']: return - ip = socket.gethostbyname(socket.gethostname()) if self.getConfig("extern") else "127.0.0.1" + ip = "0.0.0.0" if self.getConfig("extern") else "127.0.0.1" webport = int(self.config['webinterface']['port']) cnlport = self.getConfig('port') @@ -85,6 +82,7 @@ class ClickAndLoad(Hook): @threaded def proxy(self, ip, webport, cnlport): + self.logInfo(_("Proxy listening on %s:%s") % (ip, cnlport)) self.manager.startThread(self._server, ip, webport, cnlport) lock = Lock() lock.acquire() -- cgit v1.2.3 From b6e0ba974abe0d6bf712ecf5c0d116804050c9e3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 19 Feb 2015 00:55:28 +0100 Subject: [ZippyshareCom] Update get_checksum --- module/plugins/hoster/ZippyshareCom.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module/plugins/hoster/ZippyshareCom.py b/module/plugins/hoster/ZippyshareCom.py index a43f70cbd..615559989 100644 --- a/module/plugins/hoster/ZippyshareCom.py +++ b/module/plugins/hoster/ZippyshareCom.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class ZippyshareCom(SimpleHoster): __name__ = "ZippyshareCom" __type__ = "hoster" - __version__ = "0.72" + __version__ = "0.73" __pattern__ = r'http://www\d{0,2}\.zippyshare\.com/v(/|iew\.jsp.*key=)(?P[\w^_]+)' @@ -51,9 +51,9 @@ class ZippyshareCom(SimpleHoster): def get_checksum(self): try: - n = 2 - b = int(re.search(r'var b = (\d+)', self.html).group(1)) - checksum = int("%d3" % (n + n * 2 + b)) + b1 = eval(re.search(r'\.omg = (.+?);', self.html).group(1)) + b2 = eval(re.search(r'\* \((.+?)\)', self.html).group(1)) + checksum = b1 * b2 + 18 except Exception: self.error(_("Unable to calculate checksum")) -- cgit v1.2.3 From ada50c34aecdedbc0fcecb59ce941bc650ff02e5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 19 Feb 2015 01:11:30 +0100 Subject: [XFileSharingPro] Remove 180upload.com from HOSTER_BUILTIN --- module/plugins/hooks/XFileSharingPro.py | 4 ++-- module/plugins/hoster/HundredEightyUploadCom.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/module/plugins/hooks/XFileSharingPro.py b/module/plugins/hooks/XFileSharingPro.py index a1ee7f5ef..495140652 100644 --- a/module/plugins/hooks/XFileSharingPro.py +++ b/module/plugins/hooks/XFileSharingPro.py @@ -8,7 +8,7 @@ from module.plugins.Hook import Hook class XFileSharingPro(Hook): __name__ = "XFileSharingPro" __type__ = "hook" - __version__ = "0.31" + __version__ = "0.32" __config__ = [("activated" , "bool", "Activated" , True ), ("use_hoster_list" , "bool", "Load listed hosters only" , False), @@ -29,7 +29,7 @@ class XFileSharingPro(Hook): r'https?://(?:[^/]+\.)?(?P%s)/(?:user|folder)s?/\w+')} HOSTER_BUILTIN = [#WORKING HOSTERS: - "180upload.com", "backin.net", "eyesfile.ca", "file4safe.com", "fileband.com", "filedwon.com", + "backin.net", "eyesfile.ca", "file4safe.com", "fileband.com", "filedwon.com", "fileparadox.in", "filevice.com", "hostingbulk.com", "junkyvideo.com", "linestorage.com", "ravishare.com", "ryushare.com", "salefiles.com", "sendmyway.com", "sharesix.com", "thefile.me", "verzend.be", "xvidstage.com", #NOT TESTED: diff --git a/module/plugins/hoster/HundredEightyUploadCom.py b/module/plugins/hoster/HundredEightyUploadCom.py index 2f226721c..2a35a008f 100644 --- a/module/plugins/hoster/HundredEightyUploadCom.py +++ b/module/plugins/hoster/HundredEightyUploadCom.py @@ -13,7 +13,8 @@ class HundredEightyUploadCom(XFSHoster): __description__ = """180upload.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - + + OFFLINE_PATTERN = r'>File Not Found' -- cgit v1.2.3 From c61412841c8ea6c456f7d3472166d81132b56e95 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 19 Feb 2015 16:02:32 +0100 Subject: [LinkList] Rename to TXT --- module/plugins/container/LinkList.py | 69 ------------------------------------ module/plugins/container/TXT.py | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 69 deletions(-) delete mode 100644 module/plugins/container/LinkList.py create mode 100644 module/plugins/container/TXT.py diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py deleted file mode 100644 index ccb9b2fa3..000000000 --- a/module/plugins/container/LinkList.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- - -import codecs - -from module.plugins.Container import Container -from module.utils import fs_encode - - -class LinkList(Container): - __name__ = "LinkList" - __type__ = "container" - __version__ = "0.14" - - __pattern__ = r'.+\.txt$' - __config__ = [("flush" , "bool" , "Flush list after adding", False ), - ("encoding", "string", "File encoding" , "utf-8")] - - __description__ = """Read link lists in txt format""" - __license__ = "GPLv3" - __authors__ = [("spoob", "spoob@pyload.org"), - ("jeix", "jeix@hasnomail.com")] - - - def decrypt(self, pyfile): - try: - encoding = codecs.lookup(self.getConfig("encoding")).name - - except Exception: - encoding = "utf-8" - - file = fs_encode(pyfile.url.strip()) - txt = codecs.open(file, 'r', encoding) - curPack = "Parsed links from %s" % pyfile.name - packages = {curPack:[],} - - for link in txt.readlines(): - link = link.strip() - - if not link: - continue - - if link.startswith(";"): - continue - - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - - packages[curPack].append(link) - - txt.close() - - # empty packages fix - for key, value in packages.iteritems(): - if not value: - packages.pop(key, None) - - if self.getConfig("flush"): - try: - txt = open(file, 'wb') - txt.close() - - except IOError: - self.logWarning(_("Failed to flush list")) - - for name, links in packages.iteritems(): - self.packages.append((name, links, name)) diff --git a/module/plugins/container/TXT.py b/module/plugins/container/TXT.py new file mode 100644 index 000000000..585da8ac6 --- /dev/null +++ b/module/plugins/container/TXT.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import codecs + +from module.plugins.Container import Container +from module.utils import fs_encode + + +class TXT(Container): + __name__ = "TXT" + __type__ = "container" + __version__ = "0.15" + + __pattern__ = r'.+\.(txt|text)$' + __config__ = [("flush" , "bool" , "Flush list after adding", False ), + ("encoding", "string", "File encoding" , "utf-8")] + + __description__ = """Read link lists in plain text formats""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("jeix", "jeix@hasnomail.com")] + + + def decrypt(self, pyfile): + try: + encoding = codecs.lookup(self.getConfig("encoding")).name + + except Exception: + encoding = "utf-8" + + file = fs_encode(pyfile.url.strip()) + txt = codecs.open(file, 'r', encoding) + curPack = "Parsed links from %s" % pyfile.name + packages = {curPack:[],} + + for link in txt.readlines(): + link = link.strip() + + if not link: + continue + + if link.startswith(";"): + continue + + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + + packages[curPack].append(link) + + txt.close() + + # empty packages fix + for key, value in packages.iteritems(): + if not value: + packages.pop(key, None) + + if self.getConfig("flush"): + try: + txt = open(file, 'wb') + txt.close() + + except IOError: + self.logWarning(_("Failed to flush list")) + + for name, links in packages.iteritems(): + self.packages.append((name, links, name)) -- cgit v1.2.3 From 547e5991a861c68d3b87d59eda3aeb8b41116cbd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 20 Feb 2015 16:20:42 +0100 Subject: [ClickAndLoad] Improve --- module/plugins/hooks/ClickAndLoad.py | 81 ++++++++++++------------------------ 1 file changed, 27 insertions(+), 54 deletions(-) diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index a0eacd10f..0d95a801d 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import socket +import time from threading import Lock @@ -18,41 +19,11 @@ def forward(source, destination): destination.shutdown(socket.SHUT_WR) -#: create_connection wrapper for python 2.5 socket module -def create_connection(address, timeout=object(), source_address=None): - if hasattr(socket, 'create_connection'): - if type(timeout) is object: - timeout = socket._GLOBAL_DEFAULT_TIMEOUT - - return socket.create_connection(address, timeout, source_address) - - else: - host, port = address - err = None - for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - sock = None - try: - sock = socket.socket(af, socktype, proto) - if type(timeout) != object: - sock.settimeout(timeout) - if source_address: - sock.bind(source_address) - sock.connect(sa) - return sock - - except socket.error, e: - err = e - if sock is not None: - sock.close() - else: - raise socket.error(err or "getaddrinfo returns an empty list") - - +#@TODO: IPv6 support class ClickAndLoad(Hook): __name__ = "ClickAndLoad" __type__ = "hook" - __version__ = "0.36" + __version__ = "0.37" __config__ = [("activated", "bool", "Activated" , True), ("port" , "int" , "Port" , 9666), @@ -73,7 +44,7 @@ class ClickAndLoad(Hook): if not self.config['webinterface']['activated']: return - ip = "0.0.0.0" if self.getConfig("extern") else "127.0.0.1" + ip = "" if self.getConfig("extern") else "127.0.0.1" webport = int(self.config['webinterface']['port']) cnlport = self.getConfig('port') @@ -91,32 +62,34 @@ class ClickAndLoad(Hook): def _server(self, ip, webport, cnlport, thread): try: - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind((ip, cnlport)) - server_socket.listen(5) + try: + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - while True: - client_socket = server_socket.accept()[0] - dock_socket = create_connection(("127.0.0.1", webport)) + server_socket.bind((ip, cnlport)) + server_socket.listen(5) - self.manager.startThread(forward, dock_socket, client_socket) - self.manager.startThread(forward, client_socket, dock_socket) + while True: + client_socket = server_socket.accept()[0] + dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - except socket.error, e: - self.logDebug(e) - self._server(ip, webport, cnlport, thread) + dock_socket.connect(("127.0.0.1", webport)) - except Exception, e: - self.logError(e) + self.manager.startThread(forward, dock_socket, client_socket) + self.manager.startThread(forward, client_socket, dock_socket) - try: + except socket.timeout: + self.logDebug("Connection timed out, retrying...") + return self._server(ip, webport, cnlport, thread) + + finally: + server_socket.close() client_socket.close() dock_socket.close() - except Exception: - pass - try: - server_socket.close() - except Exception: - pass + except socket.error, e: + self.logError(e) + time.sleep(120) + self._server(ip, webport, cnlport, thread) + + except Exception, e: + self.logError(e) -- cgit v1.2.3 From d371d98a4a6a803efce3f1cfc779ee2b887cd4e7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 20 Feb 2015 16:21:36 +0100 Subject: [ClickAndLoad] Improve (2) --- module/plugins/hooks/ClickAndLoad.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index 0d95a801d..6691bb1fe 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -90,6 +90,3 @@ class ClickAndLoad(Hook): self.logError(e) time.sleep(120) self._server(ip, webport, cnlport, thread) - - except Exception, e: - self.logError(e) -- cgit v1.2.3 From 56caf611ab8f733ac8a7ea09d2a3d492bd485762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20H=C3=B6rnlein?= Date: Sun, 22 Feb 2015 03:09:40 +0100 Subject: Some more changes --- module/plugins/hooks/ExtractArchive.py | 161 +++++++++++++++++++-------------- module/plugins/internal/Extractor.py | 30 +++--- module/plugins/internal/SevenZip.py | 38 ++++---- module/plugins/internal/UnRar.py | 71 +++++++-------- module/plugins/internal/UnZip.py | 8 +- 5 files changed, 170 insertions(+), 138 deletions(-) diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 3ea8839dc..a1e85ba57 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -104,13 +104,14 @@ class ArchiveQueue(object): class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.29" + __version__ = "1.30" __config__ = [("activated" , "bool" , "Activated" , True ), ("fullpath" , "bool" , "Extract with full paths" , True ), ("overwrite" , "bool" , "Overwrite files" , False ), ("keepbroken" , "bool" , "Try to extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), + ("repair" , "bool" , "Repair broken archives (rar required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), ("usepasswordfile" , "bool" , "Use password file" , True ), ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), ("delete" , "bool" , "Delete archive when successfully extracted", False ), @@ -128,35 +129,32 @@ class ExtractArchive(Hook): ("Immenz" , "immenz@gmx.net" )] - event_list = ["allDownloadsProcessed"] + event_list = ["allDownloadsProcessed","packageDeleted"] NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def setup(self): self.queue = ArchiveQueue(self, "Queue") self.failed = ArchiveQueue(self, "Failed") - self.interval = 60 - self.extracting = False - self.extractors = [] - self.passwords = [] + self.interval = 60 + self.extracting = False + self.lastPackage = False + self.extractors = [] + self.passwords = [] + self.repair = False def coreReady(self): - # self.extracting = False - for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) klass = getattr(module, p) if klass.isUsable(): self.extractors.append(klass) + if klass.REPAIR: + self.repair = self.getConfig("repair") except OSError, e: if e.errno == 2: @@ -173,37 +171,49 @@ class ExtractArchive(Hook): if self.extractors: self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) - - if self.getConfig("waitall"): - self.extractPackage(*self.queue.get()) #: Resume unfinished extractions - else: - super(ExtractArchive, self).initPeriodical() - + self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) + @threaded + def extractQueued(self,thread): + packages = self.queue.get() + while packages: + if self.lastPackage: # called from allDownloadsProcessed + self.lastPackage = False + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + self.manager.dispatchEvent("all_archives_extracted") + self.manager.dispatchEvent("all_archives_processed") + else: + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + pass - def periodical(self): - if not self.extracting: - self.extractPackage(*self.queue.get()) + packages = self.queue.get() # check for packages added during extraction @Expose def extractPackage(self, *ids): """ Extract packages with given id""" - self.manager.startThread(self.extract, ids) + for id in ids: + self.queue.add(id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() + + + def packageDeleted(self, pid): + self.queue.remove(pid) def packageFinished(self, pypack): self.queue.add(pypack.id) + if not self.getConfig("waitall") and not self.extracting: + self.extractQueued() - @threaded - def allDownloadsProcessed(self, thread): - if self.extract(self.queue.get(), thread): #@NOTE: check only if all gone fine, no failed reporting for now - self.manager.dispatchEvent("all_archives_extracted") - - self.manager.dispatchEvent("all_archives_processed") + def allDownloadsProcessed(self): + self.lastPackage = True + if not self.extracting: + self.extractQueued() def extract(self, ids, thread=None): @@ -315,13 +325,17 @@ class ExtractArchive(Hook): if recursive and os.path.isfile(file): new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target - + + pyfile = self.core.files.getFile(fid) + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files) + files_ids = new_files_ids # also check extracted files if matched: if success: extracted.append(pid) self.manager.dispatchEvent("package_extracted", pypack) + else: failed.append(pid) self.manager.dispatchEvent("package_extract_failed", pypack) @@ -352,50 +366,65 @@ class ExtractArchive(Hook): encrypted = False try: - try: - archive.check() - - except CRCError, e: - self.logDebug(name, e) - self.logInfo(name, _("Header protected")) - - if self.getConfig("repair"): - self.logWarning(name, _("Repairing...")) - - pyfile.setCustomStatus(_("repairing")) - pyfile.setProgress(0) - - repaired = archive.repair() - - pyfile.setProgress(100) - - if not repaired and not self.getConfig("keepbroken"): - raise CRCError("Archive damaged") - - except PasswordError: - self.logInfo(name, _("Password protected")) - encrypted = True - - except ArchiveError, e: - raise ArchiveError(e) - - self.logDebug("Password: %s" % (password or "No provided")) + self.logDebug("Password: %s" % (password or "None provided")) + passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig("usepasswordfile") else [password] + for pw in passwords: + try: + if self.getConfig("test") or self.repair: + pyfile.setCustomStatus(_("testing")) + if pw: + self.logDebug("Testing with password: %s" % pw) + pyfile.setProgress(0) + archive.test(pw) + pyfile.setProgress(100) + else: + archive.check(pw) + + self.addPassword(pw) + break + + except PasswordError: + if not encrypted: + self.logInfo(name, _("Password protected")) + encrypted = True + + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("CRC Error")) + + if self.repair: + self.logWarning(name, _("Repairing...")) + + pyfile.setCustomStatus(_("repairing")) + pyfile.setProgress(0) + repaired = archive.repair() + pyfile.setProgress(100) + + if not repaired and not self.getConfig("keepbroken"): + raise CRCError("Archive damaged") + + self.addPassword(pw) + break + + raise CRCError("Archive damaged") + + except ArchiveError, e: + raise ArchiveError(e) pyfile.setCustomStatus(_("extracting")) pyfile.setProgress(0) if not encrypted or not self.getConfig("usepasswordfile"): + self.logDebug("Extracting using password: %s" % (password or "None")) archive.extract(password) else: for pw in filter(None, uniqify([password] + self.getPasswords(False))): try: - self.logDebug("Try password: %s" % pw) + self.logDebug("Extracting using password: %s" % pw) - ispw = archive.isPassword(pw) - if ispw or ispw is None: - archive.extract(pw) - self.addPassword(pw) - break + archive.extract(pw) + self.addPassword(pw) + break except PasswordError: self.logDebug("Password was wrong") @@ -419,9 +448,7 @@ class ExtractArchive(Hook): self.logDebug("%s does not exists" % f) self.logInfo(name, _("Extracting finished")) - extracted_files = archive.files or archive.list() - self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) return extracted_files diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index b445f1497..8bf1875cf 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -19,7 +19,7 @@ class PasswordError(Exception): class Extractor: __name__ = "Extractor" - __version__ = "0.20" + __version__ = "0.21" __description__ = """Base extractor plugin""" __license__ = "GPLv3" @@ -30,6 +30,7 @@ class Extractor: EXTENSIONS = [] VERSION = "" + REPAIR = False @classmethod @@ -90,23 +91,24 @@ class Extractor: def check(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. + """Quick Check by listing content of archive. + Raises error if password is needed, integrity is questionable or else. - :return: boolean + :raises PasswordError + :raises CRCError :raises ArchiveError """ - raise PasswordError - - - def isPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean + raise NotImplementedError + + def test(self): + """Testing with Extractors buildt-in method + Raises error if password is needed, integrity is questionable or else. + + :raises PasswordError + :raises CRCError + :raises ArchiveError """ - return None + raise NotImplementedError def repair(self): diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 7ad6b0d7a..bfa7f3943 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.08" + __version__ = "0.09" __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" @@ -32,9 +32,9 @@ class SevenZip(UnRar): #@NOTE: there are some more uncovered 7z formats re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') - re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I) - re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I) - re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) + re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I) + re_wrongcrc = re.compile(r'CRC Failed|Can not open file', re.I) + re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) @classmethod @@ -52,36 +52,38 @@ class SevenZip(UnRar): return True - def check(self): + def test(self, password): file = fs_encode(self.filename) - p = self.call_cmd("t", file) + # 7z can't distinguish crc and pw error in test + p = self.call_cmd("l", "-slt", file) out, err = p.communicate() - if p.returncode > 1: + if self.re_wrongpwd.search(out): + raise PasswordError + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): raise CRCError(err) + + + def check(self, password): + file = fs_encode(self.filename) + p = self.call_cmd("l", "-slt", file) out, err = p.communicate() - if p.returncode > 1: - raise ArchiveError(_("Process return code: %d") % p.returncode) - # check if output or error macthes the 'wrong password'-Regexp if self.re_wrongpwd.search(out): raise PasswordError - # check if output matches 'Encrypted = +' if self.re_wrongcrc.search(out): raise CRCError(_("Header protected")) - def isPassword(self, password): - p = self.call_cmd("l", fs_encode(self.filename), password=password) - p.communicate() - return p.returncode == 0 - - def repair(self): return False @@ -142,7 +144,7 @@ class SevenZip(UnRar): #set a password if "password" in kwargs and kwargs["password"]: - args.append("-p'%s'" % kwargs["password"]) + args.append("-p%s" % kwargs["password"]) else: args.append("-p-") diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 54d64c430..ca8fdd326 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.13" + __version__ = "1.14" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -33,34 +33,40 @@ class UnRar(Extractor): CMD = "unrar" VERSION = "" - EXTENSIONS = [".rar"] - re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I) + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?',re.I) re_filefixed = re.compile(r'Building (.+)') re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) re_wrongpwd = re.compile(r'password', re.I) - re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I) - re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I) + re_version = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I) @classmethod def isUsable(cls): if os.name == "nt": - cls.CMD = os.path.join(pypath, "UnRAR.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - else: try: + cls.CMD = os.path.join(pypath, "RAR.exe") p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) out, err = p.communicate() - - except OSError: #: fallback to rar - cls.CMD = "rar" + cls.__name__ = "RAR" + cls.REPAIR = True + except OSError: + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + else: + try: + p = Popen(["rar"], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + cls.__name__ = "RAR" + cls.REPAIR = True + except OSError: #: fallback to unrar p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) out, err = p.communicate() @@ -74,13 +80,25 @@ class UnRar(Extractor): multipart = cls.re_multipart.search(filename) if multipart: # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive - return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True + return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1 and not multipart.group(3)) else True return False - def check(self): - p = self.call_cmd("l", "-v", fs_encode(self.filename)) + def test(self, password): + p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password) + self._progress(p) + err = p.stderr.read().strip() + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): + raise CRCError(err) + + + def check(self, password): + p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): @@ -95,35 +113,14 @@ class UnRar(Extractor): raise PasswordError - def isPassword(self, password): - # at this point we can only verify header protected files - p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) - out, err = p.communicate() - return False if self.re_wrongpwd.search(err) else True - - def repair(self): p = self.call_cmd("rc", fs_encode(self.filename)) # communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() - if err or p.returncode: - p = self.call_cmd("r", fs_encode(self.filename)) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err or p.returncode: - return False - else: - dir = os.path.dirname(filename) - name = re_filefixed.search(out).group(1) - - self.filename = os.path.join(dir, name) - + return False return True diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 704b49ca0..d95afbc70 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.10" + __version__ = "1.11" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" @@ -34,7 +34,11 @@ class UnZip(Extractor): return z.namelist() - def check(self): + def check(self, password): + pass + + + def test(self): with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: badfile = z.testzip() -- cgit v1.2.3 From 8701d26fed036f70f99fa212dff172c3fd0bce28 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 22 Feb 2015 16:42:31 +0100 Subject: [HotFolder] Fixup (thx zapp-brannigan) --- module/plugins/hooks/HotFolder.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/module/plugins/hooks/HotFolder.py b/module/plugins/hooks/HotFolder.py index 1ff02c319..302bc909f 100644 --- a/module/plugins/hooks/HotFolder.py +++ b/module/plugins/hooks/HotFolder.py @@ -2,10 +2,9 @@ from __future__ import with_statement +import os import time -from os import listdir, makedirs -from os.path import exists, isfile, join from shutil import move from module.plugins.Hook import Hook @@ -15,12 +14,12 @@ from module.utils import fs_encode, save_join class HotFolder(Hook): __name__ = "HotFolder" __type__ = "hook" - __version__ = "0.12" + __version__ = "0.13" - __config__ = [("folder", "str", "Folder to observe", "container"), - ("watch_file", "bool", "Observe link file", False), - ("keep", "bool", "Keep added containers", True), - ("file", "str", "Link file", "links.txt")] + __config__ = [("folder" , "str" , "Folder to observe" , "container"), + ("watch_file", "bool", "Observe link file" , False ), + ("keep" , "bool", "Keep added containers", True ), + ("file" , "str" , "Link file" , "links.txt")] __description__ = """Observe folder and file for changes and add container and links""" __license__ = "GPLv3" @@ -28,36 +27,40 @@ class HotFolder(Hook): def setup(self): - self.interval = 10 + self.interval = 30 def periodical(self): folder = fs_encode(self.getConfig("folder")) + file = fs_encode(self.getConfig("file")) try: - if not exists(join(folder, "finished")): - makedirs(join(folder, "finished")) + if not os.path.isdir(os.path.join(folder, "finished")): + os.makedirs(os.path.join(folder, "finished")) if self.getConfig("watch_file"): - file = fs_encode(self.getConfig("file")) with open(file, "a+") as f: + f.seek(0) content = f.read().strip() if content: - name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y")) + f = open(file, "wb") + f.close() + + name = "%s_%s.txt" % (file, time.strftime("%H-%M-%S_%d%b%Y")) with open(save_join(folder, "finished", name), "wb") as f: f.write(content) self.core.api.addPackage(f.name, [f.name], 1) - for f in listdir(folder): - path = join(folder, f) + for f in os.listdir(folder): + path = os.path.join(folder, f) - if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): + if not os.path.isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): continue - newpath = join(folder, "finished", f if self.getConfig("keep") else "tmp_" + f) + newpath = os.path.join(folder, "finished", f if self.getConfig("keep") else "tmp_" + f) move(path, newpath) self.logInfo(_("Added %s from HotFolder") % f) -- cgit v1.2.3