Some file encoding fixup + optimizations

11 files changed, 103 insertions, 89 deletions

diff --git a/module/plugins/captcha/captcha.py b/module/plugins/captcha/captcha.py
index e89ec2e81..1874ba07d 100644
--- a/module/plugins/captcha/captcha.py
+++ b/module/plugins/captcha/captcha.py
@@ -4,6 +4,7 @@ from __future__ import with_statement
 
 try:
     from PIL import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin
+
 except ImportError:
     import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin
 
@@ -12,13 +13,13 @@ import os
 import subprocess
 #import tempfile
 
-from os.path import abspath, join
+from module.utils import save_join
 
 
 class OCR(object):
     __name__    = "OCR"
     __type__    = "ocr"
-    __version__ = "0.10"
+    __version__ = "0.11"
 
     __description__ = """OCR base plugin"""
     __license__     = "GPLv3"
@@ -58,11 +59,11 @@ class OCR(object):
     def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True):
         #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
         try:
-            tmpTif = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb")
+            tmpTif = open(save_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb")
             tmpTif.close()
 
             #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
-            tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb")
+            tmpTxt = open(save_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb")
             tmpTxt.close()
 
         except IOError, e:
@@ -73,15 +74,15 @@ class OCR(object):
         self.image.save(tmpTif.name, 'TIFF')
 
         if os.name == "nt":
-            tessparams = [join(pypath, "tesseract", "tesseract.exe")]
+            tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")]
         else:
             tessparams = ["tesseract"]
 
-        tessparams.extend( [abspath(tmpTif.name), abspath(tmpTxt.name).replace(".txt", "")] )
+        tessparams.extend( [os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")] )
 
         if subset and (digits or lowercase or uppercase):
             #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset")
-            with open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub:
+            with open(save_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub:
                 tmpSub.write("tessedit_char_whitelist ")
 
                 if digits:
@@ -93,7 +94,7 @@ class OCR(object):
 
                 tmpSub.write("\n")
                 tessparams.append("nobatch")
-                tessparams.append(abspath(tmpSub.name))
+                tessparams.append(os.path.abspath(tmpSub.name))
 
         self.logger.debug("run tesseract")
         self.run(tessparams)
diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py
index c6e642cca..e7ad8f761 100644
--- a/module/plugins/container/CCF.py
+++ b/module/plugins/container/CCF.py
@@ -11,12 +11,13 @@ from urllib2 import build_opener
 from MultipartPostHandler import MultipartPostHandler
 
 from module.plugins.Container import Container
-from module.utils import save_join
+from module.utils import fs_encode, save_join
 
 
 class CCF(Container):
     __name__    = "CCF"
-    __version__ = "0.21"
+    __type__    = "container"
+    __version__ = "0.22"
 
     __pattern__ = r'.+\.ccf$'
 
@@ -26,17 +27,17 @@ class CCF(Container):
 
 
     def decrypt(self, pyfile):
-        infile = pyfile.url.replace("\n", "")
-
+        file   = fs_encode(pyfile.url.strip())
         opener = build_opener(MultipartPostHandler)
-        params = {"src": "ccf",
-            "filename": "test.ccf",
-            "upload": open(infile, "rb")}
-        tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', params).read()
+
+        tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php',
+                                      {'src'     : "ccf",
+                                       'filename': "test.ccf",
+                                       'upload'  : open(file, "rb")}).read()
 
         download_folder = self.config['general']['download_folder']
+        tempdlc_name    = save_join(download_folder, "tmp_%s.dlc" % pyfile.name)
 
-        tempdlc_name = save_join(download_folder, "tmp_%s.dlc" % pyfile.name)
         with open(tempdlc_name, "w") as tempdlc:
             tempdlc.write(re.search(r'<dlc>(.*)</dlc>', tempdlc_content, re.S).group(1))
 
diff --git a/module/plugins/container/DLC.py b/module/plugins/container/DLC.py
index 53349c5c7..184a7b25a 100644
--- a/module/plugins/container/DLC.py
+++ b/module/plugins/container/DLC.py
@@ -9,13 +9,15 @@ from base64 import standard_b64decode
 from Crypto.Cipher import AES
 
 from module.plugins.Container import Container
-from module.utils import decode
+from module.utils import decode, fs_encode
 
 
 class DLC(Container):
-    __name__        = "DLC"
-    __version__     = "0.22"
-    __pattern__     = r'.+\.dlc$'
+    __name__    = "DLC"
+    __type__    = "container"
+    __version__ = "0.23"
+
+    __pattern__ = r'.+\.dlc$'
 
     __description__ = """DLC container decrypter plugin"""
     __license__     = "GPLv3"
@@ -26,31 +28,33 @@ class DLC(Container):
                        ("Walter Purcaro", "vuolter@gmail.com")]
 
 
-    def setup(self):
-        self.key     = "cb99b5cbc24db398"
-        self.iv      = "9bc24cb995cb8db3"
-        self.api_url = "http://service.jdownloader.org/dlcrypt/service.php?srcType=dlc&destType=pylo&data="
+    KEY     = "cb99b5cbc24db398"
+    IV      = "9bc24cb995cb8db3"
+    API_URL = "http://service.jdownloader.org/dlcrypt/service.php?srcType=dlc&destType=pylo&data=%s"
 
 
     def decrypt(self, pyfile):
-        with open(pyfile.url.replace("\n", "")) as dlc:
+        file = fs_encode(pyfile.url.strip())
+        with open(file) as dlc:
             data = dlc.read().strip()
 
         data += '=' * (-len(data) % 4)
 
         dlckey  = data[-88:]
-        dlcdata = data[:-88]
-        dlcdata = standard_b64decode(dlcdata)
+        dlcdata = standard_b64decode(data[:-88])
+
+        try:
+            rc = re.search(r'<rc>(.+)</rc>', self.load(self.API_URL % dlckey)).group(1)
+
+        except Exception:
+            self.fail(_("DLC file is corrupted"))
 
-        rc = self.load(self.api_url + dlckey)
-        rc = re.search(r'<rc>(.+)</rc>', rc).group(1)
-        rc = standard_b64decode(rc)
+        else:
+            rc = standard_b64decode(rc)
 
-        obj    = AES.new(self.key, AES.MODE_CBC, self.iv)
-        dlckey = obj.decrypt(rc)
-        obj    = AES.new(dlckey, AES.MODE_CBC, dlckey)
+        dlckey = AES.new(self.KEY, AES.MODE_CBC, self.IV).decrypt(rc)
 
-        self.data     = standard_b64decode(obj.decrypt(dlcdata))
+        self.data     = standard_b64decode(AES.new(dlckey, AES.MODE_CBC, dlckey).decrypt(dlcdata))
         self.packages = [(entry[0] if entry[0] else pyfile.name, entry[1], entry[0] if entry[0] else pyfile.name) \
                          for entry in self.getPackages()]
 
diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py
index 9c76c4341..cff1b9915 100644
--- a/module/plugins/container/LinkList.py
+++ b/module/plugins/container/LinkList.py
@@ -8,11 +8,12 @@ from module.utils import fs_encode
 
 class LinkList(Container):
     __name__    = "LinkList"
-    __version__ = "0.13"
+    __type__    = "container"
+    __version__ = "0.14"
 
     __pattern__ = r'.+\.txt$'
-    __config__ = [("clear", "bool", "Clear Linklist after adding", False),
-                  ("encoding", "string", "File encoding (default utf-8)", "")]
+    __config__ = [("clear"   , "bool"  , "Clear Linklist after adding"  , False),
+                  ("encoding", "string", "File encoding (default utf-8)", ""   )]
 
     __description__ = """Read link lists in txt format"""
     __license__     = "GPLv3"
@@ -22,38 +23,42 @@ class LinkList(Container):
 
     def decrypt(self, pyfile):
         try:
-            file_enc = codecs.lookup(self.getConfig("encoding")).name
-        except Exception:
-            file_enc = "utf-8"
+            encoding = codecs.lookup(self.getConfig("encoding")).name
 
-        file_name = fs_encode(pyfile.url)
+        except Exception:
+            encoding = "utf-8"
 
-        txt = codecs.open(file_name, 'r', file_enc)
-        links = txt.readlines()
+        file    = fs_encode(pyfile.url.strip())
+        txt     = codecs.open(file, 'r', encoding)
+        links   = txt.readlines()
         curPack = "Parsed links from %s" % pyfile.name
 
         packages = {curPack:[],}
 
         for link in links:
             link = link.strip()
+
             if not link:
                 continue
 
             if link.startswith(";"):
                 continue
+
             if link.startswith("[") and link.endswith("]"):
                 # new package
                 curPack = link[1:-1]
                 packages[curPack] = []
                 continue
+
             packages[curPack].append(link)
+
         txt.close()
 
         # empty packages fix
 
         delete = []
 
-        for key,value in packages.iteritems():
+        for key, value in packages.iteritems():
             if not value:
                 delete.append(key)
 
@@ -62,8 +67,9 @@ class LinkList(Container):
 
         if self.getConfig("clear"):
             try:
-                txt = open(file_name, 'wb')
+                txt = open(file, 'wb')
                 txt.close()
+
             except Exception:
                 self.logWarning(_("LinkList could not be cleared"))
 
diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py
index 0c36293ec..20a0d34cd 100644
--- a/module/plugins/container/RSDF.py
+++ b/module/plugins/container/RSDF.py
@@ -6,13 +6,16 @@ import base64
 import binascii
 import re
 
+from Crypto.Cipher import AES
+
 from module.plugins.Container import Container
 from module.utils import fs_encode
 
 
 class RSDF(Container):
     __name__    = "RSDF"
-    __version__ = "0.25"
+    __type__    = "container"
+    __version__ = "0.26"
 
     __pattern__ = r'.+\.rsdf$'
 
@@ -22,35 +25,30 @@ class RSDF(Container):
                        ("spoob", "spoob@pyload.org")]
 
 
-    def decrypt(self, pyfile):
-
-        from Crypto.Cipher import AES
+    KEY = "8C35192D964DC3182C6F84F3252239EB4A320D2500000000"
+    IV  = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
 
-        infile = fs_encode(pyfile.url.replace("\n", ""))
-        Key = binascii.unhexlify('8C35192D964DC3182C6F84F3252239EB4A320D2500000000')
 
-        IV = binascii.unhexlify('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
+    def decrypt(self, pyfile):
+        Key       = binascii.unhexlify(self.KEY)
         IV_Cipher = AES.new(Key, AES.MODE_ECB)
-        IV = IV_Cipher.encrypt(IV)
+        IV        = IV_Cipher.encrypt(binascii.unhexlify(self.IV))
 
         obj = AES.new(Key, AES.MODE_CFB, IV)
 
         try:
-            with open(infile, 'r') as rsdf:
+            file = fs_encode(pyfile.url.strip())
+            with open(file, 'r') as rsdf:
                 data = rsdf.read()
+
         except IOError, e:
             self.fail(e)
 
-        if re.search(r"<title>404 - Not Found</title>", data) is None:
-            data = binascii.unhexlify(''.join(data.split()))
-            data = data.splitlines()
+        if re.search(r"<title>404 - Not Found</title>", data):
+            return
 
-            for link in data:
-                if not link:
-                    continue
-                link = base64.b64decode(link)
-                link = obj.decrypt(link)
+        for link in binascii.unhexlify(''.join(data.split())).splitlines():
+            if not link:
+                link         = obj.decrypt(base64.b64decode(link))
                 decryptedUrl = link.replace('CCF: ', '')
                 self.urls.append(decryptedUrl)
-
-            self.logDebug("Adding package %s with %d links" % (pyfile.package().name, len(self.urls)))
diff --git a/module/plugins/crypter/RelinkUs.py b/module/plugins/crypter/RelinkUs.py
index a6014c866..2165445b6 100644
--- a/module/plugins/crypter/RelinkUs.py
+++ b/module/plugins/crypter/RelinkUs.py
@@ -9,12 +9,13 @@ import os
 
 from Crypto.Cipher import AES
 from module.plugins.Crypter import Crypter
+from module.utils import save_join
 
 
 class RelinkUs(Crypter):
     __name__    = "RelinkUs"
     __type__    = "crypter"
-    __version__ = "3.11"
+    __version__ = "3.12"
 
     __pattern__ = r'http://(?:www\.)?relink\.us/(f/|((view|go)\.php\?id=))(?P<ID>.+)'
     __config__  = [("use_subfolder", "bool", "Save package to subfolder", True),
@@ -219,7 +220,7 @@ class RelinkUs(Crypter):
             try:
                 dlc = self.load(container_url)
                 dlc_filename = self.fileid + ".dlc"
-                dlc_filepath = os.path.join(self.config['general']['download_folder'], dlc_filename)
+                dlc_filepath = save_join(self.config['general']['download_folder'], dlc_filename)
                 with open(dlc_filepath, "wb") as f:
                     f.write(dlc)
                 package_links.append(dlc_filepath)
diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py
index b2b4548a2..3d9a1e811 100644
--- a/module/plugins/hooks/ExternalScripts.py
+++ b/module/plugins/hooks/ExternalScripts.py
@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 
+import os
 import subprocess
 
 from itertools import chain
-from os import listdir, access, X_OK, makedirs
-from os.path import join, exists, basename, abspath
 
 from module.plugins.Hook import Hook
 from module.utils import save_join
@@ -47,40 +46,40 @@ class ExternalScripts(Hook):
         for folder in folders:
             self.scripts[folder] = []
 
-            self.initPluginType(folder, join(pypath, 'scripts', folder))
-            self.initPluginType(folder, join('scripts', folder))
+            self.initPluginType(folder, os.path.join(pypath, 'scripts', folder))
+            self.initPluginType(folder, os.path.join('scripts', folder))
 
         for script_type, names in self.scripts.iteritems():
             if names:
-                self.logInfo(_("Installed scripts for"), script_type, ", ".join([basename(x) for x in names]))
+                self.logInfo(_("Installed scripts for"), script_type, ", ".join([os.path.basename(x) for x in names]))
 
 
     def initPluginType(self, folder, path):
-        if not exists(path):
+        if not os.path.exists(path):
             try:
-                makedirs(path)
+                os.makedirs(path)
             except Exception:
                 self.logDebug("Script folder %s not created" % folder)
                 return
 
-        for f in listdir(path):
+        for f in os.listdir(path):
             if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"):
                 continue
 
-            if not access(join(path, f), X_OK):
+            if not os.access(os.path.join(path, f), os.X_OK):
                 self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f))
 
-            self.scripts[folder].append(join(path, f))
+            self.scripts[folder].append(os.path.join(path, f))
 
 
     def callScript(self, script, *args):
         try:
             cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args]
-            self.logDebug("Executing", abspath(script), " ".join(cmd))
+            self.logDebug("Executing", os.path.abspath(script), " ".join(cmd))
             #output goes to pyload
             subprocess.Popen(cmd, bufsize=-1)
         except Exception, e:
-            self.logError(_("Error in %(script)s: %(error)s") % {"script": basename(script), "error": e})
+            self.logError(_("Error in %(script)s: %(error)s") % {"script": os.path.basename(script), "error": e})
 
 
     def downloadPreparing(self, pyfile):
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
index c24173964..11427109b 100644
--- a/module/plugins/hooks/ExtractArchive.py
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -49,7 +49,7 @@ if os.name != "nt":
 
 from module.plugins.Hook import Hook, threaded, Expose
 from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError
-from module.utils import fs_decode, save_join, uniqify
+from module.utils import fs_decode, fs_encode, save_join, uniqify
 
 
 class ExtractArchive(Hook):
@@ -371,7 +371,9 @@ class ExtractArchive(Hook):
     def reloadPasswords(self):
         try:
             passwords = []
-            with open(self.getConfig("passwordfile")) as f:
+
+            file = fs_encode(self.getConfig("passwordfile"))
+            with open(file) as f:
                 for pw in f.read().splitlines():
                     passwords.append(pw)
 
@@ -388,7 +390,8 @@ class ExtractArchive(Hook):
         try:
             self.passwords = uniqify([password] + self.passwords)
 
-            with open(self.getConfig("passwordfile"), "wb") as f:
+            file = fs_encode(self.getConfig("passwordfile"))
+            with open(file, "wb") as f:
                 for pw in self.passwords:
                     f.write(pw + '\n')
 
diff --git a/module/plugins/hooks/HotFolder.py b/module/plugins/hooks/HotFolder.py
index b0b59e2ba..1ff02c319 100644
--- a/module/plugins/hooks/HotFolder.py
+++ b/module/plugins/hooks/HotFolder.py
@@ -39,7 +39,8 @@ class HotFolder(Hook):
                 makedirs(join(folder, "finished"))
 
             if self.getConfig("watch_file"):
-                with open(fs_encode(self.getConfig("file")), "a+") as f:
+                file = fs_encode(self.getConfig("file"))
+                with open(file, "a+") as f:
                     content = f.read().strip()
 
                 if content:
diff --git a/module/plugins/hooks/MergeFiles.py b/module/plugins/hooks/MergeFiles.py
index 4de45f958..9f1348485 100644
--- a/module/plugins/hooks/MergeFiles.py
+++ b/module/plugins/hooks/MergeFiles.py
@@ -8,13 +8,13 @@ import re
 from traceback import print_exc
 
 from module.plugins.Hook import Hook, threaded
-from module.utils import save_join, fs_encode
+from module.utils import save_join
 
 
 class MergeFiles(Hook):
     __name__    = "MergeFiles"
     __type__    = "hook"
-    __version__ = "0.13"
+    __version__ = "0.14"
 
     __config__ = [("activated", "bool", "Activated", True)]
 
@@ -65,7 +65,7 @@ class MergeFiles(Hook):
                     pyfile.setStatus("processing")
 
                     try:
-                        with open(os.path.join(download_folder, splitted_file), "rb") as s_file:
+                        with open(save_join(download_folder, splitted_file), "rb") as s_file:
                             size_written = 0
                             s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file)))
                             while True:
diff --git a/module/plugins/hoster/PremiumTo.py b/module/plugins/hoster/PremiumTo.py
index 39e7522e2..b66b9a505 100644
--- a/module/plugins/hoster/PremiumTo.py
+++ b/module/plugins/hoster/PremiumTo.py
@@ -41,10 +41,10 @@ class PremiumTo(MultiHoster):
         err = ''
         if self.req.http.code == '420':
             # Custom error code send - fail
-            lastDownload = fs_encode(self.lastDownload)
-            with open(lastDownload, "rb") as f:
+            file = fs_encode(self.lastDownload)
+            with open(file, "rb") as f:
                 err = f.read(256).strip()
-            remove(lastDownload)
+            remove(file)
 
         if err:
             self.fail(err)