plugin cleanups

12 files changed, 38 insertions, 60 deletions

diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 720d64091..9417a4bfd 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -402,7 +402,7 @@ class Plugin(Base):
         return result
 
 
-    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, utf8=False, decode=False):
+    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
         """Load content at url and returns it
 
         :param url:
@@ -411,7 +411,6 @@ class Plugin(Base):
         :param ref:
         :param cookies:
         :param just_header: if True only the header will be retrieved and returned as dict
-        :param utf8: Deprecated
         :param decode: Wether to decode the output according to http header, should be True in most cases
         :return: Loaded content
         """
@@ -419,7 +418,7 @@ class Plugin(Base):
         #utf8 vs decode -> please use decode attribute in all future plugins
         if type(url) == unicode: url = str(url)
 
-        res = self.req.load(url, get, post, ref, cookies, just_header, decode=utf8 or decode)
+        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
 
         if self.core.debug:
             from inspect import currentframe
diff --git a/module/plugins/accounts/FileserveCom.py b/module/plugins/accounts/FileserveCom.py
index 316c2446e..5e5068f22 100644
--- a/module/plugins/accounts/FileserveCom.py
+++ b/module/plugins/accounts/FileserveCom.py
@@ -20,11 +20,7 @@
 from time import mktime, strptime
 
 from module.plugins.Account import Account
-
-try:
-    from json import loads as json_loads
-except ImportError:
-    from module.lib.simplejson import loads as json_loads
+from module.common.json_layer import json_loads
 
 class FileserveCom(Account):
     __name__ = "FileserveCom"
diff --git a/module/plugins/accounts/FilesonicCom.py b/module/plugins/accounts/FilesonicCom.py
index ff4e066e2..1207f1b56 100644
--- a/module/plugins/accounts/FilesonicCom.py
+++ b/module/plugins/accounts/FilesonicCom.py
@@ -17,15 +17,10 @@
     @author: RaNaN
 """
 
-import re
 from time import mktime, strptime
 
 from module.plugins.Account import Account
-
-try:
-    from json import loads as json_loads
-except ImportError: # pragma: no cover
-    from module.lib.simplejson import loads as json_loads
+from module.common.json_layer import json_loads
 
 class FilesonicCom(Account):
     __name__ = "FilesonicCom"
diff --git a/module/plugins/accounts/WuploadCom.py b/module/plugins/accounts/WuploadCom.py
index c6710a657..b3d73df27 100644
--- a/module/plugins/accounts/WuploadCom.py
+++ b/module/plugins/accounts/WuploadCom.py
@@ -20,11 +20,7 @@
 from types import MethodType
 
 from module.plugins.Account import Account
-
-try:
-    from json import loads as json_loads
-except ImportError: # pragma: no cover
-    from module.lib.simplejson import loads as json_loads
+from module.common.json_layer import json_loads
 
 class WuploadCom(Account):
     __name__ = "WuploadCom"
diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py
index d48900f51..794e978b2 100644
--- a/module/plugins/hoster/BitshareCom.py
+++ b/module/plugins/hoster/BitshareCom.py
@@ -71,7 +71,7 @@ class BitshareCom(Hoster):
 
         # Load main page
         self.req.cj.setCookie(self.HOSTER_DOMAIN, "language_selection", "EN")
-        self.html = self.load(self.pyfile.url, ref=False, utf8=True, cookies=True)
+        self.html = self.load(self.pyfile.url, ref=False, decode=True)
 
         # Check offline
         if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None:
diff --git a/module/plugins/hoster/FilefactoryCom.py b/module/plugins/hoster/FilefactoryCom.py
index 55bdeb5bf..37b2bb7ce 100644
--- a/module/plugins/hoster/FilefactoryCom.py
+++ b/module/plugins/hoster/FilefactoryCom.py
@@ -61,7 +61,7 @@ class FilefactoryCom(Hoster):
         self.req.cj.setCookie("filefactory.com", "ff_locale","")
 
         # Load main page
-        self.html = self.load(self.pyfile.url, ref=False, utf8=True, cookies=True)
+        self.html = self.load(self.pyfile.url, ref=False, decode=True)
 
         # Check offline
         if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None:
@@ -105,7 +105,7 @@ class FilefactoryCom(Hoster):
         
         # This will take us to a wait screen
         self.log.debug("%s: fetching wait with url [%s]" % (self.__name__, waiturl))
-        waithtml = self.load(waiturl, ref=True, utf8=True, cookies=True)
+        waithtml = self.load(waiturl, decode=True)
 
         # Find the wait value and wait     
         wait = int(re.search(self.WAIT_PATTERN, waithtml).group('wait'))
diff --git a/module/plugins/hoster/FileserveCom.py b/module/plugins/hoster/FileserveCom.py
index ba76f0e39..7f34621aa 100644
--- a/module/plugins/hoster/FileserveCom.py
+++ b/module/plugins/hoster/FileserveCom.py
@@ -6,13 +6,10 @@ import re
 from module.plugins.Hoster import Hoster

 from module.plugins.ReCaptcha import ReCaptcha

 

+from module.common.json_layer import json_loads

 from module.network.RequestFactory import getURL

 from module.utils import parseFileSize

 

-try:

-    from json import loads as json_loads

-except ImportError: # pragma: no cover

-    from module.lib.simplejson import loads as json_loads

 

 def getInfo(urls):

     reg = r"<td>(http://(?:www\.)?fileserve\.com/file/.+(?:[\r\n\t]+)?)</td>[\r\n\t ]+<td>(.*?)</td>[\r\n\t ]+<td>(.*?)</td>[\r\n\t ]+<td>(Available|Not available)(?:\&nbsp;)?(?:<img|</td>)"

@@ -83,7 +80,7 @@ class FileserveCom(Hoster):
         # TODO: handle login timeouts

         self.download(self.pyfile.url)

 

-        check = self.checkDownload({"login" : '<form action="/login.php" method="POST">'})

+        check = self.checkDownload({"login": '<form action="/login.php" method="POST">'})

 

         if check == "login":

             self.account.relogin(self.user)

diff --git a/module/plugins/hoster/FilesonicCom.py b/module/plugins/hoster/FilesonicCom.py
index 6535b04f1..a74bcc1c8 100644
--- a/module/plugins/hoster/FilesonicCom.py
+++ b/module/plugins/hoster/FilesonicCom.py
@@ -9,11 +9,8 @@ from module.plugins.ReCaptcha import ReCaptcha
 from module.plugins.Plugin import chunks

 

 from module.network.RequestFactory import getURL

+from module.common.json_layer import json_loads

 

-try:

-    from json import loads as json_loads

-except ImportError: # pragma: no cover

-    from module.lib.simplejson import loads as json_loads

 

 def getInfo(urls):

     for chunk in chunks(urls, 20):

@@ -123,7 +120,7 @@ class FilesonicCom(Hoster):
         self.logDebug("Premium download")

 

         api = self.API_ADDRESS + "/link?method=getDownloadLink&u=%s&p=%s&ids=%s" % (

-        self.user, self.account.getAccountData(self.user)["password"], getId(self.pyfile.url))

+            self.user, self.account.getAccountData(self.user)["password"], getId(self.pyfile.url))

 

         result = json_loads(self.load(api))

         links = result["FSApi_Link"]["getDownloadLink"]["response"]["links"]

diff --git a/module/plugins/hoster/IfileIt.py b/module/plugins/hoster/IfileIt.py
index 77961e51e..2b70729a9 100644
--- a/module/plugins/hoster/IfileIt.py
+++ b/module/plugins/hoster/IfileIt.py
@@ -17,7 +17,8 @@
 """
 
 import re
-from json import loads as json_loads
+
+from module.common.json_layer import json_loads
 from module.common.JsEngine import JsEngine
 from module.plugins.ReCaptcha import ReCaptcha
 from module.plugins.Hoster import Hoster
@@ -42,6 +43,7 @@ def getInfo(urls):
                 result.append((name, size, 2, url))
     yield result
 
+
 class IfileIt(Hoster):
     __name__ = "IfileIt"
     __type__ = "hoster"
@@ -49,44 +51,43 @@ class IfileIt(Hoster):
     __version__ = "0.2"
     __description__ = """Ifile.it"""
     __author_name__ = ("zoidberg")
-    
+
     EVAL_PATTERN = r'(eval\(function\(p,a,c,k,e,d\).*)'
     DEC_PATTERN = r"function requestBtn_clickEvent[^}]*url:\s*([^,]+)"
     DOWNLOAD_LINK_PATTERN = r'</span> If it doesn\'t, <a target="_blank" href="([^"]+)">'
     RECAPTCHA_KEY_PATTERN = r"var __recaptcha_public\s*=\s*'([^']+)';"
     FILE_INFO_PATTERN = r'<span style="cursor: default;[^>]*>\s*(.*?)\s*&nbsp;\s*<strong>\s*([0-9.]+)\s*(kB|MB|GB)\s*</strong>\s*</span>'
     FILE_OFFLINE_PATTERN = r'$\("#errorPnl"\)\.empty\(\)\.append\( "no such file" \);'
-    
+
     def process(self, pyfile):
-        self.html = self.load(pyfile.url, cookies = True)
-        
+        self.html = self.load(pyfile.url)
+
         found = re.search(self.FILE_INFO_PATTERN, self.html)
         pyfile.name = found.group(1)
-        pyfile.size = pyfile.size = float(found.group(2)) * 1024 ** {'kB': 1, 'MB': 2, 'GB': 3}[found.group(3)] 
-        
+        pyfile.size = pyfile.size = float(found.group(2)) * 1024 ** {'kB': 1, 'MB': 2, 'GB': 3}[found.group(3)]
+
         eval_string = re.search(self.EVAL_PATTERN, self.html).group(1)
         dec_string = re.search(self.DEC_PATTERN, self.html).group(1)
-        recaptcha_key = re.search(self.RECAPTCHA_KEY_PATTERN, self.html).group(1)  
-        
+
         js = JsEngine()
-        json_url = js.eval(eval_string+";"+dec_string)
+        json_url = js.eval(eval_string + ";" + dec_string)
         self.logDebug(json_url)
-        
-        json_response = json_loads(self.load(json_url, cookies = True))
+
+        json_response = json_loads(self.load(json_url))
         self.logDebug(json_response)
         if json_response["captcha"]:
             captcha_key = re.search(self.RECAPTCHA_KEY_PATTERN, self.html).group(1)
             recaptcha = ReCaptcha(self)
-    
+
             for i in range(5):
                 captcha_challenge, captcha_response = recaptcha.challenge(captcha_key)
-    
+
                 json_response = json_loads(self.load(json_url, post={
                     "ctype": "recaptcha",
                     "recaptcha_challenge": captcha_challenge,
                     "recaptcha_response": captcha_response
-                    }))
-                    
+                }))
+
                 self.logDebug(json_response)
                 if json_response["retry"]:
                     self.invalidCaptcha()
@@ -95,10 +96,10 @@ class IfileIt(Hoster):
                     break
             else:
                 self.fail("Incorrect captcha")
-        
+
         # load twice
-        self.html = self.load(pyfile.url, cookies = True)                     
-        self.html = self.load(pyfile.url, cookies = True)
+        self.html = self.load(pyfile.url)
+        self.html = self.load(pyfile.url)
         download_url = re.search(self.DOWNLOAD_LINK_PATTERN, self.html).group(1)
-        
+
         self.download(download_url)
\ No newline at end of file
diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py
index 59786d996..b2bec873d 100644
--- a/module/plugins/hoster/NetloadIn.py
+++ b/module/plugins/hoster/NetloadIn.py
@@ -133,7 +133,7 @@ class NetloadIn(Hoster):
 
     def download_html(self):
         self.log.debug("Netload: Entering download_html")
-        page = self.load(self.url, utf8=True)
+        page = self.load(self.url, decode=True)
         t = time() + 30
         
         if "/share/templates/download_hddcrash.tpl" in page:
diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py
index 22af20923..f89ef3ad1 100644
--- a/module/plugins/hoster/UploadStationCom.py
+++ b/module/plugins/hoster/UploadStationCom.py
@@ -62,7 +62,7 @@ class UploadStationCom(Hoster):
     def process(self, pyfile):

         

         # Get URL

-        self.html = self.load(self.pyfile.url, ref=False, cookies=True, utf8=True)

+        self.html = self.load(self.pyfile.url, ref=False, decode=True)

 

         # Is offline?

         m = re.search(UploadStationCom.FILE_OFFLINE_PATTERN, self.html) 

@@ -84,7 +84,7 @@ class UploadStationCom(Hoster):
         # self.jsPage = self.load("http://uploadstation.com" + jsPage)

         

         # Check download

-        response = self.load(self.pyfile.url, post={"checkDownload" : "check"}, utf8=True)

+        response = self.load(self.pyfile.url, post={"checkDownload" : "check"}, decode=True)

         self.logDebug("Checking download, response [%s]" % response.encode('ascii', 'ignore'))

         self.handleErrors(response)

         

diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py
index 93b45fbfe..11b61ae59 100644
--- a/module/plugins/hoster/WuploadCom.py
+++ b/module/plugins/hoster/WuploadCom.py
@@ -10,11 +10,8 @@ from module.plugins.Hoster import Hoster
 from module.plugins.Plugin import chunks
 
 from module.network.RequestFactory import getURL
+from module.common.json_layer import json_loads
 
-try:
-    from json import loads as json_loads
-except ImportError: # pragma: no cover
-    from module.lib.simplejson import loads as json_loads
 
 def getInfo(urls):
     for chunk in chunks(urls, 20):
@@ -84,6 +81,6 @@ class WuploadCom(Hoster):
 
     def getDomain(self):
         result = json_loads(
-            self.load(self.API_ADDRESS + "/utility?method=getWuploadDomainForCurrentIp&format=json", utf8=True))
+            self.load(self.API_ADDRESS + "/utility?method=getWuploadDomainForCurrentIp&format=json", decode=True))
         self.log.debug("%s: response to get domain %s" % (self.__name__, result))
         return result["FSApi_Utility"]["getWuploadDomainForCurrentIp"]["response"]
\ No newline at end of file