diff options
Diffstat (limited to 'module/plugins')
171 files changed, 1246 insertions, 1279 deletions
diff --git a/module/plugins/accounts/AlldebridCom.py b/module/plugins/accounts/AlldebridCom.py index e02983e1c..a109faf4f 100644 --- a/module/plugins/accounts/AlldebridCom.py +++ b/module/plugins/accounts/AlldebridCom.py @@ -2,7 +2,7 @@ import re import time -import xml.dom.minidom +import xml.dom.minidom as dom from BeautifulSoup import BeautifulSoup @@ -21,7 +21,7 @@ class AlldebridCom(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - html = req.load("http://www.alldebrid.com/account/") + html = self.load("http://www.alldebrid.com/account/", req=req) soup = BeautifulSoup(html) #Try to parse expiration date directly from the control panel page (better accuracy) @@ -38,12 +38,12 @@ class AlldebridCom(Account): #Get expiration date from API except Exception: data = self.getAccountData(user) - html = req.load("https://www.alldebrid.com/api.php", - get={'action': "info_user", 'login': user, 'pw': data['password']}) + html = self.load("https://www.alldebrid.com/api.php", + get={'action': "info_user", 'login': user, 'pw': data['password']}, req=req) self.logDebug(html) - xml = xml.dom.minidom.parseString(html) + xml = dom.parseString(html) exp_time = time.time() + int(xml.getElementsByTagName("date")[0].childNodes[0].nodeValue) * 24 * 60 * 60 return {'validuntil' : exp_time, @@ -52,11 +52,10 @@ class AlldebridCom(Account): def login(self, user, data, req): - html = req.load("https://www.alldebrid.com/register/", + html = self.load("https://www.alldebrid.com/register/", get={'action' : "login", 'login_login' : user, - 'login_password': data['password']}, - decode=True) + 'login_password': data['password']}, req=req) if "This login doesn't exist" in html \ or "The password is not valid" in html \ diff --git a/module/plugins/accounts/BitshareCom.py b/module/plugins/accounts/BitshareCom.py index afbed2920..8f748b9e9 100644 --- a/module/plugins/accounts/BitshareCom.py +++ b/module/plugins/accounts/BitshareCom.py @@ -14,7 +14,7 @@ class BitshareCom(Account): def loadAccountInfo(self, user, req): - html = req.load("http://bitshare.com/mysettings.html") + html = self.load("http://bitshare.com/mysettings.html", req=req) if "\"http://bitshare.com/myupgrade.html\">Free" in html: return {"validuntil": -1, "trafficleft": -1, "premium": False} @@ -26,9 +26,8 @@ class BitshareCom(Account): def login(self, user, data, req): - html = req.load("https://bitshare.com/login.html", - post={"user": user, "password": data['password'], "submit": "Login"}, - decode=True) + html = self.load("https://bitshare.com/login.html", + post={"user": user, "password": data['password'], "submit": "Login"}, req=req) if "login" in req.lastEffectiveURL: self.wrongPassword() diff --git a/module/plugins/accounts/CatShareNet.py b/module/plugins/accounts/CatShareNet.py index ae2b69dfa..79e94cc17 100644 --- a/module/plugins/accounts/CatShareNet.py +++ b/module/plugins/accounts/CatShareNet.py @@ -26,7 +26,7 @@ class CatShareNet(Account): validuntil = -1 trafficleft = -1 - html = req.load("http://catshare.net/", decode=True) + html = self.load("http://catshare.net/", req=req) if re.search(self.PREMIUM_PATTERN, html): premium = True @@ -50,12 +50,11 @@ class CatShareNet(Account): def login(self, user, data, req): - html = req.load("http://catshare.net/login", #@TODO: Revert to `https` in 0.4.10 + html = self.load("http://catshare.net/login", #@TODO: Revert to `https` in 0.4.10 post={'user_email': user, 'user_password': data['password'], 'remindPassword': 0, - 'user[submit]': "Login"}, - decode=True) + 'user[submit]': "Login"}, req=req) if not '<a href="/logout">Wyloguj</a>' in html: self.wrongPassword() diff --git a/module/plugins/accounts/CloudzillaTo.py b/module/plugins/accounts/CloudzillaTo.py index 48e0fd5c3..2328b0a86 100644 --- a/module/plugins/accounts/CloudzillaTo.py +++ b/module/plugins/accounts/CloudzillaTo.py @@ -19,7 +19,7 @@ class CloudzillaTo(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.cloudzilla.to/") + html = self.load("http://www.cloudzilla.to/", req=req) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -27,11 +27,10 @@ class CloudzillaTo(Account): def login(self, user, data, req): - html = req.load("https://www.cloudzilla.to/", + html = self.load("https://www.cloudzilla.to/", post={'lusername': user, 'lpassword': data['password'], - 'w' : "dologin"}, - decode=True) + 'w' : "dologin"}, req=req) if "ERROR" in html: self.wrongPassword() diff --git a/module/plugins/accounts/CzshareCom.py b/module/plugins/accounts/CzshareCom.py index 67869551a..c53c1807b 100644 --- a/module/plugins/accounts/CzshareCom.py +++ b/module/plugins/accounts/CzshareCom.py @@ -25,7 +25,7 @@ class CzshareCom(Account): validuntil = None trafficleft = None - html = req.load("http://sdilej.cz/prehled_kreditu/") + html = self.load("http://sdilej.cz/prehled_kreditu/", req=req) try: m = re.search(self.CREDIT_LEFT_PATTERN, html) @@ -44,11 +44,10 @@ class CzshareCom(Account): def login(self, user, data, req): - html = req.load('https://sdilej.cz/index.php', + html = self.load('https://sdilej.cz/index.php', post={"Prihlasit": "Prihlasit", "login-password": data['password'], - "login-name": user}, - decode=True) + "login-name": user}, req=req) if '<div class="login' in html: self.wrongPassword() diff --git a/module/plugins/accounts/DebridItaliaCom.py b/module/plugins/accounts/DebridItaliaCom.py index 4fe4fcecc..defa1a313 100644 --- a/module/plugins/accounts/DebridItaliaCom.py +++ b/module/plugins/accounts/DebridItaliaCom.py @@ -22,7 +22,7 @@ class DebridItaliaCom(Account): def loadAccountInfo(self, user, req): info = {"premium": False, "validuntil": None, "trafficleft": None} - html = req.load("http://debriditalia.com/") + html = self.load("http://debriditalia.com/", req=req) if 'Account premium not activated' not in html: m = re.search(self.WALID_UNTIL_PATTERN, html) @@ -36,9 +36,8 @@ class DebridItaliaCom(Account): def login(self, user, data, req): - html = req.load("https://debriditalia.com/login.php", - get={'u': user, 'p': data['password']}, - decode=True) + html = self.load("https://debriditalia.com/login.php", + get={'u': user, 'p': data['password']}, req=req) if 'NO' in html: self.wrongPassword() diff --git a/module/plugins/accounts/DepositfilesCom.py b/module/plugins/accounts/DepositfilesCom.py index 234ad0b7a..e89f0add8 100644 --- a/module/plugins/accounts/DepositfilesCom.py +++ b/module/plugins/accounts/DepositfilesCom.py @@ -19,7 +19,7 @@ class DepositfilesCom(Account): def loadAccountInfo(self, user, req): - html = req.load("https://dfiles.eu/de/gold/") + html = self.load("https://dfiles.eu/de/gold/", req=req) validuntil = re.search(r"Sie haben Gold Zugang bis: <b>(.*?)</b></div>", html).group(1) validuntil = time.mktime(time.strptime(validuntil, "%Y-%m-%d %H:%M:%S")) @@ -28,9 +28,8 @@ class DepositfilesCom(Account): def login(self, user, data, req): - html = req.load("https://dfiles.eu/de/login.php", get={"return": "/de/gold/payment.php"}, - post={"login": user, "password": data['password']}, - decode=True) + html = self.load("https://dfiles.eu/de/login.php", get={"return": "/de/gold/payment.php"}, + post={"login": user, "password": data['password']}, req=req) if r'<div class="error_message">Sie haben eine falsche Benutzername-Passwort-Kombination verwendet.</div>' in html: self.wrongPassword() diff --git a/module/plugins/accounts/EuroshareEu.py b/module/plugins/accounts/EuroshareEu.py index 0568b9c4f..4af9803b7 100644 --- a/module/plugins/accounts/EuroshareEu.py +++ b/module/plugins/accounts/EuroshareEu.py @@ -18,7 +18,7 @@ class EuroshareEu(Account): def loadAccountInfo(self, user, req): self.relogin(user) - html = req.load("http://euroshare.eu/customer-zone/settings/") + html = self.load("http://euroshare.eu/customer-zone/settings/", req=req) m = re.search('id="input_expire_date" value="(\d+\.\d+\.\d+ \d+:\d+)"', html) if m is None: @@ -32,11 +32,10 @@ class EuroshareEu(Account): def login(self, user, data, req): - html = req.load('http://euroshare.eu/customer-zone/login/', + html = self.load('http://euroshare.eu/customer-zone/login/', post={"trvale": "1", "login": user, - "password": data['password']}, - decode=True) + "password": data['password']}, req=req) if u">Nesprávne prihlasovacie meno alebo heslo" in html: self.wrongPassword() diff --git a/module/plugins/accounts/FastixRu.py b/module/plugins/accounts/FastixRu.py index c83964feb..dedbc0d5d 100644 --- a/module/plugins/accounts/FastixRu.py +++ b/module/plugins/accounts/FastixRu.py @@ -16,7 +16,7 @@ class FastixRu(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - html = json_loads(req.load("http://fastix.ru/api_v2/", get={'apikey': data['api'], 'sub': "getaccountdetails"})) + html = json_loads(self.load("http://fastix.ru/api_v2/", get={'apikey': data['api'], 'sub': "getaccountdetails"}), req=req) points = html['points'] kb = float(points) * 1024 ** 2 / 1000 @@ -29,8 +29,8 @@ class FastixRu(Account): def login(self, user, data, req): - html = req.load("https://fastix.ru/api_v2/", - get={'sub': "get_apikey", 'email': user, 'password': data['password']}) + html = self.load("https://fastix.ru/api_v2/", + get={'sub': "get_apikey", 'email': user, 'password': data['password']}, req=req) api = json_loads(html) api = api['apikey'] diff --git a/module/plugins/accounts/FastshareCz.py b/module/plugins/accounts/FastshareCz.py index e291d5859..d1d79f7b1 100644 --- a/module/plugins/accounts/FastshareCz.py +++ b/module/plugins/accounts/FastshareCz.py @@ -24,7 +24,7 @@ class FastshareCz(Account): trafficleft = None premium = False - html = req.load("http://www.fastshare.cz/user", decode=True) + html = self.load("http://www.fastshare.cz/user", req=req) m = re.search(self.CREDIT_PATTERN, html) if m: @@ -40,11 +40,10 @@ class FastshareCz(Account): def login(self, user, data, req): req.cj.setCookie("fastshare.cz", "lang", "en") - req.load('http://www.fastshare.cz/login') # Do not remove or it will not login + self.load('http://www.fastshare.cz/login', req=req) # Do not remove or it will not login - html = req.load("https://www.fastshare.cz/sql.php", - post={'login': user, 'heslo': data['password']}, - decode=True) + html = self.load("https://www.fastshare.cz/sql.php", + post={'login': user, 'heslo': data['password']}, req=req) if ">Wrong username or password" in html: self.wrongPassword() diff --git a/module/plugins/accounts/FilecloudIo.py b/module/plugins/accounts/FilecloudIo.py index c5239405e..300159acf 100644 --- a/module/plugins/accounts/FilecloudIo.py +++ b/module/plugins/accounts/FilecloudIo.py @@ -18,7 +18,7 @@ class FilecloudIo(Account): def loadAccountInfo(self, user, req): # It looks like the first API request always fails, so we retry 5 times, it should work on the second try for _i in xrange(5): - rep = req.load("https://secure.filecloud.io/api-fetch_apikey.api", + rep = self.load("https://secure.filecloud.io/api-fetch_apikey.api", post={"username": user, "password": self.getAccountData(user)['password']}) rep = json_loads(rep) if rep['status'] == 'ok': @@ -31,8 +31,8 @@ class FilecloudIo(Account): akey = rep['akey'] self.accounts[user]['akey'] = akey # Saved for hoster plugin - rep = req.load("http://api.filecloud.io/api-fetch_account_details.api", - post={"akey": akey}) + rep = self.load("http://api.filecloud.io/api-fetch_account_details.api", + post={"akey": akey}, req=req) rep = json_loads(rep) if rep['is_premium'] == 1: @@ -43,7 +43,7 @@ class FilecloudIo(Account): def login(self, user, data, req): req.cj.setCookie("secure.filecloud.io", "lang", "en") - html = req.load('https://secure.filecloud.io/user-login.html') + html = self.load('https://secure.filecloud.io/user-login.html', req=req) if not hasattr(self, "form_data"): self.form_data = {} @@ -51,9 +51,9 @@ class FilecloudIo(Account): self.form_data['username'] = user self.form_data['password'] = data['password'] - html = req.load('https://secure.filecloud.io/user-login_p.html', - post=self.form_data, - multipart=True) + html = self.load('https://secure.filecloud.io/user-login_p.html', + post=self.form_data, + req=req) if "you have successfully logged in" not in html: self.wrongPassword() diff --git a/module/plugins/accounts/FilefactoryCom.py b/module/plugins/accounts/FilefactoryCom.py index e56708831..9d61ea927 100644 --- a/module/plugins/accounts/FilefactoryCom.py +++ b/module/plugins/accounts/FilefactoryCom.py @@ -22,7 +22,7 @@ class FilefactoryCom(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.filefactory.com/account/") + html = self.load("http://www.filefactory.com/account/", req=req) m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -39,10 +39,10 @@ class FilefactoryCom(Account): def login(self, user, data, req): req.http.c.setopt(pycurl.REFERER, "http://www.filefactory.com/member/login.php") - html = req.load("https://www.filefactory.com/member/signin.php", + html = self.load("https://www.filefactory.com/member/signin.php", post={"loginEmail" : user, "loginPassword": data['password'], - "Submit" : "Sign In"}) + "Submit" : "Sign In"}, req=req) if req.lastEffectiveURL != "http://www.filefactory.com/account/": self.wrongPassword() diff --git a/module/plugins/accounts/FilejungleCom.py b/module/plugins/accounts/FilejungleCom.py index efeeaceb7..459c70c95 100644 --- a/module/plugins/accounts/FilejungleCom.py +++ b/module/plugins/accounts/FilejungleCom.py @@ -24,7 +24,7 @@ class FilejungleCom(Account): def loadAccountInfo(self, user, req): - html = req.load(self.URL + "dashboard.php") + html = self.load(self.URL + "dashboard.php", req=req) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: premium = True @@ -37,14 +37,13 @@ class FilejungleCom(Account): def login(self, user, data, req): - html = req.load(self.URL + "login.php", + html = self.load(self.URL + "login.php", post={"loginUserName": user, "loginUserPassword": data['password'], "loginFormSubmit": "Login", "recaptcha_challenge_field": "", "recaptcha_response_field": "", - "recaptcha_shortencode_field": ""}, - decode=True) + "recaptcha_shortencode_field": ""}, req=req) if re.search(self.LOGIN_FAILED_PATTERN, html): self.wrongPassword() diff --git a/module/plugins/accounts/FilerNet.py b/module/plugins/accounts/FilerNet.py index f77ac5197..aca09a7ed 100644 --- a/module/plugins/accounts/FilerNet.py +++ b/module/plugins/accounts/FilerNet.py @@ -23,7 +23,7 @@ class FilerNet(Account): def loadAccountInfo(self, user, req): - html = req.load("https://filer.net/profile") + html = self.load("https://filer.net/profile", req=req) # Free user if re.search(self.FREE_PATTERN, html): @@ -43,17 +43,16 @@ class FilerNet(Account): def login(self, user, data, req): - html = req.load("https://filer.net/login", decode=True) + html = self.load("https://filer.net/login", req=req) token = re.search(self.TOKEN_PATTERN, html).group(1) - html = req.load("https://filer.net/login_check", + html = self.load("https://filer.net/login_check", post={"_username" : user, "_password" : data['password'], "_remember_me": "on", "_csrf_token" : token, - "_target_path": "https://filer.net/"}, - decode=True) + "_target_path": "https://filer.net/"}, req=req) if 'Logout' not in html: self.wrongPassword() diff --git a/module/plugins/accounts/FilesMailRu.py b/module/plugins/accounts/FilesMailRu.py index 3f9068fff..a1f86f7b5 100644 --- a/module/plugins/accounts/FilesMailRu.py +++ b/module/plugins/accounts/FilesMailRu.py @@ -20,12 +20,11 @@ class FilesMailRu(Account): def login(self, user, data, req): user, domain = user.split("@") - html = req.load("https://swa.mail.ru/cgi-bin/auth", + html = self.load("https://swa.mail.ru/cgi-bin/auth", post={"Domain": domain, "Login": user, "Password": data['password'], - "Page": "http://files.mail.ru/"}, - decode=True) + "Page": "http://files.mail.ru/"}, req=req) if "ÐевеÑМПе ÐžÐŒÑ Ð¿ÐŸÐ»ÑзПваÑÐµÐ»Ñ ÐžÐ»Ðž паÑПлÑ" in html: self.wrongPassword() diff --git a/module/plugins/accounts/FileserveCom.py b/module/plugins/accounts/FileserveCom.py index 38e21e246..bda6efc8d 100644 --- a/module/plugins/accounts/FileserveCom.py +++ b/module/plugins/accounts/FileserveCom.py @@ -19,8 +19,8 @@ class FileserveCom(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - html = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], - "submit": "Submit+Query"}) + html = self.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], + "submit": "Submit+Query"}, req=req) res = json_loads(html) if res['type'] == "premium": @@ -31,14 +31,14 @@ class FileserveCom(Account): def login(self, user, data, req): - html = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], - "submit": "Submit+Query"}) + html = self.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], + "submit": "Submit+Query"}, req=req) res = json_loads(html) if not res['type']: self.wrongPassword() #login at fileserv html - req.load("http://www.fileserve.com/login.php", + self.load("http://www.fileserve.com/login.php", post={"loginUserName": user, "loginUserPassword": data['password'], "autoLogin": "checked", - "loginFormSubmit": "Login"}) + "loginFormSubmit": "Login"}, req=req) diff --git a/module/plugins/accounts/FourSharedCom.py b/module/plugins/accounts/FourSharedCom.py index 054f0d3a1..041671b9a 100644 --- a/module/plugins/accounts/FourSharedCom.py +++ b/module/plugins/accounts/FourSharedCom.py @@ -22,13 +22,12 @@ class FourSharedCom(Account): def login(self, user, data, req): req.cj.setCookie("4shared.com", "4langcookie", "en") - res = req.load("https://www.4shared.com/web/login", + res = self.load("https://www.4shared.com/web/login", post={'login' : user, 'password' : data['password'], 'remember' : "on", '_remember': "on", - 'returnTo' : "http://www.4shared.com/account/home.jsp"}, - decode=True) + 'returnTo' : "http://www.4shared.com/account/home.jsp"}, req=req) if 'Please log in to access your 4shared account' in res: self.wrongPassword() diff --git a/module/plugins/accounts/FreakshareCom.py b/module/plugins/accounts/FreakshareCom.py index 83d26fbe5..b33790071 100644 --- a/module/plugins/accounts/FreakshareCom.py +++ b/module/plugins/accounts/FreakshareCom.py @@ -21,7 +21,7 @@ class FreakshareCom(Account): validuntil = None trafficleft = None - html = req.load("http://freakshare.com/") + html = self.load("http://freakshare.com/", req=req) try: m = re.search(r'ltig bis:</td>\s*<td><b>([\d.:-]+)</b></td>', html, re.M) @@ -41,11 +41,10 @@ class FreakshareCom(Account): def login(self, user, data, req): - req.load("http://freakshare.com/index.php?language=EN") + self.load("http://freakshare.com/index.php?language=EN", req=req) - html = req.load("https://freakshare.com/login.html", - post={"submit": "Login", "user": user, "pass": data['password']}, - decode=True) + html = self.load("https://freakshare.com/login.html", + post={"submit": "Login", "user": user, "pass": data['password']}, req=req) if ">Wrong Username or Password" in html: self.wrongPassword() diff --git a/module/plugins/accounts/FreeWayMe.py b/module/plugins/accounts/FreeWayMe.py index 66f93e0bb..5a1f75bf6 100644 --- a/module/plugins/accounts/FreeWayMe.py +++ b/module/plugins/accounts/FreeWayMe.py @@ -41,7 +41,7 @@ class FreeWayMe(Account): def getAccountStatus(self, user, req): - answer = req.load("http://www.free-way.bz/ajax/jd.php", #@TODO: Revert to `https` in 0.4.10 + answer = self.load("http://www.free-way.bz/ajax/jd.php", #@TODO: Revert to `https` in 0.4.10 get={"id": 4, "user": user, "pass": self.getAccountData(user)['password']}) self.logDebug("Login: %s" % answer) diff --git a/module/plugins/accounts/FshareVn.py b/module/plugins/accounts/FshareVn.py index 08d5fe555..67aafae86 100644 --- a/module/plugins/accounts/FshareVn.py +++ b/module/plugins/accounts/FshareVn.py @@ -24,7 +24,7 @@ class FshareVn(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.fshare.vn/account_info.php", decode=True) + html = self.load("http://www.fshare.vn/account_info.php", req=req) if re.search(self.LIFETIME_PATTERN, html): self.logDebug("Lifetime membership detected") @@ -45,13 +45,12 @@ class FshareVn(Account): def login(self, user, data, req): - html = req.load("https://www.fshare.vn/login.php", + html = self.load("https://www.fshare.vn/login.php", post={'LoginForm[email]' : user, 'LoginForm[password]' : data['password'], 'LoginForm[rememberMe]': 1, 'yt0' : "Login"}, - referer=True, - decode=True) + referer=True, req=req) if not re.search(r'<img\s+alt="VIP"', html): self.wrongPassword() diff --git a/module/plugins/accounts/HellshareCz.py b/module/plugins/accounts/HellshareCz.py index 69b4b8443..a1f50eb8b 100644 --- a/module/plugins/accounts/HellshareCz.py +++ b/module/plugins/accounts/HellshareCz.py @@ -21,7 +21,7 @@ class HellshareCz(Account): def loadAccountInfo(self, user, req): self.relogin(user) - html = req.load("http://www.hellshare.com/") + html = self.load("http://www.hellshare.com/", req=req) m = re.search(self.CREDIT_LEFT_PATTERN, html) if m is None: @@ -52,28 +52,27 @@ class HellshareCz(Account): def login(self, user, data, req): - html = req.load('http://www.hellshare.com/', decode=True) + html = self.load('http://www.hellshare.com/', req=req) if req.lastEffectiveURL != 'http://www.hellshare.com/': #Switch to English self.logDebug("Switch lang - URL: %s" % req.lastEffectiveURL) - json = req.load("%s?do=locRouter-show" % req.lastEffectiveURL) + json = self.load("%s?do=locRouter-show" % req.lastEffectiveURL, req=req) hash = re.search(r"(\-\-[0-9a-f]+\-)", json).group(1) self.logDebug("Switch lang - HASH: %s" % hash) - html = req.load('http://www.hellshare.com/%s/' % hash, decode=True) + html = self.load('http://www.hellshare.com/%s/' % hash, req=req) if re.search(self.CREDIT_LEFT_PATTERN, html): self.logDebug("Already logged in") return - html = req.load('https://www.hellshare.com/login?do=loginForm-submit', + html = self.load('https://www.hellshare.com/login?do=loginForm-submit', post={"login": "Log in", "password": data['password'], "username": user, - "perm_login": "on"}, - decode=True) + "perm_login": "on"}, req=req) if "<p>You input a wrong user name or wrong password</p>" in html: self.wrongPassword() diff --git a/module/plugins/accounts/HighWayMe.py b/module/plugins/accounts/HighWayMe.py index eba1b4692..d85db9769 100644 --- a/module/plugins/accounts/HighWayMe.py +++ b/module/plugins/accounts/HighWayMe.py @@ -19,7 +19,7 @@ class HighWayMe(Account): validuntil = -1 trafficleft = None - json_data = req.load('https://high-way.me/api.php?user') + json_data = self.load('https://high-way.me/api.php?user', req=req) self.logDebug("JSON data: %s" % json_data) @@ -40,9 +40,8 @@ class HighWayMe(Account): def login(self, user, data, req): - html = req.load("https://high-way.me/api.php?login", - post={'login': '1', 'user': user, 'pass': data['password']}, - decode=True) + html = self.load("https://high-way.me/api.php?login", + post={'login': '1', 'user': user, 'pass': data['password']}, req=req) if 'UserOrPassInvalid' in html: self.wrongPassword() diff --git a/module/plugins/accounts/Keep2ShareCc.py b/module/plugins/accounts/Keep2ShareCc.py index 8cd23aed6..617e18f25 100644 --- a/module/plugins/accounts/Keep2ShareCc.py +++ b/module/plugins/accounts/Keep2ShareCc.py @@ -28,7 +28,7 @@ class Keep2ShareCc(Account): trafficleft = -1 premium = False - html = req.load("http://keep2share.cc/site/profile.html", decode=True) + html = self.load("http://keep2share.cc/site/profile.html", req=req) m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -62,12 +62,11 @@ class Keep2ShareCc(Account): def login(self, user, data, req): req.cj.setCookie("keep2share.cc", "lang", "en") - html = req.load("https://keep2share.cc/login.html", + html = self.load("https://keep2share.cc/login.html", post={'LoginForm[username]' : user, 'LoginForm[password]' : data['password'], 'LoginForm[rememberMe]': 1, - 'yt0' : ""}, - decode=True) + 'yt0' : ""}, req=req) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrongPassword() diff --git a/module/plugins/accounts/LetitbitNet.py b/module/plugins/accounts/LetitbitNet.py index e5c61e35f..6b538c1f0 100644 --- a/module/plugins/accounts/LetitbitNet.py +++ b/module/plugins/accounts/LetitbitNet.py @@ -18,7 +18,7 @@ class LetitbitNet(Account): ## DISABLED BECAUSE IT GET 'key exausted' EVEN IF VALID ## # api_key = self.getAccountData(user)['password'] # json_data = [api_key, ['key/info']] - # api_rep = req.load('http://api.letitbit.net/json', post={'r': json_dumps(json_data)}) + # api_rep = self.load('http://api.letitbit.net/json', post={'r': json_dumps(json_data)}, req=req) # self.logDebug("API Key Info: " + api_rep) # api_rep = json_loads(api_rep) # diff --git a/module/plugins/accounts/LinksnappyCom.py b/module/plugins/accounts/LinksnappyCom.py index 2b1d1aebf..4a475bd24 100644 --- a/module/plugins/accounts/LinksnappyCom.py +++ b/module/plugins/accounts/LinksnappyCom.py @@ -18,8 +18,8 @@ class LinksnappyCom(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - r = req.load('http://gen.linksnappy.com/lseAPI.php', - get={'act': 'USERDETAILS', 'username': user, 'password': hashlib.md5(data['password']).hexdigest()}) + r = self.load('http://gen.linksnappy.com/lseAPI.php', + get={'act': 'USERDETAILS', 'username': user, 'password': hashlib.md5(data['password'], req=req).hexdigest()}) self.logDebug("JSON data: " + r) @@ -50,11 +50,10 @@ class LinksnappyCom(Account): def login(self, user, data, req): - html = req.load("https://gen.linksnappy.com/lseAPI.php", + html = self.load("https://gen.linksnappy.com/lseAPI.php", get={'act' : 'USERDETAILS', 'username': user, - 'password': hashlib.md5(data['password']).hexdigest()}, - decode=True) + 'password': hashlib.md5(data['password'], req=req).hexdigest()}) if "Invalid Account Details" in html: self.wrongPassword() diff --git a/module/plugins/accounts/MegaDebridEu.py b/module/plugins/accounts/MegaDebridEu.py index fc7a5e5a3..4767b9b05 100644 --- a/module/plugins/accounts/MegaDebridEu.py +++ b/module/plugins/accounts/MegaDebridEu.py @@ -20,8 +20,8 @@ class MegaDebridEu(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - jsonResponse = req.load(self.API_URL, - get={'action': 'connectUser', 'login': user, 'password': data['password']}) + jsonResponse = self.load(self.API_URL, + get={'action': 'connectUser', 'login': user, 'password': data['password']}, req=req) res = json_loads(jsonResponse) if res['response_code'] == "ok": @@ -32,8 +32,8 @@ class MegaDebridEu(Account): def login(self, user, data, req): - jsonResponse = req.load(self.API_URL, - get={'action': 'connectUser', 'login': user, 'password': data['password']}) + jsonResponse = self.load(self.API_URL, + get={'action': 'connectUser', 'login': user, 'password': data['password']}, req=req) res = json_loads(jsonResponse) if res['response_code'] != "ok": self.wrongPassword() diff --git a/module/plugins/accounts/MegaRapidCz.py b/module/plugins/accounts/MegaRapidCz.py index ffb0024ca..af23b7bf3 100644 --- a/module/plugins/accounts/MegaRapidCz.py +++ b/module/plugins/accounts/MegaRapidCz.py @@ -25,7 +25,7 @@ class MegaRapidCz(Account): def loadAccountInfo(self, user, req): - htmll = req.load("http://megarapid.cz/mujucet/", decode=True) + htmll = self.load("http://megarapid.cz/mujucet/", req=req) m = re.search(self.LIMITDL_PATTERN, htmll) if m: @@ -46,15 +46,15 @@ class MegaRapidCz(Account): def login(self, user, data, req): - html = req.load("http://megarapid.cz/prihlaseni/", decode=True) + html = self.load("http://megarapid.cz/prihlaseni/", req=req) if "Heslo:" in html: start = html.index('id="inp_hash" name="hash" value="') html = html[start + 33:] hashes = html[0:32] - html = req.load("https://megarapid.cz/prihlaseni/", + html = self.load("https://megarapid.cz/prihlaseni/", post={"hash": hashes, "login": user, "pass1": data['password'], "remember": 0, - "sbmt": u"PÅihlásit"}) + "sbmt": u"PÅihlásit"}, req=req) diff --git a/module/plugins/accounts/MegaRapidoNet.py b/module/plugins/accounts/MegaRapidoNet.py index 93afd3578..de1c775c7 100644 --- a/module/plugins/accounts/MegaRapidoNet.py +++ b/module/plugins/accounts/MegaRapidoNet.py @@ -25,7 +25,7 @@ class MegaRapidoNet(Account): trafficleft = None premium = False - html = req.load("http://megarapido.net/gerador", decode=True) + html = self.load("http://megarapido.net/gerador", req=req) validuntil = re.search(self.VALID_UNTIL_PATTERN, html) if validuntil: @@ -40,12 +40,11 @@ class MegaRapidoNet(Account): def login(self, user, data, req): - req.load("http://megarapido.net/login") - req.load("http://megarapido.net/painel_user/ajax/logar.php", - post={'login': user, 'senha': data['password']}, - decode=True) + self.load("http://megarapido.net/login", req=req) + self.load("http://megarapido.net/painel_user/ajax/logar.php", + post={'login': user, 'senha': data['password']}, req=req) - html = req.load("http://megarapido.net/gerador") + html = self.load("http://megarapido.net/gerador", req=req) if "sair" not in html.lower(): self.wrongPassword() diff --git a/module/plugins/accounts/MegasharesCom.py b/module/plugins/accounts/MegasharesCom.py index d9ef7abb9..af0987ed3 100644 --- a/module/plugins/accounts/MegasharesCom.py +++ b/module/plugins/accounts/MegasharesCom.py @@ -21,7 +21,7 @@ class MegasharesCom(Account): def loadAccountInfo(self, user, req): #self.relogin(user) - html = req.load("http://d01.megashares.com/myms.php", decode=True) + html = self.load("http://d01.megashares.com/myms.php", req=req) premium = False if '>Premium Upgrade<' in html else True @@ -37,12 +37,11 @@ class MegasharesCom(Account): def login(self, user, data, req): - html = req.load('http://d01.megashares.com/myms_login.php', + html = self.load('http://d01.megashares.com/myms_login.php', post={"httpref" : "", "myms_login" : "Login", "mymslogin_name": user, - "mymspassword" : data['password']}, - decode=True) + "mymspassword" : data['password']}, req=req) if not '<span class="b ml">%s</span>' % user in html: self.wrongPassword() diff --git a/module/plugins/accounts/MultishareCz.py b/module/plugins/accounts/MultishareCz.py index df498e6a6..be947f7f2 100644 --- a/module/plugins/accounts/MultishareCz.py +++ b/module/plugins/accounts/MultishareCz.py @@ -21,24 +21,23 @@ class MultishareCz(Account): def loadAccountInfo(self, user, req): #self.relogin(user) - html = req.load("http://www.multishare.cz/profil/", decode=True) + html = self.load("http://www.multishare.cz/profil/", req=req) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) trafficleft = self.parseTraffic(m.group('S') + m.group('U')) if m else 0 self.premium = True if trafficleft else False - html = req.load("http://www.multishare.cz/", decode=True) + html = self.load("http://www.multishare.cz/", req=req) mms_info = dict(re.findall(self.ACCOUNT_INFO_PATTERN, html)) return dict(mms_info, **{"validuntil": -1, "trafficleft": trafficleft}) def login(self, user, data, req): - html = req.load('https://www.multishare.cz/html/prihlaseni_process.php', + html = self.load('https://www.multishare.cz/html/prihlaseni_process.php', post={"akce" : "PÅihlásit", "heslo": data['password'], - "jmeno": user}, - decode=True) + "jmeno": user}, req=req) if '<div class="akce-chyba akce">' in html: self.wrongPassword() diff --git a/module/plugins/accounts/MyfastfileCom.py b/module/plugins/accounts/MyfastfileCom.py index 00de6d959..a854dd826 100644 --- a/module/plugins/accounts/MyfastfileCom.py +++ b/module/plugins/accounts/MyfastfileCom.py @@ -26,8 +26,8 @@ class MyfastfileCom(Account): def login(self, user, data, req): # Password to use is the API-Password written in http://myfastfile.com/myaccount - html = req.load("https://myfastfile.com/api.php", - get={"user": user, "pass": data['password']}) + html = self.load("https://myfastfile.com/api.php", + get={"user": user, "pass": data['password']}, req=req) self.logDebug("JSON data: " + html) diff --git a/module/plugins/accounts/NitroflareCom.py b/module/plugins/accounts/NitroflareCom.py index 0397b113d..cc2f4e232 100644 --- a/module/plugins/accounts/NitroflareCom.py +++ b/module/plugins/accounts/NitroflareCom.py @@ -28,9 +28,8 @@ class NitroflareCom(Account): trafficleft = None premium = False - html = req.load("https://nitroflare.com/member", - get={'s': "premium"}, - decode=True) + html = self.load("https://nitroflare.com/member", + get={'s': "premium"}, req=req) m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -68,16 +67,15 @@ class NitroflareCom(Account): def login(self, user, data, req): - html = req.load("https://nitroflare.com/login", decode=True) + html = self.load("https://nitroflare.com/login", req=req) token = re.search(self.TOKEN_PATTERN, html).group(1) - html = req.load("https://nitroflare.com/login", + html = self.load("https://nitroflare.com/login", post={'login' : "", 'email' : user, 'password': data['password'], - 'token' : token}, - decode=True) + 'token' : token}, req=req) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrongPassword() diff --git a/module/plugins/accounts/NoPremiumPl.py b/module/plugins/accounts/NoPremiumPl.py index 4944ad4df..024cdfebe 100644 --- a/module/plugins/accounts/NoPremiumPl.py +++ b/module/plugins/accounts/NoPremiumPl.py @@ -78,4 +78,4 @@ class NoPremiumPl(Account): def runAuthQuery(self): - return self._req.load(self.API_URL, post=self.createAuthQuery()) + return self._self.load(self.API_URL, post=self.createAuthQuery(), req=req) diff --git a/module/plugins/accounts/NowVideoSx.py b/module/plugins/accounts/NowVideoSx.py index a0cc90790..81267d413 100644 --- a/module/plugins/accounts/NowVideoSx.py +++ b/module/plugins/accounts/NowVideoSx.py @@ -24,7 +24,7 @@ class NowVideoSx(Account): trafficleft = -1 premium = None - html = req.load("http://www.nowvideo.sx/premium.php") + html = self.load("http://www.nowvideo.sx/premium.php", req=req) m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -48,9 +48,8 @@ class NowVideoSx(Account): def login(self, user, data, req): - html = req.load("http://www.nowvideo.sx/login.php", - post={'user': user, 'pass': data['password']}, - decode=True) + html = self.load("http://www.nowvideo.sx/login.php", + post={'user': user, 'pass': data['password']}, req=req) if re.search(r'>Log In<', html): self.wrongPassword() diff --git a/module/plugins/accounts/OboomCom.py b/module/plugins/accounts/OboomCom.py index b1217cff5..96ef51f1e 100644 --- a/module/plugins/accounts/OboomCom.py +++ b/module/plugins/accounts/OboomCom.py @@ -35,7 +35,7 @@ class OboomCom(Account): salt = passwd[::-1] pbkdf2 = PBKDF2(passwd, salt, 1000).hexread(16) - result = json_loads(req.load("https://www.oboom.com/1/login", get={"auth": user, "pass": pbkdf2})) + result = json_loads(self.load("https://www.oboom.com/1/login", get={"auth": user, "pass": pbkdf2}), req=req) if not result[0] == 200: self.logWarning(_("Failed to log in: %s") % result[1]) diff --git a/module/plugins/accounts/OneFichierCom.py b/module/plugins/accounts/OneFichierCom.py index a9c6829b1..3a345bd9f 100644 --- a/module/plugins/accounts/OneFichierCom.py +++ b/module/plugins/accounts/OneFichierCom.py @@ -26,7 +26,7 @@ class OneFichierCom(Account): trafficleft = -1 premium = None - html = req.load("https://1fichier.com/console/abo.pl") + html = self.load("https://1fichier.com/console/abo.pl", req=req) m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -46,13 +46,12 @@ class OneFichierCom(Account): def login(self, user, data, req): req.http.c.setopt(pycurl.REFERER, "https://1fichier.com/login.pl?lg=en") - html = req.load("https://1fichier.com/login.pl?lg=en", + html = self.load("https://1fichier.com/login.pl?lg=en", post={'mail' : user, 'pass' : data['password'], 'It' : "on", 'purge' : "off", - 'valider': "Send"}, - decode=True) + 'valider': "Send"}, req=req) if '>Invalid email address' in html or '>Invalid password' in html: self.wrongPassword() diff --git a/module/plugins/accounts/OverLoadMe.py b/module/plugins/accounts/OverLoadMe.py index 82f4d9240..e7e1b71e1 100644 --- a/module/plugins/accounts/OverLoadMe.py +++ b/module/plugins/accounts/OverLoadMe.py @@ -16,9 +16,9 @@ class OverLoadMe(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) - html = req.load("https://api.over-load.me/account.php", + html = self.load("https://api.over-load.me/account.php", get={'user': user, - 'auth': data['password']}).strip() + 'auth': data['password']}, req=req).strip() data = json_loads(html) self.logDebug(data) @@ -31,9 +31,9 @@ class OverLoadMe(Account): def login(self, user, data, req): - jsondata = req.load("https://api.over-load.me/account.php", + jsondata = self.load("https://api.over-load.me/account.php", get={'user': user, - 'auth': data['password']}).strip() + 'auth': data['password']}, req=req).strip() data = json_loads(jsondata) diff --git a/module/plugins/accounts/PremiumTo.py b/module/plugins/accounts/PremiumTo.py index 322225d36..fb99cfd59 100644 --- a/module/plugins/accounts/PremiumTo.py +++ b/module/plugins/accounts/PremiumTo.py @@ -16,8 +16,8 @@ class PremiumTo(Account): def loadAccountInfo(self, user, req): - traffic = req.load("http://premium.to/api/straffic.php", #@TODO: Revert to `https` in 0.4.10 - get={'username': self.username, 'password': self.password}) + traffic = self.load("http://premium.to/api/straffic.php", #@TODO: Revert to `https` in 0.4.10 + get={'username': self.username, 'password': self.password}, req=req) if "wrong username" not in traffic: trafficleft = sum(map(float, traffic.split(';'))) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 @@ -29,9 +29,8 @@ class PremiumTo(Account): def login(self, user, data, req): self.username = user self.password = data['password'] - authcode = req.load("http://premium.to/api/getauthcode.php", #@TODO: Revert to `https` in 0.4.10 - get={'username': user, 'password': self.password}, - decode=True) + authcode = self.load("http://premium.to/api/getauthcode.php", #@TODO: Revert to `https` in 0.4.10 + get={'username': user, 'password': self.password}, req=req) if "wrong username" in authcode: self.wrongPassword() diff --git a/module/plugins/accounts/PremiumizeMe.py b/module/plugins/accounts/PremiumizeMe.py index 17470b4a2..686898ca4 100644 --- a/module/plugins/accounts/PremiumizeMe.py +++ b/module/plugins/accounts/PremiumizeMe.py @@ -41,7 +41,7 @@ class PremiumizeMe(Account): def getAccountStatus(self, user, req): # Use premiumize.me API v1 (see https://secure.premiumize.me/?show=api) # to retrieve account info and return the parsed json answer - answer = req.load("http://api.premiumize.me/pm-api/v1.php", #@TODO: Revert to `https` in 0.4.10 + answer = self.load("http://api.premiumize.me/pm-api/v1.php", #@TODO: Revert to `https` in 0.4.10 get={'method' : "accountstatus", 'params[login]': user, 'params[pass]' : self.getAccountData(user)['password']}) diff --git a/module/plugins/accounts/QuickshareCz.py b/module/plugins/accounts/QuickshareCz.py index 0e52d5f79..c3cfe6f6f 100644 --- a/module/plugins/accounts/QuickshareCz.py +++ b/module/plugins/accounts/QuickshareCz.py @@ -19,7 +19,7 @@ class QuickshareCz(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.quickshare.cz/premium", decode=True) + html = self.load("http://www.quickshare.cz/premium", req=req) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: @@ -33,11 +33,10 @@ class QuickshareCz(Account): def login(self, user, data, req): - html = req.load('http://www.quickshare.cz/html/prihlaseni_process.php', + html = self.load('http://www.quickshare.cz/html/prihlaseni_process.php', post={"akce": u'PÅihlásit', "heslo": data['password'], - "jmeno": user}, - decode=True) + "jmeno": user}, req=req) if u'>TakovÃœ uÅŸivatel neexistuje.<' in html or u'>Å patné heslo.<' in html: self.wrongPassword() diff --git a/module/plugins/accounts/RPNetBiz.py b/module/plugins/accounts/RPNetBiz.py index b188a34d9..509febde6 100644 --- a/module/plugins/accounts/RPNetBiz.py +++ b/module/plugins/accounts/RPNetBiz.py @@ -43,7 +43,7 @@ class RPNetBiz(Account): def getAccountStatus(self, user, req): # Using the rpnet API, check if valid premium account - res = req.load("https://premium.rpnet.biz/client_api.php", + res = self.load("https://premium.rpnet.biz/client_api.php", get={"username": user, "password": self.getAccountData(user)['password'], "action": "showAccountInformation"}) self.logDebug("JSON data: %s" % res) diff --git a/module/plugins/accounts/RapideoPl.py b/module/plugins/accounts/RapideoPl.py index 5039d1494..8b868ee07 100644 --- a/module/plugins/accounts/RapideoPl.py +++ b/module/plugins/accounts/RapideoPl.py @@ -78,4 +78,4 @@ class RapideoPl(Account): def runAuthQuery(self): - return self._req.load(self.API_URL, post=self.createAuthQuery()) + return self._self.load(self.API_URL, post=self.createAuthQuery(), req=req) diff --git a/module/plugins/accounts/RapidgatorNet.py b/module/plugins/accounts/RapidgatorNet.py index 94692da76..c14490dd5 100644 --- a/module/plugins/accounts/RapidgatorNet.py +++ b/module/plugins/accounts/RapidgatorNet.py @@ -27,7 +27,7 @@ class RapidgatorNet(Account): sid = self.getAccountData(user).get('sid', None) assert sid - html = req.load("%s/info" % self.API_URL, get={'sid': sid}) + html = self.load("%s/info" % self.API_URL, get={'sid': sid}, req=req) self.logDebug("API:USERINFO", html) @@ -54,7 +54,7 @@ class RapidgatorNet(Account): def login(self, user, data, req): try: - html = req.load('%s/login' % self.API_URL, post={"username": user, "password": data['password']}) + html = self.load('%s/login' % self.API_URL, post={"username": user, "password": data['password']}, req=req) self.logDebug("API:LOGIN", html) diff --git a/module/plugins/accounts/RapiduNet.py b/module/plugins/accounts/RapiduNet.py index a158e9ce2..5e74e18bb 100644 --- a/module/plugins/accounts/RapiduNet.py +++ b/module/plugins/accounts/RapiduNet.py @@ -30,7 +30,7 @@ class RapiduNet(Account): trafficleft = -1 premium = False - html = req.load("https://rapidu.net/", decode=True) + html = self.load("https://rapidu.net/", req=req) if re.search(self.PREMIUM_PATTERN, html): premium = True @@ -47,17 +47,17 @@ class RapiduNet(Account): def login(self, user, data, req): - req.load("https://rapidu.net/ajax.php", + self.load("https://rapidu.net/ajax.php", get={'a': "getChangeLang"}, post={'_go' : "", - 'lang': "en"}) + 'lang': "en"}, req=req) - json = json_loads(req.load("https://rapidu.net/ajax.php", + json = json_loads(self.load("https://rapidu.net/ajax.php", get={'a': "getUserLogin"}, post={'_go' : "", 'login' : user, 'pass' : data['password'], - 'remember': "1"})) + 'remember': "1"}, req=req)) self.logDebug(json) diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py index b29ad33d3..83fdc5dbc 100644 --- a/module/plugins/accounts/RealdebridCom.py +++ b/module/plugins/accounts/RealdebridCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import xml.dom.minidom +import xml.dom.minidom as dom from module.plugins.internal.Account import Account @@ -19,10 +19,10 @@ class RealdebridCom(Account): if self.pin_code: return - html = req.load("https://real-debrid.com/api/account.php") - account = xml.dom.minidom.parseString(html) + html = self.load("https://real-debrid.com/api/account.php", req=req) + xml = dom.parseString(html) - validuntil = float(account.getElementsByTagName("expiration")[0].childNodes[0].nodeValue) + validuntil = float(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue) return {'validuntil' : validuntil, 'trafficleft': -1 , @@ -32,9 +32,8 @@ class RealdebridCom(Account): def login(self, user, data, req): self.pin_code = False - html = req.load("https://real-debrid.com/ajax/login.php", - get={"user": user, "pass": data['password']}, - decode=True) + html = self.load("https://real-debrid.com/ajax/login.php", + get={"user": user, "pass": data['password']}, req=req) if "Your login informations are incorrect" in html: self.wrongPassword() diff --git a/module/plugins/accounts/RehostTo.py b/module/plugins/accounts/RehostTo.py index 728b2e84e..5f0c1253e 100644 --- a/module/plugins/accounts/RehostTo.py +++ b/module/plugins/accounts/RehostTo.py @@ -19,14 +19,14 @@ class RehostTo(Account): validuntil = -1 session = "" - html = req.load("https://rehost.to/api.php", + html = self.load("https://rehost.to/api.php", get={'cmd' : "login", 'user': user, 'pass': self.getAccountData(user)['password']}) try: session = html.split(",")[1].split("=")[1] - html = req.load("http://rehost.to/api.php", - get={'cmd': "get_premium_credits", 'long_ses': session}) + html = self.load("http://rehost.to/api.php", + get={'cmd': "get_premium_credits", 'long_ses': session}, req=req) if html.strip() == "0,0" or "ERROR" in html: self.logDebug(html) @@ -45,9 +45,8 @@ class RehostTo(Account): def login(self, user, data, req): - html = req.load("https://rehost.to/api.php", - get={'cmd': "login", 'user': user, 'pass': data['password']}, - decode=True) + html = self.load("https://rehost.to/api.php", + get={'cmd': "login", 'user': user, 'pass': data['password']}, req=req) if "ERROR" in html: self.logDebug(html) diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 9269cf2a1..5a3359496 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -16,7 +16,7 @@ class ShareonlineBiz(Account): def api_response(self, user, req): - return req.load("http://api.share-online.biz/cgi-bin", + return self.load("http://api.share-online.biz/cgi-bin", get={'q' : "userdetails", 'aux' : "traffic", 'username': user, diff --git a/module/plugins/accounts/SimplyPremiumCom.py b/module/plugins/accounts/SimplyPremiumCom.py index 5c87cbbfb..053f2c33b 100644 --- a/module/plugins/accounts/SimplyPremiumCom.py +++ b/module/plugins/accounts/SimplyPremiumCom.py @@ -19,7 +19,7 @@ class SimplyPremiumCom(Account): validuntil = -1 trafficleft = None - json_data = req.load('http://www.simply-premium.com/api/user.php?format=json') + json_data = self.load('http://www.simply-premium.com/api/user.php?format=json', req=req) self.logDebug("JSON data: %s" % json_data) @@ -40,9 +40,8 @@ class SimplyPremiumCom(Account): def login(self, user, data, req): req.cj.setCookie("simply-premium.com", "lang", "EN") - html = req.load("https://www.simply-premium.com/login.php", - post={'key': user} if not data['password'] else {'login_name': user, 'login_pass': data['password']}, - decode=True) + html = self.load("https://www.simply-premium.com/login.php", + post={'key': user} if not data['password'] else {'login_name': user, 'login_pass': data['password']}, req=req) if 'logout' not in html: self.wrongPassword() diff --git a/module/plugins/accounts/SimplydebridCom.py b/module/plugins/accounts/SimplydebridCom.py index 97b2b8ecd..e9fda682b 100644 --- a/module/plugins/accounts/SimplydebridCom.py +++ b/module/plugins/accounts/SimplydebridCom.py @@ -17,7 +17,7 @@ class SimplydebridCom(Account): def loadAccountInfo(self, user, req): get_data = {'login': 2, 'u': self.loginname, 'p': self.password} - res = req.load("http://simply-debrid.com/api.php", get=get_data, decode=True) + res = self.load("http://simply-debrid.com/api.php", get=get_data, req=req) data = [x.strip() for x in res.split(";")] if str(data[0]) != "1": return {"premium": False} @@ -30,6 +30,6 @@ class SimplydebridCom(Account): self.password = data['password'] get_data = {'login': 1, 'u': self.loginname, 'p': self.password} - res = req.load("https://simply-debrid.com/api.php", get=get_data, decode=True) + res = self.load("https://simply-debrid.com/api.php", get=get_data, req=req) if res != "02: loggin success": self.wrongPassword() diff --git a/module/plugins/accounts/SmoozedCom.py b/module/plugins/accounts/SmoozedCom.py index b51e5cef3..da81a6b80 100644 --- a/module/plugins/accounts/SmoozedCom.py +++ b/module/plugins/accounts/SmoozedCom.py @@ -74,5 +74,5 @@ class SmoozedCom(Account): salt = hashlib.sha256(password).hexdigest() encrypted = PBKDF2(password, salt, iterations=1000).hexread(32) - return json_loads(req.load("http://www2.smoozed.com/api/login", - get={'auth': user, 'password': encrypted})) + return json_loads(self.load("http://www2.smoozed.com/api/login", + get={'auth': user, 'password': encrypted}, req=req)) diff --git a/module/plugins/accounts/TurbobitNet.py b/module/plugins/accounts/TurbobitNet.py index 43aaf6aa3..7e3bd628f 100644 --- a/module/plugins/accounts/TurbobitNet.py +++ b/module/plugins/accounts/TurbobitNet.py @@ -17,7 +17,7 @@ class TurbobitNet(Account): def loadAccountInfo(self, user, req): - html = req.load("http://turbobit.net") + html = self.load("http://turbobit.net", req=req) m = re.search(r'<u>Turbo Access</u> to ([\d.]+)', html) if m: @@ -33,11 +33,10 @@ class TurbobitNet(Account): def login(self, user, data, req): req.cj.setCookie("turbobit.net", "user_lang", "en") - html = req.load("http://turbobit.net/user/login", + html = self.load("http://turbobit.net/user/login", post={"user[login]": user, "user[pass]": data['password'], - "user[submit]": "Login"}, - decode=True) + "user[submit]": "Login"}, req=req) if not '<div class="menu-item user-name">' in html: self.wrongPassword() diff --git a/module/plugins/accounts/UlozTo.py b/module/plugins/accounts/UlozTo.py index b4f393bfc..3763e2a16 100644 --- a/module/plugins/accounts/UlozTo.py +++ b/module/plugins/accounts/UlozTo.py @@ -21,7 +21,7 @@ class UlozTo(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.ulozto.net/", decode=True) + html = self.load("http://www.ulozto.net/", req=req) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) @@ -32,18 +32,17 @@ class UlozTo(Account): def login(self, user, data, req): - login_page = req.load('http://www.ulozto.net/?do=web-login', decode=True) + login_page = self.load('http://www.ulozto.net/?do=web-login', req=req) action = re.findall('<form action="(.+?)"', login_page)[1].replace('&', '&') token = re.search('_token_" value="(.+?)"', login_page).group(1) - html = req.load(urlparse.urljoin("https://www.ulozto.net/", action), + html = self.load(urlparse.urljoin("https://www.ulozto.net/", action, req=req), post={'_token_' : token, 'do' : "loginForm-submit", 'login' : u"PÅihlásit", 'password': data['password'], 'username': user, - 'remember': "on"}, - decode=True) + 'remember': "on"}) if '<div class="flash error">' in html: self.wrongPassword() diff --git a/module/plugins/accounts/UploadableCh.py b/module/plugins/accounts/UploadableCh.py index 9543feb6d..b0def5600 100644 --- a/module/plugins/accounts/UploadableCh.py +++ b/module/plugins/accounts/UploadableCh.py @@ -14,7 +14,7 @@ class UploadableCh(Account): def loadAccountInfo(self, user, req): - html = req.load("http://www.uploadable.ch/login.php") + html = self.load("http://www.uploadable.ch/login.php", req=req) premium = '<a href="/logout.php"' in html trafficleft = -1 if premium else None @@ -23,12 +23,11 @@ class UploadableCh(Account): def login(self, user, data, req): - html = req.load("http://www.uploadable.ch/login.php", + html = self.load("http://www.uploadable.ch/login.php", post={'userName' : user, 'userPassword' : data["password"], 'autoLogin' : "1", - 'action__login': "normalLogin"}, - decode=True) + 'action__login': "normalLogin"}, req=req) if "Login failed" in html: self.wrongPassword() diff --git a/module/plugins/accounts/UploadedTo.py b/module/plugins/accounts/UploadedTo.py index dc4c9fb73..8cdfab0c6 100644 --- a/module/plugins/accounts/UploadedTo.py +++ b/module/plugins/accounts/UploadedTo.py @@ -26,7 +26,7 @@ class UploadedTo(Account): trafficleft = None premium = None - html = req.load("http://uploaded.net/me") + html = self.load("http://uploaded.net/me", req=req) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -63,9 +63,8 @@ class UploadedTo(Account): def login(self, user, data, req): # req.cj.setCookie("uploaded.net", "lang", "en") - html = req.load("https://uploaded.net/io/login", - post={'id': user, 'pw': data['password'], '_': ""}, - decode=True) + html = self.load("https://uploaded.net/io/login", + post={'id': user, 'pw': data['password'], '_': ""}, req=req) if '"err"' in html: self.wrongPassword() diff --git a/module/plugins/accounts/UploadheroCom.py b/module/plugins/accounts/UploadheroCom.py index f8f2fbc1d..598b7ac57 100644 --- a/module/plugins/accounts/UploadheroCom.py +++ b/module/plugins/accounts/UploadheroCom.py @@ -21,7 +21,7 @@ class UploadheroCom(Account): premium_pattern = re.compile('Il vous reste <span class="bleu">(\d+)</span> jours premium') data = self.getAccountData(user) - html = req.load("http://uploadhero.co/my-account") + html = self.load("http://uploadhero.co/my-account", req=req) if premium_pattern.search(html): end_date = datetime.date.today() + datetime.timedelta(days=int(premium_pattern.search(html).group(1))) @@ -34,9 +34,8 @@ class UploadheroCom(Account): def login(self, user, data, req): - html = req.load("http://uploadhero.co/lib/connexion.php", - post={"pseudo_login": user, "password_login": data['password']}, - decode=True) + html = self.load("http://uploadhero.co/lib/connexion.php", + post={"pseudo_login": user, "password_login": data['password']}, req=req) if "mot de passe invalide" in html: self.wrongPassword() diff --git a/module/plugins/accounts/UploadingCom.py b/module/plugins/accounts/UploadingCom.py index 967f79678..c701fb8fa 100644 --- a/module/plugins/accounts/UploadingCom.py +++ b/module/plugins/accounts/UploadingCom.py @@ -4,7 +4,7 @@ import re import time from module.plugins.internal.Account import Account -from module.plugins.internal.SimpleHoster import set_cookies +from module.plugins.internal.Plugin import set_cookies class UploadingCom(Account): @@ -26,7 +26,7 @@ class UploadingCom(Account): trafficleft = None premium = None - html = req.load("http://uploading.com/") + html = self.load("http://uploading.com/", req=req) premium = False if re.search(self.PREMIUM_PATTERN, html) else True @@ -60,6 +60,6 @@ class UploadingCom(Account): ("uploading.com", "setlang" , "en"), ("uploading.com", "_lang" , "en")]) - req.load("http://uploading.com/") - req.load("https://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time.time() * 1000), + self.load("http://uploading.com/", req=req) + self.load("https://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time.time() * 1000, req=req), post={'email': user, 'password': data['password'], 'remember': "on"}) diff --git a/module/plugins/accounts/WebshareCz.py b/module/plugins/accounts/WebshareCz.py index c88e86aba..d0f5524a2 100644 --- a/module/plugins/accounts/WebshareCz.py +++ b/module/plugins/accounts/WebshareCz.py @@ -25,9 +25,8 @@ class WebshareCz(Account): def loadAccountInfo(self, user, req): - html = req.load("https://webshare.cz/api/user_data/", - post={'wst': self.getAccountData(user).get('wst', None)}, - decode=True) + html = self.load("https://webshare.cz/api/user_data/", + post={'wst': self.getAccountData(user).get('wst', None)}) self.logDebug("Response: " + html) @@ -42,10 +41,9 @@ class WebshareCz(Account): def login(self, user, data, req): - salt = req.load("https://webshare.cz/api/salt/", + salt = self.load("https://webshare.cz/api/salt/", post={'username_or_email': user, - 'wst' : ""}, - decode=True) + 'wst' : ""}, req=req) if "<status>OK</status>" not in salt: self.wrongPassword() @@ -54,13 +52,12 @@ class WebshareCz(Account): password = hashlib.sha1(md5_crypt.encrypt(data["password"], salt=salt)).hexdigest() digest = hashlib.md5(user + ":Webshare:" + password).hexdigest() - login = req.load("https://webshare.cz/api/login/", + login = self.load("https://webshare.cz/api/login/", post={'digest' : digest, 'keep_logged_in' : 1, 'password' : password, 'username_or_email': user, - 'wst' : ""}, - decode=True) + 'wst' : ""}, req=req) if "<status>OK</status>" not in login: self.wrongPassword() diff --git a/module/plugins/accounts/YibaishiwuCom.py b/module/plugins/accounts/YibaishiwuCom.py index 2f1c990ec..9d1d1d38f 100644 --- a/module/plugins/accounts/YibaishiwuCom.py +++ b/module/plugins/accounts/YibaishiwuCom.py @@ -20,7 +20,7 @@ class YibaishiwuCom(Account): def loadAccountInfo(self, user, req): #self.relogin(user) - html = req.load("http://115.com/", decode=True) + html = self.load("http://115.com/", req=req) m = re.search(self.ACCOUNT_INFO_PATTERN, html, re.S) premium = True if m and 'is_vip: 1' in m.group(1) else False @@ -29,12 +29,11 @@ class YibaishiwuCom(Account): def login(self, user, data, req): - html = req.load("https://passport.115.com/?ac=login", + html = self.load("https://passport.115.com/?ac=login", post={"back": "http://www.115.com/", "goto": "http://115.com/", "login[account]": user, - "login[passwd]": data['password']}, - decode=True) + "login[passwd]": data['password']}, req=req) if not 'var USER_PERMISSION = {' in html: self.wrongPassword() diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py index 220a6600e..0ca7fba45 100644 --- a/module/plugins/accounts/ZeveraCom.py +++ b/module/plugins/accounts/ZeveraCom.py @@ -62,10 +62,9 @@ class ZeveraCom(Account): get_data.update(kwargs) - res = req.load(self.API_URL, + res = self.load(self.API_URL, get=get_data, - just_header=just_header, - decode=True) + just_header=just_header, req=req) self.logDebug(res) diff --git a/module/plugins/crypter/DataHuFolder.py b/module/plugins/crypter/DataHuFolder.py index c0fe26869..fcbc5b2af 100644 --- a/module/plugins/crypter/DataHuFolder.py +++ b/module/plugins/crypter/DataHuFolder.py @@ -35,7 +35,7 @@ class DataHuFolder(SimpleCrypter): self.logDebug("The folder is password protected', 'Using password: " + password) - self.html = self.load(self.pyfile.url, post={'mappa_pass': password}, decode=True) + self.html = self.load(self.pyfile.url, post={'mappa_pass': password}) if u'Hib\xe1s jelsz\xf3' in self.html: # Wrong password self.fail(_("Wrong password")) diff --git a/module/plugins/crypter/EmbeduploadCom.py b/module/plugins/crypter/EmbeduploadCom.py index 221e365fa..3a830331f 100644 --- a/module/plugins/crypter/EmbeduploadCom.py +++ b/module/plugins/crypter/EmbeduploadCom.py @@ -25,7 +25,7 @@ class EmbeduploadCom(Crypter): def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) tmp_links = [] m = re.findall(self.LINK_PATTERN, self.html) diff --git a/module/plugins/crypter/FilecryptCc.py b/module/plugins/crypter/FilecryptCc.py index 2271a7b6d..7ca5659fb 100644 --- a/module/plugins/crypter/FilecryptCc.py +++ b/module/plugins/crypter/FilecryptCc.py @@ -96,8 +96,7 @@ class FilecryptCc(Crypter): imgtype="gif") self.siteWithLinks = self.load(self.pyfile.url, - post={'recaptcha_response_field': captcha_code}, - decode=True) + post={'recaptcha_response_field': captcha_code}) elif m2: #: circle captcha self.logDebug("Captcha-URL: %s" % m2.group(1)) @@ -105,8 +104,7 @@ class FilecryptCc(Crypter): result_type='positional') self.siteWithLinks = self.load(self.pyfile.url, - post={'button.x': captcha_code[0], 'button.y': captcha_code[1]}, - decode=True) + post={'button.x': captcha_code[0], 'button.y': captcha_code[1]}) else: recaptcha = ReCaptcha(self) @@ -115,8 +113,7 @@ class FilecryptCc(Crypter): if captcha_key: response, challenge = recaptcha.challenge(captcha_key) self.siteWithLinks = self.load(self.pyfile.url, - post={'g-recaptcha-response': response}, - decode=True) + post={'g-recaptcha-response': response}) else: self.logInfo(_("No captcha found")) self.siteWithLinks = self.html diff --git a/module/plugins/crypter/FreakhareComFolder.py b/module/plugins/crypter/FreakhareComFolder.py index 173660668..59fa9cf32 100644 --- a/module/plugins/crypter/FreakhareComFolder.py +++ b/module/plugins/crypter/FreakhareComFolder.py @@ -36,7 +36,7 @@ class FreakhareComFolder(SimpleCrypter): 'f_md5': self.f_md5, 'entrys': '20', 'page': page_n - 1, - 'order': ''}, decode=True) + 'order': ''}) getInfo = create_getInfo(FreakhareComFolder) diff --git a/module/plugins/crypter/LinkdecrypterCom.py b/module/plugins/crypter/LinkdecrypterCom.py index e12bf3359..c8de44157 100644 --- a/module/plugins/crypter/LinkdecrypterCom.py +++ b/module/plugins/crypter/LinkdecrypterCom.py @@ -35,7 +35,7 @@ class LinkdecrypterCom(Crypter): retries = 5 post_dict = {"link_cache": "on", "pro_links": pyfile.url, "modo_links": "text"} - self.html = self.load('http://linkdecrypter.com/', post=post_dict, decode=True) + self.html = self.load('http://linkdecrypter.com/', post=post_dict) while retries: m = re.search(self.TEXTAREA_PATTERN, self.html, re.S) @@ -54,16 +54,16 @@ class LinkdecrypterCom(Crypter): captcha = self.decryptCaptcha(captcha_url, result_type=result_type) if result_type == "positional": captcha = "%d|%d" % captcha - self.html = self.load('http://linkdecrypter.com/', post={"captcha": captcha}, decode=True) + self.html = self.load('http://linkdecrypter.com/', post={"captcha": captcha}) retries -= 1 elif self.PASSWORD_PATTERN in self.html: if self.password: self.logInfo(_("Password protected link")) - self.html = self.load('http://linkdecrypter.com/', post={'password': self.password}, decode=True) + self.html = self.load('http://linkdecrypter.com/', post={'password': self.password}) else: self.fail(_("Missing password")) else: retries -= 1 - self.html = self.load('http://linkdecrypter.com/', decode=True) + self.html = self.load('http://linkdecrypter.com/') diff --git a/module/plugins/crypter/LixIn.py b/module/plugins/crypter/LixIn.py index a4bd9df1e..2b4c2071c 100644 --- a/module/plugins/crypter/LixIn.py +++ b/module/plugins/crypter/LixIn.py @@ -35,7 +35,7 @@ class LixIn(Crypter): id = m.group('ID') self.logDebug("File id is %s" % id) - self.html = self.load(url, decode=True) + self.html = self.load(url) m = re.search(self.SUBMIT_PATTERN, self.html) if m is None: @@ -48,12 +48,12 @@ class LixIn(Crypter): if m: self.logDebug("Trying captcha") captcharesult = self.decryptCaptcha(urlparse.urljoin("http://lix.in/", m.group(1))) - self.html = self.load(url, decode=True, + self.html = self.load(url, post={"capt": captcharesult, "submit": "submit", "tiny": id}) else: self.logDebug("No captcha/captcha solved") else: - self.html = self.load(url, decode=True, post={"submit": "submit", "tiny": id}) + self.html = self.load(url, post={"submit": "submit", "tiny": id}) m = re.search(self.LINK_PATTERN, self.html) if m is None: diff --git a/module/plugins/crypter/MegaCoNzFolder.py b/module/plugins/crypter/MegaCoNzFolder.py index 517a6b239..a462b7bca 100644 --- a/module/plugins/crypter/MegaCoNzFolder.py +++ b/module/plugins/crypter/MegaCoNzFolder.py @@ -27,6 +27,3 @@ class MegaCoNzFolder(Crypter): url = "https://mega.co.nz/#F!%s!%s" % re.match(self.__pattern__, pyfile.url).groups() self.html = self.load("http://rapidgen.org/linkfinder", post={'linklisturl': url}) self.urls = re.findall(r'(https://mega(\.co)?\.nz/#N!.+?)<', self.html) - - if not self.urls: #@TODO: Remove in 0.4.10 - self.fail(_("No link grabbed")) diff --git a/module/plugins/crypter/MultiloadCz.py b/module/plugins/crypter/MultiloadCz.py index 10b5770fa..6b1171a54 100644 --- a/module/plugins/crypter/MultiloadCz.py +++ b/module/plugins/crypter/MultiloadCz.py @@ -25,7 +25,7 @@ class MultiloadCz(Crypter): def decrypt(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) if re.match(self.__pattern__, pyfile.url).group(1) == "slozka": m = re.search(self.FOLDER_PATTERN, self.html) diff --git a/module/plugins/crypter/NCryptIn.py b/module/plugins/crypter/NCryptIn.py index e85120f92..f92f47e99 100644 --- a/module/plugins/crypter/NCryptIn.py +++ b/module/plugins/crypter/NCryptIn.py @@ -84,7 +84,7 @@ class NCryptIn(Crypter): def requestFolderHome(self): - return self.load(self.pyfile.url, decode=True) + return self.load(self.pyfile.url) def removeHtmlCrap(self, content): @@ -170,7 +170,7 @@ class NCryptIn(Crypter): # Unlock protection postData['submit_protected'] = 'Continue to folder' - return self.load(self.pyfile.url, post=postData, decode=True) + return self.load(self.pyfile.url, post=postData) def handleErrors(self): diff --git a/module/plugins/crypter/NitroflareComFolder.py b/module/plugins/crypter/NitroflareComFolder.py index 43572e7f8..1db2378e3 100644 --- a/module/plugins/crypter/NitroflareComFolder.py +++ b/module/plugins/crypter/NitroflareComFolder.py @@ -23,8 +23,7 @@ class NitroflareComFolder(SimpleCrypter): post={'userId' : self.info['pattern']['USER'], 'folder' : self.info['pattern']['ID'], 'page' : 1, - 'perPage': 10000}, - decode=True)) + 'perPage': 10000})) if res['name']: self.pyfile.name = res['name'] else: diff --git a/module/plugins/crypter/RelinkUs.py b/module/plugins/crypter/RelinkUs.py index 316d44b4c..6e7c80561 100644 --- a/module/plugins/crypter/RelinkUs.py +++ b/module/plugins/crypter/RelinkUs.py @@ -104,7 +104,7 @@ class RelinkUs(Crypter): def requestPackage(self): - self.html = self.load(self.pyfile.url, decode=True) + self.html = self.load(self.pyfile.url) def isOnline(self): @@ -135,7 +135,7 @@ class RelinkUs(Crypter): if password: passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.fileid passwd_data = {'id': self.fileid, 'password': password, 'pw': 'submit'} - self.html = self.load(passwd_url, post=passwd_data, decode=True) + self.html = self.load(passwd_url, post=passwd_data) def unlockCaptchaProtection(self): @@ -145,7 +145,7 @@ class RelinkUs(Crypter): self.logDebug("Captcha resolved, coords [%s]" % str(coords)) captcha_post_url = self.CAPTCHA_SUBMIT_URL + "?id=%s" % self.fileid captcha_post_data = {'button.x': coords[0], 'button.y': coords[1], 'captcha': 'submit'} - self.html = self.load(captcha_post_url, post=captcha_post_data, decode=True) + self.html = self.load(captcha_post_url, post=captcha_post_data) def getPackageInfo(self): @@ -244,7 +244,7 @@ class RelinkUs(Crypter): self.logDebug("Decrypting Web link %d, %s" % (index + 1, url)) - res = self.load(url, decode=True) + res = self.load(url) link = re.search(self.WEB_LINK_REGEX, res).group(1) package_links.append(link) diff --git a/module/plugins/crypter/SexuriaCom.py b/module/plugins/crypter/SexuriaCom.py index b2101e92a..4ee4eb0b3 100644 --- a/module/plugins/crypter/SexuriaCom.py +++ b/module/plugins/crypter/SexuriaCom.py @@ -60,7 +60,7 @@ class SexuriaCom(Crypter): elif re.match(self.PATTERN_SUPPORTED_CRYPT, url, re.I): # Extract info from main file id = re.search(self.PATTERN_SUPPORTED_CRYPT, url, re.I).group('ID') - html = self.load("http://sexuria.com/v1/Pornos_Kostenlos_info_%s.html" % id, decode=True) + html = self.load("http://sexuria.com/v1/Pornos_Kostenlos_info_%s.html" % id) title = re.search(self.PATTERN_TITLE, html, re.I).group('TITLE').strip() if title: diff --git a/module/plugins/crypter/ShareLinksBiz.py b/module/plugins/crypter/ShareLinksBiz.py index aab898813..2d72b513e 100644 --- a/module/plugins/crypter/ShareLinksBiz.py +++ b/module/plugins/crypter/ShareLinksBiz.py @@ -34,7 +34,7 @@ class ShareLinksBiz(Crypter): # Request package url = self.baseUrl + '/' + self.fileId - self.html = self.load(url, decode=True) + self.html = self.load(url) # Unblock server (load all images) self.unblockServer() @@ -104,7 +104,7 @@ class ShareLinksBiz(Crypter): self.logDebug("Submitting password [%s] for protected links" % password) post = {"password": password, 'login': 'Submit form'} url = self.baseUrl + '/' + self.fileId - self.html = self.load(url, post=post, decode=True) + self.html = self.load(url, post=post) def unlockCaptchaProtection(self): @@ -125,7 +125,7 @@ class ShareLinksBiz(Crypter): self.invalidCaptcha() self.retry(wait_time=5) url = self.baseUrl + href - self.html = self.load(url, decode=True) + self.html = self.load(url) def _getCaptchaMap(self): diff --git a/module/plugins/crypter/TnyCz.py b/module/plugins/crypter/TnyCz.py index b04c1b647..56c737969 100644 --- a/module/plugins/crypter/TnyCz.py +++ b/module/plugins/crypter/TnyCz.py @@ -25,7 +25,7 @@ class TnyCz(SimpleCrypter): def getLinks(self): m = re.search(r'<a id=\'save_paste\' href="(.+save\.php\?hash=.+)">', self.html) - return re.findall(".+", self.load(m.group(1), decode=True)) if m else None + return re.findall(".+", self.load(m.group(1))) if m else None getInfo = create_getInfo(TnyCz) diff --git a/module/plugins/crypter/TurbobitNetFolder.py b/module/plugins/crypter/TurbobitNetFolder.py index dcbd6a5d7..2ffcfae13 100644 --- a/module/plugins/crypter/TurbobitNetFolder.py +++ b/module/plugins/crypter/TurbobitNetFolder.py @@ -27,7 +27,7 @@ class TurbobitNetFolder(SimpleCrypter): def _getLinks(self, id, page=1): gridFile = self.load("http://turbobit.net/downloadfolder/gridFile", - get={"rootId": id, "rows": 200, "page": page}, decode=True) + get={"rootId": id, "rows": 200, "page": page}) grid = json_loads(gridFile) if grid['rows']: diff --git a/module/plugins/crypter/TusfilesNetFolder.py b/module/plugins/crypter/TusfilesNetFolder.py index 1cc505b81..b8840132f 100644 --- a/module/plugins/crypter/TusfilesNetFolder.py +++ b/module/plugins/crypter/TusfilesNetFolder.py @@ -28,7 +28,7 @@ class TusfilesNetFolder(XFSCrypter): def loadPage(self, page_n): - return self.load(urlparse.urljoin(self.pyfile.url, str(page_n)), decode=True) + return self.load(urlparse.urljoin(self.pyfile.url, str(page_n))) def handlePages(self, pyfile): diff --git a/module/plugins/hooks/AlldebridComHook.py b/module/plugins/hooks/AlldebridComHook.py index 092921134..c55f013a4 100644 --- a/module/plugins/hooks/AlldebridComHook.py +++ b/module/plugins/hooks/AlldebridComHook.py @@ -22,6 +22,6 @@ class AlldebridComHook(MultiHook): def getHosters(self): https = "https" if self.getConfig('ssl') else "http" - html = self.getURL(https + "://www.alldebrid.com/api.php", get={'action': "get_host"}).replace("\"", "").strip() + html = self.load(https + "://www.alldebrid.com/api.php", get={'action': "get_host"}).replace("\"", "").strip() return [x.strip() for x in html.split(",") if x.strip()] diff --git a/module/plugins/hooks/AndroidPhoneNotify.py b/module/plugins/hooks/AndroidPhoneNotify.py index e775b785b..daf4c1a27 100644 --- a/module/plugins/hooks/AndroidPhoneNotify.py +++ b/module/plugins/hooks/AndroidPhoneNotify.py @@ -2,7 +2,6 @@ import time -from module.network.RequestFactory import getURL from module.plugins.internal.Hook import Hook, Expose @@ -106,7 +105,7 @@ class AndroidPhoneNotify(Hook): elif self.notifications >= self.getConfig("sendpermin"): return - getURL("http://www.notifymyandroid.com/publicapi/notify", + self.load("http://www.notifymyandroid.com/publicapi/notify", get={'apikey' : key, 'application': "pyLoad", 'event' : event, diff --git a/module/plugins/hooks/BypassCaptcha.py b/module/plugins/hooks/BypassCaptcha.py index bbbe96b73..1651ea067 100644 --- a/module/plugins/hooks/BypassCaptcha.py +++ b/module/plugins/hooks/BypassCaptcha.py @@ -3,7 +3,7 @@ import pycurl from module.network.HTTPRequest import BadHeader -from module.network.RequestFactory import getURL, getRequest +from module.network.RequestFactory import getRequest from module.plugins.internal.Hook import Hook, threaded @@ -54,7 +54,7 @@ class BypassCaptcha(Hook): def getCredits(self): - res = getURL(self.GETCREDITS_URL, post={"key": self.getConfig('passkey')}) + res = self.load(self.GETCREDITS_URL, post={"key": self.getConfig('passkey')}) data = dict(x.split(' ', 1) for x in res.splitlines()) return int(data['Left']) @@ -67,12 +67,12 @@ class BypassCaptcha(Hook): req.c.setopt(pycurl.LOW_SPEED_TIME, 80) try: - res = req.load(self.SUBMIT_URL, - post={'vendor_key': self.PYLOAD_KEY, - 'key': self.getConfig('passkey'), - 'gen_task_id': "1", - 'file': (pycurl.FORM_FILE, captcha)}, - multipart=True) + res = self.load(self.SUBMIT_URL, + post={'vendor_key': self.PYLOAD_KEY, + 'key': self.getConfig('passkey'), + 'gen_task_id': "1", + 'file': (pycurl.FORM_FILE, captcha)}, + req=req) finally: req.close() @@ -89,7 +89,7 @@ class BypassCaptcha(Hook): def respond(self, ticket, success): try: - res = getURL(self.RESPOND_URL, post={"task_id": ticket, "key": self.getConfig('passkey'), + res = self.load(self.RESPOND_URL, post={"task_id": ticket, "key": self.getConfig('passkey'), "cv": 1 if success else 0}) except BadHeader, e: self.logError(_("Could not send response"), e) diff --git a/module/plugins/hooks/Captcha9Kw.py b/module/plugins/hooks/Captcha9Kw.py index 85bbb7924..a67e5bfc3 100644 --- a/module/plugins/hooks/Captcha9Kw.py +++ b/module/plugins/hooks/Captcha9Kw.py @@ -8,8 +8,6 @@ import time from base64 import b64encode from module.network.HTTPRequest import BadHeader -from module.network.RequestFactory import getURL - from module.plugins.internal.Hook import Hook, threaded @@ -48,7 +46,7 @@ class Captcha9Kw(Hook): def getCredits(self): - res = getURL(self.API_URL, + res = self.load(self.API_URL, get={'apikey': self.getConfig('passkey'), 'pyload': "1", 'source': "pyload", @@ -129,7 +127,7 @@ class Captcha9Kw(Hook): for _i in xrange(5): try: - res = getURL(self.API_URL, post=post_data) + res = self.load(self.API_URL, post=post_data) except BadHeader, e: time.sleep(3) else: @@ -144,7 +142,7 @@ class Captcha9Kw(Hook): task.data["ticket"] = res for _i in xrange(int(self.getConfig('timeout') / 5)): - result = getURL(self.API_URL, + result = self.load(self.API_URL, get={'apikey': self.getConfig('passkey'), 'id' : res, 'pyload': "1", @@ -186,7 +184,7 @@ class Captcha9Kw(Hook): pluginname = re.search(r'_([^_]*)_\d+.\w+', task.captchaFile).group(1) for _i in xrange(5): - servercheck = getURL("http://www.9kw.eu/grafik/servercheck.txt") + servercheck = self.load("http://www.9kw.eu/grafik/servercheck.txt") if queue < re.search(r'queue=(\d+)', servercheck).group(1): break @@ -227,7 +225,7 @@ class Captcha9Kw(Hook): passkey = self.getConfig('passkey') for _i in xrange(3): - res = getURL(self.API_URL, + res = self.load(self.API_URL, get={'action' : "usercaptchacorrectback", 'apikey' : passkey, 'api_key': passkey, diff --git a/module/plugins/hooks/CaptchaBrotherhood.py b/module/plugins/hooks/CaptchaBrotherhood.py index 1f2bbf956..3992c6ca7 100644 --- a/module/plugins/hooks/CaptchaBrotherhood.py +++ b/module/plugins/hooks/CaptchaBrotherhood.py @@ -12,7 +12,7 @@ try: except ImportError: import Image -from module.network.RequestFactory import getURL, getRequest +from module.network.RequestFactory import getRequest from module.plugins.internal.Hook import Hook, threaded @@ -59,7 +59,7 @@ class CaptchaBrotherhood(Hook): def getCredits(self): - res = getURL(self.API_URL + "askCredits.aspx", + res = self.load(self.API_URL + "askCredits.aspx", get={"username": self.getConfig('username'), "password": self.getConfig('passkey')}) if not res.startswith("OK"): raise CaptchaBrotherhoodException(res) @@ -122,7 +122,7 @@ class CaptchaBrotherhood(Hook): def api_response(self, api, ticket): - res = getURL("%s%s.aspx" % (self.API_URL, api), + res = self.load("%s%s.aspx" % (self.API_URL, api), get={"username": self.getConfig('username'), "password": self.getConfig('passkey'), "captchaID": ticket}) diff --git a/module/plugins/hooks/DeathByCaptcha.py b/module/plugins/hooks/DeathByCaptcha.py index 1ece35bdb..50331d512 100644 --- a/module/plugins/hooks/DeathByCaptcha.py +++ b/module/plugins/hooks/DeathByCaptcha.py @@ -84,9 +84,11 @@ class DeathByCaptcha(Hook): res = None try: - json = req.load("%s%s" % (self.API_URL, api), - post=post, - multipart=multipart) + json = self.load("%s%s" % (self.API_URL, api), + post=post, + multipart=multipart, + req=req) + self.logDebug(json) res = json_loads(json) diff --git a/module/plugins/hooks/DebridItaliaComHook.py b/module/plugins/hooks/DebridItaliaComHook.py index 36b307696..b9a5f1b60 100644 --- a/module/plugins/hooks/DebridItaliaComHook.py +++ b/module/plugins/hooks/DebridItaliaComHook.py @@ -23,4 +23,4 @@ class DebridItaliaComHook(MultiHook): def getHosters(self): - return self.getURL("http://debriditalia.com/api.php", get={'hosts': ""}).replace('"', '').split(',') + return self.load("http://debriditalia.com/api.php", get={'hosts': ""}).replace('"', '').split(',') diff --git a/module/plugins/hooks/EasybytezComHook.py b/module/plugins/hooks/EasybytezComHook.py index 43efb5048..951d95479 100644 --- a/module/plugins/hooks/EasybytezComHook.py +++ b/module/plugins/hooks/EasybytezComHook.py @@ -25,6 +25,6 @@ class EasybytezComHook(MultiHook): user, data = self.account.selectAccount() req = self.account.getAccountRequest(user) - html = req.load("http://www.easybytez.com") + html = self.load("http://www.easybytez.com", req=req) return re.search(r'</textarea>\s*Supported sites:(.*)', html).group(1).split(',') diff --git a/module/plugins/hooks/ExpertDecoders.py b/module/plugins/hooks/ExpertDecoders.py index 13289f8d1..af5f2cfcd 100644 --- a/module/plugins/hooks/ExpertDecoders.py +++ b/module/plugins/hooks/ExpertDecoders.py @@ -8,7 +8,7 @@ import uuid from base64 import b64encode from module.network.HTTPRequest import BadHeader -from module.network.RequestFactory import getURL, getRequest +from module.network.RequestFactory import getRequest from module.plugins.internal.Hook import Hook, threaded @@ -36,7 +36,7 @@ class ExpertDecoders(Hook): def getCredits(self): - res = getURL(self.API_URL, post={"key": self.getConfig('passkey'), "action": "balance"}) + res = self.load(self.API_URL, post={"key": self.getConfig('passkey'), "action": "balance"}) if res.isdigit(): self.logInfo(_("%s credits left") % res) @@ -60,11 +60,12 @@ class ExpertDecoders(Hook): req.c.setopt(pycurl.LOW_SPEED_TIME, 80) try: - result = req.load(self.API_URL, - post={'action' : "upload", + result = self.load(self.API_URL, + post={'action' : "upload", 'key' : self.getConfig('passkey'), 'file' : b64encode(data), - 'gen_task_id': ticket}) + 'gen_task_id': ticket}, + req=req) finally: req.close() @@ -95,7 +96,7 @@ class ExpertDecoders(Hook): if "ticket" in task.data: try: - res = getURL(self.API_URL, + res = self.load(self.API_URL, post={'action': "refund", 'key': self.getConfig('passkey'), 'gen_task_id': task.data['ticket']}) self.logInfo(_("Request refund"), res) diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py index 85df87a23..6b3d21d68 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/hooks/ExternalScripts.py @@ -82,11 +82,12 @@ class ExternalScripts(Hook): def callScript(self, script, *args): try: - cmd_args = [fs_encode(str(x) if not isinstance(x, basestring) else x) for x in args] - cmd = [script] + cmd_args + cmd_args = (fs_encode(x) if isinstande(x, basestring) else str(x) for x in args) #@NOTE: `fs_encode` -> `encode` in 0.4.10 self.logDebug("Executing: %s" % os.path.abspath(script), "Args: " + ' '.join(cmd_args)) + cmd = (script,) + cmd_args + p = subprocess.Popen(cmd, bufsize=-1) #@NOTE: output goes to pyload if self.getConfig('waitend'): p.communicate() diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 2c0d68f3e..c9f368e9f 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -54,8 +54,8 @@ if os.name != "nt": from pwd import getpwnam from module.plugins.internal.Hook import Hook, Expose, threaded +from module.plugins.internal.Plugin import replace_patterns from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.plugins.internal.SimpleHoster import replace_patterns from module.utils import fs_encode, save_join as fs_join, uniqify diff --git a/module/plugins/hooks/FastixRuHook.py b/module/plugins/hooks/FastixRuHook.py index 16e30e93a..24b18cb9f 100644 --- a/module/plugins/hooks/FastixRuHook.py +++ b/module/plugins/hooks/FastixRuHook.py @@ -21,7 +21,7 @@ class FastixRuHook(MultiHook): def getHosters(self): - html = self.getURL("http://fastix.ru/api_v2", + html = self.load("http://fastix.ru/api_v2", get={'apikey': "5182964c3f8f9a7f0b00000a_kelmFB4n1IrnCDYuIFn2y", 'sub' : "allowed_sources"}) host_list = json_loads(html) diff --git a/module/plugins/hooks/FreeWayMeHook.py b/module/plugins/hooks/FreeWayMeHook.py index b4219a953..473bd7d26 100644 --- a/module/plugins/hooks/FreeWayMeHook.py +++ b/module/plugins/hooks/FreeWayMeHook.py @@ -21,5 +21,5 @@ class FreeWayMeHook(MultiHook): def getHosters(self): user, data = self.account.selectAccount() - hostis = self.getURL("http://www.free-way.bz/ajax/jd.php", get={"id": 3, "user": user, "pass": data['password']}).replace("\"", "") #@TODO: Revert to `https` in 0.4.10 + hostis = self.load("http://www.free-way.bz/ajax/jd.php", get={"id": 3, "user": user, "pass": data['password']}).replace("\"", "") #@TODO: Revert to `https` in 0.4.10 return [x.strip() for x in hostis.split(",") if x.strip()] diff --git a/module/plugins/hooks/HighWayMeHook.py b/module/plugins/hooks/HighWayMeHook.py index ff4d3f96b..c04fc969e 100644 --- a/module/plugins/hooks/HighWayMeHook.py +++ b/module/plugins/hooks/HighWayMeHook.py @@ -21,6 +21,6 @@ class HighWayMeHook(MultiHook): def getHosters(self): - json_data = json_loads(self.getURL("https://high-way.me/api.php", + json_data = json_loads(self.load("https://high-way.me/api.php", get={'hoster': 1})) return [element['name'] for element in json_data['hoster']] diff --git a/module/plugins/hooks/IRCInterface.py b/module/plugins/hooks/IRCInterface.py index e24691bde..9176f2d2c 100644 --- a/module/plugins/hooks/IRCInterface.py +++ b/module/plugins/hooks/IRCInterface.py @@ -11,7 +11,6 @@ from select import select from threading import Thread from module.Api import PackageDoesNotExists, FileDoesNotExists -from module.network.RequestFactory import getURL from module.plugins.internal.Hook import Hook from module.utils import formatSize @@ -76,8 +75,8 @@ class IRCInterface(Thread, Hook): task.handler.append(self) task.setWaiting(60) - html = getURL("http://www.freeimagehosting.net/upload.php", - post={"attached": (pycurl.FORM_FILE, task.captchaFile)}, multipart=True) + html = self.load("http://www.freeimagehosting.net/upload.php", + post={"attached": (pycurl.FORM_FILE, task.captchaFile)}) url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", html).group(1) self.response(_("New Captcha Request: %s") % url) diff --git a/module/plugins/hooks/ImageTyperz.py b/module/plugins/hooks/ImageTyperz.py index 7fb642250..15097fd5a 100644 --- a/module/plugins/hooks/ImageTyperz.py +++ b/module/plugins/hooks/ImageTyperz.py @@ -7,7 +7,7 @@ import re from base64 import b64encode -from module.network.RequestFactory import getURL, getRequest +from module.network.RequestFactory import getRequest from module.plugins.internal.Hook import Hook, threaded @@ -56,7 +56,7 @@ class ImageTyperz(Hook): def getCredits(self): - res = getURL(self.GETCREDITS_URL, + res = self.load(self.GETCREDITS_URL, post={'action': "REQUESTBALANCE", 'username': self.getConfig('username'), 'password': self.getConfig('passkey')}) @@ -89,11 +89,12 @@ class ImageTyperz(Hook): data = f.read() data = b64encode(data) - res = req.load(self.SUBMIT_URL, - post={'action': "UPLOADCAPTCHA", - 'username': self.getConfig('username'), - 'password': self.getConfig('passkey'), "file": data}, - multipart=multipart) + res = self.load(self.SUBMIT_URL, + post={'action': "UPLOADCAPTCHA", + 'username': self.getConfig('username'), + 'password': self.getConfig('passkey'), "file": data}, + multipart=multipart, + req=req) finally: req.close() @@ -134,7 +135,7 @@ class ImageTyperz(Hook): def captcha_invalid(self, task): if task.data['service'] == self.__name__ and "ticket" in task.data: - res = getURL(self.RESPOND_URL, + res = self.load(self.RESPOND_URL, post={'action': "SETBADIMAGE", 'username': self.getConfig('username'), 'password': self.getConfig('passkey'), diff --git a/module/plugins/hooks/LinkdecrypterComHook.py b/module/plugins/hooks/LinkdecrypterComHook.py index b2eaece62..76167524b 100644 --- a/module/plugins/hooks/LinkdecrypterComHook.py +++ b/module/plugins/hooks/LinkdecrypterComHook.py @@ -23,7 +23,7 @@ class LinkdecrypterComHook(MultiHook): def getHosters(self): list = re.search(r'>Supported\(\d+\)</b>: <i>(.[\w.\-, ]+)', - self.getURL("http://linkdecrypter.com/", decode=True).replace("(g)", "")).group(1).split(', ') + self.load("http://linkdecrypter.com/").replace("(g)", "")).group(1).split(', ') try: list.remove("download.serienjunkies.org") except ValueError: diff --git a/module/plugins/hooks/LinksnappyComHook.py b/module/plugins/hooks/LinksnappyComHook.py index 72282575b..d1bd19f1e 100644 --- a/module/plugins/hooks/LinksnappyComHook.py +++ b/module/plugins/hooks/LinksnappyComHook.py @@ -21,7 +21,7 @@ class LinksnappyComHook(MultiHook): def getHosters(self): - json_data = self.getURL("http://gen.linksnappy.com/lseAPI.php", get={'act': "FILEHOSTS"}) + json_data = self.load("http://gen.linksnappy.com/lseAPI.php", get={'act': "FILEHOSTS"}) json_data = json_loads(json_data) return json_data['return'].keys() diff --git a/module/plugins/hooks/MegaDebridEuHook.py b/module/plugins/hooks/MegaDebridEuHook.py index 0de7b4dcf..0e18f6f9d 100644 --- a/module/plugins/hooks/MegaDebridEuHook.py +++ b/module/plugins/hooks/MegaDebridEuHook.py @@ -21,7 +21,7 @@ class MegaDebridEuHook(MultiHook): def getHosters(self): - reponse = self.getURL("http://www.mega-debrid.eu/api.php", get={'action': "getHosters"}) + reponse = self.load("http://www.mega-debrid.eu/api.php", get={'action': "getHosters"}) json_data = json_loads(reponse) if json_data['response_code'] == "ok": diff --git a/module/plugins/hooks/MultishareCzHook.py b/module/plugins/hooks/MultishareCzHook.py index 6052b7673..c3e923d9b 100644 --- a/module/plugins/hooks/MultishareCzHook.py +++ b/module/plugins/hooks/MultishareCzHook.py @@ -25,5 +25,5 @@ class MultishareCzHook(MultiHook): def getHosters(self): - html = self.getURL("http://www.multishare.cz/monitoring/") + html = self.load("http://www.multishare.cz/monitoring/") return re.findall(self.HOSTER_PATTERN, html) diff --git a/module/plugins/hooks/MyfastfileComHook.py b/module/plugins/hooks/MyfastfileComHook.py index 20a1cfac2..10d128a31 100644 --- a/module/plugins/hooks/MyfastfileComHook.py +++ b/module/plugins/hooks/MyfastfileComHook.py @@ -21,7 +21,7 @@ class MyfastfileComHook(MultiHook): def getHosters(self): - json_data = self.getURL("http://myfastfile.com/api.php", get={'hosts': ""}, decode=True) + json_data = self.load("http://myfastfile.com/api.php", get={'hosts': ""}) self.logDebug("JSON data", json_data) json_data = json_loads(json_data) diff --git a/module/plugins/hooks/NoPremiumPlHook.py b/module/plugins/hooks/NoPremiumPlHook.py index b5a007ff9..264f3462a 100644 --- a/module/plugins/hooks/NoPremiumPlHook.py +++ b/module/plugins/hooks/NoPremiumPlHook.py @@ -21,7 +21,7 @@ class NoPremiumPlHook(MultiHook): def getHosters(self): - hostings = json_loads(self.getURL("https://www.nopremium.pl/clipboard.php?json=3").strip()) + hostings = json_loads(self.load("https://www.nopremium.pl/clipboard.php?json=3").strip()) hostings_domains = [domain for row in hostings for domain in row["domains"] if row["sdownload"] == "0"] self.logDebug(hostings_domains) diff --git a/module/plugins/hooks/OverLoadMeHook.py b/module/plugins/hooks/OverLoadMeHook.py index d608a2ecd..a618938a4 100644 --- a/module/plugins/hooks/OverLoadMeHook.py +++ b/module/plugins/hooks/OverLoadMeHook.py @@ -22,7 +22,7 @@ class OverLoadMeHook(MultiHook): def getHosters(self): https = "https" if self.getConfig('ssl') else "http" - html = self.getURL(https + "://api.over-load.me/hoster.php", + html = self.load(https + "://api.over-load.me/hoster.php", get={'auth': "0001-cb1f24dadb3aa487bda5afd3b76298935329be7700cd7-5329be77-00cf-1ca0135f"}).replace("\"", "").strip() self.logDebug("Hosterlist", html) diff --git a/module/plugins/hooks/PremiumToHook.py b/module/plugins/hooks/PremiumToHook.py index 11f0f3c8a..63e3c72c8 100644 --- a/module/plugins/hooks/PremiumToHook.py +++ b/module/plugins/hooks/PremiumToHook.py @@ -23,6 +23,6 @@ class PremiumToHook(MultiHook): def getHosters(self): user, data = self.account.selectAccount() - html = self.getURL("http://premium.to/api/hosters.php", + html = self.load("http://premium.to/api/hosters.php", get={'username': user, 'password': data['password']}) return [x.strip() for x in html.replace("\"", "").split(";")] diff --git a/module/plugins/hooks/PremiumizeMeHook.py b/module/plugins/hooks/PremiumizeMeHook.py index 35ad70970..bc803b1f5 100644 --- a/module/plugins/hooks/PremiumizeMeHook.py +++ b/module/plugins/hooks/PremiumizeMeHook.py @@ -26,7 +26,7 @@ class PremiumizeMeHook(MultiHook): # Get supported hosters list from premiumize.me using the # json API v1 (see https://secure.premiumize.me/?show=api) - answer = self.getURL("http://api.premiumize.me/pm-api/v1.php", #@TODO: Revert to `https` in 0.4.10 + answer = self.load("http://api.premiumize.me/pm-api/v1.php", #@TODO: Revert to `https` in 0.4.10 get={'method': "hosterlist", 'params[login]': user, 'params[pass]': data['password']}) data = json_loads(answer) diff --git a/module/plugins/hooks/RPNetBizHook.py b/module/plugins/hooks/RPNetBizHook.py index 10332948d..872eb7e08 100644 --- a/module/plugins/hooks/RPNetBizHook.py +++ b/module/plugins/hooks/RPNetBizHook.py @@ -24,7 +24,7 @@ class RPNetBizHook(MultiHook): # Get account data user, data = self.account.selectAccount() - res = self.getURL("https://premium.rpnet.biz/client_api.php", + res = self.load("https://premium.rpnet.biz/client_api.php", get={'username': user, 'password': data['password'], 'action': "showHosterList"}) hoster_list = json_loads(res) diff --git a/module/plugins/hooks/RapideoPlHook.py b/module/plugins/hooks/RapideoPlHook.py index 0400f07ba..f498def2a 100644 --- a/module/plugins/hooks/RapideoPlHook.py +++ b/module/plugins/hooks/RapideoPlHook.py @@ -21,7 +21,7 @@ class RapideoPlHook(MultiHook): def getHosters(self): - hostings = json_loads(self.getURL("https://www.rapideo.pl/clipboard.php?json=3").strip()) + hostings = json_loads(self.load("https://www.rapideo.pl/clipboard.php?json=3").strip()) hostings_domains = [domain for row in hostings for domain in row["domains"] if row["sdownload"] == "0"] self.logDebug(hostings_domains) diff --git a/module/plugins/hooks/RealdebridComHook.py b/module/plugins/hooks/RealdebridComHook.py index aa0c9f640..d1a503136 100644 --- a/module/plugins/hooks/RealdebridComHook.py +++ b/module/plugins/hooks/RealdebridComHook.py @@ -22,6 +22,6 @@ class RealdebridComHook(MultiHook): def getHosters(self): https = "https" if self.getConfig('ssl') else "http" - html = self.getURL(https + "://real-debrid.com/api/hosters.php").replace("\"", "").strip() + html = self.load(https + "://real-debrid.com/api/hosters.php").replace("\"", "").strip() return [x.strip() for x in html.split(",") if x.strip()] diff --git a/module/plugins/hooks/RehostToHook.py b/module/plugins/hooks/RehostToHook.py index a2415129a..febad818a 100644 --- a/module/plugins/hooks/RehostToHook.py +++ b/module/plugins/hooks/RehostToHook.py @@ -21,7 +21,7 @@ class RehostToHook(MultiHook): def getHosters(self): user, data = self.account.selectAccount() - html = self.getURL("http://rehost.to/api.php", + html = self.load("http://rehost.to/api.php", get={'cmd' : "get_supported_och_dl", 'long_ses': self.account.getAccountInfo(user)['session']}) return [x.strip() for x in html.replace("\"", "").split(",")] diff --git a/module/plugins/hooks/SimplyPremiumComHook.py b/module/plugins/hooks/SimplyPremiumComHook.py index 116e3a76e..e211abd30 100644 --- a/module/plugins/hooks/SimplyPremiumComHook.py +++ b/module/plugins/hooks/SimplyPremiumComHook.py @@ -21,7 +21,7 @@ class SimplyPremiumComHook(MultiHook): def getHosters(self): - json_data = self.getURL("http://www.simply-premium.com/api/hosts.php", get={'format': "json", 'online': 1}) + json_data = self.load("http://www.simply-premium.com/api/hosts.php", get={'format': "json", 'online': 1}) json_data = json_loads(json_data) host_list = [element['regex'] for element in json_data['result']] diff --git a/module/plugins/hooks/SimplydebridComHook.py b/module/plugins/hooks/SimplydebridComHook.py index 01629df99..b844aad49 100644 --- a/module/plugins/hooks/SimplydebridComHook.py +++ b/module/plugins/hooks/SimplydebridComHook.py @@ -20,5 +20,5 @@ class SimplydebridComHook(MultiHook): def getHosters(self): - html = self.getURL("http://simply-debrid.com/api.php", get={'list': 1}) + html = self.load("http://simply-debrid.com/api.php", get={'list': 1}) return [x.strip() for x in html.rstrip(';').replace("\"", "").split(";")] diff --git a/module/plugins/hooks/SkipRev.py b/module/plugins/hooks/SkipRev.py index 0457f9b55..32454d255 100644 --- a/module/plugins/hooks/SkipRev.py +++ b/module/plugins/hooks/SkipRev.py @@ -8,7 +8,6 @@ from types import MethodType from module.PyFile import PyFile from module.plugins.internal.Hook import Hook -from module.plugins.internal.Plugin import Skip class SkipRev(Hook): @@ -35,7 +34,7 @@ class SkipRev(Hook): def _setup(self): self.pyfile.plugin._setup() if self.pyfile.hasStatus("skipped"): - raise Skip(self.pyfile.statusname or self.pyfile.pluginname) + self.skip(self.pyfile.statusname or self.pyfile.pluginname) def _name(self, pyfile): diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index f555e4ff3..26b0cc448 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -9,7 +9,6 @@ import time from operator import itemgetter -from module.network.RequestFactory import getURL from module.plugins.internal.Hook import Expose, Hook, threaded from module.utils import save_join as fs_join @@ -128,7 +127,8 @@ class UpdateManager(Hook): def server_response(self): try: - return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() + return self.load(self.SERVER_URL, + get={'v': self.core.api.getServerVersion()}).splitlines() except Exception: self.logWarning(_("Unable to retrieve server to get updates")) @@ -253,7 +253,7 @@ class UpdateManager(Hook): 'oldver': oldver, 'newver': newver}) try: - content = getURL(url % plugin) + content = self.load(url % plugin) m = VERSION.search(content) if m and m.group(2) == version: diff --git a/module/plugins/hooks/UserAgentSwitcher.py b/module/plugins/hooks/UserAgentSwitcher.py index 0504ec9d0..ea2d84a43 100644 --- a/module/plugins/hooks/UserAgentSwitcher.py +++ b/module/plugins/hooks/UserAgentSwitcher.py @@ -30,7 +30,7 @@ class UserAgentSwitcher(Hook): def download_preparing(self, pyfile): connecttimeout = self.getConfig('connecttimeout') maxredirs = self.getConfig('maxredirs') - useragent = self.getConfig('useragent').encode("utf8", "replace") #@TODO: Remove `encode` in 0.4.10 + useragent = self.getConfig('useragent') if connecttimeout: pyfile.plugin.req.http.c.setopt(pycurl.CONNECTTIMEOUT, connecttimeout) diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/hooks/XMPPInterface.py index 053817bef..f93273df4 100644 --- a/module/plugins/hooks/XMPPInterface.py +++ b/module/plugins/hooks/XMPPInterface.py @@ -123,7 +123,7 @@ class XMPPInterface(IRCInterface, JabberClient): subject = stanza.get_subject() body = stanza.get_body() t = stanza.get_type() - self.logDebug("Message from %s received." % unicode(stanza.get_from())) + self.logDebug("Message from %s received." % stanza.get_from()) self.logDebug("Body: %s Subject: %s Type: %s" % (body, subject, t)) if t == "headline": diff --git a/module/plugins/hoster/AndroidfilehostCom.py b/module/plugins/hoster/AndroidfilehostCom.py index 08005de0f..e5f2c7f88 100644 --- a/module/plugins/hoster/AndroidfilehostCom.py +++ b/module/plugins/hoster/AndroidfilehostCom.py @@ -46,8 +46,7 @@ class AndroidfilehostCom(SimpleHoster): html = self.load("https://www.androidfilehost.com/libs/otf/mirrors.otf.php", post={'submit': 'submit', 'action': 'getdownloadmirrors', - 'fid' : fid}, - decode=True) + 'fid' : fid}) self.link = re.findall('"url":"(.*?)"', html)[0].replace("\\", "") mirror_host = self.link.split("/")[2] @@ -57,8 +56,7 @@ class AndroidfilehostCom(SimpleHoster): html = self.load("https://www.androidfilehost.com/libs/otf/stats.otf.php", get={'fid' : fid, 'w' : 'download', - 'mirror': mirror_host}, - decode=True) + 'mirror': mirror_host}) getInfo = create_getInfo(AndroidfilehostCom) diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index e081c4fa9..de5bca207 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -5,7 +5,7 @@ import urllib import urlparse from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import create_getInfo, getFileURL +from module.plugins.internal.SimpleHoster import create_getInfo from module.plugins.internal.Hoster import Hoster @@ -50,7 +50,7 @@ class BasePlugin(Hoster): for _i in xrange(5): try: - link = getFileURL(self, urllib.unquote(pyfile.url)) + link = self.directLink(self, urllib.unquote(pyfile.url)) if link: self.download(link, ref=False, disposition=True) diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index b47c2902d..d3f25d52f 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -81,7 +81,7 @@ class BezvadataCz(SimpleHoster): def checkErrors(self): if 'images/button-download-disable.png' in self.html: - self.longWait(5 * 60, 24) #: parallel dl limit + self.wait(5 * 60, 24, _("Download limit reached")) #: parallel dl limit elif '<div class="infobox' in self.html: self.tempOffline() else: diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py index 5c709e4f7..f3d513adf 100644 --- a/module/plugins/hoster/BitshareCom.py +++ b/module/plugins/hoster/BitshareCom.py @@ -46,7 +46,7 @@ class BitshareCom(SimpleHoster): self.logDebug("File id is [%s]" % self.file_id) # Load main page - self.html = self.load(pyfile.url, ref=False, decode=True) + self.html = self.load(pyfile.url, ref=False) # Check offline if re.search(self.OFFLINE_PATTERN, self.html): diff --git a/module/plugins/hoster/CatShareNet.py b/module/plugins/hoster/CatShareNet.py index c966dbe1a..795da6229 100644 --- a/module/plugins/hoster/CatShareNet.py +++ b/module/plugins/hoster/CatShareNet.py @@ -21,8 +21,6 @@ class CatShareNet(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - TEXT_ENCODING = True - INFO_PATTERN = r'<title>(?P<N>.+) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)<' OFFLINE_PATTERN = r'<div class="alert alert-error"' diff --git a/module/plugins/hoster/CzshareCom.py b/module/plugins/hoster/CzshareCom.py index befd964bf..e3d89a2a5 100644 --- a/module/plugins/hoster/CzshareCom.py +++ b/module/plugins/hoster/CzshareCom.py @@ -44,7 +44,7 @@ class CzshareCom(SimpleHoster): m = re.search(self.USER_CREDIT_PATTERN, self.html) if m is None: self.account.relogin(self.user) - self.html = self.load(self.pyfile.url, decode=True) + self.html = self.load(self.pyfile.url) m = re.search(self.USER_CREDIT_PATTERN, self.html) if m is None: return False @@ -88,9 +88,9 @@ class CzshareCom(SimpleHoster): self.logDebug("PARSED_URL:" + parsed_url) # get download ticket and parse html - self.html = self.load(parsed_url, decode=True) + self.html = self.load(parsed_url) if re.search(self.MULTIDL_PATTERN, self.html): - self.longWait(5 * 60, 12) + self.wait(5 * 60, 12, _("Download limit reached")) try: form = re.search(self.FREE_FORM_PATTERN, self.html, re.S).group(1) @@ -105,13 +105,13 @@ class CzshareCom(SimpleHoster): captcha_url = 'http://sdilej.cz/captcha.php' for _i in xrange(5): inputs['captchastring2'] = self.decryptCaptcha(captcha_url) - self.html = self.load(parsed_url, post=inputs, decode=True) + self.html = self.load(parsed_url, post=inputs) if u"<li>ZadanÃœ ovÄÅovacà kód nesouhlasÃ!</li>" in self.html: self.invalidCaptcha() elif re.search(self.MULTIDL_PATTERN, self.html): - self.longWait(5 * 60, 12) + self.wait(5 * 60, 12, _("Download limit reached")) else: self.correctCaptcha() @@ -150,7 +150,7 @@ class CzshareCom(SimpleHoster): self.resetAccount() elif check == "multi-dl": - self.longWait(5 * 60, 12) + self.wait(5 * 60, 12, _("Download limit reached")) elif check == "captcha": self.invalidCaptcha() diff --git a/module/plugins/hoster/DailymotionCom.py b/module/plugins/hoster/DailymotionCom.py index d70b29803..f9cc0d28e 100644 --- a/module/plugins/hoster/DailymotionCom.py +++ b/module/plugins/hoster/DailymotionCom.py @@ -117,7 +117,7 @@ class DailymotionCom(Hoster): self.checkInfo(pyfile) id = re.match(self.__pattern__, pyfile.url).group('ID') - self.html = self.load("http://www.dailymotion.com/embed/video/" + id, decode=True) + self.html = self.load("http://www.dailymotion.com/embed/video/" + id) streams = self.getStreams() quality = self.getQuality() diff --git a/module/plugins/hoster/DataportCz.py b/module/plugins/hoster/DataportCz.py index ad514f5eb..1b4041c69 100644 --- a/module/plugins/hoster/DataportCz.py +++ b/module/plugins/hoster/DataportCz.py @@ -48,7 +48,7 @@ class DataportCz(SimpleHoster): elif check == "slot": self.logDebug("No free slots - wait 60s and retry") self.wait(60, False) - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) continue else: diff --git a/module/plugins/hoster/EdiskCz.py b/module/plugins/hoster/EdiskCz.py index b8485001e..96be21860 100644 --- a/module/plugins/hoster/EdiskCz.py +++ b/module/plugins/hoster/EdiskCz.py @@ -39,7 +39,7 @@ class EdiskCz(SimpleHoster): self.error(_("ACTION_PATTERN not found")) action = m.group(1) - self.html = self.load(url, decode=True) + self.html = self.load(url) self.getFileInfo() self.html = self.load(re.sub("/en/download/", "/en/download-slow/", url)) diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py index 857faebc5..c4bd54667 100644 --- a/module/plugins/hoster/EuroshareEu.py +++ b/module/plugins/hoster/EuroshareEu.py @@ -49,7 +49,7 @@ class EuroshareEu(SimpleHoster): def handleFree(self, pyfile): if re.search(self.ERR_PARDL_PATTERN, self.html): - self.longWait(5 * 60, 12) + self.wait(5 * 60, 12, _("Download limit reached")) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: @@ -60,7 +60,7 @@ class EuroshareEu(SimpleHoster): def checkFile(self): if self.checkDownload({"multi-dl": re.compile(self.ERR_PARDL_PATTERN)}) - self.longWait(5 * 60, 12) + self.wait(5 * 60, 12, _("Download limit reached")) return super(EuroshareEu, self).checkFile() diff --git a/module/plugins/hoster/FilerNet.py b/module/plugins/hoster/FilerNet.py index 3b876ed48..ecf656926 100644 --- a/module/plugins/hoster/FilerNet.py +++ b/module/plugins/hoster/FilerNet.py @@ -39,7 +39,7 @@ class FilerNet(SimpleHoster): if 'token' not in inputs: self.error(_("Unable to detect token")) - self.html = self.load(pyfile.url, post={'token': inputs['token']}, decode=True) + self.html = self.load(pyfile.url, post={'token': inputs['token']}) inputs = self.parseHtmlForm(input_names={'hash': re.compile(r'.+')})[1] if 'hash' not in inputs: diff --git a/module/plugins/hoster/FileserveCom.py b/module/plugins/hoster/FileserveCom.py index 6ea1d6541..3f4c7d6ed 100644 --- a/module/plugins/hoster/FileserveCom.py +++ b/module/plugins/hoster/FileserveCom.py @@ -12,7 +12,7 @@ from module.utils import parseFileSize def checkFile(plugin, urls): - html = getURL(plugin.URLS[1], post={"urls": "\n".join(urls)}, decode=True) + html = getURL(plugin.URLS[1], post={"urls": "\n".join(urls)}) file_info = [] for li in re.finditer(plugin.LINKCHECK_TR, html, re.S): @@ -81,14 +81,13 @@ class FileserveCom(Hoster): def handleFree(self): self.html = self.load(self.url) - action = self.load(self.url, post={"checkDownload": "check"}, decode=True) + action = self.load(self.url, post={"checkDownload": "check"}) action = json_loads(action) self.logDebug(action) if "fail" in action: if action['fail'] == "timeLimit": - self.html = self.load(self.url, post={"checkDownload": "showError", "errorType": "timeLimit"}, - decode=True) + self.html = self.load(self.url, post={"checkDownload": "showError", "errorType": "timeLimit"}) self.doLongWait(re.search(self.LONG_WAIT_PATTERN, self.html)) @@ -110,7 +109,7 @@ class FileserveCom(Hoster): self.error(_("Unknown server response")) # show download link - res = self.load(self.url, post={"downloadLink": "show"}, decode=True) + res = self.load(self.url, post={"downloadLink": "show"}) self.logDebug("Show downloadLink response: %s" % res) if "fail" in res: self.error(_("Couldn't retrieve download url")) @@ -140,7 +139,7 @@ class FileserveCom(Hoster): def doTimmer(self): - res = self.load(self.url, post={"downloadLink": "wait"}, decode=True) + res = self.load(self.url, post={"downloadLink": "wait"}) self.logDebug("Wait response: %s" % res[:80]) if "fail" in res: @@ -191,8 +190,7 @@ class FileserveCom(Hoster): res = self.load("http://app.fileserve.com/api/download/premium/", post={"username": self.user, "password": self.account.getAccountData(self.user)['password'], - "shorten": self.file_id}, - decode=True) + "shorten": self.file_id}) if res: res = json_loads(res) if res['error_code'] == "302": diff --git a/module/plugins/hoster/FshareVn.py b/module/plugins/hoster/FshareVn.py index 50128db10..717265014 100644 --- a/module/plugins/hoster/FshareVn.py +++ b/module/plugins/hoster/FshareVn.py @@ -11,8 +11,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo def getInfo(urls): for url in urls: html = getURL("http://www.fshare.vn/check_link.php", - post={'action': "check_link", 'arrlinks': url}, - decode=True) + post={'action': "check_link", 'arrlinks': url}) yield parseFileInfo(FshareVn, url, html) @@ -45,15 +44,11 @@ class FshareVn(SimpleHoster): def preload(self): self.html = self.load("http://www.fshare.vn/check_link.php", - post={'action': "check_link", 'arrlinks': pyfile.url}, - decode=True) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) + post={'action': "check_link", 'arrlinks': pyfile.url}) def handleFree(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) self.checkErrors() @@ -69,7 +64,7 @@ class FshareVn(SimpleHoster): if password: self.logInfo(_("Password protected link, trying ") + password) inputs['link_file_pwd_dl'] = password - self.html = self.load(url, post=inputs, decode=True) + self.html = self.load(url, post=inputs) if 'name="link_file_pwd_dl"' in self.html: self.fail(_("Incorrect password")) @@ -77,7 +72,7 @@ class FshareVn(SimpleHoster): self.fail(_("No password found")) else: - self.html = self.load(url, post=inputs, decode=True) + self.html = self.load(url, post=inputs) self.checkErrors() diff --git a/module/plugins/hoster/Ftp.py b/module/plugins/hoster/Ftp.py index adfb279e4..2e10de971 100644 --- a/module/plugins/hoster/Ftp.py +++ b/module/plugins/hoster/Ftp.py @@ -70,7 +70,7 @@ class Ftp(Hoster): pyfile.url += '/' self.req.http.c.setopt(48, 1) # CURLOPT_DIRLISTONLY res = self.load(pyfile.url, decode=False) - links = [pyfile.url + urllib.quote(x) for x in res.splitlines()] + links = [pyfile.url + x for x in res.splitlines()] self.logDebug("LINKS", links) self.core.api.addPackage(pkgname, links) else: diff --git a/module/plugins/hoster/GoogledriveCom.py b/module/plugins/hoster/GoogledriveCom.py index 1c33a1e4e..d658a8ba1 100644 --- a/module/plugins/hoster/GoogledriveCom.py +++ b/module/plugins/hoster/GoogledriveCom.py @@ -51,7 +51,7 @@ class GoogledriveCom(SimpleHoster): direct_link = self.directLink(link, False) if not direct_link: - self.html = self.load(link, decode=True) + self.html = self.load(link) else: self.link = direct_link break diff --git a/module/plugins/hoster/IfolderRu.py b/module/plugins/hoster/IfolderRu.py index 0f09731e4..5f3b4bb51 100644 --- a/module/plugins/hoster/IfolderRu.py +++ b/module/plugins/hoster/IfolderRu.py @@ -40,7 +40,7 @@ class IfolderRu(SimpleHoster): def handleFree(self, pyfile): url = "http://rusfolder.com/%s" % self.info['pattern']['ID'] - self.html = self.load("http://rusfolder.com/%s" % self.info['pattern']['ID'], decode=True) + self.html = self.load("http://rusfolder.com/%s" % self.info['pattern']['ID']) self.getFileInfo() session_id = re.search(self.SESSION_ID_PATTERN, self.html).groups() @@ -52,7 +52,7 @@ class IfolderRu(SimpleHoster): inputs['action'] = '1' self.logDebug(inputs) - self.html = self.load(url, decode=True, post=inputs) + self.html = self.load(url, post=inputs) if self.WRONG_CAPTCHA_PATTERN in self.html: self.invalidCaptcha() else: diff --git a/module/plugins/hoster/JumbofilesCom.py b/module/plugins/hoster/JumbofilesCom.py index a9bf14c6d..4812f0224 100644 --- a/module/plugins/hoster/JumbofilesCom.py +++ b/module/plugins/hoster/JumbofilesCom.py @@ -30,7 +30,7 @@ class JumbofilesCom(SimpleHoster): def handleFree(self, pyfile): post_data = {"id": self.info['pattern']['ID'], "op": "download3", "rand": ""} - html = self.load(self.pyfile.url, post=post_data, decode=True) + html = self.load(self.pyfile.url, post=post_data) self.link = re.search(self.LINK_FREE_PATTERN, html).group(1) diff --git a/module/plugins/hoster/KingfilesNet.py b/module/plugins/hoster/KingfilesNet.py index 42547d658..a25c39b21 100644 --- a/module/plugins/hoster/KingfilesNet.py +++ b/module/plugins/hoster/KingfilesNet.py @@ -44,7 +44,7 @@ class KingfilesNet(SimpleHoster): 'referer' : "", 'method_free': "+"} - self.html = self.load(pyfile.url, post=post_data, decode=True) + self.html = self.load(pyfile.url, post=post_data) solvemedia = SolveMedia(self) response, challenge = solvemedia.challenge() @@ -67,7 +67,7 @@ class KingfilesNet(SimpleHoster): 'adcopy_challenge': challenge, 'down_direct' : "1"} - self.html = self.load(pyfile.url, post=post_data, decode=True) + self.html = self.load(pyfile.url, post=post_data) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: diff --git a/module/plugins/hoster/LolabitsEs.py b/module/plugins/hoster/LolabitsEs.py index 61df5f0bb..8ad811248 100644 --- a/module/plugins/hoster/LolabitsEs.py +++ b/module/plugins/hoster/LolabitsEs.py @@ -40,7 +40,8 @@ class LolabitsEs(SimpleHoster): self.html = self.load("http://lolabits.es/action/License/Download", post={'fileId' : fileid, - '__RequestVerificationToken' : token}).decode('unicode-escape') + '__RequestVerificationToken' : token}, + decode="unicode-escape") self.link = HTMLParser.HTMLParser().unescape(re.search(self.LINK_PATTERN, self.html).group(1)) diff --git a/module/plugins/hoster/LuckyShareNet.py b/module/plugins/hoster/LuckyShareNet.py index f4932d93d..293dab8f9 100644 --- a/module/plugins/hoster/LuckyShareNet.py +++ b/module/plugins/hoster/LuckyShareNet.py @@ -27,7 +27,7 @@ class LuckyShareNet(SimpleHoster): def parseJson(self, rep): if 'AJAX Error' in rep: - html = self.load(self.pyfile.url, decode=True) + html = self.load(self.pyfile.url) m = re.search(r"waitingtime = (\d+);", html) if m: seconds = int(m.group(1)) @@ -43,7 +43,7 @@ class LuckyShareNet(SimpleHoster): # TODO: There should be a filesize limit for free downloads # TODO: Some files could not be downloaded in free mode def handleFree(self, pyfile): - rep = self.load(r"http://luckyshare.net/download/request/type/time/file/" + self.info['pattern']['ID'], decode=True) + rep = self.load(r"http://luckyshare.net/download/request/type/time/file/" + self.info['pattern']['ID']) self.logDebug("JSON: " + rep) @@ -55,7 +55,7 @@ class LuckyShareNet(SimpleHoster): for _i in xrange(5): response, challenge = recaptcha.challenge() rep = self.load(r"http://luckyshare.net/download/verify/challenge/%s/response/%s/hash/%s" % - (challenge, response, json['hash']), decode=True) + (challenge, response, json['hash'])) self.logDebug("JSON: " + rep) if 'link' in rep: json.update(self.parseJson(rep)) diff --git a/module/plugins/hoster/MediafireCom.py b/module/plugins/hoster/MediafireCom.py index 09eb4705d..56f13362a 100644 --- a/module/plugins/hoster/MediafireCom.py +++ b/module/plugins/hoster/MediafireCom.py @@ -43,8 +43,7 @@ class MediafireCom(SimpleHoster): response, challenge = solvemedia.challenge(captcha_key) self.html = self.load("http://www.mediafire.com/?" + self.info['pattern']['ID'], post={'adcopy_challenge': challenge, - 'adcopy_response' : response}, - decode=True) + 'adcopy_response' : response}) return recaptcha = ReCaptcha(self) @@ -53,8 +52,7 @@ class MediafireCom(SimpleHoster): if captcha_key: response, challenge = recaptcha.challenge(captcha_key) self.html = self.load(self.pyfile.url, - post={'g-recaptcha-response': response}, - decode=True) + post={'g-recaptcha-response': response}) def handleFree(self, pyfile): diff --git a/module/plugins/hoster/MegaRapidCz.py b/module/plugins/hoster/MegaRapidCz.py index e17dd4730..ed6e0a3ad 100644 --- a/module/plugins/hoster/MegaRapidCz.py +++ b/module/plugins/hoster/MegaRapidCz.py @@ -15,7 +15,7 @@ def getInfo(urls): "User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"]) for url in urls: - html = h.load(url, decode=True) + html = h.load(url) yield parseFileInfo(MegaRapidCz, url, html) diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index bbb77f525..2a5031f08 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -37,7 +37,7 @@ class MultishareCz(SimpleHoster): def handleMulti(self, pyfile): - self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post={"link": pyfile.url}, decode=True) + self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post={"link": pyfile.url}) self.checkInfo() diff --git a/module/plugins/hoster/NarodRu.py b/module/plugins/hoster/NarodRu.py index c201ac250..0a44cd6eb 100644 --- a/module/plugins/hoster/NarodRu.py +++ b/module/plugins/hoster/NarodRu.py @@ -44,7 +44,7 @@ class NarodRu(SimpleHoster): captcha_url, post_data['key'] = m.groups() post_data['rep'] = self.decryptCaptcha(captcha_url) - self.html = self.load(pyfile.url, post=post_data, decode=True) + self.html = self.load(pyfile.url, post=post_data) m = re.search(self.LINK_FREE_PATTERN, self.html) if m: diff --git a/module/plugins/hoster/NosuploadCom.py b/module/plugins/hoster/NosuploadCom.py index 241ee3a29..0b4af511c 100644 --- a/module/plugins/hoster/NosuploadCom.py +++ b/module/plugins/hoster/NosuploadCom.py @@ -26,14 +26,14 @@ class NosuploadCom(XFSHoster): def getDownloadLink(self): # stage1: press the "Free Download" button data = self.getPostParameters() - self.html = self.load(self.pyfile.url, post=data, decode=True) + self.html = self.load(self.pyfile.url, post=data) # stage2: wait some time and press the "Download File" button data = self.getPostParameters() wait_time = re.search(self.WAIT_PATTERN, self.html, re.M | re.S).group(1) self.logDebug("Hoster told us to wait %s seconds" % wait_time) self.wait(wait_time) - self.html = self.load(self.pyfile.url, post=data, decode=True) + self.html = self.load(self.pyfile.url, post=data) # stage3: get the download link return re.search(self.LINK_PATTERN, self.html, re.S).group(1) diff --git a/module/plugins/hoster/OboomCom.py b/module/plugins/hoster/OboomCom.py index 6c5d969f3..ae51d4b5a 100644 --- a/module/plugins/hoster/OboomCom.py +++ b/module/plugins/hoster/OboomCom.py @@ -48,7 +48,7 @@ class OboomCom(Hoster): def loadUrl(self, url, get=None): if get is None: get = dict() - return json_loads(self.load(url, get, decode=True)) + return json_loads(self.load(url, get)) def getFileId(self, url): diff --git a/module/plugins/hoster/PromptfileCom.py b/module/plugins/hoster/PromptfileCom.py index 3578ff7fb..4a1b07f05 100644 --- a/module/plugins/hoster/PromptfileCom.py +++ b/module/plugins/hoster/PromptfileCom.py @@ -35,7 +35,7 @@ class PromptfileCom(SimpleHoster): self.logDebug("Read chash %s" % chash) # continue to stage2 - self.html = self.load(pyfile.url, decode=True, post={'chash': chash}) + self.html = self.load(pyfile.url, post={'chash': chash}) # STAGE 2: get the direct link return super(PromptfileCom, self).handleFree(pyfile) diff --git a/module/plugins/hoster/QuickshareCz.py b/module/plugins/hoster/QuickshareCz.py index 1e0750b88..8cfb72c9a 100644 --- a/module/plugins/hoster/QuickshareCz.py +++ b/module/plugins/hoster/QuickshareCz.py @@ -25,7 +25,7 @@ class QuickshareCz(SimpleHoster): def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) self.getFileInfo() # parse js variables diff --git a/module/plugins/hoster/RapidgatorNet.py b/module/plugins/hoster/RapidgatorNet.py index bd6bb8582..e5298726c 100644 --- a/module/plugins/hoster/RapidgatorNet.py +++ b/module/plugins/hoster/RapidgatorNet.py @@ -65,7 +65,7 @@ class RapidgatorNet(SimpleHoster): try: json = self.load('%s/%s' % (self.API_URL, cmd), get={'sid': self.sid, - 'url': self.pyfile.url}, decode=True) + 'url': self.pyfile.url}) self.logDebug("API:%s" % cmd, json, "SID: %s" % self.sid) json = json_loads(json) status = json['response_status'] @@ -154,7 +154,7 @@ class RapidgatorNet(SimpleHoster): def getJsonResponse(self, url): - res = self.load(url, decode=True) + res = self.load(url) if not res.startswith('{'): self.retry() self.logDebug(url, res) diff --git a/module/plugins/hoster/RapiduNet.py b/module/plugins/hoster/RapiduNet.py index da353ec70..f2f1abfb7 100644 --- a/module/plugins/hoster/RapiduNet.py +++ b/module/plugins/hoster/RapiduNet.py @@ -43,8 +43,7 @@ class RapiduNet(SimpleHoster): jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", get={'a': "getLoadTimeToDownload"}, - post={'_go': ""}, - decode=True) + post={'_go': ""}) if str(jsvars['timeToDownload']) is "stop": t = (24 * 60 * 60) - (int(time.time()) % (24 * 60 * 60)) + time.altzone @@ -64,8 +63,7 @@ class RapiduNet(SimpleHoster): post={'_go' : "", 'captcha1': challenge, 'captcha2': response, - 'fileId' : self.info['pattern']['ID']}, - decode=True) + 'fileId' : self.info['pattern']['ID']}) if jsvars['message'] == 'success': self.link = jsvars['url'] diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index 4a7c022ed..47b6ba7da 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -41,8 +41,7 @@ class ShareonlineBiz(SimpleHoster): field = getURL("http://api.share-online.biz/linkcheck.php", get={'md5' : "1", - 'links': re.match(cls.__pattern__, url).group("ID")}, - decode=True).split(";") + 'links': re.match(cls.__pattern__, url).group("ID")}).split(";") try: if field[1] == "OK": @@ -93,8 +92,7 @@ class ShareonlineBiz(SimpleHoster): self.wait(3) self.html = self.load("%s/free/" % pyfile.url, - post={'dl_free': "1", 'choice': "free"}, - decode=True) + post={'dl_free': "1", 'choice': "free"}) self.checkErrors() diff --git a/module/plugins/hoster/ShareplaceCom.py b/module/plugins/hoster/ShareplaceCom.py index 906a8aa36..9e27aea1e 100644 --- a/module/plugins/hoster/ShareplaceCom.py +++ b/module/plugins/hoster/ShareplaceCom.py @@ -51,7 +51,7 @@ class ShareplaceCom(Hoster): def download_html(self): url = re.sub("shareplace.com\/\?", "shareplace.com//index1.php/?a=", self.pyfile.url) - self.html = self.load(url, decode=True) + self.html = self.load(url) def get_file_url(self): diff --git a/module/plugins/hoster/StreamCz.py b/module/plugins/hoster/StreamCz.py index dc32f22a1..a2fe32c5c 100644 --- a/module/plugins/hoster/StreamCz.py +++ b/module/plugins/hoster/StreamCz.py @@ -44,7 +44,7 @@ class StreamCz(Hoster): def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) if re.search(self.OFFLINE_PATTERN, self.html): self.offline() diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py index a4c36bc2a..b70f9e72b 100644 --- a/module/plugins/hoster/TurbobitNet.py +++ b/module/plugins/hoster/TurbobitNet.py @@ -43,8 +43,7 @@ class TurbobitNet(SimpleHoster): def handleFree(self, pyfile): - self.html = self.load("http://turbobit.net/download/free/%s" % self.info['pattern']['ID'], - decode=True) + self.html = self.load("http://turbobit.net/download/free/%s" % self.info['pattern']['ID']) rtUpdate = self.getRtUpdate() diff --git a/module/plugins/hoster/TusfilesNet.py b/module/plugins/hoster/TusfilesNet.py index 87cf1e0fb..3afd1da97 100644 --- a/module/plugins/hoster/TusfilesNet.py +++ b/module/plugins/hoster/TusfilesNet.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from module.network.HTTPRequest import BadHeader +from module.plugins.internal.Plugin import Retry from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo diff --git a/module/plugins/hoster/UloziskoSk.py b/module/plugins/hoster/UloziskoSk.py index ad809b47c..81fdd21ed 100644 --- a/module/plugins/hoster/UloziskoSk.py +++ b/module/plugins/hoster/UloziskoSk.py @@ -30,7 +30,7 @@ class UloziskoSk(SimpleHoster): def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) self.getFileInfo() m = re.search(self.IMG_PATTERN, self.html) diff --git a/module/plugins/hoster/UploadableCh.py b/module/plugins/hoster/UploadableCh.py index 283359927..b33c81ae7 100644 --- a/module/plugins/hoster/UploadableCh.py +++ b/module/plugins/hoster/UploadableCh.py @@ -34,13 +34,13 @@ class UploadableCh(SimpleHoster): def handleFree(self, pyfile): # Click the "free user" button and wait - a = self.load(pyfile.url, post={'downloadLink': "wait"}, decode=True) + a = self.load(pyfile.url, post={'downloadLink': "wait"}) self.logDebug(a) self.wait(30) # Make the recaptcha appear and show it the pyload interface - b = self.load(pyfile.url, post={'checkDownload': "check"}, decode=True) + b = self.load(pyfile.url, post={'checkDownload': "check"}) self.logDebug(b) #: Expected output: {"success":"showCaptcha"} recaptcha = ReCaptcha(self) @@ -51,13 +51,12 @@ class UploadableCh(SimpleHoster): self.load("http://www.uploadable.ch/checkReCaptcha.php", post={'recaptcha_challenge_field' : challenge, 'recaptcha_response_field' : response, - 'recaptcha_shortencode_field': self.info['pattern']['ID']}, - decode=True) + 'recaptcha_shortencode_field': self.info['pattern']['ID']}) self.wait(3) # Get ready for downloading - self.load(pyfile.url, post={'downloadLink': "show"}, decode=True) + self.load(pyfile.url, post={'downloadLink': "show"}) self.wait(3) diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index cb8440687..46ed863d9 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -45,8 +45,7 @@ class UploadedTo(SimpleHoster): for _i in xrange(5): html = getURL("http://uploaded.net/api/filemultiple", - get={"apikey": cls.API_KEY, 'id_0': re.match(cls.__pattern__, url).group('ID')}, - decode=True) + get={"apikey": cls.API_KEY, 'id_0': re.match(cls.__pattern__, url).group('ID')}) if html != "can't find request": api = html.split(",", 4) @@ -69,7 +68,7 @@ class UploadedTo(SimpleHoster): def handleFree(self, pyfile): self.load("http://uploaded.net/language/en", just_header=True) - self.html = self.load("http://uploaded.net/js/download.js", decode=True) + self.html = self.load("http://uploaded.net/js/download.js") recaptcha = ReCaptcha(self) response, challenge = recaptcha.challenge() diff --git a/module/plugins/hoster/UploadingCom.py b/module/plugins/hoster/UploadingCom.py index d1cafcff4..efdcd81fe 100644 --- a/module/plugins/hoster/UploadingCom.py +++ b/module/plugins/hoster/UploadingCom.py @@ -35,7 +35,7 @@ class UploadingCom(SimpleHoster): if not "/get/" in pyfile.url: pyfile.url = pyfile.url.replace("/files", "/files/get") - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url) self.getFileInfo() if self.premium: diff --git a/module/plugins/hoster/UpstoreNet.py b/module/plugins/hoster/UpstoreNet.py index 6410a2dce..e7b6d044f 100644 --- a/module/plugins/hoster/UpstoreNet.py +++ b/module/plugins/hoster/UpstoreNet.py @@ -36,7 +36,7 @@ class UpstoreNet(SimpleHoster): self.logDebug("Read hash " + chash) # continue to stage2 post_data = {'hash': chash, 'free': 'Slow download'} - self.html = self.load(pyfile.url, post=post_data, decode=True) + self.html = self.load(pyfile.url, post=post_data) # STAGE 2: solv captcha and wait # first get the infos we need: recaptcha key and wait time @@ -57,7 +57,7 @@ class UpstoreNet(SimpleHoster): post_data.update({'recaptcha_challenge_field': challenge, 'recaptcha_response_field' : response}) - self.html = self.load(pyfile.url, post=post_data, decode=True) + self.html = self.load(pyfile.url, post=post_data) # STAGE 3: get direct link m = re.search(self.LINK_FREE_PATTERN, self.html, re.S) diff --git a/module/plugins/hoster/VimeoCom.py b/module/plugins/hoster/VimeoCom.py index b263c445f..dc774ecf6 100644 --- a/module/plugins/hoster/VimeoCom.py +++ b/module/plugins/hoster/VimeoCom.py @@ -39,7 +39,7 @@ class VimeoCom(SimpleHoster): password = self.getPassword() if self.js and 'class="btn iconify_down_b"' in self.html: - html = self.js.eval(self.load(pyfile.url, get={'action': "download", 'password': password}, decode=True)) + html = self.js.eval(self.load(pyfile.url, get={'action': "download", 'password': password})) pattern = r'href="(?P<URL>http://vimeo\.com.+?)".*?\>(?P<QL>.+?) ' else: html = self.load("https://player.vimeo.com/video/" + self.info['pattern']['ID'], get={'password': password}) diff --git a/module/plugins/hoster/WebshareCz.py b/module/plugins/hoster/WebshareCz.py index c898d23ab..6aac44df6 100644 --- a/module/plugins/hoster/WebshareCz.py +++ b/module/plugins/hoster/WebshareCz.py @@ -27,8 +27,7 @@ class WebshareCz(SimpleHoster): info['pattern'] = re.match(cls.__pattern__, url).groupdict() api_data = getURL("https://webshare.cz/api/file_info/", - post={'ident': info['pattern']['ID'], 'wst': ""}, - decode=True) + post={'ident': info['pattern']['ID'], 'wst': ""}) if not re.search(r'<status>OK', api_data): info['status'] = 1 @@ -44,8 +43,7 @@ class WebshareCz(SimpleHoster): wst = self.account.getAccountData(self.user).get('wst', None) if self.account else None api_data = getURL("https://webshare.cz/api/file_link/", - post={'ident': self.info['pattern']['ID'], 'wst': wst}, - decode=True) + post={'ident': self.info['pattern']['ID'], 'wst': wst}) self.logDebug("API data: " + api_data) diff --git a/module/plugins/hoster/Xdcc.py b/module/plugins/hoster/Xdcc.py index b6f7466b5..efe3b6d6b 100644 --- a/module/plugins/hoster/Xdcc.py +++ b/module/plugins/hoster/Xdcc.py @@ -34,7 +34,7 @@ class Xdcc(Hoster): def process(self, pyfile): # change request type - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__, type="XDCC") + self.req = self.core.requestFactory.getRequest(self.__name__, type="XDCC") self.pyfile = pyfile for _i in xrange(0, 3): diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index 451a4f6c8..e53a66d00 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -6,7 +6,7 @@ import subprocess import urllib from module.plugins.internal.Hoster import Hoster -from module.plugins.internal.SimpleHoster import replace_patterns +from module.plugins.internal.Plugin import replace_patterns from module.utils import html_unescape @@ -85,7 +85,7 @@ class YoutubeCom(Hoster): def process(self, pyfile): pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - html = self.load(pyfile.url, decode=True) + html = self.load(pyfile.url) if re.search(r'<div id="player-unavailable" class="\s*player-width player-height\s*">', html): self.offline() diff --git a/module/plugins/hoster/ZDF.py b/module/plugins/hoster/ZDF.py index 76cf9ec56..2a4708a0d 100644 --- a/module/plugins/hoster/ZDF.py +++ b/module/plugins/hoster/ZDF.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from xml.etree.ElementTree import fromstring +import xml.etree.ElementTree as etree from module.plugins.internal.Hoster import Hoster @@ -42,7 +41,7 @@ class ZDF(Hoster): def process(self, pyfile): - xml = fromstring(self.load(self.XML_API % self.get_id(pyfile.url))) + xml = etree.fromstring(self.load(self.XML_API % self.get_id(pyfile.url))) status = xml.findtext("./status/statuscode") if status != "ok": diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index aa472f297..0e2996aaa 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -154,7 +154,9 @@ class Account(Plugin): if not type(infos) == dict: raise Exception("Wrong return format") except Exception, e: - infos = {"error": str(e)} + infos = super(self.__class__, self).loadAccountInfo(name, req) + infos['error'] = str(e) + if self.core.debug: traceback.print_exc() @@ -206,11 +208,8 @@ class Account(Plugin): def getAccountRequest(self, user=None): if not user: user, data = self.selectAccount() - if not user: - return None - req = self.core.requestFactory.getRequest(self.__name__, user) - return req + return self.core.requestFactory.getRequest(self.__name__, user) def getAccountCookies(self, user=None): @@ -264,7 +263,7 @@ class Account(Plugin): return self.selectAccount() != (None, None) - def parseTraffic(self, value, unit=None): #: return bytes + def parseTraffic(self, value, unit=None): #: return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" return parseFileSize(value, unit) diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index a9c194dda..b7324ef8e 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -41,7 +41,7 @@ class AdYouLike(Captcha): # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} ayl = json_loads(ayl) - html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + html = self.plugin.load("http://api-ayl.appspot.com/challenge", get={'key' : ayl['adyoulike']['key'], 'env' : ayl['all']['env'], 'callback': callback}) diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index 9cab99151..e058352dd 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -37,7 +37,7 @@ class AdsCaptcha(Captcha): def challenge(self, key=None, html=None): PublicKey, CaptchaId = key or self.retrieve_key(html) - html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", + html = self.plugin.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) try: diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 8d040515c..8dbc33da2 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -23,12 +23,6 @@ class Captcha(Plugin): #@TODO: Recheck in 0.4.10 - def fail(self, reason): - self.plugin.fail(reason) - raise AttributeError(reason) - - - #@TODO: Recheck in 0.4.10 def retrieve_key(self, html): if self.detect_key(html): return self.key diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 39b09129f..a713c7da6 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -2,14 +2,14 @@ import urlparse -from module.plugins.internal.Plugin import Plugin -from module.utils import decode, save_path as safe_filename +from module.plugins.internal.Hoster import Hoster +from module.utils import save_path as safe_filename -class Crypter(Plugin): +class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") @@ -26,18 +26,12 @@ class Crypter(Plugin): def __init__(self, pyfile): super(Crypter, self).__init__(pyfile) - #: Provide information in dict here - self.info = {} #@TODO: Move to Plugin - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] #: List of urls, pyLoad will generate packagenames self.urls = [] - self.multiDL = True - self.limitDL = 0 - def process(self, pyfile): """Main method""" @@ -80,14 +74,13 @@ class Crypter(Plugin): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - links = map(decode, links) - - pid = self.core.api.addPackage(name, links, package_queue) + pid = self.core.api.addPackage(name, self.fixurl(links), package_queue) if package_password: self.core.api.setPackageData(pid, {"password": package_password}) - setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #@NOTE: Workaround to do not break API addPackage method + # Workaround to do not break API addPackage method + setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) if use_subfolder: if not subfolder_per_package: diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 8d620e794..01ffbc5f2 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -42,9 +42,6 @@ class Hook(Plugin): #: `HookManager` self.manager = manager - #: Provide information in dict here, usable by API `getInfo` - self.info = {} - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself self.event_map = {} diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3fed8a7c6..9db22d1d7 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -1,33 +1,691 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +import inspect +import os +import random +import time +import urlparse - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. +if os.name != "nt": + import grp + import pwd - @author: mkaay -""" +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.utils import fs_decode, fs_encode, save_join as fs_join -from module.plugins.internal.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return +def getInfo(urls): + #result = [ .. (name, size, status, url) .. ] + pass + class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.02" - __pattern__ = None - __type__ = "hoster" + __name__ = "Hoster" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + __license__ = "GPLv3" + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), + ("spoob" , "spoob@pyload.org" ), + ("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, pyfile): + super(Hoster, self).__init__(pyfile.m.core) + + #: engage wan reconnection + self.wantReconnect = False + + #: enable simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + + #: chunk limit + self.chunkLimit = 1 + self.resumeDownload = False + + #: time.time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + #: captcha reader instance + self.ocr = None + + #: account handler instance, see :py:class:`Account` + self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + #: premium status + self.premium = False + + #: username/login + self.user = None + + if self.account and not self.account.canUse(): + self.account = None + + if self.account: + self.user, data = self.account.selectAccount() + + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest(self.user) + self.chunkLimit = -1 #: chunk limit, -1 for unlimited + + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = True + + #: premium status + self.premium = self.account.isPremium(self.user) + else: + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + + self.thread = None #: holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + + #: re match of the last call to `checkDownload` + self.lastCheck = None + + #: js engine, see `JsEngine` + self.js = self.core.js + + #: captcha task + self.cTask = None + + #: some plugins store html code here + self.html = None + + #: dict of the amount of retries already made + self.retries = {} + + self.init() + + + def init(self): + """ + Initialize the plugin (in addition to `__init__`) + """ + pass + + + def setup(self): + """ + Setup for enviroment and other things, called before downloading (possibly more than one time) + """ + pass + + + def preprocessing(self, thread): + """ + Handles important things to do before starting + """ + self.thread = thread + + if self.account: + self.account.checkLogin(self.user) + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """ + The 'main' method of every plugin, you **have to** overwrite it + """ + raise NotImplementedError + + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.core.config.get("download", "chunks") + return min(self.core.config.get("download", "chunks"), self.chunkLimit) + + + def resetAccount(self): + """ + Don't use account and retry download + """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + + def setWait(self, seconds, reconnect=None): + """ + Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + wait_time = int(seconds) + 1 + wait_until = time.time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) + + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.setReconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): + """ + Waits the time previously set + """ + pyfile = self.pyfile + + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) + + self.waiting = True + + status = pyfile.status + pyfile.setStatus("waiting") + + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), + _("Reconnect: %s") % self.wantReconnect) + + if self.account: + self.logDebug("Ignore reconnection due account logged") + + while pyfile.waitUntil > time.time(): + if pyfile.abort: + self.abort() + + time.sleep(1) + else: + while pyfile.waitUntil > time.time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + time.sleep(1) + + self.waiting = False + + pyfile.status = status + + + def skip(self, reason=""): + """ + Skip and give reason + """ + raise Skip(fs_encode(reason)) + + + def abort(self, reason=""): + """ + Abort and give reason + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Abort + + + def offline(self, reason=""): + """ + Fail and indicate file is offline + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("offline") + + + def tempOffline(self, reason=""): + """ + Fail and indicates file ist temporary offline, the core may take consequences + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("temp. offline") + + + def retry(self, max_tries=5, wait_time=1, reason=""): + """ + Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + id = inspect.currentframe().f_back.f_lineno + if id not in self.retries: + self.retries[id] = 0 + + if 0 < max_tries <= self.retries[id]: + self.fail(reason or _("Max retries reached"), "retry") + + self.wait(wait_time, False) + + self.retries[id] += 1 + raise Retry(reason) + + + def invalidCaptcha(self): + self.logError(_("Invalid captcha")) + if self.cTask: + self.cTask.invalid() + + + def correctCaptcha(self): + self.logInfo(_("Correct captcha")) + if self.cTask: + self.cTask.correct() + + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, + imgtype='jpg', result_type='textual'): + """ + Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time.time())[-6:].replace(".", "") + + with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + time.sleep(random.randint(3000, 5000) / 1000.0) + if self.pyfile.abort: + self.abort() + + ocr = Ocr() + result = ocr.get_captcha(tmpCaptcha.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + self.abort() + time.sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #: ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + + result = task.result + self.logDebug("Received captcha result: %s" % result) + + if not self.core.debug: + try: + os.remove(tmpCaptcha.name) + except Exception: + pass + + return result + + + def fixurl(self, url): + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + url = super(Hoster, self).fixurl(url) + + if not urlparse.urlparse(url).scheme: + url = urlparse.urljoin(baseurl, url) + + return url + + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """ + Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + if self.pyfile.abort: + self.abort() + + url = self.fixurl(url) + + if not url or not isinstance(url, basestring): + self.fail(_("No url given")) + + if self.core.debug: + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + self.correctCaptcha() + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + if disposition: + self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name + + download_folder = self.core.config.get("general", "download_folder") + + location = fs_join(download_folder, self.pyfile.package().folder) + + if not os.path.exists(location): + try: + os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(location, uid, gid) + + except Exception, e: + self.fail(e) + + # convert back to unicode + location = fs_decode(location) + name = safe_filename(self.pyfile.name) + + filename = os.path.join(location, name) + + self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if newname: + newname = urlparse.urlparse(newname).path.split('/')[-1] + + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = os.path.join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config.get("permission", "change_file"): + try: + os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + try: + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(fs_filename, uid, gid) + + except Exception, e: + self.logWarning(_("Setting User and Group failed"), e) + + self.lastDownload = filename + return self.lastDownload + + + def checkDownload(self, rules, delete=True, file_size=None, size_tolerance=1000, read_size=100000): + """ + Checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param delete: delete if matched + :param file_size: expected file size + :param size_tolerance: size check tolerance + :param read_size: amount of bytes to read from files + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + + if not self.lastDownload or not os.path.exists(lastDownload): + self.lastDownload = "" + self.fail(self.pyfile.error or _("No file downloaded")) + + download_size = os.stat(lastDownload).st_size + + if download_size < 1 or (file_size and abs(file_size - download_size) > size_tolerance): + if delete: + os.remove(lastDownload) + self.fail(_("Empty file")) + + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size) + + # produces encoding errors, better log to other file in the future? + # self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + if delete: + os.remove(lastDownload) + return name + + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + os.remove(lastDownload) + self.lastCheck = m + return name + + + def directLink(self, url, follow_location=None): + link = "" + + if follow_location is None: + redirect = 1 + + elif type(follow_location) is int: + redirect = max(follow_location, 1) + + else: + redirect = self.getConfig("maxredirs", plugin="UserAgentSwitcher") + + for i in xrange(redirect): + try: + self.logDebug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, just_header=True) + + except Exception: #: Bad bad bad... rewrite this part in 0.4.10 + req = pyreq.getHTTPRequest() + res = self.load(url, just_header=True) + + req.close() + + header = {"code": req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + + if 'content-disposition' in header: + link = url + + elif 'location' in header and header['location']: + location = header['location'] + + if not urlparse.urlparse(location).scheme: + url_p = urlparse.urlparse(url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + location = urlparse.urljoin(baseurl, location) + + if 'code' in header and header['code'] == 302: + link = location + + if follow_location: + url = location + continue + + else: + extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + + if 'content-type' in header and header['content-type']: + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" + + break + + else: + try: + self.logError(_("Too many redirects")) + except Exception: + pass + + return link + + + def parseHtmlForm(self, attr_str="", input_names={}): + return parseHtmlForm(attr_str, self.html, input_names) + + + def checkTrafficLeft(self): + if not self.account: + return True + + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + + if traffic is None: + return False + elif traffic == -1: + return True + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic + + + def getPassword(self): + """ + Get the password the user provided in the package + """ + return self.pyfile.package().password or "" + + + def checkForSameFiles(self, starting=False): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #: finished or downloading + self.skip(pyfile.pluginname) + elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before + self.skip(pyfile.pluginname) + + download_folder = self.core.config.get("general", "download_folder") + location = fs_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + self.skip("File exists") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if os.path.exists(location): + self.skip(pyfile[0]) + + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + + + def clean(self): + """ + Clean everything and remove references + """ + if hasattr(self, "pyfile"): + del self.pyfile + + if hasattr(self, "req"): + self.req.close() + del self.req + + if hasattr(self, "thread"): + del self.thread + + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 10b2e7b0e..0bcab5915 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -55,8 +55,6 @@ class MultiHook(Hook): def setup(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.plugins = [] self.supported = [] self.new_supported = [] @@ -97,19 +95,6 @@ class MultiHook(Hook): self.init_periodical(threaded=True) - def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 - """See HTTPRequest for argument list""" - h = pyreq.getHTTPRequest(timeout=120) - try: - if not 'decode' in kwargs: - kwargs['decode'] = True - rep = h.load(*args, **kwargs) - finally: - h.close() - - return rep - - def pluginsCached(self): if self.plugins: return self.plugins diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index da597ef42..0d4c3b165 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -1,65 +1,108 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +import os +import re +import urllib - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 +from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join - @author: RaNaN, spoob, mkaay -""" -from time import time, sleep -from random import randint +def replace_patterns(string, ruleslist): + for r in ruleslist: + rf, rt = r + string = re.sub(rf, rt, string) + return string -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam +def set_cookies(cj, cookies): + for cookie in cookies: + if isinstance(cookie, tuple) and len(cookie) == 3: + domain, name, value = cookie + cj.setCookie(domain, name, value) -from itertools import islice -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import save_join as fs_join, save_path as safe_filename, fs_encode, fs_decode +def parseHtmlTagAttrValue(attr_name, tag): + m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) + return m.group(2) if m else None + + +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, + html, re.S | re.I): + inputs = {} + action = parseHtmlTagAttrValue("action", form.group('TAG')) + + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): + name = parseHtmlTagAttrValue("name", inputtag.group(1)) + if name: + value = parseHtmlTagAttrValue("value", inputtag.group(1)) + if not value: + inputs[name] = inputtag.group(3) or "" + else: + inputs[name] = value + + if input_names: + # check input attributes + for key, val in input_names.iteritems(): + if key in inputs: + if isinstance(val, basestring) and inputs[key] == val: + continue + elif isinstance(val, tuple) and inputs[key] in val: + continue + elif hasattr(val, "search") and re.match(val, inputs[key]): + continue + break #: attibute value does not match + else: + break #: attibute name does not match + else: + return action, inputs #: passed attribute check + else: + # no attribute check + return action, inputs + + return {}, None #: no matching form found + def chunks(iterable, size): - it = iter(iterable) + it = iter(iterable) item = list(islice(it, size)) while item: yield item item = list(islice(it, size)) -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ +class Plugin(object): + __name__ = "Plugin" + __type__ = "hoster" + __version__ = "0.11" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), + ("spoob" , "spoob@pyload.org" ), + ("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] + def __init__(self, core): - #: Core instance self.core = core + #: Provide information in dict here + self.info = {} + - #: Log functions def _log(self, level, args): log = getattr(self.core.log, level) - msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 - str(a)).strip() for a in args if a) + msg = fs_encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or _(level.upper() + " MARK")}) def logDebug(self, *args): @@ -100,7 +143,7 @@ class Base(object): return self.setConfig(*args, **kwargs) - def getConfig(self, option, default=""): + def getConfig(self, option, default="", plugin=None): """ Returns config value for current plugin @@ -108,9 +151,10 @@ class Base(object): :return: """ try: - return self.core.config.getPlugin(self.__name__, option) + return self.core.config.getPlugin(plugin or self.__name__, option) except KeyError: + self.logWarning(_("Config option or plugin not found")) return default @@ -149,11 +193,11 @@ class Base(object): def fail(self, reason): """Fail and give reason""" - raise Fail(reason) + raise Fail(fs_encode(reason)) def error(self, reason="", type=_("Parse")): - if not reason and not type: + if not reason: type = _("Unknown") msg = _("%s error") % type.strip().capitalize() if type else _("Error") @@ -163,313 +207,73 @@ class Base(object): raise Fail(msg) -class Plugin(Base): - __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.10" + def fixurl(self, url): + return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() - __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] - __description__ = """Base plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de" )] - - - def __init__(self, pyfile): - super(Plugin, self).__init__(pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.canUse(): self.account = None - if self.account: - self.user, data = self.account.selectAccount() - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium(self.user) - else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.core.config["download"]["chunks"] - return min(self.core.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - self.account.checkLogin(self.user) - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def skip(self, reason): - raise Skip(reason) - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.core.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it + Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict + :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) + if hasattr(self, 'pyfile') and self.pyfile.abort: + self.abort() + + url = self.fixurl(url) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + if not url or not isinstance(url, basestring): + self.fail(_("No url given")) if self.core.debug: - from inspect import currentframe + self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + if req is None: + if hasattr(self, "req"): + req = self.req + else: + req = self.core.requestFactory.getRequest(self.__name__) - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) + res = req.load(url, get, post, ref, cookies, just_header, True, bool(decode)) - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned + if decode: + res = html_unescape(res) + if isinstance(decode, basestring): + res = res.decode(decode) + + if self.core.debug: + import inspect + + frame = inspect.currentframe() + framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: - tmp = res.encode("utf8") - except: - tmp = res + if not os.path.exists(os.path.join("tmp", self.__name__)): + os.makedirs(os.path.join("tmp", self.__name__)) - f.write(tmp) - f.close() + with open(framefile, "wb") as f: + del frame #: delete the frame or it wont be cleaned + f.write(res.encode('utf8')) + except IOError, e: + self.logError(e) if just_header: - #parse header - header = {"code": self.req.code} + # parse header + header = {"code": req.code} for line in res.splitlines(): line = line.strip() - if not line or ":" not in line: continue + if not line or ":" not in line: + continue key, none, value = line.partition(":") - key = key.lower().strip() + key = key.strip().lower() value = value.strip() if key in header: @@ -481,164 +285,4 @@ class Plugin(Base): header[key] = value res = header - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.core.config['general']['download_folder'] - - location = fs_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.core.config["permission"]["user"])[2] - gid = getgrnam(self.core.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = safe_filename(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.core.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.core.config["permission"]["user"])[2] - gid = getgrnam(self.core.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.core.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.core.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.core.config['general']['download_folder'] - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.core.log.debug("File %s not skipped, because it does not exists" % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html + return res
\ No newline at end of file diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index a9d0f3752..40faff5f0 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -51,7 +51,7 @@ class ReCaptcha(Captcha): def _challenge_v1(self, key): - html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", + html = self.plugin.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) try: challenge = re.search("challenge : '(.+?)',", html).group(1) @@ -66,8 +66,8 @@ class ReCaptcha(Captcha): def result(self, server, challenge, key): - self.plugin.req.load("http://www.google.com/recaptcha/api/js/recaptcha.js") - html = self.plugin.req.load("http://www.google.com/recaptcha/api/reload", + self.plugin.load("http://www.google.com/recaptcha/api/js/recaptcha.js") + html = self.plugin.load("http://www.google.com/recaptcha/api/reload", get={'c' : challenge, 'k' : key, 'reason': "i", @@ -92,7 +92,7 @@ class ReCaptcha(Captcha): def _collectApiInfo(self): - html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") + html = self.plugin.load("http://www.google.com/recaptcha/api.js") a = re.search(r'po.src = \'(.*?)\';', html).group(1) vers = a.split("/")[5] @@ -102,7 +102,7 @@ class ReCaptcha(Captcha): self.logDebug("API language: %s" % language) - html = self.plugin.req.load("https://apis.google.com/js/api.js") + html = self.plugin.load("https://apis.google.com/js/api.js") b = re.search(r'"h":"(.*?)","', html).group(1) jsh = b.decode('unicode-escape') @@ -112,7 +112,7 @@ class ReCaptcha(Captcha): def _prepareTimeAndRpc(self): - self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") + self.plugin.load("http://www.google.com/recaptcha/api2/demo") millis = int(round(time.time() * 1000)) @@ -139,7 +139,7 @@ class ReCaptcha(Captcha): vers, language, jsh = self._collectApiInfo() millis, rpc = self._prepareTimeAndRpc() - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", + html = self.plugin.load("https://www.google.com/recaptcha/api2/anchor", get={'k' : key, 'hl' : language, 'v' : vers, @@ -152,14 +152,15 @@ class ReCaptcha(Captcha): token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) self.logDebug("Token #1: %s" % token1.group(1)) - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", - get={'c' : token1.group(1), - 'hl' : language, - 'v' : vers, - 'bg' : botguardstring, - 'k' : key, - 'usegapi': "1", - 'jsh' : jsh}).decode('unicode-escape') + html = self.plugin.load("https://www.google.com/recaptcha/api2/frame", + get={'c' : token1.group(1), + 'hl' : language, + 'v' : vers, + 'bg' : botguardstring, + 'k' : key, + 'usegapi': "1", + 'jsh' : jsh}, + decode="unicode-escape") token2 = re.search(r'"finput","(.*?)",', html) self.logDebug("Token #2: %s" % token2.group(1)) @@ -179,7 +180,7 @@ class ReCaptcha(Captcha): timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading timeToSolveMore = timeToSolve + int(float("0." + str(random.randint(1, 99999999))) * 500) - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", + html = self.plugin.load("https://www.google.com/recaptcha/api2/userverify", post={'k' : key, 'c' : token3.group(1), 'response': response, diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3e8b89f79..db253d950 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -54,7 +54,7 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + account = self.core.accountManager.getAccountPlugin(account_name) if account and account.canUse(): self.user, data = account.selectAccount() @@ -68,9 +68,9 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.error = "" #@TODO: Remove in 0.4.10 self.info = {} - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.links = [] #@TODO: Move to Hoster in 0.4.10 + self.html = "" #@TODO: Recheck in 0.4.10 + self.link = "" #@TODO: Recheck in 0.4.10 + self.links = [] if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -87,11 +87,11 @@ class SimpleCrypter(Crypter, SimpleHoster): def handleDirect(self, pyfile): - for i in xrange(10): #@TODO: Use `pycurl.MAXREDIRS` value in 0.4.10 + for i in xrange(self.getConfig("maxredirs", plugin="UserAgentSwitcher")): redirect = self.link or pyfile.url self.logDebug("Redirect #%d to: %s" % (i, redirect)) - header = self.load(redirect, just_header=True, decode=True) + header = self.load(redirect, just_header=True) if 'location' in header and header['location']: self.link = header['location'] else: @@ -107,13 +107,13 @@ class SimpleCrypter(Crypter, SimpleHoster): self.handleDirect(pyfile) if self.link: - self.urls = self.fixurls([self.link]) + self.urls = [self.link] else: self.preload() self.checkInfo() - self.links = self.fixurls(self.getLinks()) or list() + self.links = self.getLinks() or list() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handlePages(pyfile) @@ -123,9 +123,6 @@ class SimpleCrypter(Crypter, SimpleHoster): if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] - elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 - self.fail(_("No link grabbed")) - def checkNameSize(self, getinfo=True): if not self.info or getinfo: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index cec5f55f1..979da24a9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -11,12 +11,11 @@ import urllib import urlparse from module.PyFile import statusMap as _statusMap -from module.network.CookieJar import CookieJar from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL from module.plugins.internal.Hoster import Hoster -from module.plugins.internal.Plugin import Fail, Retry -from module.utils import fixup, fs_encode, html_unescape, parseFileSize +from module.plugins.internal.Plugin import Fail, Retry, replace_patterns, set_cookies +from module.utils import fixup, fs_encode, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -24,73 +23,6 @@ statusMap = dict((v, k) for k, v in _statusMap.iteritems()) #@TODO: Remove in 0.4.10 -def _wait(self, seconds, reconnect): - if seconds: - self.setWait(int(seconds) + 1) - - if reconnect is not None: - self.wantReconnect = reconnect - - super(SimpleHoster, self).wait() - - -def replace_patterns(string, ruleslist): - for r in ruleslist: - rf, rt = r - string = re.sub(rf, rt, string) - return string - - -def set_cookies(cj, cookies): - for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) == 3: - domain, name, value = cookie - cj.setCookie(domain, name, value) - - -def parseHtmlTagAttrValue(attr_name, tag): - m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) - return m.group(2) if m else None - - -def parseHtmlForm(attr_str, html, input_names={}): - for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, - html, re.S | re.I): - inputs = {} - action = parseHtmlTagAttrValue("action", form.group('TAG')) - - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): - name = parseHtmlTagAttrValue("name", inputtag.group(1)) - if name: - value = parseHtmlTagAttrValue("value", inputtag.group(1)) - if not value: - inputs[name] = inputtag.group(3) or "" - else: - inputs[name] = value - - if input_names: - # check input attributes - for key, val in input_names.iteritems(): - if key in inputs: - if isinstance(val, basestring) and inputs[key] == val: - continue - elif isinstance(val, tuple) and inputs[key] in val: - continue - elif hasattr(val, "search") and re.match(val, inputs[key]): - continue - break #: attibute value does not match - else: - break #: attibute name does not match - else: - return action, inputs #: passed attribute check - else: - # no attribute check - return action, inputs - - return {}, None #: no matching form found - - -#@TODO: Remove in 0.4.10 def parseFileInfo(plugin, url="", html=""): if hasattr(plugin, "getInfo"): info = plugin.getInfo(url, html) @@ -123,91 +55,6 @@ def timestamp(): return int(time.time() * 1000) -#@TODO: Move to Hoster in 0.4.10 -def getFileURL(self, url, follow_location=None): - link = "" - redirect = 1 - - if type(follow_location) is int: - redirect = max(follow_location, 1) - else: - redirect = 10 - - for i in xrange(redirect): - try: - self.logDebug("Redirect #%d to: %s" % (i, url)) - header = self.load(url, just_header=True, decode=True) - - except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - req = pyreq.getHTTPRequest() - res = req.load(url, just_header=True, decode=True) - - req.close() - - header = {"code": req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: - continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - - if 'content-disposition' in header: - link = url - - elif 'location' in header and header['location']: - location = header['location'] - - if not urlparse.urlparse(location).scheme: - url_p = urlparse.urlparse(url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - location = urlparse.urljoin(baseurl, location) - - if 'code' in header and header['code'] == 302: - link = location - - if follow_location: - url = location - continue - - else: - extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] - - if 'content-type' in header and header['content-type']: - mimetype = header['content-type'].split(';')[0].strip() - - elif extension: - mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" - - else: - mimetype = "" - - if mimetype and (link or 'html' not in mimetype): - link = url - else: - link = "" - - break - - else: - try: - self.logError(_("Too many redirects")) - except Exception: - pass - - return link - - def secondsToMidnight(gmt=0): now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) @@ -313,14 +160,11 @@ class SimpleHoster(Hoster): LOGIN_ACCOUNT = False #: Set to True to require account login LOGIN_PREMIUM = False #: Set to True to require premium account login MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) - TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct + TEXT_ENCODING = True #: Set to encoding name if encoding value in http header is not correct LINK_PATTERN = None - directLink = getFileURL #@TODO: Remove in 0.4.10 - - @classmethod def apiInfo(cls, url): url = urllib.unquote(url) @@ -351,10 +195,7 @@ class SimpleHoster(Hoster): elif info['status'] is 3: try: - html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) - - if isinstance(cls.TEXT_ENCODING, basestring): - html = unicode(html, cls.TEXT_ENCODING) + html = getURL(url, cookies=cls.COOKIES, decode=cls.TEXT_ENCODING) except BadHeader, e: info['error'] = "%d: %s" % (e.code, e.content) @@ -424,10 +265,10 @@ class SimpleHoster(Hoster): self.pyfile.error = "" #@TODO: Remove in 0.4.10 self.info = {} - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.directDL = False #@TODO: Move to Hoster in 0.4.10 - self.multihost = False #@TODO: Move to Hoster in 0.4.10 + self.html = "" #@TODO: Recheck in 0.4.10 + self.link = "" #@TODO: Recheck in 0.4.10 + self.directDL = False + self.multihost = False if not self.getConfig('use_premium', True): self.retryFree() @@ -465,10 +306,10 @@ class SimpleHoster(Hoster): def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), ref=False, decode=not self.TEXT_ENCODING) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) + self.html = self.load(self.pyfile.url, + cookies=bool(self.COOKIES), + ref=False, + decode=self.TEXT_ENCODING) def process(self, pyfile): @@ -517,29 +358,6 @@ class SimpleHoster(Hoster): raise Fail(err) - def fixurl(self, url): - return self.fixurls([url])[0] - - - def fixurls(self, urls): - url_p = urlparse.urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - - urls = (html_unescape(url.decode('unicode-escape').strip()) for url in urls) - - return [urlparse.urljoin(baseurl, url) if not urlparse.urlparse(url).scheme else url \ - for url in urls] - - - def download(self, url, *args, **kwargs): - if not url or not isinstance(url, basestring): - return - - self.correctCaptcha() - - return super(SimpleHoster, self).download(self.fixurl(url), *args, **kwargs) - - def checkFile(self): lastDownload = fs_encode(self.lastDownload) @@ -547,15 +365,10 @@ class SimpleHoster(Hoster): self.invalidCaptcha() self.retry(10, reason=_("Wrong captcha")) - elif not self.lastDownload or not os.path.exists(lastDownload): - self.lastDownload = "" - self.error(self.pyfile.error or _("No file downloaded")) + elif self.checkDownload({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size']): + self.error(_("Empty file")) else: - #@TODO: Move to Hoster in 0.4.10 - if os.stat(lastDownload).st_size < 1 or self.checkDownload({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}): - self.error(_("Empty file")) - self.logDebug("Checking last downloaded file with built-in rules") for r, p in self.FILE_ERRORS: errmsg = self.checkDownload({r: re.compile(p)}) @@ -577,7 +390,7 @@ class SimpleHoster(Hoster): self.html = f.read(50000) #@TODO: Recheck in 0.4.10 self.checkErrors() - self.logDebug("No file errors found") + self.logDebug("No file errors found") def checkErrors(self): @@ -790,50 +603,10 @@ class SimpleHoster(Hoster): self.link = m.group(1) - def longWait(self, wait_time=None, max_tries=3): - if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) - else: - wait_time = 900 - time_str = _("(unknown time)") - max_tries = 100 - - self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) - - self.wait(wait_time, True) - self.retry(max_tries=max_tries, reason=_("Download limit reached")) - - - def parseHtmlForm(self, attr_str="", input_names={}): - return parseHtmlForm(attr_str, self.html, input_names) - - - def checkTrafficLeft(self): - if not self.account: - return True - - traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] - - if traffic is None: - return False - elif traffic == -1: - return True - else: - size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) - return size <= traffic - - def retryFree(self): if not self.premium: return self.premium = False self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retries = -1 + self.req = self.core.requestFactory.getRequest(self.__name__) raise Retry(_("Fallback to free download")) - - - #@TODO: Remove in 0.4.10 - def wait(self, seconds=0, reconnect=None): - return _wait(self, seconds, reconnect) diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index d600ef03c..dde6223ed 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -35,7 +35,7 @@ class SolveMedia(Captcha): def challenge(self, key=None, html=None): key = key or self.retrieve_key(html) - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", + html = self.plugin.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}) for i in xrange(1, 11): @@ -64,7 +64,7 @@ class SolveMedia(Captcha): self.plugin.invalidCaptcha() result = None - html = self.plugin.req.load("http://api.solvemedia.com/papi/verify.noscript", + html = self.plugin.load("http://api.solvemedia.com/papi/verify.noscript", post={'adcopy_response' : result, 'k' : key, 'l' : "en", @@ -83,7 +83,7 @@ class SolveMedia(Captcha): if "error" in html: self.logWarning("Captcha code was invalid") self.logDebug("Retry #%d" % i) - html = self.plugin.req.load(redirect) + html = self.plugin.load(redirect) else: break diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index f230cedf3..c26a91775 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -5,7 +5,7 @@ import time import urlparse from module.plugins.internal.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies +from module.plugins.internal.Plugin import parseHtmlForm, set_cookies class XFSAccount(Account): @@ -38,11 +38,6 @@ class XFSAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 - self.init() - return super(XFSAccount, self).__init__(manager, accounts) - - def init(self): if not self.HOSTER_DOMAIN: self.logError(_("Missing HOSTER_DOMAIN")) @@ -69,7 +64,7 @@ class XFSAccount(Account): 'leechtraffic': leechtraffic, 'premium' : premium} - html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) + html = self.load(self.HOSTER_URL, get={'op': "my_account"}, req=req) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -160,7 +155,7 @@ class XFSAccount(Account): if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") - html = req.load(self.LOGIN_URL, decode=True) + html = self.load(self.LOGIN_URL, req=req) action, inputs = parseHtmlForm('name="FL"', html) if not inputs: @@ -175,7 +170,7 @@ class XFSAccount(Account): else: url = self.HOSTER_URL - html = req.load(url, post=inputs, decode=True) + html = self.load(url, post=inputs, req=req) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrongPassword() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 84317c615..0f5bfd5d7 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -32,7 +32,7 @@ class XFSCrypter(SimpleCrypter): account = self.account else: account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + account = self.core.accountManager.getAccountPlugin(account_name) if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: self.HOSTER_DOMAIN = account.HOSTER_DOMAIN diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index b2642bf1a..fbce5e2fb 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -61,7 +61,7 @@ class XFSHoster(SimpleHoster): if self.account: account = self.account else: - account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + account = self.core.accountManager.getAccountPlugin(self.__name__) if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: self.HOSTER_DOMAIN = account.HOSTER_DOMAIN @@ -95,7 +95,7 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.html = self.load(pyfile.url, post=data, decode=True) + self.html = self.load(pyfile.url, post=data) self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) @@ -110,7 +110,7 @@ class XFSHoster(SimpleHoster): self.logError(data['op'] if 'op' in data else _("UNKNOWN")) return "" - self.link = m.group(1).strip() #@TODO: Remove `.strip()` in 0.4.10 + self.link = m.group(1) def handlePremium(self, pyfile): @@ -166,7 +166,7 @@ class XFSHoster(SimpleHoster): if m is None: self.error(_("LINK_LEECH_PATTERN not found")) - header = self.load(m.group(1), just_header=True, decode=True) + header = self.load(m.group(1), just_header=True) if 'location' in header: #: Direct download link self.link = header['location'] |