diff options
Diffstat (limited to 'module/plugins')
437 files changed, 7335 insertions, 5621 deletions
diff --git a/module/plugins/accounts/AlldebridCom.py b/module/plugins/accounts/AlldebridCom.py index 1f2371e28..d09086907 100644 --- a/module/plugins/accounts/AlldebridCom.py +++ b/module/plugins/accounts/AlldebridCom.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- import re -import xml.dom.minidom as dom - -from time import time +import time +import xml.dom.minidom from BeautifulSoup import BeautifulSoup @@ -33,19 +32,19 @@ class AlldebridCom(Account): p = re.compile('\d+') exp_data = p.findall(time_text) - exp_time = time() + int(exp_data[0]) * 24 * 60 * 60 + int( + exp_time = time.time() + int(exp_data[0]) * 24 * 60 * 60 + int( exp_data[1]) * 60 * 60 + (int(exp_data[2]) - 1) * 60 #Get expiration date from API - except: + except Exception: data = self.getAccountData(user) html = req.load("http://www.alldebrid.com/api.php", get={'action': "info_user", 'login': user, 'pw': data['password']}) self.logDebug(html) - xml = dom.parseString(html) - exp_time = time() + int(xml.getElementsByTagName("date")[0].childNodes[0].nodeValue) * 24 * 60 * 60 + xml = xml.dom.minidom.parseString(html) + exp_time = time.time() + int(xml.getElementsByTagName("date")[0].childNodes[0].nodeValue) * 24 * 60 * 60 account_info = {"validuntil": exp_time, "trafficleft": -1} return account_info diff --git a/module/plugins/accounts/BitshareCom.py b/module/plugins/accounts/BitshareCom.py index 412aae534..00e546f6d 100644 --- a/module/plugins/accounts/BitshareCom.py +++ b/module/plugins/accounts/BitshareCom.py @@ -28,7 +28,6 @@ class BitshareCom(Account): def login(self, user, data, req): html = req.load("http://bitshare.com/login.html", post={"user": user, "password": data['password'], "submit": "Login"}, - cookies=True, decode=True) if "login" in req.lastEffectiveURL: diff --git a/module/plugins/accounts/CatShareNet.py b/module/plugins/accounts/CatShareNet.py index bcb14bee3..3ddadca8e 100644 --- a/module/plugins/accounts/CatShareNet.py +++ b/module/plugins/accounts/CatShareNet.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -10,39 +9,44 @@ from module.plugins.Account import Account class CatShareNet(Account): __name__ = "CatShareNet" __type__ = "account" - __version__ = "0.02" + __version__ = "0.05" - __description__ = """CatShareNet account plugin""" + __description__ = """Catshare.net account plugin""" __license__ = "GPLv3" __authors__ = [("prOq", None)] - PREMIUM_PATTERN = r'class="nav-collapse collapse pull-right">[\s\w<>=-."/:]*\sz.</a></li>\s*<li><a href="/premium">.*\s*<span style="color: red">(.*?)</span>[\s\w<>/]*href="/logout"' - VALID_UNTIL_PATTERN = r'<div class="span6 pull-right">[\s\w<>=-":;]*<span style="font-size:13px;">.*?<strong>(.*?)</strong></span>' + PREMIUM_PATTERN = r'<a href="/premium">Konto:[\s\n]*Premium' + VALID_UNTIL_PATTERN = r'>Konto premium.*?<strong>(.*?)</strong></span>' + TRAFFIC_LEFT_PATTERN = r'<a href="/premium">([0-9.]+ [kMG]B)' def loadAccountInfo(self, user, req): - premium = False - validuntil = -1 + premium = False + validuntil = -1 + trafficleft = -1 html = req.load("http://catshare.net/", decode=True) + if re.search(self.PREMIUM_PATTERN, html): + premium = True + try: - m = re.search(self.PREMIUM_PATTERN, html) - if "Premium" in m.group(1): - premium = True - except: + expiredate = re.search(self.VALID_UNTIL_PATTERN, html).group(1) + self.logDebug("Expire date: " + expiredate) + + validuntil = time.mktime(time.strptime(expiredate, "%Y-%m-%d %H:%M:%S")) + + except Exception: pass try: - m = re.search(self.VALID_UNTIL_PATTERN, html) - expiredate = m.group(1) - if "-" not in expiredate: - validuntil = mktime(strptime(expiredate, "%d.%m.%Y")) - except: + trafficleft = self.parseTraffic(re.search(self.TRAFFIC_LEFT_PATTERN, html).group(1)) + + except Exception: pass - return {'premium': premium, 'trafficleft': -1, 'validuntil': validuntil} + return {'premium': premium, 'trafficleft': trafficleft, 'validuntil': validuntil} def login(self, user, data, req): diff --git a/module/plugins/accounts/CzshareCom.py b/module/plugins/accounts/CzshareCom.py index f84bc67f3..300943828 100644 --- a/module/plugins/accounts/CzshareCom.py +++ b/module/plugins/accounts/CzshareCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from time import mktime, strptime import re +import time from module.plugins.Account import Account @@ -9,7 +9,7 @@ from module.plugins.Account import Account class CzshareCom(Account): __name__ = "CzshareCom" __type__ = "account" - __version__ = "0.16" + __version__ = "0.18" __description__ = """Czshare.com account plugin, now Sdilej.cz""" __license__ = "GPLv3" @@ -21,15 +21,26 @@ class CzshareCom(Account): def loadAccountInfo(self, user, req): + premium = False + validuntil = None + trafficleft = None + html = req.load("http://sdilej.cz/prehled_kreditu/") - m = re.search(self.CREDIT_LEFT_PATTERN, html) - if m is None: - return {"validuntil": 0, "trafficleft": 0} + try: + m = re.search(self.CREDIT_LEFT_PATTERN, html) + trafficleft = self.parseTraffic(m.group(1).replace(' ', '').replace(',', '.')) + m.group(2) + validuntil = time.mktime(time.strptime(m.group(3), '%d.%m.%y %H:%M')) + + except Exception, e: + self.logError(e) + else: - trafficleft = self.parseTraffic(m.group(1).replace(' ', '').replace(',', '.')) + m.group(2)] - validuntil = mktime(strptime(m.group(3), '%d.%m.%y %H:%M')) - return {"validuntil": validuntil, "trafficleft": trafficleft} + premium = True + + return {'premium' : premium, + 'validuntil' : validuntil, + 'trafficleft': trafficleft} def login(self, user, data, req): diff --git a/module/plugins/accounts/DebridItaliaCom.py b/module/plugins/accounts/DebridItaliaCom.py index 50f9aea27..3df99101a 100644 --- a/module/plugins/accounts/DebridItaliaCom.py +++ b/module/plugins/accounts/DebridItaliaCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -28,7 +27,7 @@ class DebridItaliaCom(Account): if 'Account premium not activated' not in html: m = re.search(self.WALID_UNTIL_PATTERN, html) if m: - validuntil = mktime(strptime(m.group(1), "%d/%m/%Y %H:%M")) + validuntil = time.mktime(time.strptime(m.group(1), "%d/%m/%Y %H:%M")) info = {"premium": True, "validuntil": validuntil, "trafficleft": -1} else: self.logError(_("Unable to retrieve account information")) diff --git a/module/plugins/accounts/DepositfilesCom.py b/module/plugins/accounts/DepositfilesCom.py index dfe430276..a0bd6a37c 100644 --- a/module/plugins/accounts/DepositfilesCom.py +++ b/module/plugins/accounts/DepositfilesCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import strptime, mktime +import time from module.plugins.Account import Account @@ -23,7 +22,7 @@ class DepositfilesCom(Account): html = req.load("https://dfiles.eu/de/gold/") validuntil = re.search(r"Sie haben Gold Zugang bis: <b>(.*?)</b></div>", html).group(1) - validuntil = mktime(strptime(validuntil, "%Y-%m-%d %H:%M:%S")) + validuntil = time.mktime(time.strptime(validuntil, "%Y-%m-%d %H:%M:%S")) return {"validuntil": validuntil, "trafficleft": -1} diff --git a/module/plugins/accounts/EuroshareEu.py b/module/plugins/accounts/EuroshareEu.py index f92a4e821..db4539e2e 100644 --- a/module/plugins/accounts/EuroshareEu.py +++ b/module/plugins/accounts/EuroshareEu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from time import mktime, strptime import re +import time from module.plugins.Account import Account @@ -25,7 +25,7 @@ class EuroshareEu(Account): premium, validuntil = False, -1 else: premium = True - validuntil = mktime(strptime(m.group(1), "%d.%m.%Y %H:%M")) + validuntil = time.mktime(time.strptime(m.group(1), "%d.%m.%Y %H:%M")) return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} diff --git a/module/plugins/accounts/BillionuploadsCom.py b/module/plugins/accounts/ExashareCom.py index 11af36591..431798522 100644 --- a/module/plugins/accounts/BillionuploadsCom.py +++ b/module/plugins/accounts/ExashareCom.py @@ -3,14 +3,14 @@ from module.plugins.internal.XFSAccount import XFSAccount -class BillionuploadsCom(XFSAccount): - __name__ = "BillionuploadsCom" +class ExashareCom(XFSAccount): + __name__ = "ExashareCom" __type__ = "account" - __version__ = "0.02" + __version__ = "0.01" - __description__ = """Billionuploads.com account plugin""" + __description__ = """Exashare.com account plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = "billionuploads.com" + HOSTER_DOMAIN = "exashare.com" diff --git a/module/plugins/accounts/FastshareCz.py b/module/plugins/accounts/FastshareCz.py index d6e94f2e3..b946e29ba 100644 --- a/module/plugins/accounts/FastshareCz.py +++ b/module/plugins/accounts/FastshareCz.py @@ -8,7 +8,7 @@ from module.plugins.Account import Account class FastshareCz(Account): __name__ = "FastshareCz" __type__ = "account" - __version__ = "0.05" + __version__ = "0.06" __description__ = """Fastshare.cz account plugin""" __license__ = "GPLv3" @@ -16,13 +16,13 @@ class FastshareCz(Account): ("stickell", "l.stickell@yahoo.it")] - CREDIT_PATTERN = r'My account\s*\((.+?)\)' + CREDIT_PATTERN = r'Credit\s*:\s*</td>\s*<td>(.+?)\s*<' def loadAccountInfo(self, user, req): - validuntil = None + validuntil = -1 trafficleft = None - premium = None + premium = False html = req.load("http://www.fastshare.cz/user", decode=True) @@ -30,13 +30,11 @@ class FastshareCz(Account): if m: trafficleft = self.parseTraffic(m.group(1)) - if trafficleft: - premium = True - validuntil = -1 - else: - premium = False + premium = bool(trafficleft) - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} def login(self, user, data, req): diff --git a/module/plugins/accounts/File4safeCom.py b/module/plugins/accounts/File4SafeCom.py index 50fe1aac8..50fe1aac8 100644 --- a/module/plugins/accounts/File4safeCom.py +++ b/module/plugins/accounts/File4SafeCom.py diff --git a/module/plugins/accounts/FilefactoryCom.py b/module/plugins/accounts/FilefactoryCom.py index 426d572db..37b6f97a8 100644 --- a/module/plugins/accounts/FilefactoryCom.py +++ b/module/plugins/accounts/FilefactoryCom.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re -from time import mktime, strptime - -from pycurl import REFERER +import time from module.plugins.Account import Account @@ -29,7 +28,7 @@ class FilefactoryCom(Account): if m: premium = True validuntil = re.sub(self.VALID_UNTIL_PATTERN, '\g<D> \g<M> \g<Y>', m.group(0)) - validuntil = mktime(strptime(validuntil, "%d %b %Y")) + validuntil = time.mktime(time.strptime(validuntil, "%d %b %Y")) else: premium = False validuntil = -1 @@ -38,7 +37,7 @@ class FilefactoryCom(Account): def login(self, user, data, req): - req.http.c.setopt(REFERER, "http://www.filefactory.com/member/login.php") + req.http.c.setopt(pycurl.REFERER, "http://www.filefactory.com/member/login.php") html = req.load("http://www.filefactory.com/member/signin.php", post={"loginEmail" : user, diff --git a/module/plugins/accounts/FilejungleCom.py b/module/plugins/accounts/FilejungleCom.py index 9f7474207..b92a371a5 100644 --- a/module/plugins/accounts/FilejungleCom.py +++ b/module/plugins/accounts/FilejungleCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -28,7 +28,7 @@ class FilejungleCom(Account): m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: premium = True - validuntil = mktime(strptime(m.group(1), "%d %b %Y")) + validuntil = time.mktime(time.strptime(m.group(1), "%d %b %Y")) else: premium = False validuntil = -1 diff --git a/module/plugins/accounts/FilerNet.py b/module/plugins/accounts/FilerNet.py index 4067445af..149ff7d89 100644 --- a/module/plugins/accounts/FilerNet.py +++ b/module/plugins/accounts/FilerNet.py @@ -9,14 +9,14 @@ from module.plugins.Account import Account class FilerNet(Account): __name__ = "FilerNet" __type__ = "account" - __version__ = "0.04" + __version__ = "0.05" __description__ = """Filer.net account plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - TOKEN_PATTERN = r'_csrf_token" value="([^"]+)" />' + TOKEN_PATTERN = r'name="_csrf_token" value="(.+?)"' WALID_UNTIL_PATTERN = r'Der Premium-Zugang ist gÃŒltig bis (.+)\.\s*</td>' TRAFFIC_PATTERN = r'Traffic</th>\s*<td>([^<]+)</td>' FREE_PATTERN = r'Account Status</th>\s*<td>\s*Free' @@ -43,15 +43,15 @@ class FilerNet(Account): def login(self, user, data, req): - html = req.load("https://filer.net/login") + html = req.load("https://filer.net/login", decode=True) token = re.search(self.TOKEN_PATTERN, html).group(1) html = req.load("https://filer.net/login_check", - post={"_username": user, - "_password": data['password'], + post={"_username" : user, + "_password" : data['password'], "_remember_me": "on", - "_csrf_token": token, + "_csrf_token" : token, "_target_path": "https://filer.net/"}, decode=True) diff --git a/module/plugins/accounts/FilesMailRu.py b/module/plugins/accounts/FilesMailRu.py index 15926589e..ee309c425 100644 --- a/module/plugins/accounts/FilesMailRu.py +++ b/module/plugins/accounts/FilesMailRu.py @@ -25,7 +25,6 @@ class FilesMailRu(Account): "Login": user, "Password": data['password'], "Page": "http://files.mail.ru/"}, - cookies=True, decode=True) if "ÐевеÑМПе ÐžÐŒÑ Ð¿ÐŸÐ»ÑзПваÑÐµÐ»Ñ ÐžÐ»Ðž паÑПлÑ" in html: diff --git a/module/plugins/accounts/FileserveCom.py b/module/plugins/accounts/FileserveCom.py index 1cf2a3a3c..5eb6b844c 100644 --- a/module/plugins/accounts/FileserveCom.py +++ b/module/plugins/accounts/FileserveCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from time import mktime, strptime +import time from module.plugins.Account import Account from module.common.json_layer import json_loads @@ -24,7 +24,7 @@ class FileserveCom(Account): res = json_loads(html) if res['type'] == "premium": - validuntil = mktime(strptime(res['expireTime'], "%Y-%m-%d %H:%M:%S")) + validuntil = time.mktime(time.strptime(res['expireTime'], "%Y-%m-%d %H:%M:%S")) return {"trafficleft": res['traffic'], "validuntil": validuntil} else: return {"premium": False, "trafficleft": None, "validuntil": None} diff --git a/module/plugins/accounts/FreakshareCom.py b/module/plugins/accounts/FreakshareCom.py index 83f4a9a84..ca3602a2c 100644 --- a/module/plugins/accounts/FreakshareCom.py +++ b/module/plugins/accounts/FreakshareCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import strptime, mktime +import time from module.plugins.Account import Account @@ -26,7 +25,7 @@ class FreakshareCom(Account): try: m = re.search(r'ltig bis:</td>\s*<td><b>([\d.:-]+)</b></td>', html, re.M) - validuntil = mktime(strptime(m.group(1).strip(), "%d.%m.%Y - %H:%M")) + validuntil = time.mktime(time.strptime(m.group(1).strip(), "%d.%m.%Y - %H:%M")) except Exception: pass @@ -46,7 +45,6 @@ class FreakshareCom(Account): html = req.load("http://freakshare.com/login.html", post={"submit": "Login", "user": user, "pass": data['password']}, - cookies=True, decode=True) if ">Wrong Username or Password" in html: diff --git a/module/plugins/accounts/FshareVn.py b/module/plugins/accounts/FshareVn.py index 66d912958..7fcf88f20 100644 --- a/module/plugins/accounts/FshareVn.py +++ b/module/plugins/accounts/FshareVn.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -19,8 +18,8 @@ class FshareVn(Account): VALID_UNTIL_PATTERN = ur'<dt>Thá»i hạn dùng:</dt>\s*<dd>([^<]+)</dd>' - LIFETIME_PATTERN = ur'<dt>Lần ÄÄng nháºp trÆ°á»c:</dt>\s*<dd>[^<]+</dd>' - TRAFFIC_LEFT_PATTERN = ur'<dt>Tá»ng Dung Lượng Tà i Khoản</dt>\s*<dd[^>]*>([\d.]+) ([kKMG])B</dd>' + LIFETIME_PATTERN = ur'<dt>Lần ÄÄng nháºp trÆ°á»c:</dt>\s*<dd>.+?</dd>' + TRAFFIC_LEFT_PATTERN = ur'<dt>Tá»ng Dung Lượng Tà i Khoản</dt>\s*<dd.*?>([\d.]+) ([kKMG])B</dd>' DIRECT_DOWNLOAD_PATTERN = ur'<input type="checkbox"\s*([^=>]*)[^>]*/>KÃch hoạt download trá»±c tiếp</dt>' @@ -35,7 +34,7 @@ class FshareVn(Account): m = re.search(self.VALID_UNTIL_PATTERN, html) if m: premium = True - validuntil = mktime(strptime(m.group(1), '%I:%M:%S %p %d-%m-%Y')) + validuntil = time.mktime(time.strptime(m.group(1), '%I:%M:%S %p %d-%m-%Y')) trafficleft = self.getTrafficLeft() else: premium = False diff --git a/module/plugins/accounts/JunkyvideoCom.py b/module/plugins/accounts/JunkyvideoCom.py new file mode 100644 index 000000000..8275ff176 --- /dev/null +++ b/module/plugins/accounts/JunkyvideoCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.XFSAccount import XFSAccount + + +class JunkyvideoCom(XFSAccount): + __name__ = "JunkyvideoCom" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Junkyvideo.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "junkyvideo.com" diff --git a/module/plugins/accounts/Keep2shareCc.py b/module/plugins/accounts/Keep2ShareCc.py index e855fb977..d2ba1d237 100644 --- a/module/plugins/accounts/Keep2shareCc.py +++ b/module/plugins/accounts/Keep2ShareCc.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import gmtime, mktime, strptime +import time from module.plugins.Account import Account @@ -10,14 +9,15 @@ from module.plugins.Account import Account class Keep2ShareCc(Account): __name__ = "Keep2ShareCc" __type__ = "account" - __version__ = "0.04" + __version__ = "0.05" __description__ = """Keep2Share.cc account plugin""" __license__ = "GPLv3" - __authors__ = [("aeronaut", "aeronaut@pianoguy.de")] + __authors__ = [("aeronaut", "aeronaut@pianoguy.de"), + ("Walter Purcaro", "vuolter@gmail.com")] - VALID_UNTIL_PATTERN = r'Premium expires: <b>(.+?)</b>' + VALID_UNTIL_PATTERN = r'Premium expires:\s*<b>(.+?)<' TRAFFIC_LEFT_PATTERN = r'Available traffic \(today\):\s*<b><a href="/user/statistic.html">(.+?)<' LOGIN_FAIL_PATTERN = r'Please fix the following input errors' @@ -25,8 +25,8 @@ class Keep2ShareCc(Account): def loadAccountInfo(self, user, req): validuntil = None - trafficleft = None - premium = None + trafficleft = -1 + premium = False html = req.load("http://keep2share.cc/site/profile.html", decode=True) @@ -40,25 +40,21 @@ class Keep2ShareCc(Account): validuntil = -1 else: try: - validuntil = mktime(strptime(expiredate, "%Y.%m.%d")) + validuntil = time.mktime(time.strptime(expiredate, "%Y.%m.%d")) except Exception, e: self.logError(e) else: - if validuntil > mktime(gmtime()): - premium = True - else: - premium = False - validuntil = None + premium = True if validuntil > time.mktime(time.gmtime()) else False - m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: - try: - trafficleft = self.parseTraffic(m.group(1)) + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + trafficleft = self.parseTraffic(m.group(1)) - except Exception, e: - self.logError(e) + except Exception, e: + self.logError(e) return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} diff --git a/module/plugins/accounts/LinksnappyCom.py b/module/plugins/accounts/LinksnappyCom.py index dff28a055..34571d374 100644 --- a/module/plugins/accounts/LinksnappyCom.py +++ b/module/plugins/accounts/LinksnappyCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from hashlib import md5 +import hashlib from module.plugins.Account import Account from module.common.json_layer import json_loads @@ -9,8 +9,7 @@ from module.common.json_layer import json_loads class LinksnappyCom(Account): __name__ = "LinksnappyCom" __type__ = "account" - __version__ = "0.04" - + __version__ = "0.05" __description__ = """Linksnappy.com account plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] @@ -19,7 +18,7 @@ class LinksnappyCom(Account): def loadAccountInfo(self, user, req): data = self.getAccountData(user) r = req.load('http://gen.linksnappy.com/lseAPI.php', - get={'act': 'USERDETAILS', 'username': user, 'password': md5(data['password']).hexdigest()}) + get={'act': 'USERDETAILS', 'username': user, 'password': hashlib.md5(data['password']).hexdigest()}) self.logDebug("JSON data: " + r) @@ -42,7 +41,7 @@ class LinksnappyCom(Account): if 'trafficleft' not in j['return'] or isinstance(j['return']['trafficleft'], str): trafficleft = -1 else: - trafficleft = self.parseTraffic(float(j['return']['trafficleft'] + "MB") + trafficleft = self.parseTraffic("%d MB" % j['return']['trafficleft']) return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} @@ -51,7 +50,7 @@ class LinksnappyCom(Account): r = req.load("http://gen.linksnappy.com/lseAPI.php", get={'act' : 'USERDETAILS', 'username': user, - 'password': md5(data['password']).hexdigest()}, + 'password': hashlib.md5(data['password']).hexdigest()}, decode=True) if 'Invalid Account Details' in r: diff --git a/module/plugins/accounts/MegaRapidCz.py b/module/plugins/accounts/MegaRapidCz.py index b229fe47d..262d5a818 100644 --- a/module/plugins/accounts/MegaRapidCz.py +++ b/module/plugins/accounts/MegaRapidCz.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import re +import time -from time import mktime, strptime from module.plugins.Account import Account @@ -34,7 +34,7 @@ class MegaRapidCz(Account): m = re.search(self.VALID_UNTIL_PATTERN, htmll) if m: - validuntil = mktime(strptime(m.group(1), "%d.%m.%Y - %H:%M")) + validuntil = time.mktime(time.strptime(m.group(1), "%d.%m.%Y - %H:%M")) return {"premium": True, "trafficleft": -1, "validuntil": validuntil} m = re.search(self.TRAFFIC_LEFT_PATTERN, htmll) diff --git a/module/plugins/accounts/MegaRapidoNet.py b/module/plugins/accounts/MegaRapidoNet.py new file mode 100644 index 000000000..d061d02bc --- /dev/null +++ b/module/plugins/accounts/MegaRapidoNet.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from module.plugins.Account import Account + + +class MegaRapidoNet(Account): + __name__ = "MegaRapidoNet" + __type__ = "account" + __version__ = "0.02" + + __description__ = """MegaRapido.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + VALID_UNTIL_PATTERN = r'<\s*?div[^>]*?class\s*?=\s*?[\'"]premium_index[\'"].*?>[^<]*?<[^>]*?b.*?>\s*?TEMPO\s*?PREMIUM.*?<[^>]*?/b.*?>\s*?(\d*)[^\d]*?DIAS[^\d]*?(\d*)[^\d]*?HORAS[^\d]*?(\d*)[^\d]*?MINUTOS[^\d]*?(\d*)[^\d]*?SEGUNDOS' + USER_ID_PATTERN = r'<\s*?div[^>]*?class\s*?=\s*?["\']checkbox_compartilhar["\'].*?>.*?<\s*?input[^>]*?name\s*?=\s*?["\']usar["\'].*?>.*?<\s*?input[^>]*?name\s*?=\s*?["\']user["\'][^>]*?value\s*?=\s*?["\'](.*?)\s*?["\']' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = False + + html = req.load("http://megarapido.net/gerador", decode=True) + + validuntil = re.search(self.VALID_UNTIL_PATTERN, html) + if validuntil: + #hier weitermachen!!! (mÃŒssen umbedingt die zeit richtig machen damit! (sollte aber möglich)) + validuntil = time.time() + int(validuntil.group(1)) * 24 * 3600 + int(validuntil.group(2)) * 3600 + int(validuntil.group(3)) * 60 + int(validuntil.group(4)) + trafficleft = -1 + premium = True + + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} + + + def login(self, user, data, req): + req.load("http://megarapido.net/login") + req.load("http://megarapido.net/painel_user/ajax/logar.php", + post={'login': user, 'senha': data['password']}, + decode=True) + + html = req.load("http://megarapido.net/gerador") + + if "sair" not in html.lower(): + self.wrongPassword() + else: + m = re.search(self.USER_ID_PATTERN, html) + if m: + data['uid'] = m.group(1) + else: + self.fail("Couldn't find the user ID") diff --git a/module/plugins/accounts/MegasharesCom.py b/module/plugins/accounts/MegasharesCom.py index 127ebadc8..3d7ddbe46 100644 --- a/module/plugins/accounts/MegasharesCom.py +++ b/module/plugins/accounts/MegasharesCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -29,7 +29,7 @@ class MegasharesCom(Account): try: timestr = re.search(self.VALID_UNTIL_PATTERN, html).group(1) self.logDebug(timestr) - validuntil = mktime(strptime(timestr, "%b %d, %Y")) + validuntil = time.mktime(time.strptime(timestr, "%b %d, %Y")) except Exception, e: self.logError(e) diff --git a/module/plugins/accounts/MultishareCz.py b/module/plugins/accounts/MultishareCz.py index 0ac764ee1..3488e3288 100644 --- a/module/plugins/accounts/MultishareCz.py +++ b/module/plugins/accounts/MultishareCz.py @@ -16,7 +16,7 @@ class MultishareCz(Account): TRAFFIC_LEFT_PATTERN = r'<span class="profil-zvyrazneni">Kredit:</span>\s*<strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong>' - ACCOUNT_INFO_PATTERN = r'<input type="hidden" id="(u_ID|u_hash)" name="[^"]*" value="([^"]+)">' + ACCOUNT_INFO_PATTERN = r'<input type="hidden" id="(u_ID|u_hash)" name=".+?" value="(.+?)">' def loadAccountInfo(self, user, req): diff --git a/module/plugins/accounts/MyfastfileCom.py b/module/plugins/accounts/MyfastfileCom.py index 4c75b27f0..9a13e2e42 100644 --- a/module/plugins/accounts/MyfastfileCom.py +++ b/module/plugins/accounts/MyfastfileCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from time import time +import time from module.common.json_layer import json_loads from module.plugins.Account import Account @@ -18,7 +18,7 @@ class MyfastfileCom(Account): def loadAccountInfo(self, user, req): if 'days_left' in self.json_data: - validuntil = time() + self.json_data['days_left'] * 24 * 60 * 60 + validuntil = time.time() + self.json_data['days_left'] * 24 * 60 * 60 return {"premium": True, "validuntil": validuntil, "trafficleft": -1} else: self.logError(_("Unable to get account information")) diff --git a/module/plugins/accounts/NetloadIn.py b/module/plugins/accounts/NetloadIn.py deleted file mode 100755 index 1abd7fa84..000000000 --- a/module/plugins/accounts/NetloadIn.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from time import time - -from module.plugins.Account import Account - - -class NetloadIn(Account): - __name__ = "NetloadIn" - __type__ = "account" - __version__ = "0.23" - - __description__ = """Netload.in account plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("CryNickSystems", "webmaster@pcProfil.de")] - - - def loadAccountInfo(self, user, req): - html = req.load("http://netload.in/index.php", get={'id': 2, 'lang': "de"}) - left = r'>(\d+) (Tag|Tage), (\d+) Stunden<' - left = re.search(left, html) - if left: - validuntil = time() + int(left.group(1)) * 24 * 60 * 60 + int(left.group(3)) * 60 * 60 - trafficleft = -1 - premium = True - else: - validuntil = None - premium = False - trafficleft = None - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} - - - def login(self, user, data, req): - html = req.load("http://netload.in/index.php", - post={"txtuser" : user, - "txtpass" : data['password'], - "txtcheck": "login", - "txtlogin": "Login"}, - cookies=True, - decode=True) - if "password or it might be invalid!" in html: - self.wrongPassword() diff --git a/module/plugins/accounts/NitroflareCom.py b/module/plugins/accounts/NitroflareCom.py new file mode 100644 index 000000000..57f2b9ae8 --- /dev/null +++ b/module/plugins/accounts/NitroflareCom.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from module.plugins.Account import Account + + +class NitroflareCom(Account): + __name__ = "NitroflareCom" + __type__ = "account" + __version__ = "0.04" + + __description__ = """Nitroflare.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com" )] + + + VALID_UNTIL_PATTERN = r'>Time Left</label><strong>(.+?)</' + TRAFFIC_LEFT_PATTERN = r'>Daily Limit</label><strong>([\d.,]+)' + LOGIN_FAIL_PATTERN = r'<ul class="errors">\s*<li>' + + TOKEN_PATTERN = r'name="token" value="(.+?)"' + + + def loadAccountInfo(self, user, req): + validuntil = -1 + trafficleft = None + premium = False + + html = req.load("https://nitroflare.com/member", + get={'s': "premium"}, + decode=True) + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Time Left: " + expiredate) + + try: + validuntil = sum(int(v) * {'day': 24 * 3600, 'hour': 3600, 'minute': 60}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(day|hour|minute)', expiredate, re.I)) + except Exception, e: + self.logError(e) + + else: + self.logDebug("Valid until: %s" % validuntil) + + if validuntil: + validuntil += time.time() + premium = True + else: + validuntil = -1 + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + trafficleft = self.parseTraffic(str(max(0, 50 - float(m.group(1)))) + " GB") + + except Exception, e: + self.logError(e) + else: + self.logDebug("TRAFFIC_LEFT_PATTERN not found") + + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} + + + def login(self, user, data, req): + html = req.load("https://nitroflare.com/login", decode=True) + + token = re.search(self.TOKEN_PATTERN, html).group(1) + + html = req.load("https://nitroflare.com/login", + post={'login' : "", + 'email' : user, + 'password': data['password'], + 'token' : token}, + decode=True) + + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.wrongPassword() diff --git a/module/plugins/accounts/NoPremiumPl.py b/module/plugins/accounts/NoPremiumPl.py index f2223b7d9..6c034b31f 100644 --- a/module/plugins/accounts/NoPremiumPl.py +++ b/module/plugins/accounts/NoPremiumPl.py @@ -1,41 +1,41 @@ # -*- coding: utf-8 -*- -from datetime import datetime +import datetime import hashlib +import time +from module.common.json_layer import json_loads from module.plugins.Account import Account -from time import mktime -from module.common.json_layer import json_loads as loads class NoPremiumPl(Account): - __name__ = "NoPremiumPl" + __name__ = "NoPremiumPl" + __type__ = "account" __version__ = "0.01" - __type__ = "account" + __description__ = "NoPremium.pl account plugin" - __license__ = "GPLv3" - __authors__ = [("goddie", "dev@nopremium.pl")] + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@nopremium.pl")] - _api_url = "http://crypt.nopremium.pl" - _api_query = { - "site": "nopremium", - "username": "", - "password": "", - "output": "json", - "loc": "1", - "info": "1" - } + API_URL = "http://crypt.nopremium.pl" + API_QUERY = {'site' : "nopremium", + 'username': "" , + 'password': "" , + 'output' : "json" , + 'loc' : "1" , + 'info' : "1" } _req = None _usr = None _pwd = None + def loadAccountInfo(self, name, req): self._req = req try: - result = loads(self.runAuthQuery()) - except: + result = json_loads(self.runAuthQuery()) + except Exception: # todo: return or let it be thrown? return @@ -44,14 +44,14 @@ class NoPremiumPl(Account): if "expire" in result.keys() and result["expire"]: premium = True - valid_untill = mktime(datetime.fromtimestamp(int(result["expire"])).timetuple()) + valid_untill = time.mktime(datetime.datetime.fromtimestamp(int(result["expire"])).timetuple()) + traffic_left = result["balance"] * 1024 - return ({ - "validuntil": valid_untill, - "trafficleft": traffic_left, - "premium": premium - }) + return {'validuntil' : valid_untill, + 'trafficleft': traffic_left, + 'premium' : premium } + def login(self, user, data, req): self._usr = user @@ -59,23 +59,23 @@ class NoPremiumPl(Account): self._req = req try: - response = loads(self.runAuthQuery()) - except: + response = json_loads(self.runAuthQuery()) + except Exception: self.wrongPassword() if "errno" in response.keys(): self.wrongPassword() + data['usr'] = self._usr data['pwd'] = self._pwd + def createAuthQuery(self): - query = self._api_query + query = self.API_QUERY query["username"] = self._usr query["password"] = self._pwd - return query - def runAuthQuery(self): - data = self._req.load(self._api_url, post=self.createAuthQuery()) - return data
\ No newline at end of file + def runAuthQuery(self): + return self._req.load(self.API_URL, post=self.createAuthQuery()) diff --git a/module/plugins/accounts/NowVideoSx.py b/module/plugins/accounts/NowVideoSx.py index f44ae3865..2f7b033bd 100644 --- a/module/plugins/accounts/NowVideoSx.py +++ b/module/plugins/accounts/NowVideoSx.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import gmtime, mktime, strptime +import time from module.plugins.Account import Account @@ -33,13 +32,13 @@ class NowVideoSx(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%Y-%b-%d")) + validuntil = time.mktime(time.strptime(expiredate, "%Y-%b-%d")) except Exception, e: self.logError(e) else: - if validuntil > mktime(gmtime()): + if validuntil > time.mktime(time.gmtime()): premium = True else: premium = False diff --git a/module/plugins/accounts/OboomCom.py b/module/plugins/accounts/OboomCom.py index 0acacbb2a..020a45ba7 100644 --- a/module/plugins/accounts/OboomCom.py +++ b/module/plugins/accounts/OboomCom.py @@ -1,8 +1,20 @@ # -*- coding: utf-8 -*- -import time +try: + from beaker.crypto.pbkdf2 import PBKDF2 -from beaker.crypto.pbkdf2 import PBKDF2 +except ImportError: + from beaker.crypto.pbkdf2 import pbkdf2 + from binascii import b2a_hex + + class PBKDF2(object): + def __init__(self, passphrase, salt, iterations=1000): + self.passphrase = passphrase + self.salt = salt + self.iterations = iterations + + def hexread(self, octets): + return b2a_hex(pbkdf2(self.passphrase, self.salt, self.iterations, octets)) from module.common.json_layer import json_loads from module.plugins.Account import Account @@ -11,7 +23,7 @@ from module.plugins.Account import Account class OboomCom(Account): __name__ = "OboomCom" __type__ = "account" - __version__ = "0.23" + __version__ = "0.24" __description__ = """Oboom.com account plugin""" __license__ = "GPLv3" diff --git a/module/plugins/accounts/OneFichierCom.py b/module/plugins/accounts/OneFichierCom.py index 1fc8d994d..b19e2bc69 100644 --- a/module/plugins/accounts/OneFichierCom.py +++ b/module/plugins/accounts/OneFichierCom.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re - -from time import strptime, mktime - -from pycurl import REFERER +import time from module.plugins.Account import Account @@ -36,7 +34,7 @@ class OneFichierCom(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%d/%m/%Y")) + validuntil = time.mktime(time.strptime(expiredate, "%d/%m/%Y")) except Exception, e: self.logError(e) else: @@ -46,7 +44,7 @@ class OneFichierCom(Account): def login(self, user, data, req): - req.http.c.setopt(REFERER, "https://1fichier.com/login.pl?lg=en") + req.http.c.setopt(pycurl.REFERER, "https://1fichier.com/login.pl?lg=en") html = req.load("https://1fichier.com/login.pl?lg=en", post={'mail' : user, diff --git a/module/plugins/accounts/OverLoadMe.py b/module/plugins/accounts/OverLoadMe.py index d59944e63..64d04aded 100644 --- a/module/plugins/accounts/OverLoadMe.py +++ b/module/plugins/accounts/OverLoadMe.py @@ -15,7 +15,7 @@ class OverLoadMe(Account): def loadAccountInfo(self, user, req): - https = "https" if self.getConfig("ssl") else "http" + https = "https" if self.getConfig('ssl') else "http" data = self.getAccountData(user) html = req.load(https + "://api.over-load.me/account.php", get={'user': user, @@ -32,7 +32,7 @@ class OverLoadMe(Account): def login(self, user, data, req): - https = "https" if self.getConfig("ssl") else "http" + https = "https" if self.getConfig('ssl') else "http" jsondata = req.load(https + "://api.over-load.me/account.php", get={'user': user, 'auth': data['password']}).strip() diff --git a/module/plugins/accounts/PremiumizeMe.py b/module/plugins/accounts/PremiumizeMe.py index 7d061ec2d..da31d93ee 100644 --- a/module/plugins/accounts/PremiumizeMe.py +++ b/module/plugins/accounts/PremiumizeMe.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- -from module.plugins.Account import Account - from module.common.json_layer import json_loads +from module.plugins.Account import Account class PremiumizeMe(Account): __name__ = "PremiumizeMe" __type__ = "account" - __version__ = "0.13" + __version__ = "0.16" __description__ = """Premiumize.me account plugin""" __license__ = "GPLv3" @@ -42,7 +41,7 @@ class PremiumizeMe(Account): def getAccountStatus(self, user, req): # Use premiumize.me API v1 (see https://secure.premiumize.me/?show=api) # to retrieve account info and return the parsed json answer - answer = req.load("https://api.premiumize.me/pm-api/v1.php", + answer = req.load("http://api.premiumize.me/pm-api/v1.php", get={'method' : "accountstatus", 'params[login]': user, 'params[pass]' : self.getAccountData(user)['password']}) diff --git a/module/plugins/accounts/RapideoPl.py b/module/plugins/accounts/RapideoPl.py index 438ce7ad3..92736cba0 100644 --- a/module/plugins/accounts/RapideoPl.py +++ b/module/plugins/accounts/RapideoPl.py @@ -1,80 +1,81 @@ # -*- coding: utf-8 -*- -from datetime import datetime +import datetime import hashlib +import time +from module.common.json_layer import json_loads from module.plugins.Account import Account -from time import mktime -from module.common.json_layer import json_loads as loads class RapideoPl(Account): - __name__ = "RapideoPl" + __name__ = "RapideoPl" + __type__ = "account" __version__ = "0.01" - __type__ = "account" + __description__ = "Rapideo.pl account plugin" - __license__ = "GPLv3" - __authors__ = [("goddie", "dev@rapideo.pl")] + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@rapideo.pl")] - _api_url = "http://enc.rapideo.pl" - _api_query = { - "site": "newrd", - "username": "", - "password": "", - "output": "json", - "loc": "1", - "info": "1" - } + API_URL = "http://enc.rapideo.pl" + API_QUERY = {'site' : "newrd", + 'username': "" , + 'password': "" , + 'output' : "json" , + 'loc' : "1" , + 'info' : "1" } _req = None _usr = None _pwd = None + def loadAccountInfo(self, name, req): self._req = req try: - result = loads(self.runAuthQuery()) - except: + result = json_loads(self.runAuthQuery()) + except Exception: # todo: return or let it be thrown? return premium = False valid_untill = -1 + if "expire" in result.keys() and result["expire"]: premium = True - valid_untill = mktime(datetime.fromtimestamp(int(result["expire"])).timetuple()) + valid_untill = time.mktime(datetime.datetime.fromtimestamp(int(result["expire"])).timetuple()) traffic_left = result["balance"] - return ({ - "validuntil": valid_untill, - "trafficleft": traffic_left, - "premium": premium - }) + return {'validuntil' : valid_untill, + 'trafficleft': traffic_left, + 'premium' : premium } + def login(self, user, data, req): self._usr = user self._pwd = hashlib.md5(data["password"]).hexdigest() self._req = req + try: - response = loads(self.runAuthQuery()) - except: + response = json_loads(self.runAuthQuery()) + except Exception: self.wrongPassword() if "errno" in response.keys(): self.wrongPassword() + data['usr'] = self._usr data['pwd'] = self._pwd + def createAuthQuery(self): - query = self._api_query + query = self.API_QUERY query["username"] = self._usr query["password"] = self._pwd - return query - def runAuthQuery(self): - data = self._req.load(self._api_url, post=self.createAuthQuery()) - return data
\ No newline at end of file + def runAuthQuery(self): + return self._req.load(self.API_URL, post=self.createAuthQuery()) diff --git a/module/plugins/accounts/RapidgatorNet.py b/module/plugins/accounts/RapidgatorNet.py index b29d94228..5ff30e061 100644 --- a/module/plugins/accounts/RapidgatorNet.py +++ b/module/plugins/accounts/RapidgatorNet.py @@ -7,50 +7,61 @@ from module.common.json_layer import json_loads class RapidgatorNet(Account): __name__ = "RapidgatorNet" __type__ = "account" - __version__ = "0.06" + __version__ = "0.09" __description__ = """Rapidgator.net account plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - API_URL = 'http://rapidgator.net/api/user' + API_URL = "http://rapidgator.net/api/user" def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = False + sid = None + try: - sid = self.getAccountData(user).get('SID') + sid = self.getAccountData(user).get('sid', None) assert sid - json = req.load("%s/info?sid=%s" % (self.API_URL, sid)) - self.logDebug("API:USERINFO", json) - json = json_loads(json) + html = req.load("%s/info" % self.API_URL, get={'sid': sid}) + + self.logDebug("API:USERINFO", html) + + json = json_loads(html) if json['response_status'] == 200: if "reset_in" in json['response']: self.scheduleRefresh(user, json['response']['reset_in']) - return {"validuntil": json['response']['expire_date'], - "trafficleft": float(json['response']['traffic_left']) / 1024, #@TODO: Remove `/ 1024` in 0.4.10 - "premium": True} + validuntil = json['response']['expire_date'] + trafficleft = float(json['response']['traffic_left']) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + premium = True else: self.logError(json['response_details']) + except Exception, e: self.logError(e) - return {"validuntil": None, "trafficleft": None, "premium": False} + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium, + 'sid' : sid} def login(self, user, data, req): try: - json = req.load('%s/login' % self.API_URL, post={"username": user, "password": data['password']}) + html = req.load('%s/login' % self.API_URL, post={"username": user, "password": data['password']}) - self.logDebug("API:LOGIN", json) + self.logDebug("API:LOGIN", html) - json = json_loads(json) + json = json_loads(html) if json['response_status'] == 200: - data['SID'] = str(json['response']['session_id']) + data['sid'] = str(json['response']['session_id']) return else: self.logError(json['response_details']) diff --git a/module/plugins/accounts/RapiduNet.py b/module/plugins/accounts/RapiduNet.py index 8da698c57..70f47b673 100644 --- a/module/plugins/accounts/RapiduNet.py +++ b/module/plugins/accounts/RapiduNet.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import time +import time from module.plugins.Account import Account from module.common.json_layer import json_loads @@ -38,7 +37,7 @@ class RapiduNet(Account): m = re.search(self.VALID_UNTIL_PATTERN, html) if m: - validuntil = time() + (86400 * int(m.group(1))) + validuntil = time.time() + (86400 * int(m.group(1))) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py index 41d8a0975..ad2975aa9 100644 --- a/module/plugins/accounts/RealdebridCom.py +++ b/module/plugins/accounts/RealdebridCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import xml.dom.minidom as dom +import xml.dom.minidom from module.plugins.Account import Account @@ -19,7 +19,7 @@ class RealdebridCom(Account): if self.pin_code: return {"premium": False} html = req.load("https://real-debrid.com/api/account.php") - xml = dom.parseString(html) + xml = xml.dom.minidom.parseString(html) account_info = {"validuntil": float(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue), "trafficleft": -1} diff --git a/module/plugins/accounts/RehostTo.py b/module/plugins/accounts/RehostTo.py index 897f888b0..04e71c9ad 100644 --- a/module/plugins/accounts/RehostTo.py +++ b/module/plugins/accounts/RehostTo.py @@ -18,13 +18,13 @@ class RehostTo(Account): trafficleft = None validuntil = -1 session = "" - + html = req.load("http://rehost.to/api.php", get={'cmd' : "login", 'user': user, 'pass': self.getAccountData(user)['password']}) try: session = html.split(",")[1].split("=")[1] - + html = req.load("http://rehost.to/api.php", get={'cmd': "get_premium_credits", 'long_ses': session}) @@ -32,11 +32,11 @@ class RehostTo(Account): self.logDebug(html) else: traffic, valid = html.split(",") - + premium = True trafficleft = self.parseTraffic(traffic + "MB") validuntil = float(valid) - + finally: return {'premium' : premium, 'trafficleft': trafficleft, diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 3ee6e04af..4214b4401 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -8,7 +8,7 @@ from module.plugins.Account import Account class ShareonlineBiz(Account): __name__ = "ShareonlineBiz" __type__ = "account" - __version__ = "0.30" + __version__ = "0.33" __description__ = """Share-online.biz account plugin""" __license__ = "GPLv3" @@ -40,18 +40,24 @@ class ShareonlineBiz(Account): if api['a'].lower() != "not_available": req.cj.setCookie("share-online.biz", 'a', api['a']) - premium = api['group'] == "Premium" + premium = api['group'] in ("PrePaid", "Premium", "Penalty-Premium") validuntil = float(api['expire_date']) traffic = float(api['traffic_1d'].split(";")[0]) - maxtraffic = max(maxtraffic, traffic) - trafficleft = maxtraffic - traffic + + if maxtraffic > traffic: + trafficleft = maxtraffic - traffic + else: + trafficleft = -1 maxtraffic /= 1024 #@TODO: Remove `/ 1024` in 0.4.10 trafficleft /= 1024 #@TODO: Remove `/ 1024` in 0.4.10 - return {'premium': premium, 'validuntil': validuntil, 'trafficleft': trafficleft, 'maxtraffic': maxtraffic} + return {'premium' : premium, + 'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'maxtraffic' : maxtraffic} def login(self, user, data, req): diff --git a/module/plugins/accounts/SimplyPremiumCom.py b/module/plugins/accounts/SimplyPremiumCom.py index 79daf245e..8caf600f9 100644 --- a/module/plugins/accounts/SimplyPremiumCom.py +++ b/module/plugins/accounts/SimplyPremiumCom.py @@ -7,7 +7,7 @@ from module.plugins.Account import Account class SimplyPremiumCom(Account): __name__ = "SimplyPremiumCom" __type__ = "account" - __version__ = "0.04" + __version__ = "0.05" __description__ = """Simply-Premium.com account plugin""" __license__ = "GPLv3" @@ -15,15 +15,18 @@ class SimplyPremiumCom(Account): def loadAccountInfo(self, user, req): + premium = False validuntil = -1 trafficleft = None json_data = req.load('http://www.simply-premium.com/api/user.php?format=json') - self.logDebug("JSON data: " + json_data) + + self.logDebug("JSON data: %s" % json_data) + json_data = json_loads(json_data) - if 'vip' in json_data['result'] and json_data['result']['vip'] == 0: - return {"premium": False} + if 'vip' in json_data['result'] and json_data['result']['vip']: + premium = True if 'timeend' in json_data['result'] and json_data['result']['timeend']: validuntil = float(json_data['result']['timeend']) @@ -31,18 +34,15 @@ class SimplyPremiumCom(Account): if 'remain_traffic' in json_data['result'] and json_data['result']['remain_traffic']: trafficleft = float(json_data['result']['remain_traffic']) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 - return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} + return {"premium": premium, "validuntil": validuntil, "trafficleft": trafficleft} def login(self, user, data, req): req.cj.setCookie("simply-premium.com", "lang", "EN") - if data['password'] == '' or data['password'] == '0': - post_data = {"key": user} - else: - post_data = {"login_name": user, "login_pass": data['password']} - - html = req.load("http://www.simply-premium.com/login.php", post=post_data, decode=True) + html = req.load("http://www.simply-premium.com/login.php", + post={'key': user} if not data['password'] else {'login_name': user, 'login_pass': data['password']}, + decode=True) if 'logout' not in html: self.wrongPassword() diff --git a/module/plugins/accounts/SimplydebridCom.py b/module/plugins/accounts/SimplydebridCom.py index 29be2f73d..24108eb0b 100644 --- a/module/plugins/accounts/SimplydebridCom.py +++ b/module/plugins/accounts/SimplydebridCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -22,7 +22,7 @@ class SimplydebridCom(Account): if str(data[0]) != "1": return {"premium": False} else: - return {"trafficleft": -1, "validuntil": mktime(strptime(str(data[2]), "%d/%m/%Y"))} + return {"trafficleft": -1, "validuntil": time.mktime(time.strptime(str(data[2]), "%d/%m/%Y"))} def login(self, user, data, req): diff --git a/module/plugins/accounts/SmoozedCom.py b/module/plugins/accounts/SmoozedCom.py index e6c25752b..1c303331b 100644 --- a/module/plugins/accounts/SmoozedCom.py +++ b/module/plugins/accounts/SmoozedCom.py @@ -1,9 +1,23 @@ # -*- coding: utf-8 -*- import hashlib +import time -from beaker.crypto.pbkdf2 import PBKDF2 -from time import time +try: + from beaker.crypto.pbkdf2 import PBKDF2 + +except ImportError: + from beaker.crypto.pbkdf2 import pbkdf2 + from binascii import b2a_hex + + class PBKDF2(object): + def __init__(self, passphrase, salt, iterations=1000): + self.passphrase = passphrase + self.salt = salt + self.iterations = iterations + + def hexread(self, octets): + return b2a_hex(pbkdf2(self.passphrase, self.salt, self.iterations, octets)) from module.common.json_layer import json_loads from module.plugins.Account import Account @@ -12,7 +26,7 @@ from module.plugins.Account import Account class SmoozedCom(Account): __name__ = "SmoozedCom" __type__ = "account" - __version__ = "0.03" + __version__ = "0.05" __description__ = """Smoozed.com account plugin""" __license__ = "GPLv3" @@ -20,7 +34,6 @@ class SmoozedCom(Account): def loadAccountInfo(self, user, req): - # Get user data from premiumize.me status = self.getAccountStatus(user, req) self.logDebug(status) @@ -36,8 +49,11 @@ class SmoozedCom(Account): 'session' : status["data"]["session_key"], 'hosters' : [hoster["name"] for hoster in status["data"]["hoster"]]} - if info['validuntil'] < time(): - info['premium'] = False + if info['validuntil'] < time.time(): + if float(status["data"]["user"].get("user_trial", 0)) > time.time(): + info['premium'] = True + else: + info['premium'] = False else: info['premium'] = True diff --git a/module/plugins/accounts/TurbobitNet.py b/module/plugins/accounts/TurbobitNet.py index a857649eb..010fbc270 100644 --- a/module/plugins/accounts/TurbobitNet.py +++ b/module/plugins/accounts/TurbobitNet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -22,7 +22,7 @@ class TurbobitNet(Account): m = re.search(r'<u>Turbo Access</u> to ([\d.]+)', html) if m: premium = True - validuntil = mktime(strptime(m.group(1), "%d.%m.%Y")) + validuntil = time.mktime(time.strptime(m.group(1), "%d.%m.%Y")) else: premium = False validuntil = -1 diff --git a/module/plugins/accounts/TusfilesNet.py b/module/plugins/accounts/TusfilesNet.py index 279dfd00a..d06ba0583 100644 --- a/module/plugins/accounts/TusfilesNet.py +++ b/module/plugins/accounts/TusfilesNet.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import mktime, strptime, gmtime +import time from module.plugins.internal.XFSAccount import XFSAccount diff --git a/module/plugins/accounts/UlozTo.py b/module/plugins/accounts/UlozTo.py index 7236a4fa8..f95c8834b 100644 --- a/module/plugins/accounts/UlozTo.py +++ b/module/plugins/accounts/UlozTo.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.Account import Account @@ -18,7 +17,7 @@ class UlozTo(Account): ("pulpe", None)] - TRAFFIC_LEFT_PATTERN = r'<li class="menu-kredit"><a .*?title="[^"]*?GB = ([\d.]+) MB"' + TRAFFIC_LEFT_PATTERN = r'<li class="menu-kredit"><a .*?title=".+?GB = ([\d.]+) MB"' def loadAccountInfo(self, user, req): @@ -37,7 +36,7 @@ class UlozTo(Account): action = re.findall('<form action="(.+?)"', login_page)[1].replace('&', '&') token = re.search('_token_" value="(.+?)"', login_page).group(1) - html = req.load(urljoin("http://www.ulozto.net/", action), + html = req.load(urlparse.urljoin("http://www.ulozto.net/", action), post={'_token_' : token, 'do' : "loginForm-submit", 'login' : u"PÅihlásit", diff --git a/module/plugins/accounts/UnrestrictLi.py b/module/plugins/accounts/UnrestrictLi.py deleted file mode 100644 index 6a8187234..000000000 --- a/module/plugins/accounts/UnrestrictLi.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.Account import Account -from module.common.json_layer import json_loads - - -class UnrestrictLi(Account): - __name__ = "UnrestrictLi" - __type__ = "account" - __version__ = "0.05" - - __description__ = """Unrestrict.li account plugin""" - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it")] - - - def loadAccountInfo(self, user, req): - json_data = req.load('http://unrestrict.li/api/jdownloader/user.php?format=json') - self.logDebug("JSON data: " + json_data) - json_data = json_loads(json_data) - - if 'vip' in json_data['result'] and json_data['result']['vip'] == 0: - return {"premium": False} - - validuntil = json_data['result']['expires'] - trafficleft = float(json_data['result']['traffic'] / 1024) #@TODO: Remove `/ 1024` in 0.4.10 - - return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} - - - def login(self, user, data, req): - req.cj.setCookie("unrestrict.li", "lang", "EN") - html = req.load("https://unrestrict.li/sign_in", decode=True) - - if 'solvemedia' in html: - self.logError(_("A Captcha is required. Go to http://unrestrict.li/sign_in and login, then retry")) - return - - post_data = {"username": user, "password": data['password'], - "remember_me": "remember", "signin": "Sign in"} - html = req.load("https://unrestrict.li/sign_in", post=post_data, decode=True) - - if 'sign_out' not in html: - self.wrongPassword() diff --git a/module/plugins/accounts/UploadedTo.py b/module/plugins/accounts/UploadedTo.py index 1b6df97fb..d1556b6db 100644 --- a/module/plugins/accounts/UploadedTo.py +++ b/module/plugins/accounts/UploadedTo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from time import time +import time from module.plugins.Account import Account @@ -39,7 +39,7 @@ class UploadedTo(Account): else: m = re.findall(r'(\d+) (week|day|hour)', expiredate) if m: - validuntil = time() + validuntil = time.time() for n, u in m: validuntil += float(n) * 60 * 60 * {'week': 168, 'day': 24, 'hour': 1}[u] @@ -55,7 +55,9 @@ class UploadedTo(Account): else: trafficleft = self.parseTraffic(size + unit) - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} def login(self, user, data, req): diff --git a/module/plugins/accounts/UploadingCom.py b/module/plugins/accounts/UploadingCom.py index c70d2ec11..a20c44535 100644 --- a/module/plugins/accounts/UploadingCom.py +++ b/module/plugins/accounts/UploadingCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import time, strptime, mktime +import time from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import set_cookies @@ -11,7 +10,7 @@ from module.plugins.internal.SimpleHoster import set_cookies class UploadingCom(Account): __name__ = "UploadingCom" __type__ = "account" - __version__ = "0.11" + __version__ = "0.12" __description__ = """Uploading.com account plugin""" __license__ = "GPLv3" @@ -37,27 +36,30 @@ class UploadingCom(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%b %d, %Y")) + validuntil = time.mktime(time.strptime(expiredate, "%b %d, %Y")) except Exception, e: self.logError(e) else: - if validuntil > mktime(gmtime()): - premium = True + if validuntil > time.mktime(time.gmtime()): + premium = True else: - premium = False + premium = False validuntil = None - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} def login(self, user, data, req): - set_cookies([("uploading.com", "lang", "1"), - ("uploading.com", "language", "1"), - ("uploading.com", "setlang", "en"), - ("uploading.com", "_lang", "en")] + set_cookies(req.cj, + [("uploading.com", "lang" , "1" ), + ("uploading.com", "language", "1" ), + ("uploading.com", "setlang" , "en"), + ("uploading.com", "_lang" , "en")]) req.load("http://uploading.com/") - req.load("http://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time() * 1000), + req.load("http://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time.time() * 1000), post={'email': user, 'password': data['password'], 'remember': "on"}) diff --git a/module/plugins/accounts/UptoboxCom.py b/module/plugins/accounts/UptoboxCom.py index 299a0acc2..c40dbd6e6 100644 --- a/module/plugins/accounts/UptoboxCom.py +++ b/module/plugins/accounts/UptoboxCom.py @@ -6,7 +6,7 @@ from module.plugins.internal.XFSAccount import XFSAccount class UptoboxCom(XFSAccount): __name__ = "UptoboxCom" __type__ = "account" - __version__ = "0.07" + __version__ = "0.08" __description__ = """DDLStorage.com account plugin""" __license__ = "GPLv3" @@ -15,3 +15,4 @@ class UptoboxCom(XFSAccount): HOSTER_DOMAIN = "uptobox.com" HOSTER_URL = "https://uptobox.com/" + LOGIN_URL = "https://login.uptobox.com/" diff --git a/module/plugins/accounts/WebshareCz.py b/module/plugins/accounts/WebshareCz.py index f8e3eeb73..8c53dd41e 100644 --- a/module/plugins/accounts/WebshareCz.py +++ b/module/plugins/accounts/WebshareCz.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- +import hashlib import re +import time -from hashlib import md5, sha1 from passlib.hash import md5_crypt -from time import mktime, strptime, time from module.plugins.Account import Account @@ -12,7 +12,7 @@ from module.plugins.Account import Account class WebshareCz(Account): __name__ = "WebshareCz" __type__ = "account" - __version__ = "0.07" + __version__ = "0.08" __description__ = """Webshare.cz account plugin""" __license__ = "GPLv3" @@ -26,7 +26,7 @@ class WebshareCz(Account): def loadAccountInfo(self, user, req): html = req.load("https://webshare.cz/api/user_data/", - post={'wst': self.infos['wst']}, + post={'wst': self.getAccountData(user).get('wst', None)}, decode=True) self.logDebug("Response: " + html) @@ -34,9 +34,9 @@ class WebshareCz(Account): expiredate = re.search(self.VALID_UNTIL_PATTERN, html).group(1) self.logDebug("Expire date: " + expiredate) - validuntil = mktime(strptime(expiredate, "%Y-%m-%d %H:%M:%S")) + validuntil = time.mktime(time.strptime(expiredate, "%Y-%m-%d %H:%M:%S")) trafficleft = self.parseTraffic(re.search(self.TRAFFIC_LEFT_PATTERN, html).group(1)) - premium = validuntil > time() + premium = validuntil > time.time() return {'validuntil': validuntil, 'trafficleft': -1, 'premium': premium} @@ -51,8 +51,8 @@ class WebshareCz(Account): self.wrongPassword() salt = re.search('<salt>(.+)</salt>', salt).group(1) - password = sha1(md5_crypt.encrypt(data["password"], salt=salt)).hexdigest() - digest = md5(user + ":Webshare:" + password).hexdigest() + password = hashlib.sha1(md5_crypt.encrypt(data["password"], salt=salt)).hexdigest() + digest = hashlib.md5(user + ":Webshare:" + password).hexdigest() login = req.load("https://webshare.cz/api/login/", post={'digest' : digest, @@ -65,4 +65,4 @@ class WebshareCz(Account): if "<status>OK</status>" not in login: self.wrongPassword() - self.infos['wst'] = re.search('<token>(.+)</token>', login).group(1) + data['wst'] = re.search('<token>(.+)</token>', login).group(1) diff --git a/module/plugins/accounts/WorldbytezCom.py b/module/plugins/accounts/WorldbytezCom.py new file mode 100644 index 000000000..a90dbea56 --- /dev/null +++ b/module/plugins/accounts/WorldbytezCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.XFSAccount import XFSAccount + + +class WorldbytezCom(XFSAccount): + __name__ = "WorldbytezCom" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Worldbytez.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "worldbytez.com" diff --git a/module/plugins/accounts/YibaishiwuCom.py b/module/plugins/accounts/YibaishiwuCom.py index 863588495..33602a2fe 100644 --- a/module/plugins/accounts/YibaishiwuCom.py +++ b/module/plugins/accounts/YibaishiwuCom.py @@ -23,7 +23,7 @@ class YibaishiwuCom(Account): html = req.load("http://115.com/", decode=True) m = re.search(self.ACCOUNT_INFO_PATTERN, html, re.S) - premium = True if (m and 'is_vip: 1' in m.group(1)) else False + premium = True if m and 'is_vip: 1' in m.group(1) else False validuntil = trafficleft = (-1 if m else 0) return dict({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py index 6c69a974f..9bc6d0487 100644 --- a/module/plugins/accounts/ZeveraCom.py +++ b/module/plugins/accounts/ZeveraCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from time import mktime, strptime +import time from module.plugins.Account import Account @@ -8,7 +8,7 @@ from module.plugins.Account import Account class ZeveraCom(Account): __name__ = "ZeveraCom" __type__ = "account" - __version__ = "0.24" + __version__ = "0.26" __description__ = """Zevera.com account plugin""" __license__ = "GPLv3" @@ -39,8 +39,8 @@ class ZeveraCom(Account): api = self.api_response(req) - if api != "No trafic": - validuntil = mktime(strptime(api['endsubscriptiondate'], "%Y/%m/%d %H:%M:%S")) + if "No trafic" not in api and api['endsubscriptiondate'] != "Expired!": + validuntil = time.mktime(time.strptime(api['endsubscriptiondate'], "%Y/%m/%d %H:%M:%S")) trafficleft = float(api['availabletodaytraffic']) * 1024 if api['orondaytrafficlimit'] != '0' else -1 premium = True diff --git a/module/plugins/captcha/CircleCaptcha.py b/module/plugins/captcha/CircleCaptcha.py new file mode 100644 index 000000000..d4f08018d --- /dev/null +++ b/module/plugins/captcha/CircleCaptcha.py @@ -0,0 +1,781 @@ +# -*- coding: utf-8 -*- + +from __future__ import division + +import Image +import ImageDraw +import cStringIO +import math +import operator +import urllib + +from module.plugins.captcha.OCR import OCR + + +class ImageSequence: + + def __init__(self, im): + self.im = im + + def __getitem__(self, ix): + try: + if ix: + self.im.seek(ix) + return self.im + except EOFError: + raise IndexError # end of sequence + + +class CircleCaptcha(OCR): + __name__ = "CircleCaptcha" + __type__ = "ocr" + __version__ = "1.00" + + __description__ = """Circle captcha ocr plugin""" + __license__ = "GPLv3" + __authors__ = [("Sasch", "gsasch@gmail.com")] + + + _DEBUG = False + pointsofcirclefound = [] + + BACKGROUND = 250 + BLACKCOLOR = 5 + + + def cleanImage(self, im, pix): + cleandeep = 1 + + imageheight = range(1,int(im.size[1])) + imagewidth = range(1,int(im.size[0])) + howmany = 0 + curcolor = self.BACKGROUND + + for y in imageheight: + jump = True + howmany = 0 + for x in imagewidth: + curpix = pix[x,y] + + if curpix > self.BACKGROUND: + if howmany <= cleandeep and howmany > 0: + # clean pixel + for ic in range(1,cleandeep+1): + if x -ic > 0: + pix[x-ic,y] = self.BACKGROUND + jump = False + howmany = 0 + curcolor = curpix + # print (x, y), jump,2 + else: + if howmany == 0: + # found pixel + jump = True + howmany = howmany + 1 + curcolor = curpix + # print (x, y), jump,2 + else: + howmany = howmany + 1 + if howmany == 1: + # clean pixel + pix[x-1,y] = self.BACKGROUND + + curcolor = self.BACKGROUND + for x in imagewidth: + jump = True + howmany = 0 + for y in imageheight: + curpix = pix[x,y] + # if jump == True: + if curpix > self.BACKGROUND: + if howmany <= cleandeep and howmany > 0: + # clean pixel + for ic in range(1,cleandeep+1): + # raw_input('2'+str(ic)) + if y-ic > 0: + pix[x,y-ic] = self.BACKGROUND + jump = False + howmany = 0 + curcolor = curpix + # print (x, y), jump + else: + if howmany == 0: + # found pixel + jump = True + howmany = howmany + 1 + curcolor = curpix + # print (x, y), jump + else: + howmany = howmany + 1 + if howmany == 1: + # clean pixel + pix[x-1,y] = self.BACKGROUND + + # return -1 + + + def findFirstPixelX(self, im, pix, curx, cury, color = -1, ExitWithBlack = False): + imageheight = range(1,int(im.size[1])) + imagewidth = range(curx+1,int(im.size[0])) + jump = True + newx = (-1,-1) + blackfound = 0 + for x in imagewidth: + curpix = pix[x,cury] + + if curpix < self.BLACKCOLOR: + blackfound = blackfound + 1 + if ExitWithBlack == True and blackfound >= 3: + break; #exit if found black + else: + continue; + + if curpix >= self.BACKGROUND: + # found first pixel white + jump = False + continue; + + if (curpix < self.BACKGROUND and color == -1) or (curpix == color and color > -1): + if jump == False: + # found pixel + curcolor = curpix + newx = x, curcolor + break; + + return newx + + + def findLastPixelX(self, im, pix, curx, cury, color = -1, ExitWithBlack = False): + imageheight = range(1,int(im.size[1])) + imagewidth = range(curx+1,int(im.size[0])) + newx = (-1,-1) + blackfound = 0 + for x in imagewidth: + curpix = pix[x,cury] + + if curpix < self.BLACKCOLOR: + blackfound = blackfound + 1 + if ExitWithBlack == True and blackfound >= 3: + break; #exit if found black + else: + continue; + + if curpix >= self.BACKGROUND: + if newx != (-1,-1): + # found last pixel and the first white + break; + + if (curpix < self.BACKGROUND and color == -1) or (curpix == color and color > -1): + # found pixel + curcolor = curpix + newx = x, curcolor + + return newx + + + def findLastPixelY(self, im, pix, curx, cury, DownToUp, color = -1, ExitWithBlack = False): + if DownToUp == False: + imageheight = range(int(cury)+1,int(im.size[1])-1) + else: + imageheight = range(int(cury)-1,1,-1) + imagewidth = range(int(curx),int(im.size[0])) + newy = (-1,-1) + blackfound = 0 + for y in imageheight: + curpix = pix[curx,y] + + if curpix < self.BLACKCOLOR: + blackfound = blackfound + 1 + if ExitWithBlack == True and blackfound >= 3: + break; #exit if found black + else: + continue; + + if curpix >= self.BACKGROUND: + if newy != (-1,-1): + # found last pixel and the first white + break; + + if (curpix < self.BACKGROUND and color == -1) or (curpix == color and color > -1): + # found pixel + curcolor = curpix + newy = y, color + + return newy + + + def findCircle(self, pix, x1, y1, x2, y2, x3, y3): + # trasposizione coordinate + # A(0,0) B(x2-x1,y2-y1) C(x3-x1,y3-y1) + # x**2+y**2+ax+bx+c=0 + p1 = (0,0) + p2 = (x2-x1,y2-y1) + p3 = (x3-x1,y3-y1) + + # 1 + c=0 + # 2 + # p2[0]**2+a*p2[0]+c=0 + # a*p2[0]=-1*(p2[0]**2-c) + # a=(-1*(p2[0]**2-c))/p2[0] + a=(-1*(p2[0]**2-c))/p2[0] + # 3 + # p3[0]**2+p3[1]**2+a*p3[0]+b*p3[1]+c=0 + # b*p3[1]=-(p3[0]**2+p3[1]**2+a*p3[0]+c) + # b=(-1 * (p3[0]**2+p3[1]**2+a*p3[0]+c)) / p3[1] + b=(-1 * (p3[0]**2+p3[1]**2+a*p3[0]+c)) / p3[1] + + r=math.floor(math.sqrt((-1*(a/2))**2+(-1*(b/2))**2)) + cx=math.floor((-1*(a/2))+x1) + cy=math.floor((-1*(b/2))+y1) + + return cx,cy,r + + + def verifyCircleNew(self, im, pix, c): + """ + This is the MAIN function to recognize the circle + returns: + 1 -> Found closed circle + 0 -> Found open circle + -1 -> Not found circle + -2 -> Found black position then leave position + """ + + imageheight = range(int(c[1]-c[2]),int(c[1]+c[2])) + imagewidth = range(int(c[0]-c[2]),int(c[0]+c[2])) + + min_ray = 15 + max_ray = 30 + exactfind = False + + howmany = 0 + missing = 0 + missingconsecutive = 0 + missinglist = [] + minX = 0; maxX = 0; minY = 0; maxY = 0 + pointsofcircle = [] + + if (c[2] < min_ray) or (c[2] > max_ray): + return -1 + + # check cardinal points (at least 3) (if found i have to leave this position) + if pix[c[0] + c[2],c[1]] < self.BLACKCOLOR: + return -2; + if pix[c[0] - c[2],c[1]] < self.BLACKCOLOR: + return -2; + if pix[c[0],c[1] + c[2]] < self.BLACKCOLOR: + return -2; + if pix[c[0],c[1] - c[2]] < self.BLACKCOLOR: + return -2; + + cardinalpoints = 0 + if self.verifyPoint(im, pix,c[0] + c[2],c[1],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0] + c[2],c[1],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0] - c[2],c[1],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0] - c[2],c[1],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0],c[1] + c[2],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0],c[1] + c[2],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0],c[1] - c[2],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0],c[1] - c[2],False) == -1: + return -2; + if cardinalpoints < 3: + return -1; + + for x in imagewidth: + # Pitagora + y = int(round(c[1]- math.sqrt(c[2]**2-(c[0]-x)**2))) + y2= int(round(c[1]+ math.sqrt(c[2]**2-(c[0]-x)**2))) + + howmany = howmany + 2 + if self.verifyPoint(im, pix, x,y,exactfind) == 0: + missing = missing + 1 + missinglist.append((x,y)) + else: + pointsofcircle.append((x,y)) + + if self.verifyPoint(im, pix, x,y,False) == -1: + return -2; + + if self.verifyPoint(im, pix, x,y2,exactfind) == 0: + missing = missing + 1 + missinglist.append((x,y2)) + else: + pointsofcircle.append((x,y2)) + + if self.verifyPoint(im, pix, x,y2,False) == -1: + return -2; + + + def verifyCircle(self, im, pix, c): + """ + This is the MAIN function to recognize the circle + returns: + 1 -> Found closed circle + 0 -> Found open circle + -1 -> Not found circle + -2 -> Found black position then leave position + """ + + imageheight = range(int(c[1]-c[2]),int(c[1]+c[2])) + imagewidth = range(int(c[0]-c[2]),int(c[0]+c[2])) + + min_ray = 15 + max_ray = 30 + exactfind = False + + howmany = 0 + missing = 0 + missingconsecutive = 0 + missinglist = [] + minX = 0; maxX = 0; minY = 0; maxY = 0 + pointsofcircle = [] + + if (c[2] < min_ray) or (c[2] > max_ray): + return -1 + + # check cardinal points (at least 3) (if found i have to leave this position) + if pix[c[0] + c[2],c[1]] < self.BLACKCOLOR: + return -2; + if pix[c[0] - c[2],c[1]] < self.BLACKCOLOR: + return -2; + if pix[c[0],c[1] + c[2]] < self.BLACKCOLOR: + return -2; + if pix[c[0],c[1] - c[2]] < self.BLACKCOLOR: + return -2; + + cardinalpoints = 0 + if self.verifyPoint(im, pix,c[0] + c[2],c[1],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0] + c[2],c[1],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0] - c[2],c[1],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0] - c[2],c[1],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0],c[1] + c[2],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0],c[1] + c[2],False) == -1: + return -2; + if self.verifyPoint(im, pix,c[0],c[1] - c[2],True) == 1: + cardinalpoints = cardinalpoints + 1 + if self.verifyPoint(im, pix,c[0],c[1] - c[2],False) == -1: + return -2; + if cardinalpoints < 3: + return -1; + + for x in imagewidth: + # Pitagora + y = int(round(c[1]- math.sqrt(c[2]**2-(c[0]-x)**2))) + y2= int(round(c[1]+ math.sqrt(c[2]**2-(c[0]-x)**2))) + + howmany = howmany + 2 + if self.verifyPoint(im, pix, x,y,exactfind) == 0: + missing = missing + 1 + missinglist.append((x,y)) + else: + pointsofcircle.append((x,y)) + + if self.verifyPoint(im, pix, x,y,False) == -1: + return -2; + + if self.verifyPoint(im, pix, x,y2,exactfind) == 0: + missing = missing + 1 + missinglist.append((x,y2)) + else: + pointsofcircle.append((x,y2)) + + if self.verifyPoint(im, pix, x,y2,False) == -1: + return -2; + + for y in imageheight: + # Pitagora + x = int(round(c[0]- math.sqrt(c[2]**2-(c[1]-y)**2))) + x2= int(round(c[0]+ math.sqrt(c[2]**2-(c[1]-y)**2))) + + howmany = howmany + 2 + if self.verifyPoint(im, pix, x,y,exactfind) == 0: + missing = missing + 1 + missinglist.append((x,y)) + else: + pointsofcircle.append((x,y)) + + if self.verifyPoint(im, pix, x,y,False) == -1: + return -2; + + if self.verifyPoint(im, pix, x2,y,exactfind) == 0: + missing = missing + 1 + missinglist.append((x2,y)) + else: + pointsofcircle.append((x2,y)) + + if self.verifyPoint(im, pix, x2,y,exactfind) == -1: + return -2; + + for p in missinglist: + # left and bottom + if (self.verifyPoint(im, pix, p[0]-1, p[1],exactfind) == 1 and \ + self.verifyPoint(im, pix, p[0], p[1]+1,exactfind) == 1): + missing = missing - 1 + elif (self.verifyPoint(im, pix, p[0]-1, p[1],exactfind) == 1 and \ + self.verifyPoint(im, pix, p[0], p[1]-1,exactfind) == 1): + missing = missing - 1 + # right and bottom + elif (self.verifyPoint(im, pix, p[0]+1, p[1],exactfind) == 1 and \ + self.verifyPoint(im, pix, p[0], p[1]+1,exactfind) == 1): + missing = missing - 1 + # right and up + elif (self.verifyPoint(im, pix, p[0]+1, p[1],exactfind) == 1 and \ + self.verifyPoint(im, pix, p[0], p[1]-1,exactfind) == 1): + missing = missing - 1 + + if (p[0], p[1]+1) in missinglist or \ + (p[0], p[1]-1) in missinglist or \ + (p[0]+1, p[1]) in missinglist or \ + (p[0]-1, p[1]) in missinglist or \ + (p[0]+1, p[1]+1) in missinglist or \ + (p[0]-1, p[1]+1) in missinglist or \ + (p[0]+1, p[1]-1) in missinglist or \ + (p[0]-1, p[1]-1) in missinglist or \ + self.verifyPoint(im, pix, p[0], p[1],False) == 1: + missingconsecutive = missingconsecutive + 1 + # else: + # pix[p[0], p[1]] = 0 + + if missing / howmany > 0: + indice = c[2] * (missing / howmany) + else: + indice = 0 + + if len(missinglist) > 0: + minX = min(missinglist, key=operator.itemgetter(0))[0] + maxX = max(missinglist, key=operator.itemgetter(0))[0] + + minY = min(missinglist, key=operator.itemgetter(1))[1] + maxY = max(missinglist, key=operator.itemgetter(1))[1] + + # Assial Simmetric + if self._DEBUG == True: + print "Center: " + str(c) + print "Missing: " + str(missing) + print "Howmany: " + str(howmany) + print "Ratio: " + str(missing / howmany) + print "Missing consecutives: " + str(missingconsecutive) + print "Missing X lenght: " + str(minX) + ":" + str(maxX) + print "Missing Y lenght: " + str(minY) + ":" + str(maxY) + print "Ratio without consecutives: " + str((missing - missingconsecutive) / howmany) + print "List missing: " + str(missinglist) + + # Lenght of missing cannot be over 75% of diameter + + if maxX - minX >= c[2] * 2 * 0.75: + return -1; + if maxY - minY >= c[2] * 2 * 0.75: + # raw_input('tro') + return -1; + """ + # Lenght of missing cannot be less 10% of diameter + if maxX - minX < c[2] * 2 * 0.10 and maxY - minY < c[2] * 2 * 0.10: + return -1; + """ + + if missing / howmany > 0.25 or \ + missingconsecutive >= (howmany / 4) * 2 or \ + howmany < 80: + return -1; + # elif missing / howmany < 0.10: + elif missing == 0: + self.pointsofcirclefound.extend(pointsofcircle) + return 1; + elif (missing - missingconsecutive) / howmany < 0.20: + return 0; + else: + self.pointsofcirclefound.extend(pointsofcircle) + return 1; + + + def verifyPoint(self, im, pix, x,y,exact,color = -1): + # Verify point + result = 0 + + if x < 0 or x >= im.size[0]: + return result; + if y < 0 or y >= im.size[1]: + return result; + + curpix = pix[x,y] + if (curpix == color and color > -1) or (curpix < self.BACKGROUND and color == -1): + if curpix > self.BLACKCOLOR: + result = 1 + else: + result = -1 + + # Verify around + if (exact == False): + if x + 1 < im.size[0]: + curpix = pix[x+1,y] + if (curpix == color and color > -1) or (curpix < self.BACKGROUND and color == -1): + if curpix > self.BLACKCOLOR: + result = 1 + if curpix <= self.BLACKCOLOR: + result = -1 + + if x > 0: + curpix = pix[x-1,y] + if (curpix == color and color > -1) or (curpix < self.BACKGROUND and color == -1): + if curpix > self.BLACKCOLOR: + result = 1 + if curpix <= self.BLACKCOLOR: + result = -1 + # print str((x,y)) + " = " + str(result); + return result + + + def decrypt(self, img): + iDebugSaveFile = 0 + mypalette = None + for im in ImageSequence(img): + im.save("orig.png", "png") + if mypalette != None: + im.putpalette(mypalette) + mypalette = im.getpalette() + im = im.convert('L') + + if self._DEBUG == True: + iDebugSaveFile = iDebugSaveFile + 1 + # if iDebugSaveFile < 7: continue; + im.save("output" + str(iDebugSaveFile) + ".png", "png") + raw_input('frame: '+ str(im)) + + pix = im.load() + + stepheight = range(1,im.size[1],2) + # stepheight = range(45,47) + imagewidth = range(1,im.size[0]) + lstPoints = [] # Declares an empty list for the points + lstX = [] # CoordinateX + lstY = [] # CoordinateY + lstColors = [] # Declares an empty list named lst + min_distance = 10 + max_diameter = 70 + + if self._DEBUG == True: + imdebug = im.copy() + draw = ImageDraw.Draw(imdebug) + pixcopy = imdebug.load() + + # Clean image for powerfull search + self.cleanImage(im, pix) + im.save("cleaned" + str(iDebugSaveFile) + ".png", "png") + + found = set() + findnewcircle = True + + # finding all the circles + for y1 in stepheight: + x1 = 1 + curcolor = -1 + for k in range(1,100): + findnewcircle = False + retval = self.findFirstPixelX(im, pix, x1, y1, -1, False) + x1 = retval[0] + curcolor = retval[1] + if x1 == -2: + break; + if x1 == -1: + break; + if self._DEBUG == True: print "x1, y1 -> " + str((x1,y1)) + ": " + str(pix[x1,y1]) + + if (x1,y1) in self.pointsofcirclefound: + if self._DEBUG == True: print 'found ' + str((x1,y1)) + continue; + + if self._DEBUG == True: pixcopy[x1,y1] = 45 #(255,0,0,255) + # found 1 pixel, seeking x2,y2 + x2 = x1 + y2 = y1 + for i in range(1,100): + retval = self.findLastPixelX(im, pix, x2, y2, -1, True) + x2 = retval[0] + if x1 == -2: + findnewcircle = True + break; + if x2 == -1: + break; + if self._DEBUG == True: print "x2, y2 -> " + str((x2,y1)) + ": " + str(pix[x2,y1]) + if abs(x2 - x1) < min_distance: + continue; + if abs(x2 - x1) > (im.size[1] * 2 / 3): + break; + if abs(x2 - x1) > max_diameter: + break; + + if self._DEBUG == True: pixcopy[x2,y2] = 65 #(0,255,0,255) + # found 2 pixel, seeking x3,y3 + # verify cord + + for invert in range(0,2): + x3 = math.floor(x2 - ((x2 - x1) / 2)) + y3 = y1 + for j in range(1,50): + retval = self.findLastPixelY(im, pix, x3, y3, True if invert == 1 else False, -1, True) + # print (x3, y3,retval[0],invert) + y3 = retval[0] + if y3 == -2: + findnewcircle = True + break; + if y3 == -1: + break; + + if self._DEBUG == True: print "x3, y3 -> " + str((x3,y3)) + ": " + str(pix[x3,y3]) + # verify cord + if abs(y3 - y2) < min_distance: + continue; + if abs(y3 - y2) > (im.size[1] * 2 / 3): + break; + if abs(y3 - y2) > max_diameter: + break; + + if self._DEBUG == True: pixcopy[x3,y3] = 85 + # found 3 pixel. try circle + c = self.findCircle(pix, x1,y1,x2,y2,x3,y3) + + if c[0] + c[2] >= im.size[0] or c[1] + c[2] >= im.size[1] or c[0] - c[2] <= 0 or c[1] - c[2] <= 0: + continue; + + if self._DEBUG == True: pixcopy[c[0],c[1]] = 0 + # (x-r, y-r, x+r, y+r) + verified = self.verifyCircle(im, pix, c) + + if verified == -1: + verified = -1 + elif verified == 0: + found.add(((c[0],c[1],c[2]),verified)) + findnewcircle = True + elif verified == 1: + found.add(((c[0],c[1],c[2]),verified)) + findnewcircle = True + + if self._DEBUG == True: + _pause = "" + # if verified == -1: + # draw.ellipse((c[0]-c[2],c[1]-c[2],c[0]+c[2],c[1]+c[2]),outline=0) + # _pause = "NOTDOUND" + # imdebug.save("debug.png", "png") + if verified == 0: + draw.ellipse((c[0]-c[2],c[1]-c[2],c[0]+c[2],c[1]+c[2]),outline=120) + _pause = "OPENED" + + if verified == 1: + draw.ellipse((c[0]-c[2],c[1]-c[2],c[0]+c[2],c[1]+c[2]),outline=65) + _pause = "CLOSED" + + imdebug.save("debug.png", "png") + + if _pause != "": + valore = raw_input('Found ' + _pause + ' CIRCLE circle press [Enter] = continue / [q] for Quit: ' + str(verified)) + if valore == 'q': + sys.exit(); + + if findnewcircle == True: + break; + if findnewcircle == True: + break; + if findnewcircle == True: + break; + + if self._DEBUG == True: + print 'Howmany opened circle? ' + str(len(found)) + ' ' + str(found) + + # clean results + for c in found: + verify = c[1] + if verify == 0: + p = c[0] + if ( + ((p[0], p[1]+1,p[2]),1) in found or \ + ((p[0], p[1]-1,p[2]),1) in found or \ + ((p[0]+1, p[1],p[2]),1) in found or \ + ((p[0]-1, p[1],p[2]),1) in found or \ + ((p[0]+1, p[1]+1,p[2]),1) in found or \ + ((p[0]-1, p[1]+1,p[2]),1) in found or \ + ((p[0]+1, p[1]-1,p[2]),1) in found or \ + ((p[0]-1, p[1]-1,p[2]),1) in found \ + ): + + # delete nearly circle + verify = -1 + if ( + ((p[0], p[1]+1,p[2]+1),1) in found or \ + ((p[0], p[1]-1,p[2]+1),1) in found or \ + ((p[0]+1, p[1],p[2]+1),1) in found or \ + ((p[0]-1, p[1],p[2]+1),1) in found or \ + ((p[0]+1, p[1]+1,p[2]+1),1) in found or \ + ((p[0]-1, p[1]+1,p[2]+1),1) in found or \ + ((p[0]+1, p[1]-1,p[2]+1),1) in found or \ + ((p[0]-1, p[1]-1,p[2]+1),1) in found \ + ): + + # delete nearly circle + verify = -1 + if ( + ((p[0], p[1]+1,p[2]-1),1) in found or \ + ((p[0], p[1]-1,p[2]-1),1) in found or \ + ((p[0]+1, p[1],p[2]-1),1) in found or \ + ((p[0]-1, p[1],p[2]-1),1) in found or \ + ((p[0]+1, p[1]+1,p[2]-1),1) in found or \ + ((p[0]-1, p[1]+1,p[2]-1),1) in found or \ + ((p[0]+1, p[1]-1,p[2]-1),1) in found or \ + ((p[0]-1, p[1]-1,p[2]-1),1) in found \ + ): + + # delete nearly circle + verify = -1 + + # if verify == 0: + # if self._DEBUG == True: + # pix[c[0][0],c[0][1]] = 90 #(255,255,0) + # im.save("output.png", "png") + # return c[0][0],c[0][1] + # elif verify == 1: + # if self._DEBUG == True: + # pix[c[0][0],c[0][1]] = 40 #(255,0,0) + # im.save("output.png", "png") + # else: + # if self._DEBUG == True: + # pix[c[0][0],c[0][1]] = 180 #(0,0,255) + # im.save("output.png", "png") + + if self._DEBUG == True: + im.save("output.png", "png") + + + # Return coordinates of opened circle (eg (x,y)) + def decrypt_from_web(self, url): + file = cStringIO.StringIO(urllib.urlopen(url).read()) + img = Image.open(file) + coords = self.decrypt(img); + print "Coords: " + str(coords) + + + # Return coordinates of opened circle (eg (x,y)) + def decrypt_from_file(self, filename): + coords = self.decrypt(Image.open(filename)); #Can be many different formats. + print "Coords: " + str(coords) + + +##DEBUG +# import datetime +# a = datetime.datetime.now() +# x = CircleCaptcha() +# coords = x.decrypt_from_file("decripter/captx.html2.gif") +# coords = x.decrypt_from_web("http://ncrypt.in/classes/captcha/circlecaptcha.php") +# b = datetime.datetime.now() +# print 'Elapsed time: ' + str((b-a).seconds) + ' seconds' diff --git a/module/plugins/captcha/GigasizeCom.py b/module/plugins/captcha/GigasizeCom.py index 244cf6a2a..52c41729b 100644 --- a/module/plugins/captcha/GigasizeCom.py +++ b/module/plugins/captcha/GigasizeCom.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.captcha.captcha import OCR +from module.plugins.captcha.OCR import OCR class GigasizeCom(OCR): __name__ = "GigasizeCom" __type__ = "ocr" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Gigasize.com ocr plugin""" __license__ = "GPLv3" diff --git a/module/plugins/captcha/LinksaveIn.py b/module/plugins/captcha/LinksaveIn.py index 56cbd58a0..95b107977 100644 --- a/module/plugins/captcha/LinksaveIn.py +++ b/module/plugins/captcha/LinksaveIn.py @@ -5,17 +5,16 @@ try: except ImportError: import Image -from glob import glob -from os import sep -from os.path import abspath, dirname +import glob +import os -from module.plugins.captcha.captcha import OCR +from module.plugins.captcha.OCR import OCR class LinksaveIn(OCR): __name__ = "LinksaveIn" __type__ = "ocr" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Linksave.in ocr plugin""" __license__ = "GPLv3" @@ -24,7 +23,7 @@ class LinksaveIn(OCR): def __init__(self): OCR.__init__(self) - self.data_dir = dirname(abspath(__file__)) + sep + "LinksaveIn" + sep + self.data_dir = os.path.dirname(os.path.abspath(__file__)) + os.sep + "LinksaveIn" + os.sep def load_image(self, image): @@ -46,7 +45,7 @@ class LinksaveIn(OCR): pix = frame.load() for x in xrange(frame.size[0]): for y in xrange(frame.size[1]): - if lut[pix[x, y]] != (0,0,0): + if lut[pix[x, y]] != (0, 0, 0): npix[x, y] = lut[pix[x, y]] frame_nr += 1 new.save(self.data_dir+"unblacked.png") @@ -59,7 +58,7 @@ class LinksaveIn(OCR): stat = {} cstat = {} img = self.image.convert("P") - for bgpath in glob(self.data_dir+"bg/*.gif"): + for bgpath in glob.glob(self.data_dir+"bg/*.gif"): stat[bgpath] = 0 bg = Image.open(bgpath) @@ -79,7 +78,7 @@ class LinksaveIn(OCR): rgb_c = lut[pix[x, y]] try: cstat[rgb_c] += 1 - except: + except Exception: cstat[rgb_c] = 1 if rgb_bg == rgb_c: stat[bgpath] += 1 @@ -112,7 +111,7 @@ class LinksaveIn(OCR): rgb_bg = bglut[bgpix[x, y]] rgb_c = lut[pix[x, y]] if rgb_c == rgb_bg: - orgpix[x, y] = (255,255,255) + orgpix[x, y] = (255, 255, 255) def eval_black_white(self): @@ -124,15 +123,15 @@ class LinksaveIn(OCR): for y in xrange(new.size[1]): rgb = orgpix[x, y] r, g, b = rgb - pix[x, y] = (255,255,255) + pix[x, y] = (255, 255, 255) if r > max(b, g)+thresh: - pix[x, y] = (0,0,0) + pix[x, y] = (0, 0, 0) if g < min(r, b): - pix[x, y] = (0,0,0) + pix[x, y] = (0, 0, 0) if g > max(r, b)+thresh: - pix[x, y] = (0,0,0) + pix[x, y] = (0, 0, 0) if b > max(r, g)+thresh: - pix[x, y] = (0,0,0) + pix[x, y] = (0, 0, 0) self.image = new self.pixels = self.image.load() diff --git a/module/plugins/captcha/NetloadIn.py b/module/plugins/captcha/NetloadIn.py index 28eb18fb5..1fb258c47 100644 --- a/module/plugins/captcha/NetloadIn.py +++ b/module/plugins/captcha/NetloadIn.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.captcha.captcha import OCR +from module.plugins.captcha.OCR import OCR class NetloadIn(OCR): __name__ = "NetloadIn" __type__ = "ocr" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Netload.in ocr plugin""" __license__ = "GPLv3" diff --git a/module/plugins/captcha/captcha.py b/module/plugins/captcha/OCR.py index 0f233ec00..1874ba07d 100644 --- a/module/plugins/captcha/captcha.py +++ b/module/plugins/captcha/OCR.py @@ -4,6 +4,7 @@ from __future__ import with_statement try: from PIL import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin + except ImportError: import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin @@ -12,13 +13,13 @@ import os import subprocess #import tempfile -from os.path import abspath, join +from module.utils import save_join class OCR(object): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.10" + __version__ = "0.11" __description__ = """OCR base plugin""" __license__ = "GPLv3" @@ -58,11 +59,11 @@ class OCR(object): def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") try: - tmpTif = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmpTif = open(save_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") tmpTif.close() #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt = open(save_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") tmpTxt.close() except IOError, e: @@ -73,15 +74,15 @@ class OCR(object): self.image.save(tmpTif.name, 'TIFF') if os.name == "nt": - tessparams = [join(pypath, "tesseract", "tesseract.exe")] + tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] else: tessparams = ["tesseract"] - tessparams.extend( [abspath(tmpTif.name), abspath(tmpTxt.name).replace(".txt", "")] ) + tessparams.extend( [os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")] ) if subset and (digits or lowercase or uppercase): #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") - with open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: + with open(save_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: tmpSub.write("tessedit_char_whitelist ") if digits: @@ -93,7 +94,7 @@ class OCR(object): tmpSub.write("\n") tessparams.append("nobatch") - tessparams.append(abspath(tmpSub.name)) + tessparams.append(os.path.abspath(tmpSub.name)) self.logger.debug("run tesseract") self.run(tessparams) @@ -102,7 +103,7 @@ class OCR(object): try: with open(tmpTxt.name, 'r') as f: self.result_captcha = f.read().replace("\n", "") - except: + except Exception: self.result_captcha = "" self.logger.debug(self.result_captcha) @@ -111,7 +112,7 @@ class OCR(object): os.remove(tmpTxt.name) if subset and (digits or lowercase or uppercase): os.remove(tmpSub.name) - except: + except Exception: pass @@ -166,7 +167,7 @@ class OCR(object): count += 1 if pixels[x, y-1] != 255: count += 1 - except: + except Exception: pass # not enough neighbors are dark pixels so mark this pixel diff --git a/module/plugins/captcha/ShareonlineBiz.py b/module/plugins/captcha/ShareonlineBiz.py index 8210e8859..6fad66600 100644 --- a/module/plugins/captcha/ShareonlineBiz.py +++ b/module/plugins/captcha/ShareonlineBiz.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.captcha.captcha import OCR +from module.plugins.captcha.OCR import OCR class ShareonlineBiz(OCR): __name__ = "ShareonlineBiz" __type__ = "ocr" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Shareonline.biz ocr plugin""" __license__ = "GPLv3" diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py index bca535175..235d5dc1d 100644 --- a/module/plugins/container/CCF.py +++ b/module/plugins/container/CCF.py @@ -2,42 +2,46 @@ from __future__ import with_statement +import MultipartPostHandler import re - -from os import makedirs -from os.path import exists -from urllib2 import build_opener - -from MultipartPostHandler import MultipartPostHandler +import urllib2 from module.plugins.Container import Container -from module.utils import save_join +from module.utils import fs_encode, save_join class CCF(Container): __name__ = "CCF" - __version__ = "0.20" + __type__ = "container" + __version__ = "0.23" - __pattern__ = r'.+\.ccf' + __pattern__ = r'.+\.ccf$' __description__ = """CCF container decrypter plugin""" __license__ = "GPLv3" - __authors__ = [("Willnix", "Willnix@pyload.org")] + __authors__ = [("Willnix", "Willnix@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] def decrypt(self, pyfile): - infile = pyfile.url.replace("\n", "") + fs_filename = fs_encode(pyfile.url.strip()) + opener = urllib2.build_opener(MultipartPostHandler.MultipartPostHandler) - opener = build_opener(MultipartPostHandler) - params = {"src": "ccf", - "filename": "test.ccf", - "upload": open(infile, "rb")} - tempdlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', params).read() + dlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', + {'src' : "ccf", + 'filename': "test.ccf", + 'upload' : open(fs_filename, "rb")}).read() download_folder = self.config['general']['download_folder'] + dlc_file = save_join(download_folder, "tmp_%s.dlc" % pyfile.name) + + try: + dlc = re.search(r'<dlc>(.+)</dlc>', dlc_content, re.S).group(1).decode('base64') + + except AttributeError: + self.fail(_("Container is corrupted")) - tempdlc_name = save_join(download_folder, "tmp_%s.dlc" % pyfile.name) - with open(tempdlc_name, "w") as tempdlc: - tempdlc.write(re.search(r'<dlc>(.*)</dlc>', tempdlc_content, re.S).group(1)) + with open(dlc_file, "w") as tempdlc: + tempdlc.write(dlc) - self.urls = [tempdlc_name] + self.urls = [dlc_file] diff --git a/module/plugins/container/DLC.py b/module/plugins/container/DLC.py new file mode 100644 index 000000000..ff2f0104a --- /dev/null +++ b/module/plugins/container/DLC.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import xml.dom.minidom + +from Crypto.Cipher import AES + +from module.plugins.Container import Container +from module.utils import decode, fs_encode + + +class DLC(Container): + __name__ = "DLC" + __type__ = "container" + __version__ = "0.24" + + __pattern__ = r'.+\.dlc$' + + __description__ = """DLC container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de"), + ("Schnusch", "Schnusch@users.noreply.github.com"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + KEY = "cb99b5cbc24db398" + IV = "9bc24cb995cb8db3" + API_URL = "http://service.jdownloader.org/dlcrypt/service.php?srcType=dlc&destType=pylo&data=%s" + + + def decrypt(self, pyfile): + fs_filename = fs_encode(pyfile.url.strip()) + with open(fs_filename) as dlc: + data = dlc.read().strip() + + data += '=' * (-len(data) % 4) + + dlc_key = data[-88:] + dlc_data = data[:-88].decode('base64') + dlc_content = self.load(self.API_URL % dlc_key) + + try: + rc = re.search(r'<rc>(.+)</rc>', dlc_content, re.S).group(1).decode('base64') + + except AttributeError: + self.fail(_("Container is corrupted")) + + key = iv = AES.new(self.KEY, AES.MODE_CBC, self.IV).decrypt(rc) + + self.data = AES.new(key, AES.MODE_CBC, iv).decrypt(dlc_data).decode('base64') + self.packages = [(name or pyfile.name, links, name or pyfile.name) \ + for name, links in self.getPackages()] + + + def getPackages(self): + root = xml.dom.minidom.parseString(self.data).documentElement + content = root.getElementsByTagName("content")[0] + return self.parsePackages(content) + + + def parsePackages(self, startNode): + return [(decode(node.getAttribute("name")).decode('base64'), self.parseLinks(node)) \ + for node in startNode.getElementsByTagName("package")] + + + def parseLinks(self, startNode): + return [node.getElementsByTagName("url")[0].firstChild.data.decode('base64') \ + for node in startNode.getElementsByTagName("file")] diff --git a/module/plugins/container/DLC_25.pyc b/module/plugins/container/DLC_25.pyc Binary files differdeleted file mode 100644 index b8fde0051..000000000 --- a/module/plugins/container/DLC_25.pyc +++ /dev/null diff --git a/module/plugins/container/DLC_26.pyc b/module/plugins/container/DLC_26.pyc Binary files differdeleted file mode 100644 index 41a4e0cb8..000000000 --- a/module/plugins/container/DLC_26.pyc +++ /dev/null diff --git a/module/plugins/container/DLC_27.pyc b/module/plugins/container/DLC_27.pyc Binary files differdeleted file mode 100644 index a6bffaf74..000000000 --- a/module/plugins/container/DLC_27.pyc +++ /dev/null diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py deleted file mode 100644 index c6173ad73..000000000 --- a/module/plugins/container/LinkList.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- - -import codecs - -from module.plugins.Container import Container -from module.utils import fs_encode - - -class LinkList(Container): - __name__ = "LinkList" - __version__ = "0.12" - - __pattern__ = r'.+\.txt' - __config__ = [("clear", "bool", "Clear Linklist after adding", False), - ("encoding", "string", "File encoding (default utf-8)", "")] - - __description__ = """Read link lists in txt format""" - __license__ = "GPLv3" - __authors__ = [("spoob", "spoob@pyload.org"), - ("jeix", "jeix@hasnomail.com")] - - - def decrypt(self, pyfile): - try: - file_enc = codecs.lookup(self.getConfig("encoding")).name - except: - file_enc = "utf-8" - - file_name = fs_encode(pyfile.url) - - txt = codecs.open(file_name, 'r', file_enc) - links = txt.readlines() - curPack = "Parsed links from %s" % pyfile.name - - packages = {curPack:[],} - - for link in links: - link = link.strip() - if not link: - continue - - if link.startswith(";"): - continue - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - packages[curPack].append(link) - txt.close() - - # empty packages fix - - delete = [] - - for key,value in packages.iteritems(): - if not value: - delete.append(key) - - for key in delete: - del packages[key] - - if self.getConfig("clear"): - try: - txt = open(file_name, 'wb') - txt.close() - except: - self.logWarning(_("LinkList could not be cleared")) - - for name, links in packages.iteritems(): - self.packages.append((name, links, name)) diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py index 0c43f0e6c..dd2d14cf7 100644 --- a/module/plugins/container/RSDF.py +++ b/module/plugins/container/RSDF.py @@ -2,55 +2,60 @@ from __future__ import with_statement -import base64 import binascii import re +from Crypto.Cipher import AES + from module.plugins.Container import Container from module.utils import fs_encode class RSDF(Container): __name__ = "RSDF" - __version__ = "0.24" + __type__ = "container" + __version__ = "0.29" - __pattern__ = r'.+\.rsdf' + __pattern__ = r'.+\.rsdf$' __description__ = """RSDF container decrypter plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org")] - + ("spoob", "spoob@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] - def decrypt(self, pyfile): - from Crypto.Cipher import AES + KEY = "8C35192D964DC3182C6F84F3252239EB4A320D2500000000" + IV = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" - infile = fs_encode(pyfile.url.replace("\n", "")) - Key = binascii.unhexlify('8C35192D964DC3182C6F84F3252239EB4A320D2500000000') - IV = binascii.unhexlify('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') - IV_Cipher = AES.new(Key, AES.MODE_ECB) - IV = IV_Cipher.encrypt(IV) + def decrypt(self, pyfile): + KEY = binascii.unhexlify(self.KEY) + IV = binascii.unhexlify(self.IV) - obj = AES.new(Key, AES.MODE_CFB, IV) + iv = AES.new(KEY, AES.MODE_ECB).encrypt(IV) + cipher = AES.new(KEY, AES.MODE_CFB, iv) try: - with open(infile, 'r') as rsdf: + fs_filename = fs_encode(pyfile.url.strip()) + with open(fs_filename, 'r') as rsdf: data = rsdf.read() + except IOError, e: - self.fail(str(e)) + self.fail(e) + + if re.search(r"<title>404 - Not Found</title>", data): + pyfile.setStatus("offline") - if re.search(r"<title>404 - Not Found</title>", data) is None: - data = binascii.unhexlify(''.join(data.split())) - data = data.splitlines() + else: + try: + raw_links = binascii.unhexlify(''.join(data.split())).splitlines() - for link in data: + except TypeError: + self.fail(_("Container is corrupted")) + + for link in raw_links: if not link: continue - link = base64.b64decode(link) - link = obj.decrypt(link) - decryptedUrl = link.replace('CCF: ', '') - self.urls.append(decryptedUrl) - - self.logDebug("Adding package %s with %d links" % (pyfile.package().name, len(self.urls))) + link = cipher.decrypt(link.decode('base64')).replace('CCF: ', '') + self.urls.append(link) diff --git a/module/plugins/container/TXT.py b/module/plugins/container/TXT.py new file mode 100644 index 000000000..d419ee060 --- /dev/null +++ b/module/plugins/container/TXT.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import codecs + +from module.plugins.Container import Container +from module.utils import fs_encode + + +class TXT(Container): + __name__ = "TXT" + __type__ = "container" + __version__ = "0.15" + + __pattern__ = r'.+\.(txt|text)$' + __config__ = [("flush" , "bool" , "Flush list after adding", False ), + ("encoding", "string", "File encoding" , "utf-8")] + + __description__ = """Read link lists in plain text formats""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("jeix", "jeix@hasnomail.com")] + + + def decrypt(self, pyfile): + try: + encoding = codecs.lookup(self.getConfig('encoding')).name + + except Exception: + encoding = "utf-8" + + fs_filename = fs_encode(pyfile.url.strip()) + txt = codecs.open(fs_filename, 'r', encoding) + curPack = "Parsed links from %s" % pyfile.name + packages = {curPack:[],} + + for link in txt.readlines(): + link = link.strip() + + if not link: + continue + + if link.startswith(";"): + continue + + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + + packages[curPack].append(link) + + txt.close() + + # empty packages fix + for key, value in packages.iteritems(): + if not value: + packages.pop(key, None) + + if self.getConfig('flush'): + try: + txt = open(fs_filename, 'wb') + txt.close() + + except IOError: + self.logWarning(_("Failed to flush list")) + + for name, links in packages.iteritems(): + self.packages.append((name, links, name)) diff --git a/module/plugins/crypter/BitshareComFolder.py b/module/plugins/crypter/BitshareComFolder.py index c70a849b6..f0dd9e570 100644 --- a/module/plugins/crypter/BitshareComFolder.py +++ b/module/plugins/crypter/BitshareComFolder.py @@ -6,11 +6,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class BitshareComFolder(SimpleCrypter): __name__ = "BitshareComFolder" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'http://(?:www\.)?bitshare\.com/\?d=\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Bitshare.com folder decrypter plugin""" __license__ = "GPLv3" @@ -18,7 +19,7 @@ class BitshareComFolder(SimpleCrypter): LINK_PATTERN = r'<a href="(http://bitshare\.com/files/.+)">.+</a></td>' - NAME_PATTERN = r'View public folder "(?P<N>.+)"</h1>' + NAME_PATTERN = r'View public folder "(?P<N>.+?)"</h1>' getInfo = create_getInfo(BitshareComFolder) diff --git a/module/plugins/crypter/C1neonCom.py b/module/plugins/crypter/C1NeonCom.py index 926633ff7..9b6093081 100644 --- a/module/plugins/crypter/C1neonCom.py +++ b/module/plugins/crypter/C1NeonCom.py @@ -9,7 +9,7 @@ class C1NeonCom(DeadCrypter): __version__ = "0.05" __pattern__ = r'http://(?:www\.)?c1neon\.com/.+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """C1neon.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/ChipDe.py b/module/plugins/crypter/ChipDe.py index 133e5a005..f535da48d 100644 --- a/module/plugins/crypter/ChipDe.py +++ b/module/plugins/crypter/ChipDe.py @@ -10,8 +10,8 @@ class ChipDe(Crypter): __version__ = "0.10" __pattern__ = r'http://(?:www\.)?chip\.de/video/.+\.html' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Chip.de decrypter plugin""" __license__ = "GPLv3" @@ -22,7 +22,7 @@ class ChipDe(Crypter): self.html = self.load(pyfile.url) try: f = re.search(r'"(http://video\.chip\.de/.+)"', self.html) - except: + except Exception: self.fail(_("Failed to find the URL")) else: self.urls = [f.group(1)] diff --git a/module/plugins/crypter/CloudzillaToFolder.py b/module/plugins/crypter/CloudzillaToFolder.py index c156d4de4..96d7245f1 100644 --- a/module/plugins/crypter/CloudzillaToFolder.py +++ b/module/plugins/crypter/CloudzillaToFolder.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -10,7 +9,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class CloudzillaToFolder(SimpleHoster): __name__ = "CloudzillaToFolder" __type__ = "crypter" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = r'http://(?:www\.)?cloudzilla\.to/share/folder/(?P<ID>[\w^_]+)' @@ -36,8 +35,4 @@ class CloudzillaToFolder(SimpleHoster): self.retry(reason="Wrong password") - def getLinks(self): - return [urljoin("http://www.cloudzilla.to", link) for link in super(CloudzillaToFolder, self).getLinks()] - - getInfo = create_getInfo(CloudzillaToFolder) diff --git a/module/plugins/crypter/CrockoComFolder.py b/module/plugins/crypter/CrockoComFolder.py index 57bb339ff..f56cc449a 100644 --- a/module/plugins/crypter/CrockoComFolder.py +++ b/module/plugins/crypter/CrockoComFolder.py @@ -9,15 +9,16 @@ class CrockoComFolder(SimpleCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?crocko\.com/f/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Crocko.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - LINK_PATTERN = r'<td class="last"><a href="([^"]+)">download</a>' + LINK_PATTERN = r'<td class="last"><a href="(.+?)">download</a>' getInfo = create_getInfo(CrockoComFolder) diff --git a/module/plugins/crypter/CryptItCom.py b/module/plugins/crypter/CryptItCom.py index 2cf4e9f62..1c9e841b7 100644 --- a/module/plugins/crypter/CryptItCom.py +++ b/module/plugins/crypter/CryptItCom.py @@ -9,7 +9,7 @@ class CryptItCom(DeadCrypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?crypt-it\.com/(s|e|d|c)/\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Crypt-it.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/CzshareComFolder.py b/module/plugins/crypter/CzshareComFolder.py index 5623a4093..c317b1b49 100644 --- a/module/plugins/crypter/CzshareComFolder.py +++ b/module/plugins/crypter/CzshareComFolder.py @@ -10,8 +10,8 @@ class CzshareComFolder(Crypter): __version__ = "0.20" __pattern__ = r'http://(?:www\.)?(czshare|sdilej)\.(com|cz)/folders/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Czshare.com folder decrypter plugin, now Sdilej.cz""" __license__ = "GPLv3" @@ -19,7 +19,7 @@ class CzshareComFolder(Crypter): FOLDER_PATTERN = r'<tr class="subdirectory">\s*<td>\s*<table>(.*?)</table>' - LINK_PATTERN = r'<td class="col2"><a href="([^"]+)">info</a></td>' + LINK_PATTERN = r'<td class="col2"><a href="(.+?)">info</a></td>' def decrypt(self, pyfile): diff --git a/module/plugins/crypter/DDLMusicOrg.py b/module/plugins/crypter/DDLMusicOrg.py deleted file mode 100644 index 55181e9ad..000000000 --- a/module/plugins/crypter/DDLMusicOrg.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from time import sleep - -from module.plugins.Crypter import Crypter - - -class DDLMusicOrg(Crypter): - __name__ = "DDLMusicOrg" - __type__ = "crypter" - __version__ = "0.30" - - __pattern__ = r'http://(?:www\.)?ddl-music\.org/captcha/ddlm_cr\d\.php\?\d+\?\d+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Ddl-music.org decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] - - - def setup(self): - self.multiDL = False - - - def decrypt(self, pyfile): - html = self.load(pyfile.url, cookies=True) - - if re.search(r"Wer dies nicht rechnen kann", html) is not None: - self.offline() - - math = re.search(r"(\d+) ([+-]) (\d+) =\s+<inp", self.html) - id = re.search(r"name=\"id\" value=\"(\d+)\"", self.html).group(1) - linknr = re.search(r"name=\"linknr\" value=\"(\d+)\"", self.html).group(1) - - solve = "" - if math.group(2) == "+": - solve = int(math.group(1)) + int(math.group(3)) - else: - solve = int(math.group(1)) - int(math.group(3)) - sleep(3) - htmlwithlink = self.load(pyfile.url, cookies=True, - post={"calc%s" % linknr: solve, "send%s" % linknr: "Send", "id": id, - "linknr": linknr}) - m = re.search(r"<form id=\"ff\" action=\"(.*?)\" method=\"post\">", htmlwithlink) - if m: - self.urls = [m.group(1)] - else: - self.retry() diff --git a/module/plugins/crypter/DailymotionBatch.py b/module/plugins/crypter/DailymotionComFolder.py index 8d4cb64df..01caa0bb8 100644 --- a/module/plugins/crypter/DailymotionBatch.py +++ b/module/plugins/crypter/DailymotionComFolder.py @@ -1,22 +1,21 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.common.json_layer import json_loads from module.plugins.Crypter import Crypter from module.utils import save_join -class DailymotionBatch(Crypter): - __name__ = "DailymotionBatch" +class DailymotionComFolder(Crypter): + __name__ = "DailymotionComFolder" __type__ = "crypter" __version__ = "0.01" __pattern__ = r'https?://(?:www\.)?dailymotion\.com/((playlists/)?(?P<TYPE>playlist|user)/)?(?P<ID>[\w^_]+)(?(TYPE)|#)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Dailymotion.com channel & playlist decrypter""" __license__ = "GPLv3" @@ -24,9 +23,9 @@ class DailymotionBatch(Crypter): def api_response(self, ref, req=None): - url = urljoin("https://api.dailymotion.com/", ref) - page = self.load(url, get=req) - return json_loads(page) + url = urlparse.urljoin("https://api.dailymotion.com/", ref) + html = self.load(url, get=req) + return json_loads(html) def getPlaylistInfo(self, id): diff --git a/module/plugins/crypter/DataHuFolder.py b/module/plugins/crypter/DataHuFolder.py index a5602d6c6..c0fe26869 100644 --- a/module/plugins/crypter/DataHuFolder.py +++ b/module/plugins/crypter/DataHuFolder.py @@ -8,11 +8,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class DataHuFolder(SimpleCrypter): __name__ = "DataHuFolder" __type__ = "crypter" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'http://(?:www\.)?data\.hu/dir/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Data.hu folder decrypter plugin""" __license__ = "GPLv3" @@ -21,7 +22,7 @@ class DataHuFolder(SimpleCrypter): LINK_PATTERN = r'<a href=\'(http://data\.hu/get/.+)\' target=\'_blank\'>\1</a>' - NAME_PATTERN = ur'<title>(?P<N>.+) Let\xf6lt\xe9se</title>' + NAME_PATTERN = ur'<title>(?P<N>.+?) Let\xf6lt\xe9se</title>' def prepare(self): diff --git a/module/plugins/crypter/DdlstorageComFolder.py b/module/plugins/crypter/DdlstorageComFolder.py index e02e77fda..dc6ec1da5 100644 --- a/module/plugins/crypter/DdlstorageComFolder.py +++ b/module/plugins/crypter/DdlstorageComFolder.py @@ -9,7 +9,7 @@ class DdlstorageComFolder(DeadCrypter): __version__ = "0.03" __pattern__ = r'https?://(?:www\.)?ddlstorage\.com/folder/\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """DDLStorage.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/DepositfilesComFolder.py b/module/plugins/crypter/DepositfilesComFolder.py index 147f093c3..46ec265c3 100644 --- a/module/plugins/crypter/DepositfilesComFolder.py +++ b/module/plugins/crypter/DepositfilesComFolder.py @@ -9,15 +9,16 @@ class DepositfilesComFolder(SimpleCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?depositfiles\.com/folders/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Depositfiles.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - LINK_PATTERN = r'<div class="progressName"[^>]*>\s*<a href="([^"]+)" title="[^"]*" target="_blank">' + LINK_PATTERN = r'<div class="progressName".*?>\s*<a href="(.+?)" title=".+?" target="_blank">' getInfo = create_getInfo(DepositfilesComFolder) diff --git a/module/plugins/crypter/Dereferer.py b/module/plugins/crypter/Dereferer.py index 0729c8cb6..85b84a242 100644 --- a/module/plugins/crypter/Dereferer.py +++ b/module/plugins/crypter/Dereferer.py @@ -1,17 +1,42 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleDereferer import SimpleDereferer +import re +from module.plugins.internal.SimpleCrypter import SimpleCrypter -class Dereferer(SimpleDereferer): + +class Dereferer(SimpleCrypter): __name__ = "Dereferer" __type__ = "crypter" - __version__ = "0.11" + __version__ = "0.14" - __pattern__ = r'https?://([^/]+)/.*?(?P<LINK>(ht|f)tps?(://|%3A%2F%2F).+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/.*?(?P<LINK>(?:ht|f)tps?://.+)' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Crypter for dereferers""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + HOSTER_NAME = None + + + def _log(self, type, args): + msg = " | ".join(str(a).strip() for a in args if a) + logger = getattr(self.log, type) + logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + + + def init(self): + super(Dereferer, self).init() + + self.__pattern__ = self.core.pluginManager.crypterPlugins[self.__name__]['pattern'] #@TODO: Recheck in 0.4.10 + + self.HOSTER_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() + self.HOSTER_NAME = "".join(part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.') + + + def getLinks(self): + return [re.match(self.__pattern__, self.pyfile.url).group('LINK').strip()] diff --git a/module/plugins/crypter/DevhostStFolder.py b/module/plugins/crypter/DevhostStFolder.py index 5ac9d4faf..4d15e2058 100644 --- a/module/plugins/crypter/DevhostStFolder.py +++ b/module/plugins/crypter/DevhostStFolder.py @@ -4,8 +4,7 @@ # http://d-h.st/users/shine/?fld_id=37263#files import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -13,11 +12,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class DevhostStFolder(SimpleCrypter): __name__ = "DevhostStFolder" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.05" __pattern__ = r'http://(?:www\.)?d-h\.st/users/(?P<USER>\w+)(/\?fld_id=(?P<ID>\d+))?' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """d-h.st folder decrypter plugin""" __license__ = "GPLv3" @@ -25,37 +25,41 @@ class DevhostStFolder(SimpleCrypter): ("Walter Purcaro", "vuolter@gmail.com")] - LINK_PATTERN = r'(?:/> |;">)<a href="(.+?)"(?!>Back to \w+<)' + LINK_PATTERN = r'(?:/> |;">)<a href="(.+?)"(?!>Back to \w+<)' OFFLINE_PATTERN = r'"/cHP">test\.png<' - def getFileInfo(self): - if re.search(self.OFFLINE_PATTERN, self.html): - self.offline() + def checkNameSize(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("File info (BEFORE): %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("File info (AFTER): %s" % self.info) try: - id = re.match(self.__pattern__, self.pyfile.url).group('ID') - if id == "0": + if self.info['pattern']['ID'] == "0": raise p = r'href="(.+?)">Back to \w+<' m = re.search(p, self.html) - html = self.load(urljoin("http://d-h.st", m.group(1)), + html = self.load(urlparse.urljoin("http://d-h.st", m.group(1)), cookies=False) - p = '\?fld_id=%s.*?">(.+?)<' % id + p = '\?fld_id=%s.*?">(.+?)<' % self.info['pattern']['ID'] m = re.search(p, html) - name = folder = m.group(1) + self.pyfile.name = m.group(1) except Exception, e: self.logDebug(e) - name = folder = re.match(self.__pattern__, self.pyfile.url).group('USER') + self.pyfile.name = self.info['pattern']['USER'] - return {'name': name, 'folder': folder} + try: + folder = self.info['folder'] = self.pyfile.name + except Exception: + pass - def getLinks(self): - return [urljoin("http://d-h.st", link) for link in re.findall(self.LINK_PATTERN, self.html)] + self.logDebug("File name: %s" % self.pyfile.name, + "File folder: %s" % self.pyfile.name) getInfo = create_getInfo(DevhostStFolder) diff --git a/module/plugins/crypter/DlProtectCom.py b/module/plugins/crypter/DlProtectCom.py index 4a188aa83..a9f39c6f3 100644 --- a/module/plugins/crypter/DlProtectCom.py +++ b/module/plugins/crypter/DlProtectCom.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import re +import time from base64 import urlsafe_b64encode -from time import time from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -14,8 +14,9 @@ class DlProtectCom(SimpleCrypter): __version__ = "0.03" __pattern__ = r'https?://(?:www\.)?dl-protect\.com/((en|fr)/)?\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Dl-protect.com decrypter plugin""" __license__ = "GPLv3" @@ -40,7 +41,7 @@ class DlProtectCom(SimpleCrypter): self.wait(2) else: - mstime = int(round(time() * 1000)) + mstime = int(round(time.time() * 1000)) b64time = "_" + urlsafe_b64encode(str(mstime)).replace("=", "%3D") post_req.update({'i' : b64time, diff --git a/module/plugins/crypter/DontKnowMe.py b/module/plugins/crypter/DontKnowMe.py deleted file mode 100644 index d656cde4c..000000000 --- a/module/plugins/crypter/DontKnowMe.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleDereferer import SimpleDereferer - - -class DontKnowMe(SimpleDereferer): - __name__ = "DontKnowMe" - __type__ = "crypter" - __version__ = "0.11" - - __pattern__ = r'http://(?:www\.)?dontknow\.me/at/\?(?P<LINK>.+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """DontKnow.me decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("selaux", "")] diff --git a/module/plugins/crypter/DuckCryptInfo.py b/module/plugins/crypter/DuckCryptInfo.py index 07cc5cdc4..cc108d101 100644 --- a/module/plugins/crypter/DuckCryptInfo.py +++ b/module/plugins/crypter/DuckCryptInfo.py @@ -13,8 +13,8 @@ class DuckCryptInfo(Crypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?duckcrypt\.info/(folder|wait|link)/(\w+)/?(\w*)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """DuckCrypt.info decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/DuploadOrgFolder.py b/module/plugins/crypter/DuploadOrgFolder.py index 066fbe3d7..d505806b7 100644 --- a/module/plugins/crypter/DuploadOrgFolder.py +++ b/module/plugins/crypter/DuploadOrgFolder.py @@ -9,7 +9,7 @@ class DuploadOrgFolder(DeadCrypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?dupload\.org/folder/\d+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Dupload.org folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/EasybytezComFolder.py b/module/plugins/crypter/EasybytezComFolder.py index 04f9b853b..fdd3d4ff4 100644 --- a/module/plugins/crypter/EasybytezComFolder.py +++ b/module/plugins/crypter/EasybytezComFolder.py @@ -9,16 +9,14 @@ class EasybytezComFolder(XFSCrypter): __version__ = "0.10" __pattern__ = r'http://(?:www\.)?easybytez\.com/users/\d+/\d+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Easybytez.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "easybytez.com" - LOGIN_ACCOUNT = True diff --git a/module/plugins/crypter/EmbeduploadCom.py b/module/plugins/crypter/EmbeduploadCom.py index be3181793..28633f634 100644 --- a/module/plugins/crypter/EmbeduploadCom.py +++ b/module/plugins/crypter/EmbeduploadCom.py @@ -11,17 +11,17 @@ class EmbeduploadCom(Crypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?embedupload\.com/\?d=.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True), - ("preferedHoster", "str", "Prefered hoster list (bar-separated)", "embedupload"), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated)", "")] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True ), + ("subfolder_per_pack", "bool", "Create a subfolder for each package" , True ), + ("preferedHoster" , "str" , "Prefered hoster list (bar-separated)", "embedupload"), + ("ignoredHoster" , "str" , "Ignored hoster list (bar-separated)" , "" )] __description__ = """EmbedUpload.com decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - LINK_PATTERN = r'<div id="([^"]+)"[^>]*>\s*<a href="([^"]+)" target="_blank" (?:class="DownloadNow"|style="color:red")>' + LINK_PATTERN = r'<div id="(.+?)".*?>\s*<a href="(.+?)" target="_blank" (?:class="DownloadNow"|style="color:red")>' def decrypt(self, pyfile): @@ -30,21 +30,21 @@ class EmbeduploadCom(Crypter): m = re.findall(self.LINK_PATTERN, self.html) if m: - prefered_set = set(self.getConfig("preferedHoster").split('|')) + prefered_set = set(self.getConfig('preferedHoster').split('|')) prefered_set = map(lambda s: s.lower().split('.')[0], prefered_set) self.logDebug("PF: %s" % prefered_set) - tmp_links.extend([x[1] for x in m if x[0] in prefered_set]) + tmp_links.extend(x[1] for x in m if x[0] in prefered_set) self.urls = self.getLocation(tmp_links) if not self.urls: - ignored_set = set(self.getConfig("ignoredHoster").split('|')) + ignored_set = set(self.getConfig('ignoredHoster').split('|')) ignored_set = map(lambda s: s.lower().split('.')[0], ignored_set) self.logDebug("IG: %s" % ignored_set) - tmp_links.extend([x[1] for x in m if x[0] not in ignored_set]) + tmp_links.extend(x[1] for x in m if x[0] not in ignored_set) self.urls = self.getLocation(tmp_links) diff --git a/module/plugins/crypter/FilebeerInfoFolder.py b/module/plugins/crypter/FilebeerInfoFolder.py index a3c7ee74c..5b401c555 100644 --- a/module/plugins/crypter/FilebeerInfoFolder.py +++ b/module/plugins/crypter/FilebeerInfoFolder.py @@ -9,7 +9,7 @@ class FilebeerInfoFolder(DeadCrypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?filebeer\.info/\d*~f\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Filebeer.info folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FilecloudIoFolder.py b/module/plugins/crypter/FilecloudIoFolder.py index 83cce352d..f900f4910 100644 --- a/module/plugins/crypter/FilecloudIoFolder.py +++ b/module/plugins/crypter/FilecloudIoFolder.py @@ -9,8 +9,9 @@ class FilecloudIoFolder(SimpleCrypter): __version__ = "0.03" __pattern__ = r'https?://(?:www\.)?(filecloud\.io|ifile\.it)/_\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Filecloud.io folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FilecryptCc.py b/module/plugins/crypter/FilecryptCc.py index af31dc743..4d2b42f90 100644 --- a/module/plugins/crypter/FilecryptCc.py +++ b/module/plugins/crypter/FilecryptCc.py @@ -1,20 +1,22 @@ # -*- coding: utf-8 -*- +# +# Test links: +# http://filecrypt.cc/Container/64E039F859.html -import base64 import binascii import re +import urlparse from Crypto.Cipher import AES -from urlparse import urljoin from module.plugins.Crypter import Crypter -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha class FilecryptCc(Crypter): __name__ = "FilecryptCc" __type__ = "crypter" - __version__ = "0.09" + __version__ = "0.15" __pattern__ = r'https?://(?:www\.)?filecrypt\.cc/Container/\w+' @@ -31,7 +33,7 @@ class FilecryptCc(Crypter): CAPTCHA_PATTERN = r'<img id="nc" src="(.+?)"' CIRCLE_CAPTCHA_PATTERN = r'<input type="image" src="(.+?)"' - MIRROR_PAGE_PATTERN = r'"[\w]*" href="(http://filecrypt.cc/Container/\w+\.html\?mirror=\d+)">' + MIRROR_PAGE_PATTERN = r'"[\w]*" href="(https?://(?:www\.)?filecrypt.cc/Container/\w+\.html\?mirror=\d+)">' def setup(self): @@ -39,7 +41,8 @@ class FilecryptCc(Crypter): def decrypt(self, pyfile): - self.html = self.load(pyfile.url, cookies=True) + self.html = self.load(pyfile.url) + self.base_url = self.pyfile.url.split("Container")[0] if "content notfound" in self.html: #@NOTE: "content notfound" is NOT a typo self.offline() @@ -64,7 +67,7 @@ class FilecryptCc(Crypter): self.logInfo(_("Found %d mirrors") % len(mirror)) for i in mirror[1:]: - self.siteWithLinks = self.siteWithLinks + self.load(i, cookies=True).decode("utf-8", "replace") + self.siteWithLinks = self.siteWithLinks + self.load(i).decode("utf-8", "replace") def handlePasswordProtection(self): @@ -78,7 +81,7 @@ class FilecryptCc(Crypter): if not password: self.fail(_("Please enter the password in package section and try again")) - self.html = self.load(self.pyfile.url, post={"password": password}, cookies=True) + self.html = self.load(self.pyfile.url, post={"password": password}) def handleCaptcha(self): @@ -88,40 +91,37 @@ class FilecryptCc(Crypter): if m: #: normal captcha self.logDebug("Captcha-URL: %s" % m.group(1)) - captcha_code = self.decryptCaptcha(urljoin("http://filecrypt.cc", m.group(1)), + captcha_code = self.decryptCaptcha(urlparse.urljoin(self.base_url, m.group(1)), forceUser=True, imgtype="gif") self.siteWithLinks = self.load(self.pyfile.url, post={'recaptcha_response_field': captcha_code}, - cookies=True, decode=True) elif m2: #: circle captcha self.logDebug("Captcha-URL: %s" % m2.group(1)) - captcha_code = self.decryptCaptcha(urljoin("http://filecrypt.cc", m2.group(1)), - forceUser=True, - imgtype="gif", + captcha_code = self.decryptCaptcha('%s%s?c=abc' %(self.base_url, m2.group(1)), result_type='positional') self.siteWithLinks = self.load(self.pyfile.url, post={'button.x': captcha_code[0], 'button.y': captcha_code[1]}, - cookies=True, decode=True) + else: recaptcha = ReCaptcha(self) captcha_key = recaptcha.detect_key() if captcha_key: - self.siteWithLinks = self.load(self.pyfile.url, - post={'g-recaptcha-response': recaptcha.challenge(captcha_key, True)}, - cookies=True, - decode=True) + response, challenge = recaptcha.challenge(captcha_key) + self.siteWithLinks = self.load(self.pyfile.url, + post={'g-recaptcha-response': response}, + decode=True) else: - self.logDebug("No captcha found") + self.logInfo(_("No captcha found")) self.siteWithLinks = self.html - if "recaptcha_image" in self.siteWithLinks: + if "recaptcha_image" in self.siteWithLinks or "data-sitekey" in self.siteWithLinks: self.invalidCaptcha() self.retry() @@ -133,7 +133,7 @@ class FilecryptCc(Crypter): return for i in dlc: - self.links.append("http://filecrypt.cc/DLC/%s.dlc" % i) + self.links.append("%s/DLC/%s.dlc" % (self.base_url, i)) def handleWeblinks(self): @@ -141,9 +141,9 @@ class FilecryptCc(Crypter): weblinks = re.findall(self.WEBLINK_PATTERN, self.siteWithLinks) for link in weblinks: - res = self.load("http://filecrypt.cc/Link/%s.html" % link, cookies=True) + res = self.load("%s/Link/%s.html" % (self.base_url, link)) link2 = re.search('<iframe noresize src="(.*)"></iframe>', res) - res2 = self.load(link2.group(1), just_header=True, cookies=True) + res2 = self.load(link2.group(1), just_header=True) self.links.append(res2['location']) except Exception, e: @@ -166,17 +166,14 @@ class FilecryptCc(Crypter): # Get key key = binascii.unhexlify(str(jk)) - # Decode crypted - crypted = base64.standard_b64decode(crypted) - # Decrypt Key = key IV = key obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) + text = obj.decrypt(crypted.decode('base64')) # Extract links - links = filter(lambda x: x != "", - text.replace("\x00", "").replace("\r", "").split("\n")) + text = text.replace("\x00", "").replace("\r", "") + links = filter(bool, text.split('\n')) return links diff --git a/module/plugins/crypter/FilefactoryComFolder.py b/module/plugins/crypter/FilefactoryComFolder.py index 261c7e01a..c0cd028cc 100644 --- a/module/plugins/crypter/FilefactoryComFolder.py +++ b/module/plugins/crypter/FilefactoryComFolder.py @@ -9,8 +9,9 @@ class FilefactoryComFolder(SimpleCrypter): __version__ = "0.32" __pattern__ = r'https?://(?:www\.)?filefactory\.com/(?:f|folder)/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Filefactory.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FilerNetFolder.py b/module/plugins/crypter/FilerNetFolder.py index 00db173bb..e2b44e868 100644 --- a/module/plugins/crypter/FilerNetFolder.py +++ b/module/plugins/crypter/FilerNetFolder.py @@ -6,11 +6,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class FilerNetFolder(SimpleCrypter): __name__ = "FilerNetFolder" __type__ = "crypter" - __version__ = "0.41" + __version__ = "0.42" __pattern__ = r'https?://filer\.net/folder/\w{16}' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Filer.net decrypter plugin""" __license__ = "GPLv3" @@ -19,11 +20,9 @@ class FilerNetFolder(SimpleCrypter): LINK_PATTERN = r'href="(/get/\w{16})">(?!<)' - NAME_PATTERN = r'<h3>(?P<N>.+?) - <small' - - def getLinks(self): - return ['http://filer.net%s' % link for link in re.findall(self.LINK_PATTERN, self.html)] + NAME_PATTERN = r'<h3>(?P<N>.+?) - <small' + OFFLINE_PATTERN = r'Nicht gefunden' getInfo = create_getInfo(FilerNetFolder) diff --git a/module/plugins/crypter/FileserveComFolder.py b/module/plugins/crypter/FileserveComFolder.py index e6b35fd36..1363e2d45 100644 --- a/module/plugins/crypter/FileserveComFolder.py +++ b/module/plugins/crypter/FileserveComFolder.py @@ -11,8 +11,8 @@ class FileserveComFolder(Crypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?fileserve\.com/list/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """FileServe.com folder decrypter plugin""" __license__ = "GPLv3" @@ -20,7 +20,7 @@ class FileserveComFolder(Crypter): FOLDER_PATTERN = r'<table class="file_list">(.*?)</table>' - LINK_PATTERN = r'<a href="([^"]+)" class="sheet_icon wbold">' + LINK_PATTERN = r'<a href="(.+?)" class="sheet_icon wbold">' def decrypt(self, pyfile): diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index d58516986..6cd4d8b8c 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -9,6 +9,7 @@ class FilesonicComFolder(DeadCrypter): __version__ = "0.12" __pattern__ = r'http://(?:www\.)?filesonic\.com/folder/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Filesonic.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FilestubeCom.py b/module/plugins/crypter/FilestubeCom.py index 16ebdda37..9e6b5c114 100644 --- a/module/plugins/crypter/FilestubeCom.py +++ b/module/plugins/crypter/FilestubeCom.py @@ -6,11 +6,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class FilestubeCom(SimpleCrypter): __name__ = "FilestubeCom" __type__ = "crypter" - __version__ = "0.05" + __version__ = "0.06" __pattern__ = r'http://(?:www\.)?filestube\.(?:com|to)/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Filestube.com decrypter plugin""" __license__ = "GPLv3" @@ -18,7 +19,7 @@ class FilestubeCom(SimpleCrypter): LINK_PATTERN = r'<a class=\"file-link-main(?: noref)?\" [^>]* href=\"(http://[^\"]+)' - NAME_PATTERN = r'<h1\s*> (?P<N>.+) download\s*</h1>' + NAME_PATTERN = r'<h1\s*> (?P<N>.+?) download\s*</h1>' getInfo = create_getInfo(FilestubeCom) diff --git a/module/plugins/crypter/FiletramCom.py b/module/plugins/crypter/FiletramCom.py index 76530c589..b012d35bc 100644 --- a/module/plugins/crypter/FiletramCom.py +++ b/module/plugins/crypter/FiletramCom.py @@ -9,8 +9,9 @@ class FiletramCom(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?filetram\.com/[^/]+/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Filetram.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FiredriveComFolder.py b/module/plugins/crypter/FiredriveComFolder.py index 7d3a357fd..792f3198b 100644 --- a/module/plugins/crypter/FiredriveComFolder.py +++ b/module/plugins/crypter/FiredriveComFolder.py @@ -9,7 +9,7 @@ class FiredriveComFolder(DeadCrypter): __version__ = "0.03" __pattern__ = r'https?://(?:www\.)?(firedrive|putlocker)\.com/share/.+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Firedrive.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FourChanOrg.py b/module/plugins/crypter/FourChanOrg.py index c3fe3db4b..366adef14 100644 --- a/module/plugins/crypter/FourChanOrg.py +++ b/module/plugins/crypter/FourChanOrg.py @@ -3,6 +3,7 @@ # Based on 4chandl by Roland Beermann (https://gist.github.com/enkore/3492599) import re +import urlparse from module.plugins.Crypter import Crypter @@ -13,8 +14,8 @@ class FourChanOrg(Crypter): __version__ = "0.31" __pattern__ = r'http://(?:www\.)?boards\.4chan\.org/\w+/res/(\d+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """4chan.org folder decrypter plugin""" __license__ = "GPLv3" @@ -23,5 +24,5 @@ class FourChanOrg(Crypter): def decrypt(self, pyfile): pagehtml = self.load(pyfile.url) - images = set(re.findall(r'(images\.4chan\.org/[^/]*/src/[^"<]*)', pagehtml)) - self.urls = ["http://" + image for image in images] + images = set(re.findall(r'(images\.4chan\.org/[^/]*/src/[^"<]+)', pagehtml)) + self.urls = [urlparse.urljoin("http://", image) for image in images] diff --git a/module/plugins/crypter/FreakhareComFolder.py b/module/plugins/crypter/FreakhareComFolder.py index 7c1b7de2b..173660668 100644 --- a/module/plugins/crypter/FreakhareComFolder.py +++ b/module/plugins/crypter/FreakhareComFolder.py @@ -11,15 +11,16 @@ class FreakhareComFolder(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?freakshare\.com/folder/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Freakhare.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - LINK_PATTERN = r'<a href="(http://freakshare\.com/files/[^"]+)" target="_blank">' + LINK_PATTERN = r'<a href="(http://freakshare\.com/files/.+?)" target="_blank">' NAME_PATTERN = r'Folder:</b> (?P<N>.+)' PAGES_PATTERN = r'Pages: +(\d+)' diff --git a/module/plugins/crypter/FreetexthostCom.py b/module/plugins/crypter/FreetexthostCom.py index c33c9ff64..cf6fbb8a4 100644 --- a/module/plugins/crypter/FreetexthostCom.py +++ b/module/plugins/crypter/FreetexthostCom.py @@ -11,8 +11,9 @@ class FreetexthostCom(SimpleCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?freetexthost\.com/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Freetexthost.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/FshareVnFolder.py b/module/plugins/crypter/FshareVnFolder.py index 474364e40..bbee53337 100644 --- a/module/plugins/crypter/FshareVnFolder.py +++ b/module/plugins/crypter/FshareVnFolder.py @@ -9,15 +9,16 @@ class FshareVnFolder(SimpleCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?fshare\.vn/folder/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Fshare.vn folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - LINK_PATTERN = r'<li class="w_80pc"><a href="([^"]+)" target="_blank">' + LINK_PATTERN = r'<li class="w_80pc"><a href="(.+?)" target="_blank">' getInfo = create_getInfo(FshareVnFolder) diff --git a/module/plugins/crypter/FurLy.py b/module/plugins/crypter/FurLy.py new file mode 100644 index 000000000..b8e9332e2 --- /dev/null +++ b/module/plugins/crypter/FurLy.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo + + +class FurLy(SimpleCrypter): + __name__ = "FurLy" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?fur\.ly/(\d/)?\w+' + + __description__ = """Fur.ly decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [(r'fur\.ly/', r'fur\.ly/bar/')] + + LINK_PATTERN = r'urls\[\d+\] = "(.+?)"' + OFFLINE_PATTERN = r'var output;\s*var total' + + +getInfo = create_getInfo(FurLy) diff --git a/module/plugins/crypter/Go4UpCom.py b/module/plugins/crypter/Go4UpCom.py index 102bc32b5..22f31f6f6 100644 --- a/module/plugins/crypter/Go4UpCom.py +++ b/module/plugins/crypter/Go4UpCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -10,9 +9,12 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class Go4UpCom(SimpleCrypter): __name__ = "Go4UpCom" __type__ = "crypter" - __version__ = "0.11" + __version__ = "0.12" __pattern__ = r'http://go4up\.com/(dl/\w{12}|rd/\w{12}/\d+)' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Go4Up.com decrypter plugin""" __license__ = "GPLv3" @@ -27,12 +29,12 @@ class Go4UpCom(SimpleCrypter): OFFLINE_PATTERN = r'>\s*(404 Page Not Found|File not Found|Mirror does not exist)' - def getLinks(self + def getLinks(self): links = [] - m = re.search(r'(/download/gethosts/.+?)"') + m = re.search(r'(/download/gethosts/.+?)"', self.html) if m: - self.html = self.load(urljoin("http://go4up.com/", m.group(1))) + self.html = self.load(urlparse.urljoin("http://go4up.com/", m.group(1))) pages = [self.load(url) for url in re.findall(self.LINK_PATTERN, self.html)] else: pages = [self.html] @@ -40,7 +42,7 @@ class Go4UpCom(SimpleCrypter): for html in pages: try: links.append(re.search(r'<b><a href="(.+?)"', html).group(1)) - except: + except Exception: continue return links diff --git a/module/plugins/crypter/GooGl.py b/module/plugins/crypter/GooGl.py index d548a3375..b36e22884 100644 --- a/module/plugins/crypter/GooGl.py +++ b/module/plugins/crypter/GooGl.py @@ -1,32 +1,32 @@ # -*- coding: utf-8 -*- -from module.plugins.Crypter import Crypter +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo from module.common.json_layer import json_loads -class GooGl(Crypter): +class GooGl(SimpleCrypter): __name__ = "GooGl" __type__ = "crypter" - __version__ = "0.01" + __version__ = "0.03" - __pattern__ = r'https?://(?:www\.)?goo\.gl/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __pattern__ = r'https?://(?:www\.)?goo\.gl/([a-zA-Z]+/)?\w+' __description__ = """Goo.gl decrypter plugin""" __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it")] + __authors__ = [("stickell" , "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com" )] API_URL = "https://www.googleapis.com/urlshortener/v1/url" + OFFLINE_PATTERN = r'has been disabled|does not exist' - def decrypt(self, pyfile): - rep = self.load(self.API_URL, get={'shortUrl': pyfile.url}) + + def getLinks(self): + rep = self.load(self.API_URL, get={'shortUrl': self.pyfile.url}) self.logDebug("JSON data: " + rep) rep = json_loads(rep) + return [rep['longUrl']] if "longUrl" in rep else None + - if 'longUrl' in rep: - self.urls = [rep['longUrl']] - else: - self.fail(_("Unable to expand shortened link")) +getInfo = create_getInfo(GooGl) diff --git a/module/plugins/crypter/GoogledriveComFolder.py b/module/plugins/crypter/GoogledriveComFolder.py new file mode 100644 index 000000000..70898b313 --- /dev/null +++ b/module/plugins/crypter/GoogledriveComFolder.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -* + +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo + + +class GoogledriveComFolder(SimpleCrypter): + __name__ = "GoogledriveCom" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?drive\.google\.com/folderview\?.*id=\w+' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] + + __description__ = """Drive.google.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r"folderName: '(?P<N>.+?)'" + LINK_PATTERN = r'\[,"\w+"(?:,,".+?")?,"(.+?)"' + OFFLINE_PATTERN = r'<TITLE>' + + +getInfo = create_getInfo(GoogledriveComFolder) diff --git a/module/plugins/crypter/HoerbuchIn.py b/module/plugins/crypter/HoerbuchIn.py index a347e4232..ccc3ab664 100644 --- a/module/plugins/crypter/HoerbuchIn.py +++ b/module/plugins/crypter/HoerbuchIn.py @@ -13,8 +13,8 @@ class HoerbuchIn(Crypter): __version__ = "0.60" __pattern__ = r'http://(?:www\.)?hoerbuch\.in/(wp/horbucher/\d+/.+/|tp/out\.php\?.+|protection/folder_\d+\.html)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Hoerbuch.in decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/HotfileComFolder.py b/module/plugins/crypter/HotfileComFolder.py index 4f40587ad..f1dd56228 100644 --- a/module/plugins/crypter/HotfileComFolder.py +++ b/module/plugins/crypter/HotfileComFolder.py @@ -9,7 +9,7 @@ class HotfileComFolder(DeadCrypter): __version__ = "0.30" __pattern__ = r'https?://(?:www\.)?hotfile\.com/list/\w+/\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Hotfile.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/ILoadTo.py b/module/plugins/crypter/ILoadTo.py index f3415706d..4c7387739 100644 --- a/module/plugins/crypter/ILoadTo.py +++ b/module/plugins/crypter/ILoadTo.py @@ -9,7 +9,7 @@ class ILoadTo(DeadCrypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?iload\.to/go/\d+-[\w.-]+/' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Iload.to decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/ImgurComAlbum.py b/module/plugins/crypter/ImgurComAlbum.py index 6c074f5f1..9ef7797af 100644 --- a/module/plugins/crypter/ImgurComAlbum.py +++ b/module/plugins/crypter/ImgurComAlbum.py @@ -10,8 +10,9 @@ class ImgurComAlbum(SimpleCrypter): __version__ = "0.51" __pattern__ = r'https?://(?:www\.|m\.)?imgur\.com/(a|gallery|)/?\w{5,7}' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Imgur.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/JunocloudMeFolder.py b/module/plugins/crypter/JunocloudMeFolder.py deleted file mode 100644 index 990f25902..000000000 --- a/module/plugins/crypter/JunocloudMeFolder.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo - - -class JunocloudMeFolder(XFSCrypter): - __name__ = "JunocloudMeFolder" - __type__ = "crypter" - __version__ = "0.03" - - __pattern__ = r'http://(?:www\.)?junocloud\.me/folders/(?P<ID>\d+/\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Junocloud.me folder decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("guidobelix", "guidobelix@hotmail.it")] - - - HOSTER_DOMAIN = "junocloud.me" - - -getInfo = create_getInfo(JunocloudMeFolder) diff --git a/module/plugins/crypter/LetitbitNetFolder.py b/module/plugins/crypter/LetitbitNetFolder.py index 56ecbc7f8..b60d754a7 100644 --- a/module/plugins/crypter/LetitbitNetFolder.py +++ b/module/plugins/crypter/LetitbitNetFolder.py @@ -10,8 +10,8 @@ class LetitbitNetFolder(Crypter): __version__ = "0.10" __pattern__ = r'http://(?:www\.)?letitbit\.net/folder/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Letitbit.net folder decrypter plugin""" __license__ = "GPLv3" @@ -20,7 +20,7 @@ class LetitbitNetFolder(Crypter): FOLDER_PATTERN = r'<table>(.*)</table>' - LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' + LINK_PATTERN = r'<a href="(.+?)" target="_blank">' def decrypt(self, pyfile): diff --git a/module/plugins/crypter/LinkCryptWs.py b/module/plugins/crypter/LinkCryptWs.py index 996a92c7b..5c65e726d 100644 --- a/module/plugins/crypter/LinkCryptWs.py +++ b/module/plugins/crypter/LinkCryptWs.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -import base64 import binascii import re @@ -92,7 +91,7 @@ class LinkCryptWs(Crypter): def isOnline(self): if "<title>Linkcrypt.ws // Error 404</title>" in self.html: - self.logDebug("folder doesen't exist anymore") + self.logDebug("Folder doesn't exist anymore") return False else: return True @@ -132,7 +131,7 @@ class LinkCryptWs(Crypter): def unlockCaptchaProtection(self): - captcha_url = re.search(r'<form.*?id\s*?=\s*?"captcha"[^>]*?>.*?<\s*?input.*?src="([^"]*?)"', self.html, re.I | re.S).group(1) + captcha_url = re.search(r'<form.*?id\s*?=\s*?"captcha"[^>]*?>.*?<\s*?input.*?src="(.+?)"', self.html, re.I | re.S).group(1) captcha_code = self.decryptCaptcha(captcha_url, forceUser=True, imgtype="gif", result_type='positional') self.html = self.load(self.pyfile.url, post={"x": captcha_code[0], "y": captcha_code[1]}) @@ -191,15 +190,13 @@ class LinkCryptWs(Crypter): self.logDebug("Search for Web links ") package_links = [] - pattern = r'<form action="http://linkcrypt.ws/out.html"[^>]*?>.*?<input[^>]*?value="([^"]*?)"[^>]*?name="file"' + pattern = r'<form action="http://linkcrypt.ws/out.html"[^>]*?>.*?<input[^>]*?value="(.+?)"[^>]*?name="file"' ids = re.findall(pattern, self.html, re.I | re.S) self.logDebug("Decrypting %d Web links" % len(ids)) for idx, weblink_id in enumerate(ids): try: - self.logDebug("Decrypting Web link %d, %s" % (idx + 1, weblink_id)) - res = self.load("http://linkcrypt.ws/out.html", post = {'file':weblink_id}) indexs = res.find("window.location =") + 19 @@ -207,8 +204,6 @@ class LinkCryptWs(Crypter): link2 = res[indexs:indexe] - self.logDebug(link2) - link2 = html_unescape(link2) package_links.append(link2) @@ -245,7 +240,7 @@ class LinkCryptWs(Crypter): for line in self.container_html: if type in line: jseval = self.handle_javascript(line) - clink = re.search(r'href=["\']([^"\']*?)["\']',jseval,re.I) + clink = re.search(r'href=["\'](["\']+)', jseval, re.I) if not clink: continue @@ -279,7 +274,7 @@ class LinkCryptWs(Crypter): (vcrypted, vjk) = self._getCipherParams(cnl_section) for (crypted, jk) in zip(vcrypted, vjk): package_links.extend(self._getLinks(crypted, jk)) - except: + except Exception: self.logError(_("Unable to decrypt CNL links (JS Error) try to get over links")) return self.handleWebLinks() @@ -307,19 +302,15 @@ class LinkCryptWs(Crypter): self.logDebug("JsEngine returns value [%s]" % jreturn) - # Decode crypted - crypted = base64.standard_b64decode(crypted) - # Decrypt Key = key IV = key obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) + text = obj.decrypt(crypted.decode('base64')) # Extract links text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) + links = filter(bool, text.split('\n')) # Log and return self.logDebug("Package has %d links" % len(links)) diff --git a/module/plugins/crypter/LinkSaveIn.py b/module/plugins/crypter/LinkSaveIn.py index e96d0438e..dce74b689 100644 --- a/module/plugins/crypter/LinkSaveIn.py +++ b/module/plugins/crypter/LinkSaveIn.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleDereferer import SimpleDereferer +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo -class LinkSaveIn(SimpleDereferer): +class LinkSaveIn(SimpleCrypter): __name__ = "LinkSaveIn" __type__ = "crypter" - __version__ = "2.03" + __version__ = "2.05" __pattern__ = r'https?://(?:www\.)?linksave\.in/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """LinkSave.in decrypter plugin""" __license__ = "GPLv3" @@ -20,3 +20,6 @@ class LinkSaveIn(SimpleDereferer): COOKIES = [("linksave.in", "Linksave_Language", "english")] OFFLINE_PATTERN = r'>(Error )?404 -' + + +getInfo = create_getInfo(LinkSaveIn) diff --git a/module/plugins/crypter/LinkdecrypterCom.py b/module/plugins/crypter/LinkdecrypterCom.py index 7f24784c7..0704214d0 100644 --- a/module/plugins/crypter/LinkdecrypterCom.py +++ b/module/plugins/crypter/LinkdecrypterCom.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re + from module.plugins.Crypter import Crypter @@ -10,8 +11,8 @@ class LinkdecrypterCom(Crypter): __version__ = "0.29" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Linkdecrypter.com decrypter plugin""" __license__ = "GPLv3" @@ -34,10 +35,10 @@ class LinkdecrypterCom(Crypter): retries = 5 post_dict = {"link_cache": "on", "pro_links": pyfile.url, "modo_links": "text"} - self.html = self.load('http://linkdecrypter.com/', post=post_dict, cookies=True, decode=True) + self.html = self.load('http://linkdecrypter.com/', post=post_dict, decode=True) while retries: - m = re.search(self.TEXTAREA_PATTERN, self.html, flags=re.S) + m = re.search(self.TEXTAREA_PATTERN, self.html, re.S) if m: self.urls = [x for x in m.group(1).splitlines() if '[LINK-ERROR]' not in x] @@ -65,4 +66,4 @@ class LinkdecrypterCom(Crypter): else: retries -= 1 - self.html = self.load('http://linkdecrypter.com/', cookies=True, decode=True) + self.html = self.load('http://linkdecrypter.com/', decode=True) diff --git a/module/plugins/crypter/LixIn.py b/module/plugins/crypter/LixIn.py index d899d58c7..5c77eae5e 100644 --- a/module/plugins/crypter/LixIn.py +++ b/module/plugins/crypter/LixIn.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re +import urlparse from module.plugins.Crypter import Crypter @@ -11,8 +12,8 @@ class LixIn(Crypter): __version__ = "0.22" __pattern__ = r'http://(?:www\.)?lix\.in/(?P<ID>.+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Lix.in decrypter plugin""" __license__ = "GPLv3" @@ -46,7 +47,7 @@ class LixIn(Crypter): m = re.search(self.CAPTCHA_PATTERN, self.html) if m: self.logDebug("Trying captcha") - captcharesult = self.decryptCaptcha("http://lix.in/" + m.group(1)) + captcharesult = self.decryptCaptcha(urlparse.urljoin("http://lix.in/", m.group(1))) self.html = self.load(url, decode=True, post={"capt": captcharesult, "submit": "submit", "tiny": id}) else: diff --git a/module/plugins/crypter/LofCc.py b/module/plugins/crypter/LofCc.py index 3cac0fbf2..62e05f299 100644 --- a/module/plugins/crypter/LofCc.py +++ b/module/plugins/crypter/LofCc.py @@ -9,7 +9,7 @@ class LofCc(DeadCrypter): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?lof\.cc/(.+)' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Lof.cc decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/MBLinkInfo.py b/module/plugins/crypter/MBLinkInfo.py index 82c2d9719..7ff400f7e 100644 --- a/module/plugins/crypter/MBLinkInfo.py +++ b/module/plugins/crypter/MBLinkInfo.py @@ -9,7 +9,7 @@ class MBLinkInfo(DeadCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?mblink\.info/?\?id=(\d+)' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """MBLink.info decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/MediafireComFolder.py b/module/plugins/crypter/MediafireComFolder.py index d1dc89518..c1612bb6a 100644 --- a/module/plugins/crypter/MediafireComFolder.py +++ b/module/plugins/crypter/MediafireComFolder.py @@ -12,8 +12,8 @@ class MediafireComFolder(Crypter): __version__ = "0.14" __pattern__ = r'http://(?:www\.)?mediafire\.com/(folder/|\?sharekey=|\?\w{13}($|[/#]))' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Mediafire.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/MegaCoNzFolder.py b/module/plugins/crypter/MegaCoNzFolder.py index bec4eba22..0c6f224a4 100644 --- a/module/plugins/crypter/MegaCoNzFolder.py +++ b/module/plugins/crypter/MegaCoNzFolder.py @@ -2,17 +2,17 @@ import re -from module.plugins.internal.Crypter import Crypter +from module.plugins.Crypter import Crypter class MegaCoNzFolder(Crypter): __name__ = "MegaCoNzFolder" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" - __pattern__ = r'(?:https?://(?:www\.)?mega\.co\.nz/|mega:|chrome:.+?)#F!(?P<ID>[\w+^_])!(?P<KEY>[\w,\\-]+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __pattern__ = r'(?:https?://(?:www\.)?mega(\.co)?\.nz/|mega:|chrome:.+?)#F!(?P<ID>[\w^_]+)!(?P<KEY>[\w,\\-]+)' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Mega.co.nz folder decrypter plugin""" __license__ = "GPLv3" @@ -26,7 +26,7 @@ class MegaCoNzFolder(Crypter): def decrypt(self, pyfile): url = "https://mega.co.nz/#F!%s!%s" % re.match(self.__pattern__, pyfile.url).groups() self.html = self.load("http://rapidgen.org/linkfinder", post={'linklisturl': url}) - self.urls = re.findall(r'(https://mega.co.nz/#N!.+?)<', self.html) + self.urls = re.findall(r'(https://mega(\.co)?\.nz/#N!.+?)<', self.html) if not self.urls: #@TODO: Remove in 0.4.10 - self.fail("No link grabbed") + self.fail(_("No link grabbed")) diff --git a/module/plugins/crypter/MegaRapidCzFolder.py b/module/plugins/crypter/MegaRapidCzFolder.py index e7dff6c8a..fadd6dbed 100644 --- a/module/plugins/crypter/MegaRapidCzFolder.py +++ b/module/plugins/crypter/MegaRapidCzFolder.py @@ -9,15 +9,16 @@ class MegaRapidCzFolder(SimpleCrypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?(share|mega)rapid\.cz/slozka/\d+/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Share-Rapid.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - LINK_PATTERN = r'<td class="soubor"[^>]*><a href="([^"]+)">' + LINK_PATTERN = r'<td class="soubor".*?><a href="(.+?)">' getInfo = create_getInfo(MegaRapidCzFolder) diff --git a/module/plugins/crypter/MegauploadComFolder.py b/module/plugins/crypter/MegauploadComFolder.py index 08f96700d..60e4af57e 100644 --- a/module/plugins/crypter/MegauploadComFolder.py +++ b/module/plugins/crypter/MegauploadComFolder.py @@ -9,6 +9,7 @@ class MegauploadComFolder(DeadCrypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?megaupload\.com/(\?f|xml/folderfiles\.php\?.*&?folderid)=\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Megaupload.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/Movie2kTo.py b/module/plugins/crypter/Movie2KTo.py index 76bf702ac..919d77c86 100644 --- a/module/plugins/crypter/Movie2kTo.py +++ b/module/plugins/crypter/Movie2KTo.py @@ -9,7 +9,7 @@ class Movie2KTo(DeadCrypter): __version__ = "0.51" __pattern__ = r'http://(?:www\.)?movie2k\.to/(.+)\.html' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Movie2k.to decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/MultiUpOrg.py b/module/plugins/crypter/MultiUpOrg.py index 5209ebf09..b676c3029 100644 --- a/module/plugins/crypter/MultiUpOrg.py +++ b/module/plugins/crypter/MultiUpOrg.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -12,8 +12,9 @@ class MultiUpOrg(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?multiup\.org/(en|fr)/(?P<TYPE>project|download|miror)/\w+(/\w+)?' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """MultiUp.org crypter plugin""" __license__ = "GPLv3" @@ -32,7 +33,7 @@ class MultiUpOrg(SimpleCrypter): pattern = r'style="width:97%;text-align:left".*\n.*href="(.*)"' if m_type == "download": dl_pattern = r'href="(.*)">.*\n.*<h5>DOWNLOAD</h5>' - miror_page = urljoin("http://www.multiup.org", re.search(dl_pattern, self.html).group(1)) + miror_page = urlparse.urljoin("http://www.multiup.org", re.search(dl_pattern, self.html).group(1)) self.html = self.load(miror_page) return re.findall(pattern, self.html) diff --git a/module/plugins/crypter/MultiloadCz.py b/module/plugins/crypter/MultiloadCz.py index fa1eb02d7..eea689a09 100644 --- a/module/plugins/crypter/MultiloadCz.py +++ b/module/plugins/crypter/MultiloadCz.py @@ -10,18 +10,18 @@ class MultiloadCz(Crypter): __version__ = "0.40" __pattern__ = r'http://(?:[^/]*\.)?multiload\.cz/(stahnout|slozka)/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True), - ("usedHoster", "str", "Prefered hoster list (bar-separated)", ""), - ("ignoredHoster", "str", "Ignored hoster list (bar-separated)", "")] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package" , True), + ("usedHoster" , "str" , "Prefered hoster list (bar-separated)", "" ), + ("ignoredHoster" , "str" , "Ignored hoster list (bar-separated)" , "" )] __description__ = """Multiload.cz decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - FOLDER_PATTERN = r'<form action="" method="get"><textarea[^>]*>([^>]*)</textarea></form>' - LINK_PATTERN = r'<p class="manager-server"><strong>([^<]+)</strong></p><p class="manager-linky"><a href="([^"]+)">' + FOLDER_PATTERN = r'<form action="" method="get"><textarea.*?>([^>]*)</textarea></form>' + LINK_PATTERN = r'<p class="manager-server"><strong>([^<]+)</strong></p><p class="manager-linky"><a href="(.+?)">' def decrypt(self, pyfile): @@ -34,9 +34,9 @@ class MultiloadCz(Crypter): else: m = re.findall(self.LINK_PATTERN, self.html) if m: - prefered_set = set(self.getConfig("usedHoster").split('|')) - self.urls.extend([x[1] for x in m if x[0] in prefered_set]) + prefered_set = set(self.getConfig('usedHoster').split('|')) + self.urls.extend(x[1] for x in m if x[0] in prefered_set) if not self.urls: - ignored_set = set(self.getConfig("ignoredHoster").split('|')) - self.urls.extend([x[1] for x in m if x[0] not in ignored_set]) + ignored_set = set(self.getConfig('ignoredHoster').split('|')) + self.urls.extend(x[1] for x in m if x[0] not in ignored_set) diff --git a/module/plugins/crypter/MultiuploadCom.py b/module/plugins/crypter/MultiuploadCom.py index 347b7e5af..41794445a 100644 --- a/module/plugins/crypter/MultiuploadCom.py +++ b/module/plugins/crypter/MultiuploadCom.py @@ -9,7 +9,7 @@ class MultiuploadCom(DeadCrypter): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?multiupload\.(com|nl)/\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """ MultiUpload.com decrypter plugin """ __license__ = "GPLv3" diff --git a/module/plugins/crypter/NCryptIn.py b/module/plugins/crypter/NCryptIn.py index 8b7214157..593e9b506 100644 --- a/module/plugins/crypter/NCryptIn.py +++ b/module/plugins/crypter/NCryptIn.py @@ -1,23 +1,22 @@ # -*- coding: utf-8 -*- -import base64 import binascii import re from Crypto.Cipher import AES from module.plugins.Crypter import Crypter -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha class NCryptIn(Crypter): __name__ = "NCryptIn" __type__ = "crypter" - __version__ = "1.33" + __version__ = "1.35" __pattern__ = r'http://(?:www\.)?ncrypt\.in/(?P<TYPE>folder|link|frame)-([^/\?]+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """NCrypt.in decrypter plugin""" __license__ = "GPLv3" @@ -28,7 +27,7 @@ class NCryptIn(Crypter): JK_KEY = "jk" CRYPTED_KEY = "crypted" - NAME_PATTERN = r'<meta name="description" content="(?P<N>[^"]+)"' + NAME_PATTERN = r'<meta name="description" content="(?P<N>.+?)"' def setup(self): @@ -109,7 +108,7 @@ class NCryptIn(Crypter): def isProtected(self): form = re.search(r'<form.*?name.*?protected.*?>(.*?)</form>', self.cleanedHtml, re.S) - if form is not None: + if form: content = form.group(1) for keyword in ("password", "captcha"): if keyword in content: @@ -145,7 +144,7 @@ class NCryptIn(Crypter): # Resolve anicaptcha if "anicaptcha" in form: self.logDebug("Captcha protected") - captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1) + captchaUri = re.search(r'src="(/temp/anicaptcha/.+?)"', form).group(1) captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri) self.logDebug("Captcha resolved [%s]" % captcha) postData['captcha'] = captcha @@ -156,7 +155,7 @@ class NCryptIn(Crypter): captcha_key = re.search(r'\?k=(.*?)"', form).group(1) self.logDebug("Resolving ReCaptcha with key [%s]" % captcha_key) recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge(captcha_key) + response, challenge = recaptcha.challenge(captcha_key) postData['recaptcha_challenge_field'] = challenge postData['recaptcha_response_field'] = response @@ -205,7 +204,7 @@ class NCryptIn(Crypter): elif link_source_type == "web": return self.handleWebLinks() else: - self.error('Unknown source type "%s" (this is probably a bug)' % link_source_type) + self.error(_('Unknown source type "%s"') % link_source_type) def handleSingleLink(self): @@ -229,7 +228,7 @@ class NCryptIn(Crypter): (vcrypted, vjk) = self._getCipherParams() for (crypted, jk) in zip(vcrypted, vjk): package_links.extend(self._getLinks(crypted, jk)) - except: + except Exception: self.fail(_("Unable to decrypt CNL2 links")) return package_links @@ -296,19 +295,15 @@ class NCryptIn(Crypter): self.logDebug("JsEngine returns value [%s]" % jreturn) key = binascii.unhexlify(jreturn) - # Decode crypted - crypted = base64.standard_b64decode(crypted) - # Decrypt Key = key IV = key obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) + text = obj.decrypt(crypted.decode('base64')) # Extract links text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) + links = filter(bool, text.split('\n')) # Log and return self.logDebug("Block has %d links" % len(links)) diff --git a/module/plugins/crypter/NetfolderIn.py b/module/plugins/crypter/NetfolderIn.py index 62dc5c914..32d59af03 100644 --- a/module/plugins/crypter/NetfolderIn.py +++ b/module/plugins/crypter/NetfolderIn.py @@ -1,18 +1,15 @@ # -*- coding: utf-8 -*- -import re +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo -from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo - -class NetfolderIn(SimpleCrypter): +class NetfolderIn(DeadCrypter): __name__ = "NetfolderIn" __type__ = "crypter" - __version__ = "0.72" + __version__ = "0.73" - __pattern__ = r'http://(?:www\.)?netfolder\.in/(folder\.php\?folder_id=)?(?P<ID>\w+)(?(1)|/\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __pattern__ = r'http://(?:www\.)?netfolder\.(in|me)/(folder\.php\?folder_id=)?(?P<ID>\w+)(?(1)|/\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """NetFolder.in decrypter plugin""" __license__ = "GPLv3" @@ -20,54 +17,4 @@ class NetfolderIn(SimpleCrypter): ("fragonib", "fragonib[AT]yahoo[DOT]es")] - NAME_PATTERN = r'<div class="Text">Inhalt des Ordners <span.*>(?P<N>.+)</span></div>' - - - def prepare(self): - super(NetfolderIn, self).prepare() - - # Check for password protection - if self.isPasswordProtected(): - self.html = self.submitPassword() - if not self.html: - self.fail(_("Incorrect password, please set right password on Add package form and retry")) - - - def isPasswordProtected(self): - if '<input type="password" name="password"' in self.html: - self.logDebug("Links are password protected") - return True - return False - - - def submitPassword(self): - # Gather data - try: - m = re.match(self.__pattern__, self.pyfile.url) - id = m.group('ID') - except AttributeError: - self.logDebug("Unable to get package id from url [%s]" % self.pyfile.url) - return - url = "http://netfolder.in/folder.php?folder_id=" + id - password = self.getPassword() - - # Submit package password - post = {'password': password, 'save': 'Absenden'} - self.logDebug("Submitting password [%s] for protected links with id [%s]" % (password, id)) - html = self.load(url, {}, post) - - # Check for invalid password - if '<div class="InPage_Error">' in html: - self.logDebug("Incorrect password, please set right password on Edit package form and retry") - return None - - return html - - - def getLinks(self): - links = re.search(r'name="list" value="(.*?)"', self.html).group(1).split(",") - self.logDebug("Package has %d links" % len(links)) - return links - - getInfo = create_getInfo(NetfolderIn) diff --git a/module/plugins/crypter/NosvideoCom.py b/module/plugins/crypter/NosvideoCom.py index 1e280abd2..9808837c4 100644 --- a/module/plugins/crypter/NosvideoCom.py +++ b/module/plugins/crypter/NosvideoCom.py @@ -9,8 +9,9 @@ class NosvideoCom(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?nosvideo\.com/\?v=\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Nosvideo.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/OneKhDe.py b/module/plugins/crypter/OneKhDe.py index 3e75d97b5..4dcb416f5 100644 --- a/module/plugins/crypter/OneKhDe.py +++ b/module/plugins/crypter/OneKhDe.py @@ -13,8 +13,8 @@ class OneKhDe(Crypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?1kh\.de/f/' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """1kh.de decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py index 9e06bdf32..048af7d82 100755..100644 --- a/module/plugins/crypter/OronComFolder.py +++ b/module/plugins/crypter/OronComFolder.py @@ -9,7 +9,7 @@ class OronComFolder(DeadCrypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?oron\.com/folder/\w+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Oron.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/PastebinCom.py b/module/plugins/crypter/PastebinCom.py index b3d5a4bea..751b47cc9 100644 --- a/module/plugins/crypter/PastebinCom.py +++ b/module/plugins/crypter/PastebinCom.py @@ -9,8 +9,9 @@ class PastebinCom(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?pastebin\.com/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Pastebin.com decrypter plugin""" __license__ = "GPLv3" @@ -18,7 +19,7 @@ class PastebinCom(SimpleCrypter): LINK_PATTERN = r'<div class="de\d+">(https?://[^ <]+)(?:[^<]*)</div>' - NAME_PATTERN = r'<div class="paste_box_line1" title="(?P<N>[^"]+)">' + NAME_PATTERN = r'<div class="paste_box_line1" title="(?P<N>.+?)">' getInfo = create_getInfo(PastebinCom) diff --git a/module/plugins/crypter/QuickshareCzFolder.py b/module/plugins/crypter/QuickshareCzFolder.py index 70666b55a..3e38d36b4 100644 --- a/module/plugins/crypter/QuickshareCzFolder.py +++ b/module/plugins/crypter/QuickshareCzFolder.py @@ -10,15 +10,15 @@ class QuickshareCzFolder(Crypter): __version__ = "0.10" __pattern__ = r'http://(?:www\.)?quickshare\.cz/slozka-\d+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Quickshare.cz folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - FOLDER_PATTERN = r'<textarea[^>]*>(.*?)</textarea>' + FOLDER_PATTERN = r'<textarea.*?>(.*?)</textarea>' LINK_PATTERN = r'(http://www\.quickshare\.cz/\S+)' diff --git a/module/plugins/crypter/RSLayerCom.py b/module/plugins/crypter/RSLayerCom.py index cc3b23bbc..6f078e9e4 100644 --- a/module/plugins/crypter/RSLayerCom.py +++ b/module/plugins/crypter/RSLayerCom.py @@ -9,7 +9,7 @@ class RSLayerCom(DeadCrypter): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?rs-layer\.com/directory-' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """RS-Layer.com decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/RapidfileshareNetFolder.py b/module/plugins/crypter/RapidfileshareNetFolder.py deleted file mode 100644 index fc3d4241e..000000000 --- a/module/plugins/crypter/RapidfileshareNetFolder.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo - - -class RapidfileshareNetFolder(XFSCrypter): - __name__ = "RapidfileshareNetFolder" - __type__ = "crypter" - __version__ = "0.03" - - __pattern__ = r'http://(?:www\.)?rapidfileshare\.net/users/\w+/\d+/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Rapidfileshare.net folder decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("guidobelix", "guidobelix@hotmail.it")] - - - HOSTER_DOMAIN = "rapidfileshare.net" - - -getInfo = create_getInfo(RapidfileshareNetFolder) diff --git a/module/plugins/crypter/RelinkUs.py b/module/plugins/crypter/RelinkUs.py index c5a3f1011..ed3da1f9b 100644 --- a/module/plugins/crypter/RelinkUs.py +++ b/module/plugins/crypter/RelinkUs.py @@ -2,23 +2,23 @@ from __future__ import with_statement -import base64 import binascii import re import os from Crypto.Cipher import AES from module.plugins.Crypter import Crypter +from module.utils import save_join class RelinkUs(Crypter): __name__ = "RelinkUs" __type__ = "crypter" - __version__ = "3.11" + __version__ = "3.12" __pattern__ = r'http://(?:www\.)?relink\.us/(f/|((view|go)\.php\?id=))(?P<ID>.+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Relink.us decrypter plugin""" __license__ = "GPLv3" @@ -153,7 +153,7 @@ class RelinkUs(Crypter): # Try to get info from web m = re.search(self.FILE_TITLE_REGEX, self.html) - if m is not None: + if m: title = m.group(1).strip() if not self.FILE_NOTITLE in title: name = folder = title @@ -191,20 +191,20 @@ class RelinkUs(Crypter): elif source == 'web': return self.handleWEBLinks() else: - self.error('Unknown source type "%s" (this is probably a bug)' % source) + self.error(_('Unknown source type "%s"') % source) def handleCNL2Links(self): self.logDebug("Search for CNL2 links") package_links = [] m = re.search(self.CNL2_FORM_REGEX, self.html, re.S) - if m is not None: + if m: cnl2_form = m.group(1) try: (vcrypted, vjk) = self._getCipherParams(cnl2_form) for (crypted, jk) in zip(vcrypted, vjk): package_links.extend(self._getLinks(crypted, jk)) - except: + except Exception: self.logDebug("Unable to decrypt CNL2 links") return package_links @@ -213,18 +213,20 @@ class RelinkUs(Crypter): self.logDebug("Search for DLC links") package_links = [] m = re.search(self.DLC_LINK_REGEX, self.html) - if m is not None: + if m: container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.fileid self.logDebug("Downloading DLC container link [%s]" % container_url) try: dlc = self.load(container_url) dlc_filename = self.fileid + ".dlc" - dlc_filepath = os.path.join(self.config['general']['download_folder'], dlc_filename) + dlc_filepath = save_join(self.config['general']['download_folder'], dlc_filename) with open(dlc_filepath, "wb") as f: f.write(dlc) package_links.append(dlc_filepath) - except: - self.fail("Unable to download DLC container") + + except Exception: + self.fail(_("Unable to download DLC container")) + return package_links @@ -276,19 +278,15 @@ class RelinkUs(Crypter): self.logDebug("JsEngine returns value [%s]" % jreturn) key = binascii.unhexlify(jreturn) - # Decode crypted - crypted = base64.standard_b64decode(crypted) - # Decrypt Key = key IV = key obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) + text = obj.decrypt(crypted.decode('base64')) # Extract links text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) + links = filter(bool, text.split('\n')) # Log and return self.logDebug("Package has %d links" % len(links)) diff --git a/module/plugins/crypter/SafelinkingNet.py b/module/plugins/crypter/SafelinkingNet.py index a56a0a44c..5d822af11 100644 --- a/module/plugins/crypter/SafelinkingNet.py +++ b/module/plugins/crypter/SafelinkingNet.py @@ -2,23 +2,21 @@ import re -from pycurl import FOLLOWLOCATION - from BeautifulSoup import BeautifulSoup from module.common.json_layer import json_loads from module.plugins.Crypter import Crypter -from module.plugins.internal.CaptchaService import SolveMedia +from module.plugins.internal.SolveMedia import SolveMedia class SafelinkingNet(Crypter): __name__ = "SafelinkingNet" __type__ = "crypter" - __version__ = "0.13" + __version__ = "0.15" __pattern__ = r'https?://(?:www\.)?safelinking\.net/([pd])/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Safelinking.net decrypter plugin""" __license__ = "GPLv3" @@ -57,7 +55,7 @@ class SafelinkingNet(Crypter): else: self.fail(_("Error parsing captcha")) - challenge, response = captcha.challenge(captchaKey) + response, challenge = captcha.challenge(captchaKey) postData['adcopy_challenge'] = challenge postData['adcopy_response'] = response diff --git a/module/plugins/crypter/SecuredIn.py b/module/plugins/crypter/SecuredIn.py index cbfa919ac..a2eac4f13 100644 --- a/module/plugins/crypter/SecuredIn.py +++ b/module/plugins/crypter/SecuredIn.py @@ -9,7 +9,7 @@ class SecuredIn(DeadCrypter): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?secured\.in/download-[\d]+-\w{8}\.html' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Secured.in decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/SexuriaCom.py b/module/plugins/crypter/SexuriaCom.py index 3c952fd6b..06dcf547e 100644 --- a/module/plugins/crypter/SexuriaCom.py +++ b/module/plugins/crypter/SexuriaCom.py @@ -11,21 +11,21 @@ class SexuriaCom(Crypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?sexuria\.com/(v1/)?(Pornos_Kostenlos_.+?_(\d+)\.html|dl_links_\d+_\d+\.html|id=\d+\&part=\d+\&link=\d+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Sexuria.com decrypter plugin""" __license__ = "GPLv3" __authors__ = [("NETHead", "NETHead.AT.gmx.DOT.net")] - PATTERN_SUPPORTED_MAIN = re.compile(r'http://(www\.)?sexuria\.com/(v1/)?Pornos_Kostenlos_.+?_(\d+)\.html', flags=re.I) - PATTERN_SUPPORTED_CRYPT = re.compile(r'http://(www\.)?sexuria\.com/(v1/)?dl_links_\d+_(?P<ID>\d+)\.html', flags=re.I) - PATTERN_SUPPORTED_REDIRECT = re.compile(r'http://(www\.)?sexuria\.com/out\.php\?id=(?P<ID>\d+)\&part=\d+\&link=\d+', flags=re.I) - PATTERN_TITLE = re.compile(r'<title> - (?P<TITLE>.*) Sexuria - Kostenlose Pornos - Rapidshare XXX Porn</title>', flags=re.I) - PATTERN_PASSWORD = re.compile(r'<strong>Passwort: </strong></div></td>.*?bgcolor="#EFEFEF">(?P<PWD>.*?)</td>', flags=re.I | re.S) - PATTERN_DL_LINK_PAGE = re.compile(r'"(dl_links_\d+_\d+\.html)"', flags=re.I) - PATTERN_REDIRECT_LINKS = re.compile(r'value="(http://sexuria\.com/out\.php\?id=\d+\&part=\d+\&link=\d+)" readonly', flags=re.I) + PATTERN_SUPPORTED_MAIN = re.compile(r'http://(www\.)?sexuria\.com/(v1/)?Pornos_Kostenlos_.+?_(\d+)\.html', re.I) + PATTERN_SUPPORTED_CRYPT = re.compile(r'http://(www\.)?sexuria\.com/(v1/)?dl_links_\d+_(?P<ID>\d+)\.html', re.I) + PATTERN_SUPPORTED_REDIRECT = re.compile(r'http://(www\.)?sexuria\.com/out\.php\?id=(?P<ID>\d+)\&part=\d+\&link=\d+', re.I) + PATTERN_TITLE = re.compile(r'<title> - (?P<TITLE>.*) Sexuria - Kostenlose Pornos - Rapidshare XXX Porn</title>', re.I) + PATTERN_PASSWORD = re.compile(r'<strong>Passwort: </strong></div></td>.*?bgcolor="#EFEFEF">(?P<PWD>.*?)</td>', re.I | re.S) + PATTERN_DL_LINK_PAGE = re.compile(r'"(dl_links_\d+_\d+\.html)"', re.I) + PATTERN_REDIRECT_LINKS = re.compile(r'value="(http://sexuria\.com/out\.php\?id=\d+\&part=\d+\&link=\d+)" readonly', re.I) def decrypt(self, pyfile): diff --git a/module/plugins/crypter/ShareLinksBiz.py b/module/plugins/crypter/ShareLinksBiz.py index d2e8138f6..f17ba54d7 100644 --- a/module/plugins/crypter/ShareLinksBiz.py +++ b/module/plugins/crypter/ShareLinksBiz.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -import base64 import binascii import re @@ -14,8 +13,8 @@ class ShareLinksBiz(Crypter): __version__ = "1.14" __pattern__ = r'http://(?:www\.)?(share-links|s2l)\.biz/(?P<ID>_?\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Share-Links.biz decrypter plugin""" __license__ = "GPLv3" @@ -165,7 +164,7 @@ class ShareLinksBiz(Crypter): # Extract from web package header title_re = r'<h2><img.*?/>(.*)</h2>' m = re.search(title_re, self.html, re.S) - if m is not None: + if m: title = m.group(1).strip() if 'unnamed' not in title: name = folder = title @@ -235,7 +234,7 @@ class ShareLinksBiz(Crypter): try: (crypted, jk) = self._getCipherParams() package_links.extend(self._getLinks(crypted, jk)) - except: + except Exception: self.fail(_("Unable to decrypt CNL2 links")) return package_links @@ -248,14 +247,12 @@ class ShareLinksBiz(Crypter): params = res.split(";;") # Get jk - strlist = list(base64.standard_b64decode(params[1])) - strlist.reverse() - jk = ''.join(strlist) + strlist = list(params[1].decode('base64')) + jk = ''.join(strlist[::-1]) # Get crypted - strlist = list(base64.standard_b64decode(params[2])) - strlist.reverse() - crypted = ''.join(strlist) + strlist = list(params[2].decode('base64')) + crypted = ''.join(strlist[::-1]) # Log and return return crypted, jk @@ -267,19 +264,15 @@ class ShareLinksBiz(Crypter): self.logDebug("JsEngine returns value [%s]" % jreturn) key = binascii.unhexlify(jreturn) - # Decode crypted - crypted = base64.standard_b64decode(crypted) - # Decrypt Key = key IV = key obj = AES.new(Key, AES.MODE_CBC, IV) - text = obj.decrypt(crypted) + text = obj.decrypt(crypted.decode('base64')) # Extract links text = text.replace("\x00", "").replace("\r", "") - links = text.split("\n") - links = filter(lambda x: x != "", links) + links = filter(bool, text.split('\n')) # Log and return self.logDebug("Block has %d links" % len(links)) diff --git a/module/plugins/crypter/SharingmatrixComFolder.py b/module/plugins/crypter/SharingmatrixComFolder.py index e16bdf814..eb1b4d8a6 100644 --- a/module/plugins/crypter/SharingmatrixComFolder.py +++ b/module/plugins/crypter/SharingmatrixComFolder.py @@ -9,6 +9,7 @@ class SharingmatrixComFolder(DeadCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?sharingmatrix\.com/folder/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Sharingmatrix.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/SpeedLoadOrgFolder.py b/module/plugins/crypter/SpeedLoadOrgFolder.py index ddde7dec2..af768ef7b 100644 --- a/module/plugins/crypter/SpeedLoadOrgFolder.py +++ b/module/plugins/crypter/SpeedLoadOrgFolder.py @@ -9,7 +9,7 @@ class SpeedLoadOrgFolder(DeadCrypter): __version__ = "0.30" __pattern__ = r'http://(?:www\.)?speedload\.org/(\d+~f$|folder/\d+/)' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Speedload decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/StealthTo.py b/module/plugins/crypter/StealthTo.py index 5173421f1..551468980 100644 --- a/module/plugins/crypter/StealthTo.py +++ b/module/plugins/crypter/StealthTo.py @@ -9,7 +9,7 @@ class StealthTo(DeadCrypter): __version__ = "0.20" __pattern__ = r'http://(?:www\.)?stealth\.to/folder/.+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Stealth.to decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/TinyurlCom.py b/module/plugins/crypter/TinyurlCom.py new file mode 100644 index 000000000..e2cf50358 --- /dev/null +++ b/module/plugins/crypter/TinyurlCom.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo + + +class TinyurlCom(SimpleCrypter): + __name__ = "TinyurlCom" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'https?://(?:www\.)?(preview\.)?tinyurl\.com/[\w-]+' + + __description__ = """Tinyurl.com decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [(r'preview\.', r'')] + + OFFLINE_PATTERN = r">Error: Unable to find site's URL to redirect to" + + +getInfo = create_getInfo(TinyurlCom) diff --git a/module/plugins/crypter/TnyCz.py b/module/plugins/crypter/TnyCz.py index d36128550..f3b245118 100644 --- a/module/plugins/crypter/TnyCz.py +++ b/module/plugins/crypter/TnyCz.py @@ -8,18 +8,19 @@ import re class TnyCz(SimpleCrypter): __name__ = "TnyCz" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'http://(?:www\.)?tny\.cz/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Tny.cz decrypter plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<title>(?P<N>.+) - .+</title>' + NAME_PATTERN = r'<title>(?P<N>.+?) - .+</title>' def getLinks(self): diff --git a/module/plugins/crypter/TrailerzoneInfo.py b/module/plugins/crypter/TrailerzoneInfo.py index 10780dd45..6f1c90612 100644 --- a/module/plugins/crypter/TrailerzoneInfo.py +++ b/module/plugins/crypter/TrailerzoneInfo.py @@ -9,7 +9,7 @@ class TrailerzoneInfo(DeadCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?trailerzone\.info/.+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """TrailerZone.info decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/TurbobitNetFolder.py b/module/plugins/crypter/TurbobitNetFolder.py index c6734c997..dcbd6a5d7 100644 --- a/module/plugins/crypter/TurbobitNetFolder.py +++ b/module/plugins/crypter/TurbobitNetFolder.py @@ -12,8 +12,9 @@ class TurbobitNetFolder(SimpleCrypter): __version__ = "0.05" __pattern__ = r'http://(?:www\.)?turbobit\.net/download/folder/(?P<ID>\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Turbobit.net folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/TusfilesNetFolder.py b/module/plugins/crypter/TusfilesNetFolder.py index 20bed0ab1..1cc505b81 100644 --- a/module/plugins/crypter/TusfilesNetFolder.py +++ b/module/plugins/crypter/TusfilesNetFolder.py @@ -2,7 +2,7 @@ import math import re -from urlparse import urljoin +import urlparse from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo @@ -13,8 +13,8 @@ class TusfilesNetFolder(XFSCrypter): __version__ = "0.08" __pattern__ = r'https?://(?:www\.)?tusfiles\.net/go/(?P<ID>\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Tusfiles.net folder decrypter plugin""" __license__ = "GPLv3" @@ -22,15 +22,13 @@ class TusfilesNetFolder(XFSCrypter): ("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "tusfiles.net" - PAGES_PATTERN = r'>\((\d+) \w+\)<' URL_REPLACEMENTS = [(__pattern__ + ".*", r'https://www.tusfiles.net/go/\g<ID>/')] def loadPage(self, page_n): - return self.load(urljoin(self.pyfile.url, str(page_n)), decode=True) + return self.load(urlparse.urljoin(self.pyfile.url, str(page_n)), decode=True) def handlePages(self, pyfile): diff --git a/module/plugins/crypter/UlozToFolder.py b/module/plugins/crypter/UlozToFolder.py index a1f3ed5ea..3f2a60cf7 100644 --- a/module/plugins/crypter/UlozToFolder.py +++ b/module/plugins/crypter/UlozToFolder.py @@ -10,8 +10,8 @@ class UlozToFolder(Crypter): __version__ = "0.20" __pattern__ = r'http://(?:www\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj\.cz|zachowajto\.pl)/(m|soubory)/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Uloz.to folder decrypter plugin""" __license__ = "GPLv3" @@ -19,8 +19,8 @@ class UlozToFolder(Crypter): FOLDER_PATTERN = r'<ul class="profile_files">(.*?)</ul>' - LINK_PATTERN = r'<br /><a href="/([^"]+)">[^<]+</a>' - NEXT_PAGE_PATTERN = r'<a class="next " href="/([^"]+)"> </a>' + LINK_PATTERN = r'<br /><a href="/(.+?)">.+?</a>' + NEXT_PAGE_PATTERN = r'<a class="next " href="/(.+?)"> </a>' def decrypt(self, pyfile): diff --git a/module/plugins/crypter/UploadableChFolder.py b/module/plugins/crypter/UploadableChFolder.py index 22f9ca2ed..f1eb93323 100644 --- a/module/plugins/crypter/UploadableChFolder.py +++ b/module/plugins/crypter/UploadableChFolder.py @@ -9,8 +9,9 @@ class UploadableChFolder(SimpleCrypter): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?uploadable\.ch/list/\w+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """ Uploadable.ch folder decrypter plugin """ __license__ = "GPLv3" diff --git a/module/plugins/crypter/UploadedToFolder.py b/module/plugins/crypter/UploadedToFolder.py index 0a71add70..d54ed47eb 100644 --- a/module/plugins/crypter/UploadedToFolder.py +++ b/module/plugins/crypter/UploadedToFolder.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -13,8 +12,9 @@ class UploadedToFolder(SimpleCrypter): __version__ = "0.42" __pattern__ = r'http://(?:www\.)?(uploaded|ul)\.(to|net)/(f|folder|list)/(?P<ID>\w+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """UploadedTo decrypter plugin""" __license__ = "GPLv3" @@ -30,7 +30,7 @@ class UploadedToFolder(SimpleCrypter): if m is None: self.error(_("PLAIN_PATTERN not found")) - plain_link = urljoin("http://uploaded.net/", m.group(1)) + plain_link = urlparse.urljoin("http://uploaded.net/", m.group(1)) return self.load(plain_link).split('\n')[:-1] diff --git a/module/plugins/crypter/WiiReloadedOrg.py b/module/plugins/crypter/WiiReloadedOrg.py index c3c5b8222..eddc0e5d1 100644 --- a/module/plugins/crypter/WiiReloadedOrg.py +++ b/module/plugins/crypter/WiiReloadedOrg.py @@ -9,7 +9,7 @@ class WiiReloadedOrg(DeadCrypter): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?wii-reloaded\.org/protect/get\.php\?i=.+' - __config__ = [] + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Wii-Reloaded.org decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/WuploadComFolder.py b/module/plugins/crypter/WuploadComFolder.py index 873c71fad..dece467a2 100644 --- a/module/plugins/crypter/WuploadComFolder.py +++ b/module/plugins/crypter/WuploadComFolder.py @@ -9,6 +9,7 @@ class WuploadComFolder(DeadCrypter): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?wupload\.com/folder/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Wupload.com folder decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/XFileSharingProFolder.py b/module/plugins/crypter/XFileSharingProFolder.py index 55cd41931..2e1a2f022 100644 --- a/module/plugins/crypter/XFileSharingProFolder.py +++ b/module/plugins/crypter/XFileSharingProFolder.py @@ -8,11 +8,11 @@ from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo class XFileSharingProFolder(XFSCrypter): __name__ = "XFileSharingProFolder" __type__ = "crypter" - __version__ = "0.04" + __version__ = "0.06" - __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:user|folder)s?/\w+' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """XFileSharingPro dummy folder decrypter plugin for hook""" __license__ = "GPLv3" @@ -20,7 +20,7 @@ class XFileSharingProFolder(XFSCrypter): def _log(self, type, args): - msg = " | ".join([str(a).strip() for a in args if a]) + msg = " | ".join(str(a).strip() for a in args if a) logger = getattr(self.log, type) logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) @@ -31,10 +31,7 @@ class XFileSharingProFolder(XFSCrypter): self.__pattern__ = self.core.pluginManager.crypterPlugins[self.__name__]['pattern'] self.HOSTER_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() - self.HOSTER_NAME = "".join([part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.']) - - if self.HOSTER_NAME[0].isdigit(): - self.HOSTER_NAME = 'X' + self.HOSTER_NAME + self.HOSTER_NAME = "".join(part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.') account = self.core.accountManager.getAccountPlugin(self.HOSTER_NAME) diff --git a/module/plugins/crypter/XupPl.py b/module/plugins/crypter/XupPl.py index f2ecbee5a..5ab6750c4 100644 --- a/module/plugins/crypter/XupPl.py +++ b/module/plugins/crypter/XupPl.py @@ -9,8 +9,8 @@ class XupPl(Crypter): __version__ = "0.10" __pattern__ = r'https?://(?:[^/]*\.)?xup\.pl/.+' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Xup.pl decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/crypter/YoutubeBatch.py b/module/plugins/crypter/YoutubeComFolder.py index 73ebf5fb3..a2b02b2f1 100644 --- a/module/plugins/crypter/YoutubeBatch.py +++ b/module/plugins/crypter/YoutubeComFolder.py @@ -1,25 +1,24 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.common.json_layer import json_loads from module.plugins.Crypter import Crypter from module.utils import save_join -class YoutubeBatch(Crypter): - __name__ = "YoutubeBatch" +class YoutubeComFolder(Crypter): + __name__ = "YoutubeComFolder" __type__ = "crypter" __version__ = "1.01" __pattern__ = r'https?://(?:www\.|m\.)?youtube\.com/(?P<TYPE>user|playlist|view_play_list)(/|.*?[?&](?:list|p)=)(?P<ID>[\w-]+)' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True), - ("likes", "bool", "Grab user (channel) liked videos", False), - ("favorites", "bool", "Grab user (channel) favorite videos", False), - ("uploads", "bool", "Grab channel unplaylisted videos", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True ), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True ), + ("likes" , "bool", "Grab user (channel) liked videos" , False), + ("favorites" , "bool", "Grab user (channel) favorite videos", False), + ("uploads" , "bool", "Grab channel unplaylisted videos" , True )] __description__ = """Youtube.com channel & playlist decrypter plugin""" __license__ = "GPLv3" @@ -31,9 +30,9 @@ class YoutubeBatch(Crypter): def api_response(self, ref, req): req.update({"key": self.API_KEY}) - url = urljoin("https://www.googleapis.com/youtube/v3/", ref) - page = self.load(url, get=req) - return json_loads(page) + url = urlparse.urljoin("https://www.googleapis.com/youtube/v3/", ref) + html = self.load(url, get=req) + return json_loads(html) def getChannel(self, user): diff --git a/module/plugins/hooks/AlldebridCom.py b/module/plugins/hooks/AlldebridComHook.py index dddd97c0c..092921134 100644 --- a/module/plugins/hooks/AlldebridCom.py +++ b/module/plugins/hooks/AlldebridComHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class AlldebridCom(MultiHook): - __name__ = "AlldebridCom" +class AlldebridComHook(MultiHook): + __name__ = "AlldebridComHook" __type__ = "hook" __version__ = "0.16" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 ), ("ssl" , "bool" , "Use HTTPS" , True )] @@ -23,7 +21,7 @@ class AlldebridCom(MultiHook): def getHosters(self): - https = "https" if self.getConfig("ssl") else "http" - page = self.getURL(https + "://www.alldebrid.com/api.php", get={'action': "get_host"}).replace("\"", "").strip() + https = "https" if self.getConfig('ssl') else "http" + html = self.getURL(https + "://www.alldebrid.com/api.php", get={'action': "get_host"}).replace("\"", "").strip() - return [x.strip() for x in page.split(",") if x.strip()] + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/module/plugins/hooks/AndroidPhoneNotify.py b/module/plugins/hooks/AndroidPhoneNotify.py index 18e1cce66..a8a1cff72 100644 --- a/module/plugins/hooks/AndroidPhoneNotify.py +++ b/module/plugins/hooks/AndroidPhoneNotify.py @@ -1,59 +1,80 @@ # -*- coding: utf-8 -*- -from time import time +import time from module.network.RequestFactory import getURL -from module.plugins.Hook import Hook +from module.plugins.Hook import Hook, Expose class AndroidPhoneNotify(Hook): __name__ = "AndroidPhoneNotify" __type__ = "hook" - __version__ = "0.02" + __version__ = "0.08" __config__ = [("apikey" , "str" , "API key" , "" ), ("notifycaptcha" , "bool", "Notify captcha request" , True ), ("notifypackage" , "bool", "Notify package finished" , True ), - ("notifyprocessed", "bool", "Notify processed packages status" , True ), - ("timeout" , "int" , "Timeout between captchas in seconds" , 5 ), - ("force" , "bool", "Send notifications if client is connected", False)] - - __description__ = """Send push notifications to your Android Phone using notifymyandroid.com""" + ("notifyprocessed", "bool", "Notify packages processed" , True ), + ("notifyupdate" , "bool", "Notify plugin updates" , True ), + ("notifyexit" , "bool", "Notify pyLoad shutdown" , True ), + ("sendtimewait" , "int" , "Timewait in seconds between notifications", 5 ), + ("sendpermin" , "int" , "Max notifications per minute" , 12 ), + ("ignoreclient" , "bool", "Send notifications if client is connected", False)] + + __description__ = """Send push notifications to your Android Phone (using notifymyandroid.com)""" __license__ = "GPLv3" - __authors__ = [("Steven Kosyra", "steven.kosyra@gmail.com"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - event_list = ["allDownloadsProcessed"] + __authors__ = [("Steven Kosyra" , "steven.kosyra@gmail.com"), + ("Walter Purcaro", "vuolter@gmail.com" )] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 def setup(self): self.info = {} #@TODO: Remove in 0.4.10 + self.event_list = ["allDownloadsProcessed", "plugin_updated"] + + self.last_notify = 0 + self.notifications = 0 + + + def plugin_updated(self, type_plugins): + if not self.getConfig('notifyupdate'): + return + + self.notify(_("Plugins updated"), str(type_plugins)) + + + def coreReady(self): + self.key = self.getConfig('apikey') - def newCaptchaTask(self, task): - if not self.getConfig("notifycaptcha"): - return False - if time() - float(self.getStorage("AndroidPhoneNotify", 0)) < self.getConf("timeout"): - return False + def coreExiting(self): + if not self.getConfig('notifyexit'): + return + + if self.core.do_restart: + self.notify(_("Restarting pyLoad")) + else: + self.notify(_("Exiting pyLoad")) + + + def newCaptchaTask(self, task): + if not self.getConfig('notifycaptcha'): + return self.notify(_("Captcha"), _("New request waiting user input")) def packageFinished(self, pypack): - if self.getConfig("notifypackage"): + if self.getConfig('notifypackage'): self.notify(_("Package finished"), pypack.name) - def allDownloadsProcessed(self, thread): - if not self.getConfig("notifyprocessed"): - return False + def allDownloadsProcessed(self): + if not self.getConfig('notifyprocessed'): + return if any(True for pdata in self.core.api.getQueue() if pdata.linksdone < pdata.linkstotal): self.notify(_("Package failed"), _("One or more packages was not completed successfully")) @@ -61,19 +82,37 @@ class AndroidPhoneNotify(Hook): self.notify(_("All packages finished")) - def notify(self, event, msg=""): - apikey = self.getConfig("apikey") + @Expose + def notify(self, + event, + msg="", + key=None): - if not apikey: - return False + key = key or self.key + if not key: + return - if self.core.isClientConnected() and not self.getConfig("force"): - return False + if self.core.isClientConnected() and not self.getConfig('ignoreclient'): + return + + elapsed_time = time.time() - self.last_notify + + if elapsed_time < self.getConf("sendtimewait"): + return + + if elapsed_time > 60: + self.notifications = 0 + + elif self.notifications >= self.getConf("sendpermin"): + return getURL("http://www.notifymyandroid.com/publicapi/notify", - get={'apikey' : apikey, + get={'apikey' : key, 'application': "pyLoad", 'event' : event, 'description': msg}) - self.setStorage("AndroidPhoneNotify", time()) + self.last_notify = time.time() + self.notifications += 1 + + return True diff --git a/module/plugins/hooks/AntiVirus.py b/module/plugins/hooks/AntiVirus.py new file mode 100644 index 000000000..c620f556d --- /dev/null +++ b/module/plugins/hooks/AntiVirus.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +import os +import shutil +import subprocess + +try: + import send2trash +except ImportError: + pass + +from module.plugins.Hook import Hook, Expose, threaded +from module.utils import fs_encode, save_join + + +class AntiVirus(Hook): + __name__ = "AntiVirus" + __type__ = "hook" + __version__ = "0.09" + + #@TODO: add trash option (use Send2Trash lib) + __config__ = [("action" , "Antivirus default;Delete;Quarantine", "Manage infected files" , "Antivirus default"), + ("quardir" , "folder" , "Quarantine folder" , "" ), + ("deltotrash", "bool" , "Move to trash (recycle bin) instead delete", True ), + ("scanfailed", "bool" , "Scan incompleted files (failed downloads)" , False ), + ("cmdfile" , "file" , "Antivirus executable" , "" ), + ("cmdargs" , "str" , "Scan options" , "" ), + ("ignore-err", "bool" , "Ignore scan errors" , False )] + + __description__ = """Scan downloaded files with antivirus program""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + + + @Expose + @threaded + def scan(self, pyfile, thread): + file = fs_encode(pyfile.plugin.lastDownload) + filename = os.path.basename(pyfile.plugin.lastDownload) + cmdfile = fs_encode(self.getConfig('cmdfile')) + cmdargs = fs_encode(self.getConfig('cmdargs').strip()) + + if not os.path.isfile(file) or not os.path.isfile(cmdfile): + return + + thread.addActive(pyfile) + pyfile.setCustomStatus(_("virus scanning")) + pyfile.setProgress(0) + + try: + p = subprocess.Popen([cmdfile, cmdargs, file], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + out, err = map(str.strip, p.communicate()) + + if out: + self.logInfo(filename, out) + + if err: + self.logWarning(filename, err) + if not self.getConfig('ignore-err'): + self.logDebug("Delete/Quarantine task is aborted") + return + + if p.returncode: + pyfile.error = _("infected file") + action = self.getConfig('action') + try: + if action == "Delete": + if not self.getConfig('deltotrash'): + os.remove(file) + + else: + try: + send2trash.send2trash(file) + + except NameError: + self.logWarning(_("Send2Trash lib not found, moving to quarantine instead")) + pyfile.setCustomStatus(_("file moving")) + shutil.move(file, self.getConfig('quardir')) + + except Exception, e: + self.logWarning(_("Unable to move file to trash: %s, moving to quarantine instead") % e.message) + pyfile.setCustomStatus(_("file moving")) + shutil.move(file, self.getConfig('quardir')) + + else: + self.logDebug(_("Successfully moved file to trash")) + + elif action == "Quarantine": + pyfile.setCustomStatus(_("file moving")) + shutil.move(file, self.getConfig('quardir')) + + except (IOError, shutil.Error), e: + self.logError(filename, action + " action failed!", e) + + elif not out and not err: + self.logDebug(filename, "No infected file found") + + finally: + pyfile.setProgress(100) + thread.finishFile(pyfile) + + + def downloadFinished(self, pyfile): + return self.scan(pyfile) + + + def downloadFailed(self, pyfile): + #: Check if pyfile is still "failed", + # maybe might has been restarted in meantime + if pyfile.status == 8 and self.getConfig('scanfailed'): + return self.scan(pyfile) diff --git a/module/plugins/hooks/BypassCaptcha.py b/module/plugins/hooks/BypassCaptcha.py index 8e0d9d36e..9e01edfa2 100644 --- a/module/plugins/hooks/BypassCaptcha.py +++ b/module/plugins/hooks/BypassCaptcha.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pycurl import FORM_FILE, LOW_SPEED_TIME +import pycurl from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL, getRequest @@ -35,11 +35,13 @@ class BypassCaptcha(Hook): __description__ = """Send captchas to BypassCaptcha.com""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("Godofdream", "soilfcition@gmail.com"), - ("zoidberg", "zoidberg@mujmail.cz")] + ("zoidberg" , "zoidberg@mujmail.cz" )] + interval = 0 #@TODO: Remove in 0.4.10 + PYLOAD_KEY = "4f771155b640970d5607f919a615bdefc67e7d32" SUBMIT_URL = "http://bypasscaptcha.com/upload.php" @@ -47,19 +49,14 @@ class BypassCaptcha(Hook): GETCREDITS_URL = "http://bypasscaptcha.com/ex_left.php" - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def setup(self): self.info = {} #@TODO: Remove in 0.4.10 def getCredits(self): - res = getURL(self.GETCREDITS_URL, post={"key": self.getConfig("passkey")}) + res = getURL(self.GETCREDITS_URL, post={"key": self.getConfig('passkey')}) - data = dict([x.split(' ', 1) for x in res.splitlines()]) + data = dict(x.split(' ', 1) for x in res.splitlines()) return int(data['Left']) @@ -67,19 +64,19 @@ class BypassCaptcha(Hook): req = getRequest() #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) + req.c.setopt(pycurl.LOW_SPEED_TIME, 80) try: res = req.load(self.SUBMIT_URL, post={'vendor_key': self.PYLOAD_KEY, - 'key': self.getConfig("passkey"), + 'key': self.getConfig('passkey'), 'gen_task_id': "1", - 'file': (FORM_FILE, captcha)}, + 'file': (pycurl.FORM_FILE, captcha)}, multipart=True) finally: req.close() - data = dict([x.split(' ', 1) for x in res.splitlines()]) + data = dict(x.split(' ', 1) for x in res.splitlines()) if not data or "Value" not in data: raise BypassCaptchaException(res) @@ -92,7 +89,7 @@ class BypassCaptcha(Hook): def respond(self, ticket, success): try: - res = getURL(self.RESPOND_URL, post={"task_id": ticket, "key": self.getConfig("passkey"), + res = getURL(self.RESPOND_URL, post={"task_id": ticket, "key": self.getConfig('passkey'), "cv": 1 if success else 0}) except BadHeader, e: self.logError(_("Could not send response"), e) @@ -105,10 +102,10 @@ class BypassCaptcha(Hook): if not task.isTextual(): return False - if not self.getConfig("passkey"): + if not self.getConfig('passkey'): return False - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return False if self.getCredits() > 0: diff --git a/module/plugins/hooks/Captcha9kw.py b/module/plugins/hooks/Captcha9Kw.py index 600694e78..60482b8fc 100755..100644 --- a/module/plugins/hooks/Captcha9kw.py +++ b/module/plugins/hooks/Captcha9Kw.py @@ -3,9 +3,9 @@ from __future__ import with_statement import re +import time from base64 import b64encode -from time import sleep from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL @@ -16,10 +16,9 @@ from module.plugins.Hook import Hook, threaded class Captcha9Kw(Hook): __name__ = "Captcha9Kw" __type__ = "hook" - __version__ = "0.27" + __version__ = "0.28" - __config__ = [("activated" , "bool" , "Activated" , True ), - ("ssl" , "bool" , "Use HTTPS" , True ), + __config__ = [("ssl" , "bool" , "Use HTTPS" , True ), ("force" , "bool" , "Force captcha resolving even if client is connected" , True ), ("confirm" , "bool" , "Confirm Captcha (cost +6 credits)" , False ), ("captchaperhour", "int" , "Captcha per hour" , "9999" ), @@ -33,27 +32,24 @@ class Captcha9Kw(Hook): __description__ = """Send captchas to 9kw.eu""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("Walter Purcaro", "vuolter@gmail.com")] - API_URL = "http://www.9kw.eu/index.cgi" - + interval = 0 #@TODO: Remove in 0.4.10 - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + API_URL = "http://www.9kw.eu/index.cgi" def setup(self): self.info = {} #@TODO: Remove in 0.4.10 - if self.getConfig("ssl"): + if self.getConfig('ssl'): self.API_URL = self.API_URL.replace("http://", "https://") def getCredits(self): res = getURL(self.API_URL, - get={'apikey': self.getConfig("passkey"), + get={'apikey': self.getConfig('passkey'), 'pyload': "1", 'source': "pyload", 'action': "usercaptchaguthaben"}) @@ -77,24 +73,21 @@ class Captcha9Kw(Hook): self.logError(e) return - data = b64encode(data) - mouse = 1 if task.isPositional() else 0 pluginname = re.search(r'_([^_]*)_\d+.\w+', task.captchaFile).group(1) - - option = {'min' : 2, - 'max' : 50, - 'phrase' : 0, - 'numeric' : 0, - 'case_sensitive': 0, - 'math' : 0, - 'prio' : min(max(self.getConfig("prio"), 0), 10), - 'confirm' : self.getConfig("confirm"), - 'timeout' : min(max(self.getConfig("timeout"), 300), 3999), - 'selfsolve' : self.getConfig("selfsolve"), - 'cph' : self.getConfig("captchaperhour"), - 'cpm' : self.getConfig("captchapermin")} - - for opt in str(self.getConfig("hoster_options").split('|')): + option = {'min' : 2, + 'max' : 50, + 'phrase' : 0, + 'numeric' : 0, + 'case_sensitive': 0, + 'math' : 0, + 'prio' : min(max(self.getConfig('prio'), 0), 10), + 'confirm' : self.getConfig('confirm'), + 'timeout' : min(max(self.getConfig('timeout'), 300), 3999), + 'selfsolve' : self.getConfig('selfsolve'), + 'cph' : self.getConfig('captchaperhour'), + 'cpm' : self.getConfig('captchapermin')} + + for opt in str(self.getConfig('hoster_options').split('|')): details = map(str.strip, opt.split(':')) @@ -113,7 +106,7 @@ class Captcha9Kw(Hook): break - post_data = {'apikey' : self.getConfig("passkey"), + post_data = {'apikey' : self.getConfig('passkey'), 'prio' : option['prio'], 'confirm' : option['confirm'], 'maxtimeout' : option['timeout'], @@ -130,15 +123,15 @@ class Captcha9Kw(Hook): 'pyload' : "1", 'source' : "pyload", 'base64' : "1", - 'mouse' : mouse, - 'file-upload-01': data, + 'mouse' : 1 if task.isPositional() else 0, + 'file-upload-01': b64encode(data), 'action' : "usercaptchaupload"} for _i in xrange(5): try: res = getURL(self.API_URL, post=post_data) except BadHeader, e: - sleep(3) + time.sleep(3) else: if res and res.isdigit(): break @@ -150,9 +143,9 @@ class Captcha9Kw(Hook): task.data["ticket"] = res - for _i in xrange(int(self.getConfig("timeout") / 5)): + for _i in xrange(int(self.getConfig('timeout') / 5)): result = getURL(self.API_URL, - get={'apikey': self.getConfig("passkey"), + get={'apikey': self.getConfig('passkey'), 'id' : res, 'pyload': "1", 'info' : "1", @@ -160,7 +153,7 @@ class Captcha9Kw(Hook): 'action': "usercaptchacorrectdata"}) if not result or result == "NO DATA": - sleep(5) + time.sleep(5) else: break else: @@ -176,10 +169,10 @@ class Captcha9Kw(Hook): if not task.isTextual() and not task.isPositional(): return - if not self.getConfig("passkey"): + if not self.getConfig('passkey'): return - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return credits = self.getCredits() @@ -188,8 +181,8 @@ class Captcha9Kw(Hook): self.logError(_("Your captcha 9kw.eu account has not enough credits")) return - queue = min(self.getConfig("queue"), 999) - timeout = min(max(self.getConfig("timeout"), 300), 3999) + queue = min(self.getConfig('queue'), 999) + timeout = min(max(self.getConfig('timeout'), 300), 3999) pluginname = re.search(r'_([^_]*)_\d+.\w+', task.captchaFile).group(1) for _i in xrange(5): @@ -197,11 +190,11 @@ class Captcha9Kw(Hook): if queue < re.search(r'queue=(\d+)', servercheck).group(1): break - sleep(10) + time.sleep(10) else: self.fail(_("Too many captchas in queue")) - for opt in str(self.getConfig("hoster_options").split('|')): + for opt in str(self.getConfig('hoster_options').split('|')): details = map(str.strip, opt.split(':')) if not details or details[0].lower() != pluginname.lower(): @@ -210,9 +203,9 @@ class Captcha9Kw(Hook): for d in details: hosteroption = d.split("=") - if (len(hosteroption) > 1 - and hosteroption[0].lower() == 'timeout' - and hosteroption[1].isdigit()): + if len(hosteroption) > 1 \ + and hosteroption[0].lower() == 'timeout' \ + and hosteroption[1].isdigit(): timeout = int(hosteroption[1]) break @@ -231,7 +224,7 @@ class Captcha9Kw(Hook): self.logDebug("No CaptchaID for %s request (task: %s)" % (type, task)) return - passkey = self.getConfig("passkey") + passkey = self.getConfig('passkey') for _i in xrange(3): res = getURL(self.API_URL, @@ -248,7 +241,7 @@ class Captcha9Kw(Hook): if res == "OK": break - sleep(5) + time.sleep(5) else: self.logDebug("Could not send %s request: %s" % (type, res)) diff --git a/module/plugins/hooks/CaptchaBrotherhood.py b/module/plugins/hooks/CaptchaBrotherhood.py index 3c08f5e36..bda080037 100644 --- a/module/plugins/hooks/CaptchaBrotherhood.py +++ b/module/plugins/hooks/CaptchaBrotherhood.py @@ -4,15 +4,14 @@ from __future__ import with_statement import StringIO import pycurl +import time +import urllib try: from PIL import Image except ImportError: import Image -from time import sleep -from urllib import urlencode - from module.network.RequestFactory import getURL, getRequest from module.plugins.Hook import Hook, threaded @@ -46,16 +45,13 @@ class CaptchaBrotherhood(Hook): __description__ = """Send captchas to CaptchaBrotherhood.com""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("zoidberg", "zoidberg@mujmail.cz")] - API_URL = "http://www.captchabrotherhood.com/" - + interval = 0 #@TODO: Remove in 0.4.10 - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + API_URL = "http://www.captchabrotherhood.com/" def setup(self): @@ -64,7 +60,7 @@ class CaptchaBrotherhood(Hook): def getCredits(self): res = getURL(self.API_URL + "askCredits.aspx", - get={"username": self.getConfig("username"), "password": self.getConfig("passkey")}) + get={"username": self.getConfig('username'), "password": self.getConfig('passkey')}) if not res.startswith("OK"): raise CaptchaBrotherhoodException(res) else: @@ -93,10 +89,10 @@ class CaptchaBrotherhood(Hook): req = getRequest() url = "%ssendNewCaptcha.aspx?%s" % (self.API_URL, - urlencode({'username' : self.getConfig("username"), - 'password' : self.getConfig("passkey"), - 'captchaSource': "pyLoad", - 'timeout' : "80"})) + urllib.urlencode({'username' : self.getConfig('username'), + 'password' : self.getConfig('passkey'), + 'captchaSource': "pyLoad", + 'timeout' : "80"})) req.c.setopt(pycurl.URL, url) req.c.setopt(pycurl.POST, 1) @@ -117,7 +113,7 @@ class CaptchaBrotherhood(Hook): ticket = res[3:] for _i in xrange(15): - sleep(5) + time.sleep(5) res = self.api_response("askCaptchaResult", ticket) if res.startswith("OK-answered"): return ticket, res[12:] @@ -127,8 +123,8 @@ class CaptchaBrotherhood(Hook): def api_response(self, api, ticket): res = getURL("%s%s.aspx" % (self.API_URL, api), - get={"username": self.getConfig("username"), - "password": self.getConfig("passkey"), + get={"username": self.getConfig('username'), + "password": self.getConfig('passkey'), "captchaID": ticket}) if not res.startswith("OK"): raise CaptchaBrotherhoodException("Unknown response: %s" % res) @@ -143,10 +139,10 @@ class CaptchaBrotherhood(Hook): if not task.isTextual(): return False - if not self.getConfig("username") or not self.getConfig("passkey"): + if not self.getConfig('username') or not self.getConfig('passkey'): return False - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return False if self.getCredits() > 10: diff --git a/module/plugins/hooks/Checksum.py b/module/plugins/hooks/Checksum.py index 8d9f8f981..ab7daf701 100644 --- a/module/plugins/hooks/Checksum.py +++ b/module/plugins/hooks/Checksum.py @@ -3,12 +3,10 @@ from __future__ import with_statement import hashlib +import os import re import zlib -from os import remove -from os.path import getsize, isfile, splitext - from module.plugins.Hook import Hook from module.utils import save_join, fs_encode @@ -40,7 +38,7 @@ def computeChecksum(local_file, algorithm): class Checksum(Hook): __name__ = "Checksum" __type__ = "hook" - __version__ = "0.15" + __version__ = "0.16" __config__ = [("check_checksum", "bool", "Check checksum? (If False only size will be verified)", True), ("check_action", "fail;retry;nothing", "What to do if check fails?", "retry"), @@ -50,32 +48,33 @@ class Checksum(Hook): __description__ = """Verify downloaded file size and checksum""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com"), - ("stickell", "l.stickell@yahoo.it")] - + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com" ), + ("stickell" , "l.stickell@yahoo.it")] - methods = {'sfv': 'crc32', 'crc': 'crc32', 'hash': 'md5'} - regexps = {'sfv': r'^(?P<NAME>[^;].+)\s+(?P<HASH>[0-9A-Fa-f]{8})$', - 'md5': r'^(?P<NAME>[0-9A-Fa-f]{32}) (?P<FILE>.+)$', - 'crc': r'filename=(?P<NAME>.+)\nsize=(?P<SIZE>\d+)\ncrc32=(?P<HASH>[0-9A-Fa-f]{8})$', - 'default': r'^(?P<HASH>[0-9A-Fa-f]+)\s+\*?(?P<NAME>.+)$'} - - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 + methods = {'sfv' : 'crc32', + 'crc' : 'crc32', + 'hash': 'md5'} + regexps = {'sfv' : r'^(?P<NAME>[^;].+)\s+(?P<HASH>[0-9A-Fa-f]{8})$', + 'md5' : r'^(?P<NAME>[0-9A-Fa-f]{32}) (?P<FILE>.+)$', + 'crc' : r'filename=(?P<NAME>.+)\nsize=(?P<SIZE>\d+)\ncrc32=(?P<HASH>[0-9A-Fa-f]{8})$', + 'default': r'^(?P<HASH>[0-9A-Fa-f]+)\s+\*?(?P<NAME>.+)$'} def coreReady(self): - if not self.getConfig("check_checksum"): + if not self.getConfig('check_checksum'): self.logInfo(_("Checksum validation is disabled in plugin configuration")) def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 self.algorithms = sorted( getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")), reverse=True) + self.algorithms.extend(["crc32", "adler32"]) + self.formats = self.algorithms + ["sfv", "crc", "hash"] @@ -92,8 +91,9 @@ class Checksum(Hook): elif hasattr(pyfile.plugin, "api_data") and isinstance(pyfile.plugin.api_data, dict): data = pyfile.plugin.api_data.copy() - # elif hasattr(pyfile.plugin, "info") and isinstance(pyfile.plugin.info, dict): - # data = pyfile.plugin.info.copy() + elif hasattr(pyfile.plugin, "info") and isinstance(pyfile.plugin.info, dict): + data = pyfile.plugin.info.copy() + data.pop('size', None) #@NOTE: Don't check file size until a similary matcher will be implemented else: return @@ -107,22 +107,28 @@ class Checksum(Hook): #download_folder = self.config['general']['download_folder'] #local_file = fs_encode(save_join(download_folder, pyfile.package().folder, pyfile.name)) - if not isfile(local_file): + if not os.path.isfile(local_file): self.checkFailed(pyfile, None, "File does not exist") - # validate file size + # validate file size if "size" in data: - api_size = int(data['size']) - file_size = getsize(local_file) + api_size = int(data['size']) + file_size = os.path.getsize(local_file) + if api_size != file_size: self.logWarning(_("File %s has incorrect size: %d B (%d expected)") % (pyfile.name, file_size, api_size)) self.checkFailed(pyfile, local_file, "Incorrect file size") - del data['size'] + + data.pop('size', None) # validate checksum - if data and self.getConfig("check_checksum"): - if "checksum" in data: - data['md5'] = data['checksum'] + if data and self.getConfig('check_checksum'): + + if not 'md5' in data: + for type in ("checksum", "hashsum", "hash"): + if type in data: + data['md5'] = data[type] #@NOTE: What happens if it's not an md5 hash? + break for key in self.algorithms: if key in data: @@ -143,14 +149,14 @@ class Checksum(Hook): def checkFailed(self, pyfile, local_file, msg): - check_action = self.getConfig("check_action") + check_action = self.getConfig('check_action') if check_action == "retry": - max_tries = self.getConfig("max_tries") - retry_action = self.getConfig("retry_action") + max_tries = self.getConfig('max_tries') + retry_action = self.getConfig('retry_action') if pyfile.plugin.retries < max_tries: if local_file: - remove(local_file) - pyfile.plugin.retry(max_tries, self.getConfig("wait_time"), msg) + os.remove(local_file) + pyfile.plugin.retry(max_tries, self.getConfig('wait_time'), msg) elif retry_action == "nothing": return elif check_action == "nothing": @@ -162,13 +168,13 @@ class Checksum(Hook): download_folder = save_join(self.config['general']['download_folder'], pypack.folder, "") for link in pypack.getChildren().itervalues(): - file_type = splitext(link['name'])[1][1:].lower() + file_type = os.path.splitext(link['name'])[1][1:].lower() if file_type not in self.formats: continue hash_file = fs_encode(save_join(download_folder, link['name'])) - if not isfile(hash_file): + if not os.path.isfile(hash_file): self.logWarning(_("File not found"), link['name']) continue diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index 8ef31ec1e..a7862045e 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -1,43 +1,59 @@ # -*- coding: utf-8 -*- import socket +import time -from threading import Thread, Lock +try: + import ssl +except ImportError: + pass + +from threading import Lock from module.plugins.Hook import Hook, threaded def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - destination.shutdown(socket.SHUT_WR) - - + try: + bufsize = 1024 + bufdata = source.recv(bufsize) + while bufdata: + destination.sendall(bufdata) + bufdata = source.recv(bufsize) + finally: + destination.shutdown(socket.SHUT_WR) + # destination.close() + + +#@TODO: IPv6 support class ClickAndLoad(Hook): __name__ = "ClickAndLoad" __type__ = "hook" - __version__ = "0.25" + __version__ = "0.43" - __config__ = [("activated", "bool", "Activated" , True ), - ("port" , "int" , "Port" , 9666 ), - ("extern" , "bool", "Listen for requests coming from WAN (internet)", False)] + __config__ = [("activated", "bool", "Activated" , True), + ("port" , "int" , "Port" , 9666), + ("extern" , "bool", "Listen on the public network interface", True)] - __description__ = """Click'N'Load hook plugin""" + __description__ = """Click'n'Load hook plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.de"), + __authors__ = [("RaNaN" , "RaNaN@pyload.de" ), ("Walter Purcaro", "vuolter@gmail.com")] + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + + def coreReady(self): if not self.config['webinterface']['activated']: return - ip = "0.0.0.0" if self.getConfig("extern") else "127.0.0.1" - webport = int(self.config['webinterface']['port']) + ip = "" if self.getConfig('extern') else "127.0.0.1" + webport = self.config['webinterface']['port'] cnlport = self.getConfig('port') self.proxy(ip, webport, cnlport) @@ -45,37 +61,54 @@ class ClickAndLoad(Hook): @threaded def proxy(self, ip, webport, cnlport): - hookManager.startThread(self.server, ip, webport, cnlport) + time.sleep(10) #@TODO: Remove in 0.4.10 (implement addon delay on startup) + + self.logInfo(_("Proxy listening on %s:%s") % (ip or "0.0.0.0", cnlport)) + + self._server(ip, webport, cnlport) + lock = Lock() lock.acquire() lock.acquire() - def server(self, ip, webport, cnlport): + @threaded + def _server(self, ip, webport, cnlport): try: dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dock_socket.bind((ip, cnlport)) dock_socket.listen(5) while True: - server_socket = dock_socket.accept()[0] - client_socket = socket.create_connection(("127.0.0.1", webport)) + client_socket, client_addr = dock_socket.accept() + self.logDebug("Connection from %s:%s" % client_addr) + + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + if self.config['webinterface']['https']: + try: + server_socket = ssl.wrap_socket(server_socket) + + except NameError: + self.logError(_("pyLoad's webinterface is configured to use HTTPS, Please install python's ssl lib or disable HTTPS")) + client_socket.close() #: reset the connection. + continue + + except Exception, e: + self.logError(_("SSL error: %s") % e.message) + client_socket.close() #: reset the connection. + continue + + server_socket.connect(("127.0.0.1", webport)) + + self.manager.startThread(forward, client_socket, server_socket) + self.manager.startThread(forward, server_socket, client_socket) - hookManager.startThread(forward, server_socket, client_socket) - hookManager.startThread(forward, client_socket, server_socket) + except socket.timeout: + self.logDebug("Connection timed out, retrying...") + return self._server(ip, webport, cnlport) except socket.error, e: - if hasattr(e, "errno"): - errno = e.errno - else: - errno = e.args[0] - - if errno == 98: - self.logWarning(_("Port %s already in use") % cnlport) - else: - self.logError(e) - self.server(ip, webport, cnlport) - - except Exception, e: self.logError(e) - self.server(ip, webport, cnlport) + time.sleep(240) + return self._server(ip, webport, cnlport) diff --git a/module/plugins/hooks/DeathByCaptcha.py b/module/plugins/hooks/DeathByCaptcha.py index 050b6fe15..f9618f011 100644 --- a/module/plugins/hooks/DeathByCaptcha.py +++ b/module/plugins/hooks/DeathByCaptcha.py @@ -2,11 +2,11 @@ from __future__ import with_statement +import pycurl import re +import time from base64 import b64encode -from pycurl import FORM_FILE, HTTPHEADER -from time import sleep from module.common.json_layer import json_loads from module.network.HTTPRequest import BadHeader @@ -59,16 +59,13 @@ class DeathByCaptcha(Hook): __description__ = """Send captchas to DeathByCaptcha.com""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("zoidberg", "zoidberg@mujmail.cz")] - API_URL = "http://api.dbcapi.me/api/" - + interval = 0 #@TODO: Remove in 0.4.10 - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + API_URL = "http://api.dbcapi.me/api/" def setup(self): @@ -77,13 +74,13 @@ class DeathByCaptcha(Hook): def api_response(self, api="captcha", post=False, multipart=False): req = getRequest() - req.c.setopt(HTTPHEADER, ["Accept: application/json", "User-Agent: pyLoad %s" % self.core.version]) + req.c.setopt(pycurl.HTTPHEADER, ["Accept: application/json", "User-Agent: pyLoad %s" % self.core.version]) if post: if not isinstance(post, dict): post = {} - post.update({"username": self.getConfig("username"), - "password": self.getConfig("passkey")}) + post.update({"username": self.getConfig('username'), + "password": self.getConfig('passkey')}) res = None try: @@ -135,10 +132,10 @@ class DeathByCaptcha(Hook): def submit(self, captcha, captchaType="file", match=None): - #workaround multipart-post bug in HTTPRequest.py - if re.match("^\w*$", self.getConfig("passkey")): + #@NOTE: Workaround multipart-post bug in HTTPRequest.py + if re.match("^\w*$", self.getConfig('passkey')): multipart = True - data = (FORM_FILE, captcha) + data = (pycurl.FORM_FILE, captcha) else: multipart = False with open(captcha, 'rb') as f: @@ -152,7 +149,7 @@ class DeathByCaptcha(Hook): ticket = res['captcha'] for _i in xrange(24): - sleep(5) + time.sleep(5) res = self.api_response("captcha/%d" % ticket, False) if res['text'] and res['is_correct']: break @@ -172,10 +169,10 @@ class DeathByCaptcha(Hook): if not task.isTextual(): return False - if not self.getConfig("username") or not self.getConfig("passkey"): + if not self.getConfig('username') or not self.getConfig('passkey'): return False - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return False try: diff --git a/module/plugins/hooks/DebridItaliaCom.py b/module/plugins/hooks/DebridItaliaComHook.py index 719f3dd3a..36b307696 100644 --- a/module/plugins/hooks/DebridItaliaCom.py +++ b/module/plugins/hooks/DebridItaliaComHook.py @@ -5,23 +5,21 @@ import re from module.plugins.internal.MultiHook import MultiHook -class DebridItaliaCom(MultiHook): - __name__ = "DebridItaliaCom" +class DebridItaliaComHook(MultiHook): + __name__ = "DebridItaliaComHook" __type__ = "hook" __version__ = "0.12" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] __description__ = """Debriditalia.com hook plugin""" __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("stickell" , "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com" )] def getHosters(self): diff --git a/module/plugins/hooks/DeleteFinished.py b/module/plugins/hooks/DeleteFinished.py index 5d2b78d50..be17c3c0f 100644 --- a/module/plugins/hooks/DeleteFinished.py +++ b/module/plugins/hooks/DeleteFinished.py @@ -7,21 +7,26 @@ from module.plugins.Hook import Hook class DeleteFinished(Hook): __name__ = "DeleteFinished" __type__ = "hook" - __version__ = "1.11" + __version__ = "1.12" - __config__ = [('activated', 'bool', 'Activated', 'False'), - ('interval', 'int', 'Delete every (hours)', '72'), - ('deloffline', 'bool', 'Delete packages with offline links', 'False')] + __config__ = [("interval" , "int" , "Check interval in hours" , 72 ), + ("deloffline", "bool", "Delete package with offline links", False)] __description__ = """Automatically delete all finished packages from queue""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - # event_list = ["pluginConfigChanged"] + MIN_CHECK_INTERVAL = 1 * 60 * 60 #: 1 hour ## overwritten methods ## + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + # self.event_list = ["pluginConfigChanged"] + self.interval = self.MIN_CHECK_INTERVAL + + def periodical(self): if not self.info['sleep']: deloffline = self.getConfig('deloffline') @@ -33,20 +38,21 @@ class DeleteFinished(Hook): self.addEvent('packageFinished', self.wakeup) - def pluginConfigChanged(self, plugin, name, value): - if name == "interval" and value != self.interval: - self.interval = value * 3600 - self.initPeriodical() + # def pluginConfigChanged(self, plugin, name, value): + # if name == "interval" and value != self.interval: + # self.interval = value * 3600 + # self.initPeriodical() def unload(self): - self.removeEvent('packageFinished', self.wakeup) + self.manager.removeEvent('packageFinished', self.wakeup) def coreReady(self): - self.info = {'sleep': True} - interval = self.getConfig('interval') - self.pluginConfigChanged(self.__name__, 'interval', interval) + self.info['sleep'] = True + # interval = self.getConfig('interval') + # self.pluginConfigChanged(self.__name__, 'interval', interval) + self.interval = max(self.MIN_CHECK_INTERVAL, self.getConfig('interval') * 60 * 60) self.addEvent('packageFinished', self.wakeup) @@ -58,22 +64,17 @@ class DeleteFinished(Hook): def wakeup(self, pypack): - self.removeEvent('packageFinished', self.wakeup) + self.manager.removeEvent('packageFinished', self.wakeup) self.info['sleep'] = False ## event managing ## def addEvent(self, event, func): """Adds an event listener for event name""" - if event in self.m.events: - if func in self.m.events[event]: + if event in self.manager.events: + if func in self.manager.events[event]: self.logDebug("Function already registered", func) else: - self.m.events[event].append(func) + self.manager.events[event].append(func) else: - self.m.events[event] = [func] - - - def setup(self): - self.m = self.manager - self.removeEvent = self.m.removeEvent + self.manager.events[event] = [func] diff --git a/module/plugins/hooks/DownloadScheduler.py b/module/plugins/hooks/DownloadScheduler.py index 4996e212d..9c74ac1d4 100644 --- a/module/plugins/hooks/DownloadScheduler.py +++ b/module/plugins/hooks/DownloadScheduler.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import localtime +import time from module.plugins.Hook import Hook @@ -22,13 +21,12 @@ class DownloadScheduler(Hook): ("stickell", "l.stickell@yahoo.it")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 def setup(self): - self.cb = None # callback to scheduler job; will be by removed hookmanager when hook unloaded + self.info = {} #@TODO: Remove in 0.4.10 + self.cb = None # callback to scheduler job; will be by removed hookmanager when hook unloaded def coreReady(self): @@ -37,7 +35,7 @@ class DownloadScheduler(Hook): def updateSchedule(self, schedule=None): if schedule is None: - schedule = self.getConfig("timetable") + schedule = self.getConfig('timetable') schedule = re.findall("(\d{1,2}):(\d{2})[\s]*(-?\d+)", schedule.lower().replace("full", "-1").replace("none", "0")) @@ -45,7 +43,7 @@ class DownloadScheduler(Hook): self.logError(_("Invalid schedule")) return - t0 = localtime() + t0 = time.localtime() now = (t0.tm_hour, t0.tm_min, t0.tm_sec, "X") schedule = sorted([(int(x[0]), int(x[1]), 0, int(x[2])) for x in schedule] + [now]) @@ -65,7 +63,7 @@ class DownloadScheduler(Hook): def setDownloadSpeed(self, speed): if speed == 0: - abort = self.getConfig("abort") + abort = self.getConfig('abort') self.logInfo(_("Stopping download server. (Running downloads will %sbe aborted.)") % '' if abort else _('not ')) self.core.api.pauseServer() if abort: diff --git a/module/plugins/hooks/EasybytezCom.py b/module/plugins/hooks/EasybytezComHook.py index e08127514..43efb5048 100644 --- a/module/plugins/hooks/EasybytezCom.py +++ b/module/plugins/hooks/EasybytezComHook.py @@ -5,16 +5,14 @@ import re from module.plugins.internal.MultiHook import MultiHook -class EasybytezCom(MultiHook): - __name__ = "EasybytezCom" +class EasybytezComHook(MultiHook): + __name__ = "EasybytezComHook" __type__ = "hook" __version__ = "0.07" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -27,6 +25,6 @@ class EasybytezCom(MultiHook): user, data = self.account.selectAccount() req = self.account.getAccountRequest(user) - page = req.load("http://www.easybytez.com") + html = req.load("http://www.easybytez.com") - return re.search(r'</textarea>\s*Supported sites:(.*)', page).group(1).split(',') + return re.search(r'</textarea>\s*Supported sites:(.*)', html).group(1).split(',') diff --git a/module/plugins/hooks/ExpertDecoders.py b/module/plugins/hooks/ExpertDecoders.py index c80401003..ccd48f664 100644 --- a/module/plugins/hooks/ExpertDecoders.py +++ b/module/plugins/hooks/ExpertDecoders.py @@ -2,9 +2,10 @@ from __future__ import with_statement +import pycurl +import uuid + from base64 import b64encode -from pycurl import LOW_SPEED_TIME -from uuid import uuid4 from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL, getRequest @@ -21,16 +22,13 @@ class ExpertDecoders(Hook): __description__ = """Send captchas to expertdecoders.com""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("zoidberg", "zoidberg@mujmail.cz")] - API_URL = "http://www.fasttypers.org/imagepost.ashx" - + interval = 0 #@TODO: Remove in 0.4.10 - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + API_URL = "http://www.fasttypers.org/imagepost.ashx" def setup(self): @@ -38,7 +36,7 @@ class ExpertDecoders(Hook): def getCredits(self): - res = getURL(self.API_URL, post={"key": self.getConfig("passkey"), "action": "balance"}) + res = getURL(self.API_URL, post={"key": self.getConfig('passkey'), "action": "balance"}) if res.isdigit(): self.logInfo(_("%s credits left") % res) @@ -51,20 +49,22 @@ class ExpertDecoders(Hook): @threaded def _processCaptcha(self, task): - task.data['ticket'] = ticket = uuid4() + task.data['ticket'] = ticket = uuid.uuid4() result = None with open(task.captchaFile, 'rb') as f: data = f.read() - data = b64encode(data) req = getRequest() #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) + req.c.setopt(pycurl.LOW_SPEED_TIME, 80) try: - result = req.load(self.API_URL, post={"action": "upload", "key": self.getConfig("passkey"), - "file": data, "gen_task_id": ticket}) + result = req.load(self.API_URL, + post={'action' : "upload", + 'key' : self.getConfig('passkey'), + 'file' : b64encode(data), + 'gen_task_id': ticket}) finally: req.close() @@ -76,10 +76,10 @@ class ExpertDecoders(Hook): if not task.isTextual(): return False - if not self.getConfig("passkey"): + if not self.getConfig('passkey'): return False - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return False if self.getCredits() > 0: @@ -96,7 +96,7 @@ class ExpertDecoders(Hook): try: res = getURL(self.API_URL, - post={'action': "refund", 'key': self.getConfig("passkey"), 'gen_task_id': task.data['ticket']}) + post={'action': "refund", 'key': self.getConfig('passkey'), 'gen_task_id': task.data['ticket']}) self.logInfo(_("Request refund"), res) except BadHeader, e: diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py index fc0cae44f..a0815499b 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/hooks/ExternalScripts.py @@ -1,146 +1,218 @@ # -*- coding: utf-8 -*- +import os import subprocess -from itertools import chain -from os import listdir, access, X_OK, makedirs -from os.path import join, exists, basename, abspath - from module.plugins.Hook import Hook -from module.utils import save_join +from module.utils import fs_encode, save_join class ExternalScripts(Hook): __name__ = "ExternalScripts" __type__ = "hook" - __version__ = "0.26" + __version__ = "0.39" - __config__ = [("activated", "bool", "Activated", True)] + __config__ = [("activated", "bool", "Activated" , True ), + ("waitend" , "bool", "Wait script ending", False)] __description__ = """Run external scripts""" __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de"), - ("RaNaN", "ranan@pyload.org"), - ("spoob", "spoob@pyload.org"), + __authors__ = [("mkaay" , "mkaay@mkaay.de" ), + ("RaNaN" , "ranan@pyload.org" ), + ("spoob" , "spoob@pyload.org" ), ("Walter Purcaro", "vuolter@gmail.com")] - event_list = ["archive_extracted", "package_extracted", "all_archives_extracted", "all_archives_processed", - "allDownloadsFinished", "allDownloadsProcessed"] - - - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 def setup(self): + self.info = {'oldip': None} self.scripts = {} - folders = ["download_preparing", "download_finished", "all_downloads_finished", "all_downloads_processed", + self.event_list = ["archive_extract_failed", "archive_extracted" , + "package_extract_failed", "package_extracted" , + "all_archives_extracted", "all_archives_processed", + "allDownloadsFinished" , "allDownloadsProcessed" , + "packageDeleted"] + + folders = ["pyload_start", "pyload_restart", "pyload_stop", "before_reconnect", "after_reconnect", - "package_finished", "package_extracted", - "archive_extracted", "all_archives_extracted", "all_archives_processed", - # deprecated folders - "unrar_finished", "all_dls_finished", "all_dls_processed"] + "download_preparing", "download_failed", "download_finished", + "archive_extract_failed", "archive_extracted", + "package_finished", "package_deleted", "package_extract_failed", "package_extracted", + "all_downloads_processed", "all_downloads_finished", #@TODO: Invert `all_downloads_processed`, `all_downloads_finished` order in 0.4.10 + "all_archives_extracted", "all_archives_processed"] for folder in folders: self.scripts[folder] = [] - - self.initPluginType(folder, join(pypath, 'scripts', folder)) - self.initPluginType(folder, join('scripts', folder)) + for dir in (pypath, ''): + self.initPluginType(folder, os.path.join(dir, 'scripts', folder)) for script_type, names in self.scripts.iteritems(): if names: - self.logInfo(_("Installed scripts for"), script_type, ", ".join([basename(x) for x in names])) + self.logInfo(_("Installed scripts for: ") + script_type, ", ".join(map(os.path.basename, names))) + + self.pyload_start() - def initPluginType(self, folder, path): - if not exists(path): + def initPluginType(self, name, dir): + if not os.path.isdir(dir): try: - makedirs(path) - except: - self.logDebug("Script folder %s not created" % folder) + os.makedirs(dir) + + except OSError, e: + self.logDebug(e) return - for f in listdir(path): - if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): + for filename in os.listdir(dir): + file = save_join(dir, filename) + + if not os.path.isfile(file): + continue + + if filename[0] in ("#", "_") or filename.endswith("~") or filename.endswith(".swp"): continue - if not access(join(path, f), X_OK): - self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) + if not os.access(file, os.X_OK): + self.logWarning(_("Script not executable:") + " %s/%s" % (name, filename)) - self.scripts[folder].append(join(path, f)) + self.scripts[name].append(file) def callScript(self, script, *args): try: - cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args] - self.logDebug("Executing", abspath(script), " ".join(cmd)) - #output goes to pyload - subprocess.Popen(cmd, bufsize=-1) + cmd_args = [fs_encode(str(x) if not isinstance(x, basestring) else x) for x in args] + cmd = [script] + cmd_args + + self.logDebug("Executing: %s" % os.path.abspath(script), "Args: " + ' '.join(cmd_args)) + + p = subprocess.Popen(cmd, bufsize=-1) #@NOTE: output goes to pyload + if self.getConfig('waitend'): + p.communicate() + except Exception, e: - self.logError(_("Error in %(script)s: %(error)s") % {"script": basename(script), "error": e}) + try: + self.logError(_("Runtime error: %s") % os.path.abspath(script), e) + except Exception: + self.logError(_("Runtime error: %s") % os.path.abspath(script), _("Unknown error")) + + + def pyload_start(self): + for script in self.scripts['pyload_start']: + self.callScript(script) + + + def coreExiting(self): + for script in self.scripts['pyload_restart' if self.core.do_restart else 'pyload_stop']: + self.callScript(script) + + + def beforeReconnecting(self, ip): + for script in self.scripts['before_reconnect']: + self.callScript(script, ip) + self.info['oldip'] = ip + + + def afterReconnecting(self, ip): + for script in self.scripts['after_reconnect']: + self.callScript(script, ip, self.info['oldip']) #@TODO: Use built-in oldip in 0.4.10 def downloadPreparing(self, pyfile): for script in self.scripts['download_preparing']: - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id) + self.callScript(script, pyfile.id, pyfile.name, None, pyfile.pluginname, pyfile.url) + + + def downloadFailed(self, pyfile): + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pyfile.package().folder) + else: + download_folder = self.config['general']['download_folder'] + + for script in self.scripts['download_failed']: + file = save_join(download_folder, pyfile.name) + self.callScript(script, pyfile.id, pyfile.name, file, pyfile.pluginname, pyfile.url) def downloadFinished(self, pyfile): - download_folder = self.config['general']['download_folder'] + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pyfile.package().folder) + else: + download_folder = self.config['general']['download_folder'] + for script in self.scripts['download_finished']: - filename = save_join(download_folder, pyfile.package().folder, pyfile.name) - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, filename, pyfile.id) + file = save_join(download_folder, pyfile.name) + self.callScript(script, pyfile.id, pyfile.name, file, pyfile.pluginname, pyfile.url) + + + def archive_extract_failed(self, pyfile, archive): + for script in self.scripts['archive_extract_failed']: + self.callScript(script, pyfile.id, pyfile.name, archive.filename, archive.out, archive.files) + + + def archive_extracted(self, pyfile, archive): + for script in self.scripts['archive_extracted']: + self.callScript(script, pyfile.id, pyfile.name, archive.filename, archive.out, archive.files) def packageFinished(self, pypack): - download_folder = self.config['general']['download_folder'] + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pypack.folder) + else: + download_folder = self.config['general']['download_folder'] + for script in self.scripts['package_finished']: - folder = save_join(download_folder, pypack.folder) - self.callScript(script, pypack.name, folder, pypack.password, pypack.id) + self.callScript(script, pypack.id, pypack.name, download_folder, pypack.password) - def beforeReconnecting(self, ip): - for script in self.scripts['before_reconnect']: - self.callScript(script, ip) + def packageDeleted(self, pid): + pack = self.core.api.getPackageInfo(pid) + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pack.folder) + else: + download_folder = self.config['general']['download_folder'] - def afterReconnecting(self, ip): - for script in self.scripts['after_reconnect']: - self.callScript(script, ip) + for script in self.scripts['package_deleted']: + self.callScript(script, pack.id, pack.name, download_folder, pack.password) - def archive_extracted(self, pyfile, folder, filename, files): - for script in self.scripts['archive_extracted']: - self.callScript(script, folder, filename, files) - for script in self.scripts['unrar_finished']: #: deprecated - self.callScript(script, folder, filename) + def package_extract_failed(self, pypack): + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pypack.folder) + else: + download_folder = self.config['general']['download_folder'] + + for script in self.scripts['package_extract_failed']: + self.callScript(script, pypack.id, pypack.name, download_folder, pypack.password) def package_extracted(self, pypack): - download_folder = self.config['general']['download_folder'] + if self.config['general']['folder_per_package']: + download_folder = save_join(self.config['general']['download_folder'], pypack.folder) + else: + download_folder = self.config['general']['download_folder'] + for script in self.scripts['package_extracted']: - folder = save_join(download_folder, pypack.folder) - self.callScript(script, pypack.name, folder, pypack.password, pypack.id) + self.callScript(script, pypack.id, pypack.name, download_folder) - def all_archives_extracted(self): - for script in self.scripts['all_archives_extracted']: + def allDownloadsFinished(self): + for script in self.scripts['all_downloads_finished']: self.callScript(script) - def all_archives_processed(self): - for script in self.scripts['all_archives_processed']: + def allDownloadsProcessed(self): + for script in self.scripts['all_downloads_processed']: self.callScript(script) - def allDownloadsFinished(self, thread): - for script in chain(self.scripts['all_downloads_finished'], self.scripts['all_dls_finished']): + def all_archives_extracted(self): + for script in self.scripts['all_archives_extracted']: self.callScript(script) - def allDownloadsProcessed(self, thread): - for script in chain(self.scripts['all_downloads_processed'], self.scripts['all_dls_processed']): + def all_archives_processed(self): + for script in self.scripts['all_archives_processed']: self.callScript(script) diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 2e9efa2b0..3e371ec2b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -4,22 +4,19 @@ from __future__ import with_statement import os import sys - -from copy import copy -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir -from traceback import print_exc +import traceback # monkey patch bug in python 2.6 and lower # http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 if sys.version_info < (2, 7) and os.name != "nt": import errno - from subprocess import Popen + import subprocess def _eintr_retry_call(func, *args): while True: try: return func(*args) + except OSError, e: if e.errno == errno.EINTR: continue @@ -33,6 +30,7 @@ if sys.version_info < (2, 7) and os.name != "nt": if self.returncode is None: try: pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: if e.errno != errno.ECHILD: raise @@ -43,376 +41,488 @@ if sys.version_info < (2, 7) and os.name != "nt": self._handle_exitstatus(sts) return self.returncode - Popen.wait = wait + subprocess.Popen.wait = wait + +try: + import send2trash +except ImportError: + pass +from copy import copy if os.name != "nt": from grp import getgrnam - from os import chown from pwd import getpwnam -from module.plugins.Hook import Hook, threaded, Expose +from module.plugins.Hook import Hook, Expose, threaded from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError -from module.utils import save_join, uniqify +from module.plugins.internal.SimpleHoster import replace_patterns +from module.utils import fs_encode, save_join, uniqify + + +class ArchiveQueue(object): + + def __init__(self, plugin, storage): + self.plugin = plugin + self.storage = storage + + + def get(self): + try: + return [int(pid) for pid in self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split()] + except Exception: + return [] + + + def set(self, value): + if isinstance(value, list): + item = str(value)[1:-1].replace(' ', '').replace(',', ' ') + else: + item = str(value).strip() + return self.plugin.setStorage("ExtractArchive:%s" % self.storage, item.encode('base64')[:-1]) + + + def delete(self): + return self.plugin.delStorage("ExtractArchive:%s" % self.storage) + + + def add(self, item): + queue = self.get() + if item not in queue: + return self.set(queue + [item]) + else: + return True + + + def remove(self, item): + queue = self.get() + try: + queue.remove(item) + + except ValueError: + pass + + if queue == []: + return self.delete() + + return self.set(queue) class ExtractArchive(Hook): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.03" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("fullpath" , "bool" , "Extract full path" , True ), - ("overwrite" , "bool" , "Overwrite files" , False ), - ("keepbroken" , "bool" , "Extract broken archives" , False ), - ("repair" , "bool" , "Repair broken archives" , True ), - ("passwordfile" , "file" , "Store passwords in file" , "archive_password.txt" ), - ("delete" , "bool" , "Delete archive when successfully extracted", False ), - ("subfolder" , "bool" , "Create subfolder for each package" , False ), - ("destination" , "folder", "Extract files to" , "" ), - ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), - ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), - ("recursive" , "bool" , "Extract archives in archives" , True ), - ("queue" , "bool" , "Wait for all downloads to be finished" , False ), - ("renice" , "int" , "CPU Priority" , 0 )] + __version__ = "1.44" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives (RAR required)" , False ), + ("test" , "bool" , "Test archive before extracting" , False ), + ("usepasswordfile", "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive after extraction" , True ), + ("deltotrash" , "bool" , "Move to trash (recycle bin) instead delete", True ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder" , "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract archives ending with extension" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Run after all downloads was processed" , False ), + ("renice" , "int" , "CPU priority" , 0 )] __description__ = """Extract different kind of archives""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz" , "immenz@gmx.net" )] - event_list = ["allDownloadsProcessed"] + NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.event_list = ["allDownloadsProcessed","packageDeleted"] + self.queue = ArchiveQueue(self, "Queue") + self.failed = ArchiveQueue(self, "Failed") - def coreReady(self): - self.extracting = False - + self.interval = 60 + self.extracting = False + self.lastPackage = False + self.extractors = [] + self.passwords = [] + self.repair = False - def setup(self): - self.plugins = [] - self.passwords = [] - names = [] + def coreReady(self): for p in ("UnRar", "SevenZip", "UnZip"): try: module = self.core.pluginManager.loadModule("internal", p) - klass = getattr(module, p) - if klass.checkDeps(): - names.append(p) - self.plugins.append(klass) + klass = getattr(module, p) + if klass.isUsable(): + self.extractors.append(klass) + if klass.REPAIR: + self.repair = self.getConfig('repair') except OSError, e: if e.errno == 2: - self.logInfo(_("No %s installed") % p) + self.logWarning(_("No %s installed") % p) else: - self.logWarning(_("Could not activate %s") % p, e) + self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: - print_exc() + traceback.print_exc() except Exception, e: - self.logWarning(_("Could not activate %s") % p, e) + self.logWarning(_("Could not activate: %s") % p, e) if self.core.debug: - print_exc() + traceback.print_exc() - if names: - self.logInfo(_("Activated") + " " + " ".join(names)) + if self.extractors: + self.logDebug(*["Found %s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors]) + self.extractQueued() #: Resume unfinished extractions else: self.logInfo(_("No Extract plugins activated")) - # queue with package ids - self.queue = [] - - def periodical(self): - if not self.queue or self.extracting: + @threaded + def extractQueued(self, thread): + if self.extracting: #@NOTE: doing the check here for safty (called by coreReady) return - local = copy(self.queue) - self.queue[:] = [] + self.extracting = True - self.extractPackages(*local) + packages = self.queue.get() + while packages: + if self.lastPackage: #: called from allDownloadsProcessed + self.lastPackage = False + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + self.manager.dispatchEvent("all_archives_extracted") + self.manager.dispatchEvent("all_archives_processed") + else: + if self.extract(packages, thread): #@NOTE: check only if all gone fine, no failed reporting for now + pass + packages = self.queue.get() #: check for packages added during extraction - @Expose - def extractPackage(self, id): - """ Extract package wrapper""" - self.extractPackages(id) + self.extracting = False @Expose - def extractPackages(self, *ids): + def extractPackage(self, *ids): """ Extract packages with given id""" - self.manager.startThread(self.extract, ids) + for id in ids: + self.queue.add(id) + if not self.getConfig('waitall') and not self.extracting: + self.extractQueued() - def packageFinished(self, pypack): - if self.getConfig("queue") or self.extracting: - self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pypack.id) - else: - self.extractPackage(pypack.id) + def packageDeleted(self, pid): + self.queue.remove(pid) - @threaded - def allDownloadsProcessed(self, thread): - local = copy(self.queue) - self.queue[:] = [] + def packageFinished(self, pypack): + self.queue.add(pypack.id) + if not self.getConfig('waitall') and not self.extracting: + self.extractQueued() - if self.extract(local): #: check only if all gone fine, no failed reporting for now - self.manager.dispatchEvent("all_archives_extracted") - self.manager.dispatchEvent("all_archives_processed") + def allDownloadsProcessed(self): + self.lastPackage = True + if self.getConfig('waitall') and not self.extracting: + self.extractQueued() - def extract(self, ids): - self.extracting = True + @Expose + def extract(self, ids, thread=None): #@TODO: Use pypack, not pid to improve method usability + if not ids: + return False processed = [] extracted = [] failed = [] - clearlist = lambda string: [x.lstrip('.') for x in string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')] + toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') + + destination = self.getConfig('destination') + subfolder = self.getConfig('subfolder') + fullpath = self.getConfig('fullpath') + overwrite = self.getConfig('overwrite') + renice = self.getConfig('renice') + recursive = self.getConfig('recursive') + delete = self.getConfig('delete') + keepbroken = self.getConfig('keepbroken') - destination = self.getConfig("destination") - subfolder = self.getConfig("subfolder") - fullpath = self.getConfig("fullpath") - overwrite = self.getConfig("overwrite") - extensions = clearlist(self.getConfig("extensions")) - excludefiles = clearlist(self.getConfig("excludefiles")) - renice = self.getConfig("renice") - recursive = self.getConfig("recursive") - delete = self.getConfig("delete") - keepbroken = self.getConfig("keepbroken") + extensions = [x.lstrip('.').lower() for x in toList(self.getConfig('extensions'))] + excludefiles = toList(self.getConfig('excludefiles')) if extensions: - self.logDebug("Extensions: %s" % "|.".join(extensions)) + self.logDebug("Use for extensions: %s" % "|.".join(extensions)) # reload from txt file self.reloadPasswords() - # dl folder - dl = self.config['general']['download_folder'] + download_folder = self.config['general']['download_folder'] - #iterate packages -> plugins -> targets + # iterate packages -> extractors -> targets for pid in ids: - p = self.core.files.getPackage(pid) - self.logInfo(_("Check package: %s") % p.name) - if not p: + pypack = self.core.files.getPackage(pid) + + if not pypack: + self.queue.remove(pid) continue + self.logInfo(_("Check package: %s") % pypack.name) + # determine output folder - out = save_join(dl, p.folder, destination, "") #: force trailing slash + out = save_join(download_folder, pypack.folder, destination, "") #: force trailing slash if subfolder: - out = save_join(out, p.folder) + out = save_join(out, pypack.folder) - if not exists(out): - makedirs(out) + if not os.path.exists(out): + os.makedirs(out) - files_ids = [(save_join(dl, p.folder, x['name']), x['id']) for x in p.getChildren().itervalues()] matched = False success = True + files_ids = dict((pylink['name'],((save_join(download_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ + in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values() #: remove duplicates # check as long there are unseen files while files_ids: new_files_ids = [] if extensions: - files_ids = [(file, id) for file, id in files_ids if filter(lambda ext: file.endswith(ext), extensions)] - - for plugin in self.plugins: - targets = plugin.getTargets(files_ids) + files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ + if filter(lambda ext: fname.lower().endswith(ext), extensions)] + for Extractor in self.extractors: + targets = Extractor.getTargets(files_ids) if targets: - self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) + self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True - for target, fid in targets: - if target in processed: - self.logDebug(basename(target), "skipped") - continue + for fname, fid, fout in targets: + name = os.path.basename(fname) - processed.append(target) # prevent extracting same file twice + if not os.path.exists(fname): + self.logDebug(name, "File not found") + continue - self.logInfo(basename(target), _("Extract to: %s") % out) + self.logInfo(name, _("Extract to: %s") % fout) try: - klass = plugin(self, - target, - out, - p.password, - fullpath, - overwrite, - excludefiles, - renice, - delete, - keepbroken) - klass.init() - - new_files = self._extract(klass, fid) + pyfile = self.core.files.getFile(fid) + archive = Extractor(self, + fname, + fout, + fullpath, + overwrite, + excludefiles, + renice, + delete, + keepbroken, + fid) + + thread.addActive(pyfile) + archive.init() + + try: + new_files = self._extract(pyfile, archive, pypack.password) + + finally: + pyfile.setProgress(100) + thread.finishFile(pyfile) except Exception, e: - self.logError(basename(target), e) - new_files = None - - if new_files is None: - self.logWarning(basename(target), _("No files extracted")) + self.logError(name, e) success = False continue + # remove processed file and related multiparts from list + files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ + if fname not in archive.getDeleteFiles()] self.logDebug("Extracted files: %s" % new_files) self.setPermissions(new_files) - for file in new_files: - if not exists(file): - self.logDebug("New file %s does not exists" % file) + for filename in new_files: + file = fs_encode(save_join(os.path.dirname(archive.filename), filename)) + if not os.path.exists(file): + self.logDebug("New file %s does not exists" % filename) continue - if recursive and isfile(file): - new_files_ids.append((file, fid)) # append as new target - files_ids = new_files_ids # also check extracted files + if recursive and os.path.isfile(file): + new_files_ids.append((filename, fid, os.path.dirname(filename))) #: append as new target + + self.manager.dispatchEvent("archive_extracted", pyfile, archive) + + files_ids = new_files_ids #: also check extracted files if matched: if success: extracted.append(pid) - self.manager.dispatchEvent("package_extracted", p) + self.manager.dispatchEvent("package_extracted", pypack) + else: failed.append(pid) - self.manager.dispatchEvent("package_extract_failed", p) + self.manager.dispatchEvent("package_extract_failed", pypack) + + self.failed.add(pid) else: self.logInfo(_("No files found to extract")) if not matched or not success and subfolder: try: os.rmdir(out) + except OSError: pass - self.extracting = False + self.queue.remove(pid) + return True if not failed else False - def _extract(self, plugin, fid): - pyfile = self.core.files.getFile(fid) + def _extract(self, pyfile, archive, password): + name = os.path.basename(archive.filename) - pyfile.setCustomStatus(_("extracting")) + pyfile.setStatus("processing") + encrypted = False try: - progress = lambda x: pyfile.setProgress(x) - encrypted = False - passwords = self.getPasswords() - - try: - self.logInfo(basename(plugin.file), "Verifying...") + self.logDebug("Password: %s" % (password or "None provided")) + passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig('usepasswordfile') else [password] + for pw in passwords: + try: + if self.getConfig('test') or self.repair: + pyfile.setCustomStatus(_("archive testing")) + if pw: + self.logDebug("Testing with password: %s" % pw) + pyfile.setProgress(0) + archive.verify(pw) + pyfile.setProgress(100) + else: + archive.check(pw) - tmp_password = plugin.password - plugin.password = "" #: Force verifying without password + self.addPassword(pw) + break - plugin.verify() + except PasswordError: + if not encrypted: + self.logInfo(name, _("Password protected")) + encrypted = True - except PasswordError: - encrypted = True + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("CRC Error")) - except CRCError: - self.logWarning(basename(plugin.file), _("Archive damaged")) + if self.repair: + self.logWarning(name, _("Repairing...")) - if not self.getConfig("repair"): - raise CRCError + pyfile.setCustomStatus(_("archive repairing")) + pyfile.setProgress(0) + repaired = archive.repair() + pyfile.setProgress(100) - elif plugin.repair(): - self.logInfo(basename(plugin.file), _("Successfully repaired")) + if not repaired and not self.getConfig('keepbroken'): + raise CRCError("Archive damaged") - elif not self.getConfig("keepbroken"): - raise ArchiveError(_("Broken archive")) + self.addPassword(pw) + break - else: - self.logInfo(basename(plugin.file), _("All OK")) + raise CRCError("Archive damaged") - plugin.password = tmp_password + except ArchiveError, e: + raise ArchiveError(e) - if not encrypted: - plugin.extract(progress) + pyfile.setCustomStatus(_("extracting")) + pyfile.setProgress(0) + if not encrypted or not self.getConfig('usepasswordfile'): + self.logDebug("Extracting using password: %s" % (password or "None")) + archive.extract(password) else: - self.logInfo(basename(plugin.file), _("Password protected")) - - if plugin.password: - passwords.insert(0, plugin.password) - passwords = uniqify(self.passwords) - self.logDebug("Password: %s" % plugin.password) - else: - self.logDebug("No package password provided") - - for pw in passwords: + for pw in filter(None, uniqify([password] + self.getPasswords(False))): try: - self.logDebug("Try password: %s" % pw) + self.logDebug("Extracting using password: %s" % pw) - if plugin.setPassword(pw): - plugin.extract(progress) - self.addPassword(pw) - break - else: - raise PasswordError + archive.extract(pw) + self.addPassword(pw) + break except PasswordError: self.logDebug("Password was wrong") else: raise PasswordError - if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) - - if self.getConfig("delete"): - files = plugin.getDeleteFiles() - self.logInfo(_("Deleting %s files") % len(files)) - for f in files: - if exists(f): - remove(f) + pyfile.setProgress(100) + pyfile.setStatus("processing") + + delfiles = archive.getDeleteFiles() + self.logDebug("Would delete: " + ", ".join(delfiles)) + + if self.getConfig('delete'): + self.logInfo(_("Deleting %s files") % len(delfiles)) + + deltotrash = self.getConfig('deltotrash') + for f in delfiles: + file = fs_encode(f) + if not os.path.exists(file): + continue + + if not deltotrash: + os.remove(file) + else: - self.logDebug("%s does not exists" % f) + try: + send2trash.send2trash(file) - self.logInfo(basename(plugin.file), _("Extracting finished")) + except NameError: + self.logWarning(_("Unable to move %s to trash: Send2Trash lib not found") % os.path.basename(f)) - extracted_files = plugin.getExtractedFiles() - self.manager.dispatchEvent("archive_extracted", pyfile, plugin.out, plugin.file, extracted_files) + except Exception, e: + self.logWarning(_("Unable to move %s to trash: %s") % (os.path.basename(f), e.message)) + + else: + self.logDebug(_("Successfully moved %s to trash") % os.path.basename(f)) + + self.logInfo(name, _("Extracting finished")) + extracted_files = archive.files or archive.list() return extracted_files except PasswordError: - self.logError(basename(plugin.file), _("Wrong password" if passwords else "No password found")) - plugin.password = "" + self.logError(name, _("Wrong password" if password else "No password found")) - except CRCError: - self.logError(basename(plugin.file), _("CRC Mismatch")) + except CRCError, e: + self.logError(name, _("CRC mismatch"), e) except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), e) + self.logError(name, _("Archive error"), e) except Exception, e: + self.logError(name, _("Unknown error"), e) if self.core.debug: - print_exc() - self.logError(basename(plugin.file), _("Unknown Error"), e) + traceback.print_exc() - self.manager.dispatchEvent("archive_extract_failed", pyfile) + self.manager.dispatchEvent("archive_extract_failed", pyfile, archive) - self.logError(basename(plugin.file), _("Extract failed")) + raise Exception(_("Extract failed")) @Expose - def getPasswords(self): + def getPasswords(self, reload=True): """ List of saved passwords """ + if reload: + self.reloadPasswords() + return self.passwords def reloadPasswords(self): - passwordfile = self.getConfig("passwordfile") - try: passwords = [] - with open(passwordfile, "a+") as f: + + file = fs_encode(self.getConfig('passwordfile')) + with open(file) as f: for pw in f.read().splitlines(): passwords.append(pw) @@ -424,15 +534,13 @@ class ExtractArchive(Hook): @Expose - def addPassword(self, pw): + def addPassword(self, password): """ Adds a password to saved list""" - passwordfile = self.getConfig("passwordfile") - - self.passwords.insert(0, pw) - self.passwords = uniqify(self.passwords) - try: - with open(passwordfile, "wb") as f: + self.passwords = uniqify([password] + self.passwords) + + file = fs_encode(self.getConfig('passwordfile')) + with open(file, "wb") as f: for pw in self.passwords: f.write(pw + '\n') @@ -442,20 +550,21 @@ class ExtractArchive(Hook): def setPermissions(self, files): for f in files: - if not exists(f): + if not os.path.exists(f): continue try: if self.config['permission']['change_file']: - if isfile(f): - chmod(f, int(self.config['permission']['file'], 8)) - elif isdir(f): - chmod(f, int(self.config['permission']['folder'], 8)) + if os.path.isfile(f): + os.chmod(f, int(self.config['permission']['file'], 8)) + + elif os.path.isdir(f): + os.chmod(f, int(self.config['permission']['folder'], 8)) if self.config['permission']['change_dl'] and os.name != "nt": uid = getpwnam(self.config['permission']['user'])[2] gid = getgrnam(self.config['permission']['group'])[2] - chown(f, uid, gid) + os.chown(f, uid, gid) except Exception, e: self.logWarning(_("Setting User and Group failed"), e) diff --git a/module/plugins/hooks/FastixRu.py b/module/plugins/hooks/FastixRuHook.py index 25126fbd3..16e30e93a 100644 --- a/module/plugins/hooks/FastixRu.py +++ b/module/plugins/hooks/FastixRuHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class FastixRu(MultiHook): - __name__ = "FastixRu" +class FastixRuHook(MultiHook): + __name__ = "FastixRuHook" __type__ = "hook" __version__ = "0.05" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -23,9 +21,9 @@ class FastixRu(MultiHook): def getHosters(self): - page = self.getURL("http://fastix.ru/api_v2", + html = self.getURL("http://fastix.ru/api_v2", get={'apikey': "5182964c3f8f9a7f0b00000a_kelmFB4n1IrnCDYuIFn2y", 'sub' : "allowed_sources"}) - host_list = json_loads(page) + host_list = json_loads(html) host_list = host_list['allow'] return host_list diff --git a/module/plugins/hooks/FreeWayMe.py b/module/plugins/hooks/FreeWayMeHook.py index 6fec037d8..baea44540 100644 --- a/module/plugins/hooks/FreeWayMe.py +++ b/module/plugins/hooks/FreeWayMeHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class FreeWayMe(MultiHook): - __name__ = "FreeWayMe" +class FreeWayMeHook(MultiHook): + __name__ = "FreeWayMeHook" __type__ = "hook" - __version__ = "0.14" + __version__ = "0.15" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -22,6 +20,13 @@ class FreeWayMe(MultiHook): def getHosters(self): - hostis = self.getURL("https://www.free-way.me/ajax/jd.php", get={'id': 3}).replace("\"", "").strip() - self.logDebug("Hosters", hostis) + # Get account data + if not self.account or not self.account.canUse(): + hostis = self.getURL("https://www.free-way.me/ajax/jd.php", get={"id": 3}).replace("\"", "").strip() + else: + self.logDebug("AccountInfo available - Get HosterList with User Pass") + (user, data) = self.account.selectAccount() + hostis = self.getURL("https://www.free-way.me/ajax/jd.php", get={"id": 3, "user": user, "pass": data['password']}).replace("\"", "").strip() + + self.logDebug("hosters: %s" % hostis) return [x.strip() for x in hostis.split(",") if x.strip()] diff --git a/module/plugins/hooks/HotFolder.py b/module/plugins/hooks/HotFolder.py index b0b59e2ba..f771cf232 100644 --- a/module/plugins/hooks/HotFolder.py +++ b/module/plugins/hooks/HotFolder.py @@ -2,10 +2,9 @@ from __future__ import with_statement +import os import time -from os import listdir, makedirs -from os.path import exists, isfile, join from shutil import move from module.plugins.Hook import Hook @@ -15,12 +14,12 @@ from module.utils import fs_encode, save_join class HotFolder(Hook): __name__ = "HotFolder" __type__ = "hook" - __version__ = "0.12" + __version__ = "0.14" - __config__ = [("folder", "str", "Folder to observe", "container"), - ("watch_file", "bool", "Observe link file", False), - ("keep", "bool", "Keep added containers", True), - ("file", "str", "Link file", "links.txt")] + __config__ = [("folder" , "str" , "Folder to observe" , "container"), + ("watch_file", "bool", "Observe link file" , False ), + ("keep" , "bool", "Keep added containers", True ), + ("file" , "str" , "Link file" , "links.txt")] __description__ = """Observe folder and file for changes and add container and links""" __license__ = "GPLv3" @@ -28,39 +27,45 @@ class HotFolder(Hook): def setup(self): - self.interval = 10 + self.info = {} #@TODO: Remove in 0.4.10 + self.interval = 30 def periodical(self): - folder = fs_encode(self.getConfig("folder")) + folder = fs_encode(self.getConfig('folder')) + file = fs_encode(self.getConfig('file')) try: - if not exists(join(folder, "finished")): - makedirs(join(folder, "finished")) + if not os.path.isdir(os.path.join(folder, "finished")): + os.makedirs(os.path.join(folder, "finished")) - if self.getConfig("watch_file"): - with open(fs_encode(self.getConfig("file")), "a+") as f: + if self.getConfig('watch_file'): + with open(file, "a+") as f: + f.seek(0) content = f.read().strip() if content: - name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y")) + f = open(file, "wb") + f.close() + + name = "%s_%s.txt" % (file, time.strftime("%H-%M-%S_%d%b%Y")) with open(save_join(folder, "finished", name), "wb") as f: f.write(content) self.core.api.addPackage(f.name, [f.name], 1) - for f in listdir(folder): - path = join(folder, f) + for f in os.listdir(folder): + path = os.path.join(folder, f) - if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): + if not os.path.isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): continue - newpath = join(folder, "finished", f if self.getConfig("keep") else "tmp_" + f) + newpath = os.path.join(folder, "finished", f if self.getConfig('keep') else "tmp_" + f) move(path, newpath) self.logInfo(_("Added %s from HotFolder") % f) self.core.api.addPackage(f, [newpath], 1) - except IOError, e: + except (IOError, OSError), e: self.logError(e) diff --git a/module/plugins/hooks/IRCInterface.py b/module/plugins/hooks/IRCInterface.py index efd4e411d..33fde3d20 100644 --- a/module/plugins/hooks/IRCInterface.py +++ b/module/plugins/hooks/IRCInterface.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- +import pycurl import re import socket import ssl import time +import traceback -from pycurl import FORM_FILE from select import select from threading import Thread -from time import sleep -from traceback import print_exc from module.Api import PackageDoesNotExists, FileDoesNotExists from module.network.RequestFactory import getURL @@ -38,17 +37,15 @@ class IRCInterface(Thread, Hook): __authors__ = [("Jeix", "Jeix@hasnomail.com")] + interval = 0 #@TODO: Remove in 0.4.10 + + def __init__(self, core, manager): Thread.__init__(self) Hook.__init__(self, core, manager) self.setDaemon(True) - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def coreReady(self): self.abort = False self.more = [] @@ -59,30 +56,30 @@ class IRCInterface(Thread, Hook): def packageFinished(self, pypack): try: - if self.getConfig("info_pack"): + if self.getConfig('info_pack'): self.response(_("Package finished: %s") % pypack.name) - except: + except Exception: pass def downloadFinished(self, pyfile): try: - if self.getConfig("info_file"): + if self.getConfig('info_file'): self.response( _("Download finished: %(name)s @ %(plugin)s ") % {"name": pyfile.name, "plugin": pyfile.pluginname}) - except: + except Exception: pass def newCaptchaTask(self, task): - if self.getConfig("captcha") and task.isTextual(): + if self.getConfig('captcha') and task.isTextual(): task.handler.append(self) task.setWaiting(60) - page = getURL("http://www.freeimagehosting.net/upload.php", - post={"attached": (FORM_FILE, task.captchaFile)}, multipart=True) + html = getURL("http://www.freeimagehosting.net/upload.php", + post={"attached": (pycurl.FORM_FILE, task.captchaFile)}, multipart=True) - url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", page).group(1) + url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", html).group(1) self.response(_("New Captcha Request: %s") % url) self.response(_("Answer with 'c %s text on the captcha'") % task.id) @@ -90,16 +87,16 @@ class IRCInterface(Thread, Hook): def run(self): # connect to IRC etc. self.sock = socket.socket() - host = self.getConfig("host") - self.sock.connect((host, self.getConfig("port"))) + host = self.getConfig('host') + self.sock.connect((host, self.getConfig('port'))) - if self.getConfig("ssl"): + if self.getConfig('ssl'): self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE) #@TODO: support certificate - nick = self.getConfig("nick") + nick = self.getConfig('nick') self.sock.send("NICK %s\r\n" % nick) self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick)) - for t in self.getConfig("owner").split(): + for t in self.getConfig('owner').split(): if t.strip().startswith("#"): self.sock.send("JOIN %s\r\n" % t.strip()) self.logInfo(_("Connected to"), host) @@ -109,14 +106,14 @@ class IRCInterface(Thread, Hook): except IRCError, ex: self.sock.send("QUIT :byebye\r\n") - print_exc() + traceback.print_exc() self.sock.close() def main_loop(self): readbuffer = "" while True: - sleep(1) + time.sleep(1) fdset = select([self.sock], [], [], 0) if self.sock not in fdset[0]: continue @@ -153,10 +150,10 @@ class IRCInterface(Thread, Hook): def handle_events(self, msg): - if not msg['origin'].split("!", 1)[0] in self.getConfig("owner").split(): + if not msg['origin'].split("!", 1)[0] in self.getConfig('owner').split(): return - if msg['target'].split("!", 1)[0] != self.getConfig("nick"): + if msg['target'].split("!", 1)[0] != self.getConfig('nick'): return if msg['action'] != "PRIVMSG": @@ -183,7 +180,7 @@ class IRCInterface(Thread, Hook): trigger = temp[0] if len(temp) > 1: args = temp[1:] - except: + except Exception: pass handler = getattr(self, "event_%s" % trigger, self.event_pass) @@ -197,7 +194,7 @@ class IRCInterface(Thread, Hook): def response(self, msg, origin=""): if origin == "": - for t in self.getConfig("owner").split(): + for t in self.getConfig('owner').split(): self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg)) else: self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg)) @@ -347,7 +344,7 @@ class IRCInterface(Thread, Hook): return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack['name'], id)] - except: + except Exception: # create new package id = self.core.api.addPackage(pack, links, 1) return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))] diff --git a/module/plugins/hooks/ImageTyperz.py b/module/plugins/hooks/ImageTyperz.py index d448d1be9..f1fcacb71 100644 --- a/module/plugins/hooks/ImageTyperz.py +++ b/module/plugins/hooks/ImageTyperz.py @@ -2,10 +2,10 @@ from __future__ import with_statement +import pycurl import re from base64 import b64encode -from pycurl import FORM_FILE, LOW_SPEED_TIME from module.network.RequestFactory import getURL, getRequest from module.plugins.Hook import Hook, threaded @@ -40,20 +40,17 @@ class ImageTyperz(Hook): __description__ = """Send captchas to ImageTyperz.com""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("zoidberg", "zoidberg@mujmail.cz")] + interval = 0 #@TODO: Remove in 0.4.10 + SUBMIT_URL = "http://captchatypers.com/Forms/UploadFileAndGetTextNEW.ashx" RESPOND_URL = "http://captchatypers.com/Forms/SetBadImage.ashx" GETCREDITS_URL = "http://captchatypers.com/Forms/RequestBalance.ashx" - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - def setup(self): self.info = {} #@TODO: Remove in 0.4.10 @@ -61,15 +58,15 @@ class ImageTyperz(Hook): def getCredits(self): res = getURL(self.GETCREDITS_URL, post={'action': "REQUESTBALANCE", - 'username': self.getConfig("username"), - 'password': self.getConfig("passkey")}) + 'username': self.getConfig('username'), + 'password': self.getConfig('passkey')}) if res.startswith('ERROR'): raise ImageTyperzException(res) try: balance = float(res) - except: + except Exception: raise ImageTyperzException("Invalid response") self.logInfo(_("Account balance: $%s left") % res) @@ -79,13 +76,13 @@ class ImageTyperz(Hook): def submit(self, captcha, captchaType="file", match=None): req = getRequest() #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) + req.c.setopt(pycurl.LOW_SPEED_TIME, 80) try: - #workaround multipart-post bug in HTTPRequest.py - if re.match("^\w*$", self.getConfig("passkey")): + #@NOTE: Workaround multipart-post bug in HTTPRequest.py + if re.match("^\w*$", self.getConfig('passkey')): multipart = True - data = (FORM_FILE, captcha) + data = (pycurl.FORM_FILE, captcha) else: multipart = False with open(captcha, 'rb') as f: @@ -94,8 +91,8 @@ class ImageTyperz(Hook): res = req.load(self.SUBMIT_URL, post={'action': "UPLOADCAPTCHA", - 'username': self.getConfig("username"), - 'password': self.getConfig("passkey"), "file": data}, + 'username': self.getConfig('username'), + 'password': self.getConfig('passkey'), "file": data}, multipart=multipart) finally: req.close() @@ -119,10 +116,10 @@ class ImageTyperz(Hook): if not task.isTextual(): return False - if not self.getConfig("username") or not self.getConfig("passkey"): + if not self.getConfig('username') or not self.getConfig('passkey'): return False - if self.core.isClientConnected() and not self.getConfig("force"): + if self.core.isClientConnected() and not self.getConfig('force'): return False if self.getCredits() > 0: @@ -139,8 +136,8 @@ class ImageTyperz(Hook): if task.data['service'] == self.__name__ and "ticket" in task.data: res = getURL(self.RESPOND_URL, post={'action': "SETBADIMAGE", - 'username': self.getConfig("username"), - 'password': self.getConfig("passkey"), + 'username': self.getConfig('username'), + 'password': self.getConfig('passkey'), 'imageid': task.data['ticket']}) if res == "SUCCESS": diff --git a/module/plugins/hooks/JustPremium.py b/module/plugins/hooks/JustPremium.py new file mode 100644 index 000000000..f66747f82 --- /dev/null +++ b/module/plugins/hooks/JustPremium.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Hook import Hook + + +class JustPremium(Hook): + __name__ = "JustPremium" + __type__ = "hook" + __version__ = "0.22" + + __config__ = [("excluded", "str", "Exclude hosters (comma separated)", ""), + ("included", "str", "Include hosters (comma separated)", "")] + + __description__ = """Remove not-premium links from added urls""" + __license__ = "GPLv3" + __authors__ = [("mazleu" , "mazleica@gmail.com"), + ("Walter Purcaro", "vuolter@gmail.com" ), + ("immenz" , "immenz@gmx.net" )] + + + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.event_list = ["linksAdded"] + + + def linksAdded(self, links, pid): + hosterdict = self.core.pluginManager.hosterPlugins + linkdict = self.core.api.checkURLs(links) + + premiumplugins = set(account.type for account in self.core.api.getAccounts(False) \ + if account.valid and account.premium) + multihosters = set(hoster for hoster in self.core.pluginManager.hosterPlugins \ + if 'new_name' in hosterdict[hoster] \ + and hosterdict[hoster]['new_name'] in premiumplugins) + + excluded = map(lambda domain: "".join(part.capitalize() for part in re.split(r'(\.|\d+)', domain) if part != '.'), + self.getConfig('excluded').replace(' ', '').replace(',', '|').replace(';', '|').split('|')) + included = map(lambda domain: "".join(part.capitalize() for part in re.split(r'(\.|\d+)', domain) if part != '.'), + self.getConfig('included').replace(' ', '').replace(',', '|').replace(';', '|').split('|')) + + hosterlist = (premiumplugins | multihosters).union(excluded).difference(included) + + #: Found at least one hoster with account or multihoster + if not any( True for pluginname in linkdict if pluginname in hosterlist ): + return + + for pluginname in set(linkdict.keys()) - hosterlist: + self.logInfo(_("Remove links of plugin: %s") % pluginname) + for link in linkdict[pluginname]: + self.logDebug("Remove link: %s" % link) + links.remove(link) diff --git a/module/plugins/hooks/LinkdecrypterCom.py b/module/plugins/hooks/LinkdecrypterCom.py deleted file mode 100644 index d4924a687..000000000 --- a/module/plugins/hooks/LinkdecrypterCom.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.internal.MultiHook import MultiHook - - -class LinkdecrypterCom(MultiHook): - __name__ = "LinkdecrypterCom" - __type__ = "hook" - __version__ = "1.01" - - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] - - __description__ = """Linkdecrypter.com hook plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - def getCrypters(self): - try: - html = self.getURL("http://linkdecrypter.com/") - return re.search(r'>Supported\(\d+\)</b>: <i>(.+?) \+ RSDF', html).group(1).split(', ') - except Exception: - return list() diff --git a/module/plugins/hooks/LinkdecrypterComHook.py b/module/plugins/hooks/LinkdecrypterComHook.py new file mode 100644 index 000000000..b2eaece62 --- /dev/null +++ b/module/plugins/hooks/LinkdecrypterComHook.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.MultiHook import MultiHook + + +class LinkdecrypterComHook(MultiHook): + __name__ = "LinkdecrypterComHook" + __type__ = "hook" + __version__ = "1.06" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Linkdecrypter.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def getHosters(self): + list = re.search(r'>Supported\(\d+\)</b>: <i>(.[\w.\-, ]+)', + self.getURL("http://linkdecrypter.com/", decode=True).replace("(g)", "")).group(1).split(', ') + try: + list.remove("download.serienjunkies.org") + except ValueError: + pass + + return list diff --git a/module/plugins/hooks/LinksnappyCom.py b/module/plugins/hooks/LinksnappyComHook.py index 5eb0c7f6d..72282575b 100644 --- a/module/plugins/hooks/LinksnappyCom.py +++ b/module/plugins/hooks/LinksnappyComHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class LinksnappyCom(MultiHook): - __name__ = "LinksnappyCom" +class LinksnappyComHook(MultiHook): + __name__ = "LinksnappyComHook" __type__ = "hook" __version__ = "0.04" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/MegaDebridEu.py b/module/plugins/hooks/MegaDebridEuHook.py index f67fa7ac0..0de7b4dcf 100644 --- a/module/plugins/hooks/MegaDebridEu.py +++ b/module/plugins/hooks/MegaDebridEuHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class MegaDebridEu(MultiHook): - __name__ = "MegaDebridEu" +class MegaDebridEuHook(MultiHook): + __name__ = "MegaDebridEuHook" __type__ = "hook" __version__ = "0.05" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/MegaRapidoNetHook.py b/module/plugins/hooks/MegaRapidoNetHook.py new file mode 100644 index 000000000..e113b305e --- /dev/null +++ b/module/plugins/hooks/MegaRapidoNetHook.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.MultiHook import MultiHook + + +class MegaRapidoNetHook(MultiHook): + __name__ = "MegaRapidoNetHook" + __type__ = "hook" + __version__ = "0.02" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """MegaRapido.net hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + def getHosters(self): + hosters = {'1fichier' : [],#leave it there are so many possible addresses? + '1st-files' : ['1st-files.com'], + '2shared' : ['2shared.com'], + '4shared' : ['4shared.com', '4shared-china.com'], + 'asfile' : ['http://asfile.com/'], + 'bitshare' : ['bitshare.com'], + 'brupload' : ['brupload.net'], + 'crocko' : ['crocko.com','easy-share.com'], + 'dailymotion' : ['dailymotion.com'], + 'depfile' : ['depfile.com'], + 'depositfiles': ['depositfiles.com', 'dfiles.eu'], + 'dizzcloud' : ['dizzcloud.com'], + 'dl.dropbox' : [], + 'extabit' : ['extabit.com'], + 'extmatrix' : ['extmatrix.com'], + 'facebook' : [], + 'file4go' : ['file4go.com'], + 'filecloud' : ['filecloud.io','ifile.it','mihd.net'], + 'filefactory' : ['filefactory.com'], + 'fileom' : ['fileom.com'], + 'fileparadox' : ['fileparadox.in'], + 'filepost' : ['filepost.com', 'fp.io'], + 'filerio' : ['filerio.in','filerio.com','filekeen.com'], + 'filesflash' : ['filesflash.com'], + 'firedrive' : ['firedrive.com', 'putlocker.com'], + 'flashx' : [], + 'freakshare' : ['freakshare.net', 'freakshare.com'], + 'gigasize' : ['gigasize.com'], + 'hipfile' : ['hipfile.com'], + 'junocloud' : ['junocloud.me'], + 'letitbit' : ['letitbit.net','shareflare.net'], + 'mediafire' : ['mediafire.com'], + 'mega' : ['mega.co.nz'], + 'megashares' : ['megashares.com'], + 'metacafe' : ['metacafe.com'], + 'netload' : ['netload.in'], + 'oboom' : ['oboom.com'], + 'rapidgator' : ['rapidgator.net'], + 'rapidshare' : ['rapidshare.com'], + 'rarefile' : ['rarefile.net'], + 'ryushare' : ['ryushare.com'], + 'sendspace' : ['sendspace.com'], + 'turbobit' : ['turbobit.net', 'unextfiles.com'], + 'uploadable' : ['uploadable.ch'], + 'uploadbaz' : ['uploadbaz.com'], + 'uploaded' : ['uploaded.to', 'uploaded.net', 'ul.to'], + 'uploadhero' : ['uploadhero.com'], + 'uploading' : ['uploading.com'], + 'uptobox' : ['uptobox.com'], + 'xvideos' : ['xvideos.com'], + 'youtube' : ['youtube.com']} + + hoster_list = [] + + for item in hosters.itervalues(): + hoster_list.extend(item) + + return hoster_list diff --git a/module/plugins/hooks/MergeFiles.py b/module/plugins/hooks/MergeFiles.py index 4de45f958..941938920 100644 --- a/module/plugins/hooks/MergeFiles.py +++ b/module/plugins/hooks/MergeFiles.py @@ -4,17 +4,16 @@ from __future__ import with_statement import os import re - -from traceback import print_exc +import traceback from module.plugins.Hook import Hook, threaded -from module.utils import save_join, fs_encode +from module.utils import save_join class MergeFiles(Hook): __name__ = "MergeFiles" __type__ = "hook" - __version__ = "0.13" + __version__ = "0.14" __config__ = [("activated", "bool", "Activated", True)] @@ -23,17 +22,13 @@ class MergeFiles(Hook): __authors__ = [("and9000", "me@has-no-mail.com")] - BUFFER_SIZE = 4096 - + interval = 0 #@TODO: Remove in 0.4.10 - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + BUFFER_SIZE = 4096 def setup(self): - # nothing to do - pass + self.info = {} #@TODO: Remove in 0.4.10 @threaded @@ -65,7 +60,7 @@ class MergeFiles(Hook): pyfile.setStatus("processing") try: - with open(os.path.join(download_folder, splitted_file), "rb") as s_file: + with open(save_join(download_folder, splitted_file), "rb") as s_file: size_written = 0 s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file))) while True: @@ -79,7 +74,7 @@ class MergeFiles(Hook): self.logDebug("Finished merging part", splitted_file) except Exception, e: - print_exc() + traceback.print_exc() finally: pyfile.setProgress(100) diff --git a/module/plugins/hooks/MultiHome.py b/module/plugins/hooks/MultiHome.py index 105a42abd..a26d139c0 100644 --- a/module/plugins/hooks/MultiHome.py +++ b/module/plugins/hooks/MultiHome.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from time import time +import time from module.plugins.Hook import Hook @@ -17,22 +17,23 @@ class MultiHome(Hook): __authors__ = [("mkaay", "mkaay@mkaay.de")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 def setup(self): - self.register = {} + self.info = {} #@TODO: Remove in 0.4.10 + self.register = {} self.interfaces = [] - self.parseInterfaces(self.getConfig("interfaces").split(";")) + + self.parseInterfaces(self.getConfig('interfaces').split(";")) + if not self.interfaces: self.parseInterfaces([self.config['download']['interface']]) self.setConfig("interfaces", self.toConfig()) def toConfig(self): - return ";".join([i.adress for i in self.interfaces]) + return ";".join(i.adress for i in self.interfaces) def parseInterfaces(self, interfaces): @@ -80,7 +81,7 @@ class Interface(object): def useFor(self, pluginName, account): - self.history[(pluginName, account)] = time() + self.history[(pluginName, account)] = time.time() def __repr__(self): diff --git a/module/plugins/hooks/MultihostersCom.py b/module/plugins/hooks/MultihostersComHook.py index bf88cfae7..7b5e49c49 100644 --- a/module/plugins/hooks/MultihostersCom.py +++ b/module/plugins/hooks/MultihostersComHook.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from module.plugins.hooks.ZeveraCom import ZeveraCom +from module.plugins.hooks.ZeveraComHook import ZeveraComHook -class MultihostersCom(ZeveraCom): - __name__ = "MultihostersCom" +class MultihostersComHook(ZeveraComHook): + __name__ = "MultihostersComHook" __type__ = "hook" __version__ = "0.02" diff --git a/module/plugins/hooks/MultishareCz.py b/module/plugins/hooks/MultishareCzHook.py index 8349e0de8..6052b7673 100644 --- a/module/plugins/hooks/MultishareCz.py +++ b/module/plugins/hooks/MultishareCzHook.py @@ -5,16 +5,14 @@ import re from module.plugins.internal.MultiHook import MultiHook -class MultishareCz(MultiHook): - __name__ = "MultishareCz" +class MultishareCzHook(MultiHook): + __name__ = "MultishareCzHook" __type__ = "hook" __version__ = "0.07" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -23,9 +21,9 @@ class MultishareCz(MultiHook): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - HOSTER_PATTERN = r'<img class="logo-shareserveru"[^>]*?alt="([^"]+)"></td>\s*<td class="stav">[^>]*?alt="OK"' + HOSTER_PATTERN = r'<img class="logo-shareserveru"[^>]*?alt="(.+?)"></td>\s*<td class="stav">[^>]*?alt="OK"' def getHosters(self): - page = self.getURL("http://www.multishare.cz/monitoring/") - return re.findall(self.HOSTER_PATTERN, page) + html = self.getURL("http://www.multishare.cz/monitoring/") + return re.findall(self.HOSTER_PATTERN, html) diff --git a/module/plugins/hooks/MyfastfileCom.py b/module/plugins/hooks/MyfastfileComHook.py index 86408cb6d..20a1cfac2 100644 --- a/module/plugins/hooks/MyfastfileCom.py +++ b/module/plugins/hooks/MyfastfileComHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class MyfastfileCom(MultiHook): - __name__ = "MyfastfileCom" +class MyfastfileComHook(MultiHook): + __name__ = "MyfastfileComHook" __type__ = "hook" __version__ = "0.05" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/NoPremiumPl.py b/module/plugins/hooks/NoPremiumPlHook.py index 1727bec70..b5a007ff9 100644 --- a/module/plugins/hooks/NoPremiumPl.py +++ b/module/plugins/hooks/NoPremiumPlHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class NoPremiumPl(MultiHook): - __name__ = "NoPremiumPl" +class NoPremiumPlHook(MultiHook): + __name__ = "NoPremiumPlHook" __type__ = "hook" __version__ = "0.03" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/OverLoadMe.py b/module/plugins/hooks/OverLoadMeHook.py index 1872f8ccb..d608a2ecd 100644 --- a/module/plugins/hooks/OverLoadMe.py +++ b/module/plugins/hooks/OverLoadMeHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class OverLoadMe(MultiHook): - __name__ = "OverLoadMe" +class OverLoadMeHook(MultiHook): + __name__ = "OverLoadMeHook" __type__ = "hook" __version__ = "0.04" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 ), ("ssl" , "bool" , "Use HTTPS" , True )] @@ -23,9 +21,9 @@ class OverLoadMe(MultiHook): def getHosters(self): - https = "https" if self.getConfig("ssl") else "http" - page = self.getURL(https + "://api.over-load.me/hoster.php", + https = "https" if self.getConfig('ssl') else "http" + html = self.getURL(https + "://api.over-load.me/hoster.php", get={'auth': "0001-cb1f24dadb3aa487bda5afd3b76298935329be7700cd7-5329be77-00cf-1ca0135f"}).replace("\"", "").strip() - self.logDebug("Hosterlist", page) + self.logDebug("Hosterlist", html) - return [x.strip() for x in page.split(",") if x.strip()] + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/module/plugins/hooks/PremiumTo.py b/module/plugins/hooks/PremiumToHook.py index 844ecc89d..ef2a84223 100644 --- a/module/plugins/hooks/PremiumTo.py +++ b/module/plugins/hooks/PremiumToHook.py @@ -3,27 +3,25 @@ from module.plugins.internal.MultiHook import MultiHook -class PremiumTo(MultiHook): - __name__ = "PremiumTo" +class PremiumToHook(MultiHook): + __name__ = "PremiumToHook" __type__ = "hook" __version__ = "0.08" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] __description__ = """Premium.to hook plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), ("zoidberg", "zoidberg@mujmail.cz"), ("stickell", "l.stickell@yahoo.it")] def getHosters(self): - page = self.getURL("http://premium.to/api/hosters.php", + html = self.getURL("http://premium.to/api/hosters.php", get={'username': self.account.username, 'password': self.account.password}) - return [x.strip() for x in page.replace("\"", "").split(";")] + return [x.strip() for x in html.replace("\"", "").split(";")] diff --git a/module/plugins/hooks/PremiumizeMe.py b/module/plugins/hooks/PremiumizeMeHook.py index 293fcf339..f13520323 100644 --- a/module/plugins/hooks/PremiumizeMe.py +++ b/module/plugins/hooks/PremiumizeMeHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class PremiumizeMe(MultiHook): - __name__ = "PremiumizeMe" +class PremiumizeMeHook(MultiHook): + __name__ = "PremiumizeMeHook" __type__ = "hook" - __version__ = "0.17" + __version__ = "0.18" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -28,7 +26,7 @@ class PremiumizeMe(MultiHook): # Get supported hosters list from premiumize.me using the # json API v1 (see https://secure.premiumize.me/?show=api) - answer = self.getURL("https://api.premiumize.me/pm-api/v1.php", + answer = self.getURL("http://api.premiumize.me/pm-api/v1.php", get={'method': "hosterlist", 'params[login]': user, 'params[pass]': data['password']}) data = json_loads(answer) diff --git a/module/plugins/hooks/PutdriveCom.py b/module/plugins/hooks/PutdriveComHook.py index f665dabee..c3ebf4ff3 100644 --- a/module/plugins/hooks/PutdriveCom.py +++ b/module/plugins/hooks/PutdriveComHook.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from module.plugins.hooks.ZeveraCom import ZeveraCom +from module.plugins.hooks.ZeveraComHook import ZeveraComHook -class PutdriveCom(ZeveraCom): - __name__ = "PutdriveCom" +class PutdriveComHook(ZeveraComHook): + __name__ = "PutdriveComHook" __type__ = "hook" __version__ = "0.01" diff --git a/module/plugins/hooks/RPNetBiz.py b/module/plugins/hooks/RPNetBizHook.py index 0768bd6cd..10332948d 100644 --- a/module/plugins/hooks/RPNetBiz.py +++ b/module/plugins/hooks/RPNetBizHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class RPNetBiz(MultiHook): - __name__ = "RPNetBiz" +class RPNetBizHook(MultiHook): + __name__ = "RPNetBizHook" __type__ = "hook" __version__ = "0.14" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/RapideoPl.py b/module/plugins/hooks/RapideoPlHook.py index b605eca17..0400f07ba 100644 --- a/module/plugins/hooks/RapideoPl.py +++ b/module/plugins/hooks/RapideoPlHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class RapideoPl(MultiHook): - __name__ = "RapideoPl" +class RapideoPlHook(MultiHook): + __name__ = "RapideoPlHook" __type__ = "hook" __version__ = "0.03" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/RealdebridCom.py b/module/plugins/hooks/RealdebridComHook.py index cff97c2f9..aa0c9f640 100644 --- a/module/plugins/hooks/RealdebridCom.py +++ b/module/plugins/hooks/RealdebridComHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class RealdebridCom(MultiHook): - __name__ = "RealdebridCom" +class RealdebridComHook(MultiHook): + __name__ = "RealdebridComHook" __type__ = "hook" __version__ = "0.46" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 ), ("ssl" , "bool" , "Use HTTPS" , True )] @@ -23,7 +21,7 @@ class RealdebridCom(MultiHook): def getHosters(self): - https = "https" if self.getConfig("ssl") else "http" - page = self.getURL(https + "://real-debrid.com/api/hosters.php").replace("\"", "").strip() + https = "https" if self.getConfig('ssl') else "http" + html = self.getURL(https + "://real-debrid.com/api/hosters.php").replace("\"", "").strip() - return [x.strip() for x in page.split(",") if x.strip()] + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/module/plugins/hooks/RehostTo.py b/module/plugins/hooks/RehostToHook.py index ddb8b3eb0..a2415129a 100644 --- a/module/plugins/hooks/RehostTo.py +++ b/module/plugins/hooks/RehostToHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class RehostTo(MultiHook): - __name__ = "RehostTo" +class RehostToHook(MultiHook): + __name__ = "RehostToHook" __type__ = "hook" __version__ = "0.50" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -23,7 +21,7 @@ class RehostTo(MultiHook): def getHosters(self): user, data = self.account.selectAccount() - page = self.getURL("http://rehost.to/api.php", + html = self.getURL("http://rehost.to/api.php", get={'cmd' : "get_supported_och_dl", 'long_ses': self.account.getAccountInfo(user)['session']}) - return [x.strip() for x in page.replace("\"", "").split(",")] + return [x.strip() for x in html.replace("\"", "").split(",")] diff --git a/module/plugins/hooks/RestartFailed.py b/module/plugins/hooks/RestartFailed.py index 07fb80967..865af2a6b 100644 --- a/module/plugins/hooks/RestartFailed.py +++ b/module/plugins/hooks/RestartFailed.py @@ -6,29 +6,27 @@ from module.plugins.Hook import Hook class RestartFailed(Hook): __name__ = "RestartFailed" __type__ = "hook" - __version__ = "1.57" + __version__ = "1.58" __config__ = [("interval", "int", "Check interval in minutes", 90)] - __description__ = """Periodically restart all failed downloads in queue""" + __description__ = """Restart all the failed downloads in queue""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - # event_list = ["pluginConfigChanged"] + MIN_CHECK_INTERVAL = 15 * 60 #: 15 minutes - MIN_INTERVAL = 15 * 60 #: 15m minimum check interval (value is in seconds) - - def pluginConfigChanged(self, plugin, name, value): - if name == "interval": - interval = value * 60 - if self.MIN_INTERVAL <= interval != self.interval: - self.core.scheduler.removeJob(self.cb) - self.interval = interval - self.initPeriodical() - else: - self.logDebug("Invalid interval value, kept current") + # def pluginConfigChanged(self, plugin, name, value): + # if name == "interval": + # interval = value * 60 + # if self.MIN_CHECK_INTERVAL <= interval != self.interval: + # self.core.scheduler.removeJob(self.cb) + # self.interval = interval + # self.initPeriodical() + # else: + # self.logDebug("Invalid interval value, kept current") def periodical(self): @@ -37,8 +35,11 @@ class RestartFailed(Hook): def setup(self): - self.interval = self.MIN_INTERVAL + self.info = {} #@TODO: Remove in 0.4.10 + # self.event_list = ["pluginConfigChanged"] + self.interval = self.MIN_CHECK_INTERVAL def coreReady(self): - self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) + # self.pluginConfigChanged(self.__name__, "interval", self.getConfig('interval')) + self.interval = max(self.MIN_CHECK_INTERVAL, self.getConfig('interval') * 60) diff --git a/module/plugins/hooks/RestartSlow.py b/module/plugins/hooks/RestartSlow.py deleted file mode 100644 index c2fdf6f95..000000000 --- a/module/plugins/hooks/RestartSlow.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -import pycurl - -from module.plugins.Hook import Hook - - -class RestartSlow(Hook): - __name__ = "RestartSlow" - __type__ = "hook" - __version__ = "0.03" - - __config__ = [("free_limit" , "int" , "Transfer speed threshold in kilobytes" , 100 ), - ("free_time" , "int" , "Sample interval in minutes" , 5 ), - ("premium_limit", "int" , "Transfer speed threshold for premium download in kilobytes", 300 ), - ("premium_time" , "int" , "Sample interval for premium download in minutes" , 2 ), - ("safe_mode" , "bool", "Don't restart if download is not resumable" , True)] - - __description__ = """Restart slow downloads""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - event_list = ["downloadStarts"] - - - def setup(self): - self.info = {'chunk': {}} - - - def initPeriodical(self): - pass - - - def periodical(self): - if not self.pyfile.req.dl: - return - - if self.getConfig("safe_mode") and not self.pyfile.plugin.resumeDownload: - time = 30 - limit = 5 - else: - type = "premium" if self.pyfile.plugin.premium else "free" - time = max(30, self.getConfig("%s_time" % type) * 60) - limit = max(5, self.getConfig("%s_limit" % type) * 1024) - - chunks = [chunk for chunk in self.pyfile.req.dl.chunks \ - if chunk.id not in self.info['chunk'] or self.info['chunk'][chunk.id] is not (time, limit)] - - for chunk in chunks: - chunk.c.setopt(pycurl.LOW_SPEED_TIME , time) - chunk.c.setopt(pycurl.LOW_SPEED_LIMIT, limit) - - self.info['chunk'][chunk.id] = (time, limit) - - - def downloadStarts(self, pyfile, url, filename): - if self.cb or (self.getConfig("safe_mode") and not pyfile.plugin.resumeDownload): - return - - super(RestartSlow, self).initPeriodical() diff --git a/module/plugins/hooks/SimplyPremiumCom.py b/module/plugins/hooks/SimplyPremiumComHook.py index 843a3aa82..116e3a76e 100644 --- a/module/plugins/hooks/SimplyPremiumCom.py +++ b/module/plugins/hooks/SimplyPremiumComHook.py @@ -4,16 +4,14 @@ from module.common.json_layer import json_loads from module.plugins.internal.MultiHook import MultiHook -class SimplyPremiumCom(MultiHook): - __name__ = "SimplyPremiumCom" +class SimplyPremiumComHook(MultiHook): + __name__ = "SimplyPremiumComHook" __type__ = "hook" __version__ = "0.05" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/SimplydebridCom.py b/module/plugins/hooks/SimplydebridComHook.py index 10c613fb5..01629df99 100644 --- a/module/plugins/hooks/SimplydebridCom.py +++ b/module/plugins/hooks/SimplydebridComHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class SimplydebridCom(MultiHook): - __name__ = "SimplydebridCom" +class SimplydebridComHook(MultiHook): + __name__ = "SimplydebridComHook" __type__ = "hook" __version__ = "0.04" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] @@ -22,5 +20,5 @@ class SimplydebridCom(MultiHook): def getHosters(self): - page = self.getURL("http://simply-debrid.com/api.php", get={'list': 1}) - return [x.strip() for x in page.rstrip(';').replace("\"", "").split(";")] + html = self.getURL("http://simply-debrid.com/api.php", get={'list': 1}) + return [x.strip() for x in html.rstrip(';').replace("\"", "").split(";")] diff --git a/module/plugins/hooks/SkipRev.py b/module/plugins/hooks/SkipRev.py index 6b4e715da..7901ca540 100644 --- a/module/plugins/hooks/SkipRev.py +++ b/module/plugins/hooks/SkipRev.py @@ -1,43 +1,49 @@ # -*- coding: utf-8 -*- +import re +import urllib +import urlparse + from types import MethodType -from urllib import unquote -from urlparse import urlparse from module.PyFile import PyFile from module.plugins.Hook import Hook from module.plugins.Plugin import SkipDownload -def _setup(self): - self.pyfile.plugin._setup() - if self.pyfile.hasStatus("skipped"): - raise SkipDownload(self.pyfile.statusname or self.pyfile.pluginname) - - class SkipRev(Hook): __name__ = "SkipRev" __type__ = "hook" - __version__ = "0.23" + __version__ = "0.29" - __config__ = [("tokeep", "int", "Number of rev files to keep for package (-1 to auto)", -1)] + __config__ = [("mode" , "Auto;Manual", "Choose recovery archives to skip" , "Auto"), + ("revtokeep", "int" , "Number of recovery archives to keep for package", 0 )] - __description__ = """Skip files ending with extension rev""" + __description__ = """Skip recovery archives (.rev)""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + + @staticmethod + def _setup(self): + self.pyfile.plugin._setup() + if self.pyfile.hasStatus("skipped"): + raise SkipDownload(self.pyfile.statusname or self.pyfile.pluginname) - def _pyname(self, pyfile): - if hasattr(pyfile.pluginmodule, "getInfo"): - return next(getattr(pyfile.pluginmodule, "getInfo")([pyfile.url]))[0] + + def _name(self, pyfile): + if hasattr(pyfile.pluginmodule, "getInfo"): #@NOTE: getInfo is deprecated in 0.4.10 + return pyfile.pluginmodule.getInfo([pyfile.url]).next()[0] else: self.logWarning("Unable to grab file name") - return urlparse(unquote(pyfile.url)).path.split('/')[-1] + return urlparse.urlparse(urllib.unquote(pyfile.url)).path.split('/')[-1] def _pyfile(self, link): @@ -54,42 +60,52 @@ class SkipRev(Hook): def downloadPreparing(self, pyfile): - if pyfile.statusname is "unskipped" or not self._pyname(pyfile).endswith(".rev"): + name = self._name(pyfile) + + if pyfile.statusname is _("unskipped") or not name.endswith(".rev") or not ".part" in name: return - tokeep = self.getConfig("tokeep") + revtokeep = -1 if self.getConfig('mode') == "Auto" else self.getConfig('revtokeep') + + if revtokeep: + status_list = (1, 4, 8, 9, 14) if revtokeep < 0 else (1, 3, 4, 8, 9, 14) + pyname = re.compile(r'%s\.part\d+\.rev$' % name.rsplit('.', 2)[0].replace('.', '\.')) - if tokeep: - saved = [True for link in self.core.api.getPackageData(pyfile.package().id).links \ - if link.name.endswith(".rev") and link.status in (0, 12)].count(True) + queued = [True for link in self.core.api.getPackageData(pyfile.package().id).links \ + if link.status not in status_list and pyname.match(link.name)].count(True) - if not saved or saved < tokeep: #: keep one rev at least in auto mode + if not queued or queued < revtokeep: #: keep one rev at least in auto mode return pyfile.setCustomStatus("SkipRev", "skipped") - pyfile.plugin._setup = pyfile.plugin.setup - pyfile.plugin.setup = MethodType(_setup, pyfile.plugin) #: work-around: inject status checker inside the preprocessing routine of the plugin + + if not hasattr(pyfile.plugin, "_setup"): + # Work-around: inject status checker inside the preprocessing routine of the plugin + pyfile.plugin._setup = pyfile.plugin.setup + pyfile.plugin.setup = MethodType(self._setup, pyfile.plugin) def downloadFailed(self, pyfile): #: Check if pyfile is still "failed", # maybe might has been restarted in meantime - if pyfile.status != 8: + if pyfile.status != 8 or pyfile.name.rsplit('.', 1)[-1].strip() not in ("rar", "rev"): return - tokeep = self.getConfig("tokeep") + revtokeep = -1 if self.getConfig('mode') == "Auto" else self.getConfig('revtokeep') - if not tokeep: + if not revtokeep: return + pyname = re.compile(r'%s\.part\d+\.rev$' % pyfile.name.rsplit('.', 2)[0].replace('.', '\.')) + for link in self.core.api.getPackageData(pyfile.package().id).links: - if link.status is 4 and link.name.endswith(".rev"): + if link.status is 4 and pyname.match(link.name): pylink = self._pyfile(link) - if tokeep > -1 or pyfile.name.endswith(".rev"): + if revtokeep > -1 or pyfile.name.endswith(".rev"): pylink.setStatus("queued") else: - pylink.setCustomStatus("unskipped", "queued") + pylink.setCustomStatus(_("unskipped"), "queued") self.core.files.save() pylink.release() diff --git a/module/plugins/hooks/SmoozedCom.py b/module/plugins/hooks/SmoozedComHook.py index 9ba2daac9..24b7c8df0 100644 --- a/module/plugins/hooks/SmoozedCom.py +++ b/module/plugins/hooks/SmoozedComHook.py @@ -3,16 +3,14 @@ from module.plugins.internal.MultiHook import MultiHook -class SmoozedCom(MultiHook): - __name__ = "SmoozedCom" +class SmoozedComHook(MultiHook): + __name__ = "SmoozedComHook" __type__ = "hook" __version__ = "0.03" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] diff --git a/module/plugins/hooks/UnSkipOnFail.py b/module/plugins/hooks/UnSkipOnFail.py index 1becb937a..9059d0350 100644 --- a/module/plugins/hooks/UnSkipOnFail.py +++ b/module/plugins/hooks/UnSkipOnFail.py @@ -7,18 +7,20 @@ from module.plugins.Hook import Hook class UnSkipOnFail(Hook): __name__ = "UnSkipOnFail" __type__ = "hook" - __version__ = "0.05" + __version__ = "0.07" __config__ = [("activated", "bool", "Activated", True)] - __description__ = """Queue skipped duplicates when download fails""" + __description__ = """Restart skipped duplicates when download fails""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 def downloadFailed(self, pyfile): @@ -30,9 +32,9 @@ class UnSkipOnFail(Hook): msg = _("Looking for skipped duplicates of: %s (pid:%s)") self.logInfo(msg % (pyfile.name, pyfile.package().id)) - dup = self.findDuplicate(pyfile) - if dup: - self.logInfo(_("Queue found duplicate: %s (pid:%s)") % (dup.name, dup.packageID)) + link = self.findDuplicate(pyfile) + if link: + self.logInfo(_("Queue found duplicate: %s (pid:%s)") % (link.name, link.packageID)) #: Change status of "link" to "new_status". # "link" has to be a valid FileData object, @@ -41,9 +43,9 @@ class UnSkipOnFail(Hook): # It creates a temporary PyFile object using # "link" data, changes its status, and tells # the core.files-manager to save its data. - pylink = _pyfile(link) + pylink = self._pyfile(link) - pylink.setCustomStatus("UnSkipOnFail", "queued") + pylink.setCustomStatus(_("unskipped"), "queued") self.core.files.save() pylink.release() diff --git a/module/plugins/hooks/UnrestrictLi.py b/module/plugins/hooks/UnrestrictLi.py deleted file mode 100644 index cb5abb26e..000000000 --- a/module/plugins/hooks/UnrestrictLi.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.common.json_layer import json_loads -from module.plugins.internal.MultiHook import MultiHook - - -class UnrestrictLi(MultiHook): - __name__ = "UnrestrictLi" - __type__ = "hook" - __version__ = "0.05" - - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), - ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 ), - ("history" , "bool" , "Delete History" , False)] - - __description__ = """Unrestrict.li hook plugin""" - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it")] - - - def getHosters(self): - json_data = self.getURL("http://unrestrict.li/api/jdownloader/hosts.php", get={'format': "json"}) - json_data = json_loads(json_data) - - return [element['host'] for element in json_data['result']] diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index c72699228..98d602226 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -2,81 +2,98 @@ from __future__ import with_statement +import os import re import sys +import time from operator import itemgetter -from os import path, remove, stat from module.network.RequestFactory import getURL from module.plugins.Hook import Expose, Hook, threaded from module.utils import save_join +# Case-sensitive os.path.exists +def exists(path): + if os.path.exists(path): + if os.name == 'nt': + dir, name = os.path.split(path) + return name in os.listdir(dir) + else: + return True + else: + return False + + class UpdateManager(Hook): __name__ = "UpdateManager" __type__ = "hook" - __version__ = "0.42" + __version__ = "0.52" - __config__ = [("activated" , "bool" , "Activated" , True ), - ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"), - ("interval" , "int" , "Check interval in hours" , 8 ), - ("autorestart" , "bool" , "Automatically restart pyLoad when required" , True ), - ("reloadplugins", "bool" , "Monitor plugins for code changes in debug mode", True ), - ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , True )] + __config__ = [("activated" , "bool", "Activated" , True ), + ("checkinterval", "int" , "Check interval in hours" , 8 ), + ("autorestart" , "bool", "Auto-restart pyLoad when required" , True ), + ("checkonstart" , "bool", "Check for updates on startup" , True ), + ("checkperiod" , "bool", "Check for updates periodically" , True ), + ("reloadplugins", "bool", "Monitor plugin code changes in debug mode", True ), + ("nodebugupdate", "bool", "Don't update plugins in debug mode" , False)] __description__ = """ Check for updates """ __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - # event_list = ["pluginConfigChanged"] + interval = 0 - SERVER_URL = "http://updatemanager.pyload.org" - VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') - MIN_INTERVAL = 3 * 60 * 60 #: 3h minimum check interval (value is in seconds) + SERVER_URL = "http://updatemanager.pyload.org" + MIN_CHECK_INTERVAL = 3 * 60 * 60 #: 3 hours - def pluginConfigChanged(self, plugin, name, value): - if name == "interval": - interval = value * 60 * 60 - if self.MIN_INTERVAL <= interval != self.interval: - self.core.scheduler.removeJob(self.cb) - self.interval = interval - self.initPeriodical() - else: - self.logDebug("Invalid interval value, kept current") + def coreReady(self): + if self.checkonstart: + self.core.api.pauseServer() + self.update() + if self.do_restart is False: + self.core.api.unpauseServer() - elif name == "reloadplugins": - if self.cb2: - self.core.scheduler.removeJob(self.cb2) - if value is True and self.core.debug: - self.periodical2() + self.initPeriodical() - def coreReady(self): - self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) - x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins")) - self.core.scheduler.addJob(10, x, threaded=False) + def setup(self): + self.info = {'pyload': False, 'version': None, 'plugins': False, 'last_check': time.time()} + self.mtimes = {} #: store modification time for each plugin + self.event_list = ["allDownloadsProcessed"] - def unload(self): - self.pluginConfigChanged(self.__name__, "reloadplugins", False) + self.interval = 10 + if self.getConfig('checkonstart'): + self.core.api.pauseServer() + self.checkonstart = True + else: + self.checkonstart = False - def setup(self): - self.cb2 = None - self.interval = self.MIN_INTERVAL - self.updating = False - self.info = {'pyload': False, 'version': None, 'plugins': False} - self.mtimes = {} #: store modification time for each plugin + self.do_restart = False - def periodical2(self): - if not self.updating: - self.autoreloadPlugins() + def allDownloadsProcessed(self): + if self.do_restart is True: + self.logWarning(_("Downloads are done, restarting pyLoad to reload the updated plugins")) + self.core.api.restart() + + + def periodical(self): + if self.core.debug: + if self.getConfig('reloadplugins'): + self.autoreloadPlugins() + + if self.getConfig('nodebugupdate'): + return - self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False) + if self.getConfig('checkperiod') \ + and time.time() - max(self.MIN_CHECK_INTERVAL, self.getConfig('checkinterval') * 60 * 60) > self.info['last_check']: + self.update() @Expose @@ -95,10 +112,10 @@ class UpdateManager(Hook): id = (type, name) if type in self.core.pluginManager.plugins: f = m.__file__.replace(".pyc", ".py") - if not path.isfile(f): + if not os.path.isfile(f): continue - mtime = stat(f).st_mtime + mtime = os.stat(f).st_mtime if id not in self.mtimes: self.mtimes[id] = mtime @@ -109,105 +126,101 @@ class UpdateManager(Hook): return True if self.core.pluginManager.reloadPlugins(reloads) else False - def periodical(self): - if not self.info['pyload'] and not (self.getConfig("nodebugupdate") and self.core.debug): - self.updateThread() - - - def server_request(self): + def server_response(self): try: return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() - except: - self.logWarning(_("Unable to contact server to get updates")) + except Exception: + self.logWarning(_("Unable to retrieve server to get updates")) - @threaded - def updateThread(self): - self.updating = True - status = self.update(onlyplugin=self.getConfig("mode") == "plugins only") - - if status is 2 and self.getConfig("autorestart"): - self.core.api.restart() - else: - self.updating = False + @Expose + @threaded + def update(self): + """ check for updates """ + if self._update() is 2 and self.getConfig('autorestart'): + if not self.core.api.statusDownloads(): + self.core.api.restart() + else: + self.do_restart = True + self.logWarning(_("Downloads are active, will restart once the download is done")) + self.core.api.pauseServer() - @Expose - def updatePlugins(self): - """ simple wrapper for calling plugin update quickly """ - return self.update(onlyplugin=True) + def _update(self): + data = self.server_response() - @Expose - def update(self, onlyplugin=False): - """ check for updates """ - data = self.server_request() + self.info['last_check'] = time.time() if not data: exitcode = 0 elif data[0] == "None": self.logInfo(_("No new pyLoad version available")) - updates = data[1:] - exitcode = self._updatePlugins(updates) + exitcode = self._updatePlugins(data[1:]) elif onlyplugin: exitcode = 0 else: - newversion = data[0] - self.logInfo(_("*** New pyLoad Version %s available ***") % newversion) + self.logInfo(_("*** New pyLoad Version %s available ***") % data[0]) self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) + self.info['pyload'] = True + self.info['version'] = data[0] exitcode = 3 - self.info['pyload'] = True - self.info['version'] = newversion - return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available + # Exit codes: + # -1 = No plugin updated, new pyLoad version available + # 0 = No plugin updated + # 1 = Plugins updated + # 2 = Plugins updated, but restart required + return exitcode - def _updatePlugins(self, updates): + def _updatePlugins(self, data): """ check for plugin updates """ - if self.info['plugins']: - return False #: plugins were already updated - exitcode = 0 updated = [] - url = updates[0] - schema = updates[1].split('|') + url = data[0] + schema = data[1].split('|') - if "BLACKLIST" in updates: - blacklist = updates[updates.index('BLACKLIST') + 1:] - updates = updates[2:updates.index('BLACKLIST')] + VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + + if "BLACKLIST" in data: + blacklist = data[data.index('BLACKLIST') + 1:] + updatelist = data[2:data.index('BLACKLIST')] else: - blacklist = None - updates = updates[2:] + blacklist = [] + updatelist = data[2:] - upgradable = [dict(zip(schema, x.split('|'))) for x in updates] - blacklisted = [(x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]) for x in blacklist] if blacklist else [] + updatelist = [dict(zip(schema, x.split('|'))) for x in updatelist] + blacklist = [dict(zip(schema, x.split('|'))) for x in blacklist] if blacklist: + type_plugins = [(plugin['type'], plugin['name'].rsplit('.', 1)[0]) for plugin in blacklist] + # Protect UpdateManager from self-removing try: - blacklisted.remove(("hook", "UpdateManager")) - except: + type_plugins.remove(("hook", "UpdateManager")) + except ValueError: pass - for t, n in blacklisted: - for idx, plugin in enumerate(upgradable): + for t, n in type_plugins: + for idx, plugin in enumerate(updatelist): if n == plugin['name'] and t == plugin['type']: - upgradable.pop(idx) + updatelist.pop(idx) break - for t, n in self.removePlugins(sorted(blacklisted)): - self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % { + for t, n in self.removePlugins(sorted(type_plugins)): + self.logInfo(_("Removed blacklisted plugin: [%(type)s] %(name)s") % { 'type': t, 'name': n, }) - for plugin in sorted(upgradable, key=itemgetter("type", "name")): + for plugin in sorted(updatelist, key=itemgetter("type", "name")): filename = plugin['name'] prefix = plugin['type'] version = plugin['version'] @@ -217,7 +230,7 @@ class UpdateManager(Hook): else: name = filename.replace(".py", "") - #@TODO: obsolete after 0.4.10 + #@TODO: Remove in 0.4.10 if prefix.endswith("s"): type = prefix[:-1] else: @@ -241,7 +254,7 @@ class UpdateManager(Hook): 'newver': newver}) try: content = getURL(url % plugin) - m = self.VERSION.search(content) + m = VERSION.search(content) if m and m.group(2) == version: with open(save_join("userplugins", prefix, filename), "wb") as f: @@ -252,21 +265,27 @@ class UpdateManager(Hook): raise Exception, _("Version mismatch") except Exception, e: - self.logError(_("Error updating plugin: %s") % filename, str(e)) + self.logError(_("Error updating plugin: %s") % filename, e) if updated: - reloaded = self.core.pluginManager.reloadPlugins(updated) - if reloaded: - self.logInfo(_("Plugins updated and reloaded")) + self.logInfo(_("*** Plugins updated ***")) + + if self.core.pluginManager.reloadPlugins(updated): exitcode = 1 else: - self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***")) + self.logWarning(_("pyLoad restart required to reload the updated plugins")) self.info['plugins'] = True exitcode = 2 + + self.manager.dispatchEvent("plugin_updated", updated) else: self.logInfo(_("No plugin updates available")) - return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required + # Exit codes: + # 0 = No plugin updated + # 1 = Plugins updated + # 2 = Plugins updated, but restart required + return exitcode @Expose @@ -276,35 +295,36 @@ class UpdateManager(Hook): if not type_plugins: return - self.logDebug("Requested deletion of plugins: %s" % type_plugins) + removed = set() - removed = [] + self.logDebug("Requested deletion of plugins: %s" % type_plugins) for type, name in type_plugins: - err = False - file = name + ".py" - - for root in ("userplugins", path.join(pypath, "module", "plugins")): + rootplugins = os.path.join(pypath, "module", "plugins") - filename = save_join(root, type, file) - try: - remove(filename) - except Exception, e: - self.logDebug("Error removing: %s" % path.basename(filename), str(e)) - err = True + for dir in ("userplugins", rootplugins): + py_filename = save_join(dir, type, name + ".py") + pyc_filename = py_filename + "c" - filename += "c" - if path.isfile(filename): + if type == "hook": try: - if type == "hook": - self.manager.deactivateHook(name) - remove(filename) + self.manager.deactivateHook(name) + except Exception, e: - self.logDebug("Error removing: %s" % path.basename(filename), str(e)) - err = True + self.logDebug(e) + + for filename in (py_filename, pyc_filename): + if not exists(filename): + continue + + try: + os.remove(filename) + + except OSError, e: + self.logError(_("Error removing: %s") % filename, e) - if not err: - id = (type, name) - removed.append(id) + else: + id = (type, name) + removed.add(id) - return removed #: return a list of the plugins successfully removed + return list(removed) #: return a list of the plugins successfully removed diff --git a/module/plugins/hooks/UserAgentSwitcher.py b/module/plugins/hooks/UserAgentSwitcher.py new file mode 100644 index 000000000..74c43f7c4 --- /dev/null +++ b/module/plugins/hooks/UserAgentSwitcher.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +import pycurl + +from module.plugins.Hook import Hook + + +class UserAgentSwitcher(Hook): + __name__ = "UserAgentSwitcher" + __type__ = "hook" + __version__ = "0.08" + + __config__ = [("activated" , "bool", "Activated" , True ), + ("connecttimeout", "int" , "Connection timeout in seconds" , 60 ), + ("maxredirs" , "int" , "Maximum number of redirects to follow" , 10 ), + ("useragent" , "str" , "Custom user-agent string" , "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0")] + + __description__ = """Custom user-agent""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + + + def downloadPreparing(self, pyfile): + connecttimeout = self.getConfig('connecttimeout') + maxredirs = self.getConfig('maxredirs') + useragent = self.getConfig('useragent').encode("utf8", "replace") #@TODO: Remove `encode` in 0.4.10 + + if connecttimeout: + pyfile.plugin.req.http.c.setopt(pycurl.CONNECTTIMEOUT, connecttimeout) + + if maxredirs: + pyfile.plugin.req.http.c.setopt(pycurl.MAXREDIRS, maxredirs) + + if useragent: + self.logDebug("Use custom user-agent string: " + useragent) + pyfile.plugin.req.http.c.setopt(pycurl.USERAGENT, useragent) diff --git a/module/plugins/hooks/WindowsPhoneNotify.py b/module/plugins/hooks/WindowsPhoneNotify.py new file mode 100644 index 000000000..8f66761f6 --- /dev/null +++ b/module/plugins/hooks/WindowsPhoneNotify.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +import httplib +import time + +from module.plugins.Hook import Hook, Expose + + +class WindowsPhoneNotify(Hook): + __name__ = "WindowsPhoneNotify" + __type__ = "hook" + __version__ = "0.10" + + __config__ = [("push-id" , "str" , "Push ID" , "" ), + ("push-url" , "str" , "Push url" , "" ), + ("notifycaptcha" , "bool", "Notify captcha request" , True ), + ("notifypackage" , "bool", "Notify package finished" , True ), + ("notifyprocessed", "bool", "Notify packages processed" , True ), + ("notifyupdate" , "bool", "Notify plugin updates" , True ), + ("notifyexit" , "bool", "Notify pyLoad shutdown" , True ), + ("sendtimewait" , "int" , "Timewait in seconds between notifications", 5 ), + ("sendpermin" , "int" , "Max notifications per minute" , 12 ), + ("ignoreclient" , "bool", "Send notifications if client is connected", False)] + + __description__ = """Send push notifications to Windows Phone""" + __license__ = "GPLv3" + __authors__ = [("Andy Voigt" , "phone-support@hotmail.de"), + ("Walter Purcaro", "vuolter@gmail.com" )] + + + interval = 0 #@TODO: Remove in 0.4.10 + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + + self.event_list = ["allDownloadsProcessed", "plugin_updated"] + + self.last_notify = 0 + self.notifications = 0 + + + def plugin_updated(self, type_plugins): + if not self.getConfig('notifyupdate'): + return + + self.notify(_("Plugins updated"), str(type_plugins)) + + + def coreReady(self): + self.key = (self.getConfig('push-id'), self.getConfig('push-url')) + + + def coreExiting(self): + if not self.getConfig('notifyexit'): + return + + if self.core.do_restart: + self.notify(_("Restarting pyLoad")) + else: + self.notify(_("Exiting pyLoad")) + + + def newCaptchaTask(self, task): + if not self.getConfig('notifycaptcha'): + return + + self.notify(_("Captcha"), _("New request waiting user input")) + + + def packageFinished(self, pypack): + if self.getConfig('notifypackage'): + self.notify(_("Package finished"), pypack.name) + + + def allDownloadsProcessed(self): + if not self.getConfig('notifyprocessed'): + return + + if any(True for pdata in self.core.api.getQueue() if pdata.linksdone < pdata.linkstotal): + self.notify(_("Package failed"), _("One or more packages was not completed successfully")) + else: + self.notify(_("All packages finished")) + + + def getXmlData(self, msg): + return ("<?xml version='1.0' encoding='utf-8'?> <wp:Notification xmlns:wp='WPNotification'> " + "<wp:Toast> <wp:Text1>pyLoad</wp:Text1> <wp:Text2>%s</wp:Text2> " + "</wp:Toast> </wp:Notification>" % msg) + + + @Expose + def notify(self, + event, + msg="", + key=(None, None)): + + id, url = key or self.key + if not id or not url: + return + + if self.core.isClientConnected() and not self.getConfig('ignoreclient'): + return + + elapsed_time = time.time() - self.last_notify + + if elapsed_time < self.getConf("sendtimewait"): + return + + if elapsed_time > 60: + self.notifications = 0 + + elif self.notifications >= self.getConf("sendpermin"): + return + + request = self.getXmlData("%s: %s" % (event, msg) if msg else event) + webservice = httplib.HTTP(url) + + webservice.putrequest("POST", id) + webservice.putheader("Host", url) + webservice.putheader("Content-type", "text/xml") + webservice.putheader("X-NotificationClass", "2") + webservice.putheader("X-WindowsPhone-Target", "toast") + webservice.putheader("Content-length", "%d" % len(request)) + webservice.endheaders() + webservice.send(request) + webservice.close() + + self.last_notify = time.time() + self.notifications += 1 + + return True diff --git a/module/plugins/hooks/WindowsPhoneToastNotify.py b/module/plugins/hooks/WindowsPhoneToastNotify.py deleted file mode 100644 index 886d4ca6a..000000000 --- a/module/plugins/hooks/WindowsPhoneToastNotify.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- - -import httplib - -from time import time - -from module.plugins.Hook import Hook - - -class WindowsPhoneToastNotify(Hook): - __name__ = "WindowsPhoneToastNotify" - __type__ = "hook" - __version__ = "0.04" - - __config__ = [("id" , "str" , "Push ID" , "" ), - ("url" , "str" , "Push url" , "" ), - ("notifycaptcha" , "bool", "Notify captcha request" , True ), - ("notifypackage" , "bool", "Notify package finished" , True ), - ("notifyprocessed", "bool", "Notify processed packages status" , True ), - ("timeout" , "int" , "Timeout between captchas in seconds" , 5 ), - ("force" , "bool", "Send notifications if client is connected", False)] - - __description__ = """Send push notifications to Windows Phone""" - __license__ = "GPLv3" - __authors__ = [("Andy Voigt", "phone-support@hotmail.de"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - event_list = ["allDownloadsProcessed"] - - - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass - - - def setup(self): - self.info = {} #@TODO: Remove in 0.4.10 - - - def newCaptchaTask(self, task): - if not self.getConfig("notifycaptcha"): - return False - - if time() - float(self.getStorage("WindowsPhoneToastNotify", 0)) < self.getConf("timeout"): - return False - - self.notify(_("Captcha"), _("New request waiting user input")) - - - def packageFinished(self, pypack): - if self.getConfig("notifypackage"): - self.notify(_("Package finished"), pypack.name) - - - def allDownloadsProcessed(self, thread): - if not self.getConfig("notifyprocessed"): - return False - - if any(True for pdata in self.core.api.getQueue() if pdata.linksdone < pdata.linkstotal): - self.notify(_("Package failed"), _("One or more packages was not completed successfully")) - else: - self.notify(_("All packages finished")) - - - def getXmlData(self, msg): - return ("<?xml version='1.0' encoding='utf-8'?> <wp:Notification xmlns:wp='WPNotification'> " - "<wp:Toast> <wp:Text1>pyLoad</wp:Text1> <wp:Text2>%s</wp:Text2> " - "</wp:Toast> </wp:Notification>" % msg) - - - def notify(self, event, msg=""): - id = self.getConfig("id") - url = self.getConfig("url") - - if not id or not url: - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - request = self.getXmlData("%s: %s" % (event, msg) if msg else event) - webservice = httplib.HTTP(url) - - webservice.putrequest("POST", id) - webservice.putheader("Host", url) - webservice.putheader("Content-type", "text/xml") - webservice.putheader("X-NotificationClass", "2") - webservice.putheader("X-WindowsPhone-Target", "toast") - webservice.putheader("Content-length", "%d" % len(request)) - webservice.endheaders() - webservice.send(request) - webservice.close() - - self.setStorage("WindowsPhoneToastNotify", time()) diff --git a/module/plugins/hooks/XFileSharingPro.py b/module/plugins/hooks/XFileSharingPro.py index 0745a6c7e..62872992c 100644 --- a/module/plugins/hooks/XFileSharingPro.py +++ b/module/plugins/hooks/XFileSharingPro.py @@ -8,7 +8,7 @@ from module.plugins.Hook import Hook class XFileSharingPro(Hook): __name__ = "XFileSharingPro" __type__ = "hook" - __version__ = "0.30" + __version__ = "0.38" __config__ = [("activated" , "bool", "Activated" , True ), ("use_hoster_list" , "bool", "Load listed hosters only" , False), @@ -17,37 +17,37 @@ class XFileSharingPro(Hook): ("hoster_list" , "str" , "Hoster list (comma separated)" , "" ), ("crypter_list" , "str" , "Crypter list (comma separated)", "" )] - __description__ = """Load XFileSharingPro based hosters and crypter which don't need a own plugin to run""" + __description__ = """Load XFileSharingPro based hosters and crypters which don't need a own plugin to run""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - # event_list = ["pluginConfigChanged"] - regexp = {'hoster' : (r'https?://(?:www\.)?(?P<DOMAIN>[\w.^_]+(?:\.[a-zA-Z]{2,})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)', - r'https?://(?:[^/]+\.)?(?P<DOMAIN>%s)/(?:embed-)?\w+'), - 'crypter': (r'https?://(?:www\.)?(?P<DOMAIN>[\w.^_]+(?:\.[a-zA-Z]{2,})(?:\:\d+)?)/(?:user|folder)s?/\w+', - r'https?://(?:[^/]+\.)?(?P<DOMAIN>%s)/(?:user|folder)s?/\w+')} + interval = 0 #@TODO: Remove in 0.4.10 + regexp = {'hoster' : (r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)', + r'https?://(?:[^/]+\.)?(?P<DOMAIN>%s)/(?:embed-)?\w+'), + 'crypter': (r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:user|folder)s?/\w+', + r'https?://(?:[^/]+\.)?(?P<DOMAIN>%s)/(?:user|folder)s?/\w+')} HOSTER_BUILTIN = [#WORKING HOSTERS: - "180upload.com", "backin.net", "eyesfile.ca", "file4safe.com", "fileband.com", "filedwon.com", - "fileparadox.in", "filevice.com", "hostingbulk.com", "linestorage.com", "ravishare.com", "ryushare.com", - "salefiles.com", "sendmyway.com", "sharesix.com", "thefile.me", "verzend.be", "xvidstage.com", + "backin.net", "eyesfile.ca", "file4safe.com", "fileband.com", "filedwon.com", "fileparadox.in", + "filevice.com", "hostingbulk.com", "junkyvideo.com", "linestorage.com", "ravishare.com", "ryushare.com", + "salefiles.com", "sendmyway.com", "sharesix.com", "thefile.me", "verzend.be", "worldbytez.com", "xvidstage.com", #NOT TESTED: "101shared.com", "4upfiles.com", "filemaze.ws", "filenuke.com", "linkzhost.com", "mightyupload.com", "rockdizfile.com", "sharebeast.com", "sharerepo.com", "shareswift.com", "uploadbaz.com", "uploadc.com", "vidbull.com", "zalaa.com", "zomgupload.com", #NOT WORKING: "amonshare.com", "banicrazy.info", "boosterking.com", "host4desi.com", "laoupload.com", "rd-fs.com"] - CRYPTER_BUILTIN = [] + CRYPTER_BUILTIN = ["junocloud.me", "rapidfileshare.net"] - # def pluginConfigChanged(self.__name__, plugin, name, value): + # def pluginConfigChanged(self, plugin, name, value): # self.loadPattern() - #@TODO: Remove in 0.4.10 - def initPeriodical(self): - pass + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + # self.event_list = ["pluginConfigChanged"] def coreReady(self): @@ -65,22 +65,22 @@ class XFileSharingPro(Hook): self.logInfo(_("Handling any %s I can!") % type) pattern = self.regexp[type][0] else: - s = self.getConfig('%s_list' % type).replace('\\', '').replace('|', ',').replace(';', ',').lower() - plugin_list = set([x.strip() for x in s.split(',')]) + plugins = self.getConfig('%s_list' % type) + plugin_set = set(plugins.replace(' ', '').replace('\\', '').replace('|', ',').replace(';', ',').lower().split(',')) if use_builtin_list: - plugin_list |= set([x.lower() for x in getattr(self, "%s_BUILTIN" % type.upper())]) + plugin_set |= set(x.lower() for x in getattr(self, "%s_BUILTIN" % type.upper())) - plugin_list -= set(('', u'')) + plugin_set -= set(('', u'')) - if not plugin_list: + if not plugin_set: self.logInfo(_("No %s to handle") % type) self._unload(type, plugin) return - match_list = '|'.join(sorted(plugin_list)) + match_list = '|'.join(sorted(plugin_set)) - len_match_list = len(plugin_list) + len_match_list = len(plugin_set) self.logInfo(_("Handling %d %s%s: %s") % (len_match_list, type, "" if len_match_list == 1 else "s", @@ -112,11 +112,11 @@ class XFileSharingPro(Hook): hdict = self.core.pluginManager.hosterPlugins[hoster] if "new_name" in hdict and hdict['new_name'] == "XFileSharingPro": if "module" in hdict: - del hdict['module'] + hdict.pop('module', None) if "new_module" in hdict: - del hdict['new_module'] - del hdict['new_name'] + hdict.pop('new_module', None) + hdict.pop('new_name', None) return True else: @@ -126,7 +126,7 @@ class XFileSharingPro(Hook): # def downloadFailed(self, pyfile): # if pyfile.pluginname == "BasePlugin" \ # and pyfile.hasStatus("failed") \ - # and not self.getConfig("use_hoster_list") \ + # and not self.getConfig('use_hoster_list') \ # and self.unloadHoster("BasePlugin"): # self.logDebug("Unloaded XFileSharingPro from BasePlugin") # pyfile.setStatus("queued") diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/hooks/XMPPInterface.py index bbeab4341..b61428392 100644 --- a/module/plugins/hooks/XMPPInterface.py +++ b/module/plugins/hooks/XMPPInterface.py @@ -33,14 +33,14 @@ class XMPPInterface(IRCInterface, JabberClient): def __init__(self, core, manager): IRCInterface.__init__(self, core, manager) - self.jid = JID(self.getConfig("jid")) - password = self.getConfig("pw") + self.jid = JID(self.getConfig('jid')) + password = self.getConfig('pw') # if bare JID is provided add a resource -- it is required if not self.jid.resource: self.jid = JID(self.jid.node, self.jid.domain, "pyLoad") - if self.getConfig("tls"): + if self.getConfig('tls'): tls_settings = streamtls.TLSSettings(require=True, verify_peer=False) auth = ("sasl:PLAIN", "sasl:DIGEST-MD5") else: @@ -67,18 +67,18 @@ class XMPPInterface(IRCInterface, JabberClient): def packageFinished(self, pypack): try: - if self.getConfig("info_pack"): + if self.getConfig('info_pack'): self.announce(_("Package finished: %s") % pypack.name) - except: + except Exception: pass def downloadFinished(self, pyfile): try: - if self.getConfig("info_file"): + if self.getConfig('info_file'): self.announce( _("Download finished: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.pluginname}) - except: + except Exception: pass @@ -139,7 +139,7 @@ class XMPPInterface(IRCInterface, JabberClient): to_name = to_jid.as_utf8() from_name = from_jid.as_utf8() - names = self.getConfig("owners").split(";") + names = self.getConfig('owners').split(";") if to_name in names or to_jid.node + "@" + to_jid.domain in names: messages = [] @@ -152,7 +152,7 @@ class XMPPInterface(IRCInterface, JabberClient): trigger = temp[0] if len(temp) > 1: args = temp[1:] - except: + except Exception: pass handler = getattr(self, "event_%s" % trigger, self.event_pass) @@ -182,7 +182,7 @@ class XMPPInterface(IRCInterface, JabberClient): def announce(self, message): """ send message to all owners""" - for user in self.getConfig("owners").split(";"): + for user in self.getConfig('owners').split(";"): self.logDebug("Send message to", user) to_jid = JID(user) diff --git a/module/plugins/hooks/ZeveraCom.py b/module/plugins/hooks/ZeveraComHook.py index 215ec3673..21c1741d2 100644 --- a/module/plugins/hooks/ZeveraCom.py +++ b/module/plugins/hooks/ZeveraComHook.py @@ -3,23 +3,21 @@ from module.plugins.internal.MultiHook import MultiHook -class ZeveraCom(MultiHook): - __name__ = "ZeveraCom" +class ZeveraComHook(MultiHook): + __name__ = "ZeveraComHook" __type__ = "hook" __version__ = "0.05" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), ("reload" , "bool" , "Reload plugin list" , True ), ("reloadinterval", "int" , "Reload interval in hours" , 12 )] __description__ = """Zevera.com hook plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com" )] def getHosters(self): diff --git a/module/plugins/hoster/AlldebridCom.py b/module/plugins/hoster/AlldebridCom.py index 1fcb4d784..2ed09f58c 100644 --- a/module/plugins/hoster/AlldebridCom.py +++ b/module/plugins/hoster/AlldebridCom.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- import re - -from random import randrange -from urllib import unquote +import urllib from module.common.json_layer import json_loads from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -13,27 +11,16 @@ from module.utils import parseFileSize class AlldebridCom(MultiHoster): __name__ = "AlldebridCom" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.46" __pattern__ = r'https?://(?:www\.|s\d+\.)?alldebrid\.com/dl/[\w^_]+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Alldebrid.com multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("Andy Voigt", "spamsales@online.de")] - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - - if name.endswith("..."): # incomplete filename, append random stuff - name += "%s.tmp" % randrange(100, 999) - - return name - - def setup(self): self.chunkLimit = 16 @@ -58,24 +45,10 @@ class AlldebridCom(MultiHoster): pyfile.size = parseFileSize(data['filesize']) self.link = data['link'] - if self.getConfig("ssl"): + if self.getConfig('ssl'): self.link = self.link.replace("http://", "https://") else: self.link = self.link.replace("https://", "http://") - if self.link != pyfile.url: - self.logDebug("New URL: %s" % self.link) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown"): - #only use when name wasnt already set - pyfile.name = self.getFilename(self.link) - - - def checkFile(self): - if self.checkDownload({'error': "<title>An error occured while processing your request</title>"}) == "error": - self.retry(wait_time=60, reason=_("An error occured while generating link")) - - return super(AlldebridCom, self).checkFile() - getInfo = create_getInfo(AlldebridCom) diff --git a/module/plugins/hoster/AndroidfilehostCom.py b/module/plugins/hoster/AndroidfilehostCom.py new file mode 100644 index 000000000..08005de0f --- /dev/null +++ b/module/plugins/hoster/AndroidfilehostCom.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -* +# +# Test links: +# https://www.androidfilehost.com/?fid=95916177934518197 + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class AndroidfilehostCom(SimpleHoster): + __name__ = "AndroidfilehostCom" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?androidfilehost\.com/\?fid=\d+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """Androidfilehost.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'<br />(?P<N>.*?)</h1>' + SIZE_PATTERN = r'<h4>size</h4>\s*<p>(?P<S>[\d.,]+)(?P<U>[\w^_]+)</p>' + HASHSUM_PATTERN = r'<h4>(?P<T>.*?)</h4>\s*<p><code>(?P<H>.*?)</code></p>' + + OFFLINE_PATTERN = r'404 not found' + + WAIT_PATTERN = r'users must wait <strong>(\d+) secs' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + wait = re.search(self.WAIT_PATTERN, self.html) + self.logDebug("Waiting time: %s seconds" % wait.group(1)) + + fid = re.search(r'id="fid" value="(\d+)" />', self.html).group(1) + self.logDebug("fid: %s" % fid) + + html = self.load("https://www.androidfilehost.com/libs/otf/mirrors.otf.php", + post={'submit': 'submit', + 'action': 'getdownloadmirrors', + 'fid' : fid}, + decode=True) + + self.link = re.findall('"url":"(.*?)"', html)[0].replace("\\", "") + mirror_host = self.link.split("/")[2] + + self.logDebug("Mirror Host: %s" % mirror_host) + + html = self.load("https://www.androidfilehost.com/libs/otf/stats.otf.php", + get={'fid' : fid, + 'w' : 'download', + 'mirror': mirror_host}, + decode=True) + + +getInfo = create_getInfo(AndroidfilehostCom) diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 6d3132e65..2228516aa 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,19 +1,18 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote -from urlparse import urljoin, urlparse +import urllib +import urlparse from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import create_getInfo +from module.plugins.internal.SimpleHoster import create_getInfo, getFileURL from module.plugins.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.30" + __version__ = "0.43" __pattern__ = r'^unmatchable$' @@ -25,17 +24,19 @@ class BasePlugin(Hoster): @classmethod def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 - url = unquote(url) - return {'name' : (urlparse(url).path.split('/')[-1] - or urlparse(url).query.split('=', 1)[::-1][0].split('&', 1)[0] - or _("Unknown")), + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), 'size' : 0, 'status': 3 if url else 8, 'url' : url} def setup(self): - self.chunkLimit = -1 + self.chunkLimit = -1 + self.multiDL = True self.resumeDownload = True @@ -49,7 +50,12 @@ class BasePlugin(Hoster): for _i in xrange(5): try: - self.downloadFile(pyfile) + link = getFileURL(self, urllib.unquote(pyfile.url)) + + if link: + self.download(link, ref=False, disposition=True) + else: + self.fail(_("File not found")) except BadHeader, e: if e.code is 404: @@ -60,7 +66,7 @@ class BasePlugin(Hoster): account = self.core.accountManager.getAccountPlugin('Http') servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + server = urlparse.urlparse(pyfile.url).netloc if server in servers: self.logDebug("Logging on to %s" % server) @@ -78,40 +84,20 @@ class BasePlugin(Hoster): else: self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 - check = self.checkDownload({'empty file': re.compile(r'\A\Z'), - 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), - 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}) - if check: - self.fail(check.capitalize()) - - - def downloadFile(self, pyfile): - url = pyfile.url - - for i in xrange(1, 7): #@TODO: retrieve the pycurl.MAXREDIRS value set by req - header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) + errmsg = self.checkDownload({'Empty file' : re.compile(r'\A\s*\Z'), + 'Html error' : re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + 'Html file' : re.compile(r'\A\s*<!DOCTYPE html'), + 'Request error': re.compile(r'([Aa]n error occured while processing your request)')}) + if not errmsg: + return - if 'location' not in header or not header['location']: - if 'code' in header and header['code'] not in (200, 201, 203, 206): - self.logDebug("Received HTTP status code: %d" % header['code']) - self.fail(_("File not found")) - else: - break - - location = header['location'] - - self.logDebug("Redirect #%d to: %s" % (i, location)) - - if urlparse(location).scheme: - url = location - else: - p = urlparse(url) - base = "%s://%s" % (p.scheme, p.netloc) - url = urljoin(base, location) - else: - self.fail(_("Too many redirects")) + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass - self.download(unquote(url), disposition=True) + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.retry(3, 60, errmsg) getInfo = create_getInfo(BasePlugin) diff --git a/module/plugins/hoster/BasketbuildCom.py b/module/plugins/hoster/BasketbuildCom.py new file mode 100644 index 000000000..89e4d39f9 --- /dev/null +++ b/module/plugins/hoster/BasketbuildCom.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -* +# +# Test links: +# https://s.basketbuild.com/filedl/devs?dev=pacman&dl=pacman/falcon/RC-3/pac_falcon-RC-3-20141103.zip +# https://s.basketbuild.com/filedl/gapps?dl=gapps-gb-20110828-signed.zip + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class BasketbuildCom(SimpleHoster): + __name__ = "BasketbuildCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?(?:\w\.)?basketbuild\.com/filedl/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """basketbuild.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'File Name:</strong> (?P<N>.+?)<br/>' + SIZE_PATTERN = r'File Size:</strong> (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'404 - Page Not Found' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + try: + link1 = re.search(r'href="(.+dlgate/.+)"', self.html).group(1) + self.html = self.load(link1) + + except AttributeError: + self.error(_("Hop #1 not found")) + + else: + self.logDebug("Next hop: %s" % link1) + + try: + wait = re.search(r'var sec = (\d+)', self.html).group(1) + self.logDebug("Wait %s seconds" % wait) + self.wait(wait) + + except AttributeError: + self.logDebug("No wait time found") + + try: + self.link = re.search(r'id="dlLink">\s*<a href="(.+?)"', self.html).group(1) + + except AttributeError: + self.error(_("DL-Link not found")) + + +getInfo = create_getInfo(BasketbuildCom) diff --git a/module/plugins/hoster/BayfilesCom.py b/module/plugins/hoster/BayfilesCom.py index 0929ece01..cccd4acc7 100644 --- a/module/plugins/hoster/BayfilesCom.py +++ b/module/plugins/hoster/BayfilesCom.py @@ -9,6 +9,7 @@ class BayfilesCom(DeadHoster): __version__ = "0.09" __pattern__ = r'https?://(?:www\.)?bayfiles\.(com|net)/file/(?P<ID>\w+/\w+/[^/]+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Bayfiles.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index 8d56f1ff6..b47c2902d 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class BezvadataCz(SimpleHoster): __name__ = "BezvadataCz" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __pattern__ = r'http://(?:www\.)?bezvadata\.cz/stahnout/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """BezvaData.cz hoster plugin""" __license__ = "GPLv3" @@ -75,7 +76,7 @@ class BezvadataCz(SimpleHoster): wait_time = (int(m.group(1)) * 60 + int(m.group(2))) if m else 120 self.wait(wait_time, False) - self.download(url) + self.link = url def checkErrors(self): @@ -83,12 +84,12 @@ class BezvadataCz(SimpleHoster): self.longWait(5 * 60, 24) #: parallel dl limit elif '<div class="infobox' in self.html: self.tempOffline() - - self.info.pop('error', None) + else: + return super(BezvadataCz, self).checkErrors() def loadcaptcha(self, data, *args, **kwargs): - return data.decode("base64") + return data.decode('base64') getInfo = create_getInfo(BezvadataCz) diff --git a/module/plugins/hoster/BillionuploadsCom.py b/module/plugins/hoster/BillionuploadsCom.py index b20ace0f1..d0fbd7a8b 100644 --- a/module/plugins/hoster/BillionuploadsCom.py +++ b/module/plugins/hoster/BillionuploadsCom.py @@ -1,24 +1,19 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo +from module.plugins.internal.DeadHoster import DeadHoster, create_getInfo -class BillionuploadsCom(XFSHoster): +class BillionuploadsCom(DeadHoster): __name__ = "BillionuploadsCom" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.06" __pattern__ = r'http://(?:www\.)?billionuploads\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Billionuploads.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - HOSTER_DOMAIN = "billionuploads.com" - - NAME_PATTERN = r'<td class="dofir" title="(?P<N>.+?)"' - SIZE_PATTERN = r'<td class="dofir">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' - - getInfo = create_getInfo(BillionuploadsCom) diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py index f7b4d2709..5c709e4f7 100644 --- a/module/plugins/hoster/BitshareCom.py +++ b/module/plugins/hoster/BitshareCom.py @@ -4,16 +4,17 @@ from __future__ import with_statement import re -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class BitshareCom(SimpleHoster): __name__ = "BitshareCom" __type__ = "hoster" - __version__ = "0.52" + __version__ = "0.54" __pattern__ = r'http://(?:www\.)?bitshare\.com/(files/)?(?(1)|\?f=)(?P<ID>\w+)(?(1)/(?P<NAME>.+?)\.html)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Bitshare.com hoster plugin""" __license__ = "GPLv3" @@ -71,7 +72,7 @@ class BitshareCom(SimpleHoster): self.logDebug("File ajax id is [%s]" % self.ajaxid) # This may either download our file or forward us to an error page - self.download(self.getDownloadUrl()) + self.link = self.getDownloadUrl() if self.checkDownload({"error": ">Error occured<"}): self.retry(5, 5 * 60, "Bitshare host : Error occured") @@ -80,7 +81,7 @@ class BitshareCom(SimpleHoster): def getDownloadUrl(self): # Return location if direct download is active if self.premium: - header = self.load(self.pyfile.url, cookies=True, just_header=True) + header = self.load(self.pyfile.url, just_header=True) if 'location' in header: return header['location'] @@ -114,7 +115,7 @@ class BitshareCom(SimpleHoster): # Try up to 3 times for i in xrange(3): - challenge, response = recaptcha.challenge() + response, challenge = recaptcha.challenge() res = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", post={"request" : "validateCaptcha", "ajaxid" : self.ajaxid, diff --git a/module/plugins/hoster/BoltsharingCom.py b/module/plugins/hoster/BoltsharingCom.py index 924545a29..db813ba2e 100644 --- a/module/plugins/hoster/BoltsharingCom.py +++ b/module/plugins/hoster/BoltsharingCom.py @@ -9,6 +9,7 @@ class BoltsharingCom(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?boltsharing\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Boltsharing.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/CatShareNet.py b/module/plugins/hoster/CatShareNet.py index f2ddd64a0..c966dbe1a 100644 --- a/module/plugins/hoster/CatShareNet.py +++ b/module/plugins/hoster/CatShareNet.py @@ -3,15 +3,16 @@ import re from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha class CatShareNet(SimpleHoster): __name__ = "CatShareNet" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.15" __pattern__ = r'http://(?:www\.)?catshare\.net/\w{16}' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """CatShare.net hoster plugin""" __license__ = "GPLv3" @@ -23,11 +24,13 @@ class CatShareNet(SimpleHoster): TEXT_ENCODING = True INFO_PATTERN = r'<title>(?P<N>.+) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)<' - OFFLINE_PATTERN = ur'Podany plik zostaÅ usuniÄty\s*</div>' + OFFLINE_PATTERN = r'<div class="alert alert-error"' IP_BLOCKED_PATTERN = ur'>Nasz serwis wykryÅ ÅŒe Twój adres IP nie pochodzi z Polski.<' - SECONDS_PATTERN = 'var\scount\s=\s(\d+);' - LINK_FREE_PATTERN = r'<form action="(.+?)" method="GET">' + WAIT_PATTERN = r'var\scount\s=\s(\d+);' + + LINK_FREE_PATTERN = r'<form action="(.+?)" method="GET">' + LINK_PREMIUM_PATTERN = r'<form action="(.+?)" method="GET">' def setup(self): @@ -35,33 +38,17 @@ class CatShareNet(SimpleHoster): self.resumeDownload = True - def getFileInfo(self): - m = re.search(self.IP_BLOCKED_PATTERN, self.html) - if m: - self.fail(_("Only connections from Polish IP address are allowed")) - return super(CatShareNet, self).getFileInfo() - - def handleFree(self, pyfile): - m = re.search(self.SECONDS_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.wait(wait_time, True) - recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge() + response, challenge = recaptcha.challenge() self.html = self.load(pyfile.url, post={'recaptcha_challenge_field': challenge, 'recaptcha_response_field' : response}) m = re.search(self.LINK_FREE_PATTERN, self.html) - if m is None: - self.invalidCaptcha() - self.retry(reason=_("Wrong captcha entered")) - - dl_link = m.group(1) - self.download(dl_link, disposition=True) + if m: + self.link = m.group(1) getInfo = create_getInfo(CatShareNet) diff --git a/module/plugins/hoster/CloudzerNet.py b/module/plugins/hoster/CloudzerNet.py index c5440391f..af40b8d5f 100644 --- a/module/plugins/hoster/CloudzerNet.py +++ b/module/plugins/hoster/CloudzerNet.py @@ -9,6 +9,7 @@ class CloudzerNet(DeadHoster): __version__ = "0.05" __pattern__ = r'https?://(?:www\.)?(cloudzer\.net/file/|clz\.to/(file/)?)\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Cloudzer.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/CloudzillaTo.py b/module/plugins/hoster/CloudzillaTo.py index 58b1ac0dd..09453137d 100644 --- a/module/plugins/hoster/CloudzillaTo.py +++ b/module/plugins/hoster/CloudzillaTo.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class CloudzillaTo(SimpleHoster): __name__ = "CloudzillaTo" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'http://(?:www\.)?cloudzilla\.to/share/file/(?P<ID>[\w^_]+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Cloudzilla.to hoster plugin""" __license__ = "GPLv3" @@ -24,12 +25,17 @@ class CloudzillaTo(SimpleHoster): def checkErrors(self): - m = re.search(self.PASSWORD_PATTERN, self.html) - if m: - self.html = self.load(self.pyfile.url, get={'key': self.getPassword()}) + if re.search(self.PASSWORD_PATTERN, self.html): + pw = self.getPassword() + if pw: + self.html = self.load(self.pyfile.url, get={'key': pw}) + else: + self.fail(_("Missing password")) if re.search(self.PASSWORD_PATTERN, self.html): self.retry(reason="Wrong password") + else: + return super(CloudzillaTo, self).checkErrors() def handleFree(self, pyfile): @@ -47,7 +53,7 @@ class CloudzillaTo(SimpleHoster): self.fail(ticket['error']) if 'wait' in ticket: - self.wait(int(ticket['wait']), int(ticket['wait']) > 5) + self.wait(ticket['wait'], int(ticket['wait']) > 5) self.link = "http://%(server)s/download/%(file_id)s/%(ticket_id)s" % {'server' : ticket['server'], 'file_id' : self.info['pattern']['ID'], diff --git a/module/plugins/hoster/CramitIn.py b/module/plugins/hoster/CramitIn.py index f444718bc..44dac958d 100644 --- a/module/plugins/hoster/CramitIn.py +++ b/module/plugins/hoster/CramitIn.py @@ -15,8 +15,6 @@ class CramitIn(XFSHoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - HOSTER_DOMAIN = "cramit.in" - INFO_PATTERN = r'<span class=t2>\s*(?P<N>.*?)</span>.*?<small>\s*\((?P<S>.*?)\)' LINK_PATTERN = r'href="(http://cramit\.in/file_download/.*?)"' diff --git a/module/plugins/hoster/CrockoCom.py b/module/plugins/hoster/CrockoCom.py index 31d0eec95..098ba5fab 100644 --- a/module/plugins/hoster/CrockoCom.py +++ b/module/plugins/hoster/CrockoCom.py @@ -1,33 +1,35 @@ # -*- coding: utf-8 -*- import re +import urlparse -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class CrockoCom(SimpleHoster): __name__ = "CrockoCom" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.20" __pattern__ = r'http://(?:www\.)?(crocko|easy-share)\.com/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Crocko hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<span class="fz24">Download:\s*<strong>(?P<N>.*)' - SIZE_PATTERN = r'<span class="tip1"><span class="inner">(?P<S>[^<]+)</span></span>' + NAME_PATTERN = r'<span class="fz24">Download:\s*<strong>(?P<N>.*)' + SIZE_PATTERN = r'<span class="tip1"><span class="inner">(?P<S>[^<]+)</span></span>' OFFLINE_PATTERN = r'<h1>Sorry,<br />the page you\'re looking for <br />isn\'t here.</h1>|File not found' - CAPTCHA_PATTERN = re.compile(r"u='(/file_contents/captcha/\w+)';\s*w='(\d+)';") + CAPTCHA_PATTERN = r"u='(/file_contents/captcha/\w+)';\s*w='(\d+)';" - FORM_PATTERN = r'<form method="post" action="([^"]+)">(.*?)</form>' - FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' + FORM_PATTERN = r'<form method="post" action="(.+?)">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?.*?>' - NAME_REPLACEMENTS = [(r'<[^>]*>', '')] + NAME_REPLACEMENTS = [(r'<.*?>', '')] def handleFree(self, pyfile): @@ -37,8 +39,8 @@ class CrockoCom(SimpleHoster): for _i in xrange(5): m = re.search(self.CAPTCHA_PATTERN, self.html) if m: - url, wait_time = 'http://crocko.com' + m.group(1), int(m.group(2)) - self.wait(wait_time) + url = urlparse.urljoin("http://crocko.com", m.group(1)) + self.wait(m.group(2)) self.html = self.load(url) else: break @@ -52,7 +54,7 @@ class CrockoCom(SimpleHoster): recaptcha = ReCaptcha(self) for _i in xrange(5): - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge() + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge() self.download(action, post=inputs) if self.checkDownload({"captcha": recaptcha.KEY_AJAX_PATTERN}): diff --git a/module/plugins/hoster/CyberlockerCh.py b/module/plugins/hoster/CyberlockerCh.py index b26909096..9d748bf85 100644 --- a/module/plugins/hoster/CyberlockerCh.py +++ b/module/plugins/hoster/CyberlockerCh.py @@ -9,6 +9,7 @@ class CyberlockerCh(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?cyberlocker\.ch/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Cyberlocker.ch hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/CzshareCom.py b/module/plugins/hoster/CzshareCom.py index fb9e7f457..8f72f2148 100644 --- a/module/plugins/hoster/CzshareCom.py +++ b/module/plugins/hoster/CzshareCom.py @@ -12,16 +12,17 @@ from module.utils import parseFileSize class CzshareCom(SimpleHoster): __name__ = "CzshareCom" __type__ = "hoster" - __version__ = "0.97" + __version__ = "0.99" __pattern__ = r'http://(?:www\.)?(czshare|sdilej)\.(com|cz)/(\d+/|download\.php\?).+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """CZshare.com hoster plugin, now Sdilej.cz""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=[^>]*>(?P<N>[^<]+)</a>' + NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=.*?>(?P<N>[^<]+)</a>' SIZE_PATTERN = r'<div class="tab" id="category">(?:\s*<p>[^\n]*</p>)*\s*Velikost:\s*(?P<S>[\d .,]+)(?P<U>[\w^_]+)\s*</div>' OFFLINE_PATTERN = r'<div class="header clearfix">\s*<h2 class="red">' @@ -30,10 +31,10 @@ class CzshareCom(SimpleHoster): CHECK_TRAFFIC = True - FREE_URL_PATTERN = r'<a href="([^"]+)" class="page-download">[^>]*alt="([^"]+)" /></a>' + FREE_URL_PATTERN = r'<a href="(.+?)" class="page-download">[^>]*alt="(.+?)" /></a>' FREE_FORM_PATTERN = r'<form action="download\.php" method="post">\s*<img src="captcha\.php" id="captcha" />(.*?)</form>' PREMIUM_FORM_PATTERN = r'<form action="/profi_down\.php" method="post">(.*?)</form>' - FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)"[^>]*/>' + FORM_INPUT_PATTERN = r'<input[^>]* name="(.+?)" value="(.+?)"[^>]*/>' MULTIDL_PATTERN = r'<p><font color=\'red\'>Z[^<]*PROFI.</font></p>' USER_CREDIT_PATTERN = r'<div class="credit">\s*kredit: <strong>([\d .,]+)(\w+)</strong>\s*</div><!-- .credit -->' @@ -43,7 +44,7 @@ class CzshareCom(SimpleHoster): m = re.search(self.USER_CREDIT_PATTERN, self.html) if m is None: self.account.relogin(self.user) - self.html = self.load(self.pyfile.url, cookies=True, decode=True) + self.html = self.load(self.pyfile.url, decode=True) m = re.search(self.USER_CREDIT_PATTERN, self.html) if m is None: return False @@ -74,7 +75,6 @@ class CzshareCom(SimpleHoster): # download the file, destination is determined by pyLoad self.download("http://sdilej.cz/profi_down.php", post=inputs, disposition=True) - self.checkDownloadedFile() def handleFree(self, pyfile): @@ -88,7 +88,7 @@ class CzshareCom(SimpleHoster): self.logDebug("PARSED_URL:" + parsed_url) # get download ticket and parse html - self.html = self.load(parsed_url, cookies=True, decode=True) + self.html = self.load(parsed_url, decode=True) if re.search(self.MULTIDL_PATTERN, self.html): self.longWait(5 * 60, 12) @@ -105,7 +105,7 @@ class CzshareCom(SimpleHoster): captcha_url = 'http://sdilej.cz/captcha.php' for _i in xrange(5): inputs['captchastring2'] = self.decryptCaptcha(captcha_url) - self.html = self.load(parsed_url, cookies=True, post=inputs, decode=True) + self.html = self.load(parsed_url, post=inputs, decode=True) if u"<li>ZadanÃœ ovÄÅovacà kód nesouhlasÃ!</li>" in self.html: self.invalidCaptcha() @@ -129,14 +129,12 @@ class CzshareCom(SimpleHoster): if m is None: self.error(_("Download URL not found")) - url = "http://%s/download.php?%s" % (m.group(1), m.group(2)) + self.link = "http://%s/download.php?%s" % (m.group(1), m.group(2)) self.wait() - self.download(url) - self.checkDownloadedFile() - def checkDownloadedFile(self): + def checkFile(self, rules={}): # check download check = self.checkDownload({ "temp offline" : re.compile(r"^Soubor je do.*asn.* nedostupn.*$"), @@ -158,5 +156,7 @@ class CzshareCom(SimpleHoster): self.invalidCaptcha() self.retry() + return super(CzshareCom, self).checkFile(rules) + getInfo = create_getInfo(CzshareCom) diff --git a/module/plugins/hoster/DailymotionCom.py b/module/plugins/hoster/DailymotionCom.py index dc42d1f60..c6939e5e5 100644 --- a/module/plugins/hoster/DailymotionCom.py +++ b/module/plugins/hoster/DailymotionCom.py @@ -16,8 +16,8 @@ def getInfo(urls): for url in urls: id = regex.match(url).group('ID') - page = getURL(apiurl % id, get=request) - info = json_loads(page) + html = getURL(apiurl % id, get=request) + info = json_loads(html) name = info['title'] + ".mp4" if "title" in info else url @@ -72,7 +72,7 @@ class DailymotionCom(Hoster): def getQuality(self): - q = self.getConfig("quality") + q = self.getConfig('quality') if q == "Lowest": quality = 0 @@ -86,7 +86,7 @@ class DailymotionCom(Hoster): def getLink(self, streams, quality): if quality > 0: - for x, s in reversed([item for item in enumerate(streams)]): + for x, s in [item for item in enumerate(streams)][::-1]: qf = s[0][1] if qf <= quality: idx = x diff --git a/module/plugins/hoster/DataHu.py b/module/plugins/hoster/DataHu.py index f4b0692a8..955c94437 100644 --- a/module/plugins/hoster/DataHu.py +++ b/module/plugins/hoster/DataHu.py @@ -14,6 +14,7 @@ class DataHu(SimpleHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?data\.hu/get/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Data.hu hoster plugin""" __license__ = "GPLv3" @@ -23,7 +24,7 @@ class DataHu(SimpleHoster): INFO_PATTERN = ur'<title>(?P<N>.*) \((?P<S>[^)]+)\) let\xf6lt\xe9se</title>' OFFLINE_PATTERN = ur'Az adott f\xe1jl nem l\xe9tezik' - LINK_FREE_PATTERN = r'<div class="download_box_button"><a href="([^"]+)">' + LINK_FREE_PATTERN = r'<div class="download_box_button"><a href="(.+?)">' def setup(self): diff --git a/module/plugins/hoster/DataportCz.py b/module/plugins/hoster/DataportCz.py index 266199616..ad514f5eb 100644 --- a/module/plugins/hoster/DataportCz.py +++ b/module/plugins/hoster/DataportCz.py @@ -9,6 +9,7 @@ class DataportCz(SimpleHoster): __version__ = "0.41" __pattern__ = r'http://(?:www\.)?dataport\.cz/file/(.+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Dataport.cz hoster plugin""" __license__ = "GPLv3" @@ -37,7 +38,7 @@ class DataportCz(SimpleHoster): else: self.error(_("captcha")) - self.html = self.download("http://www.dataport.cz%s" % action, post=inputs) + self.download("http://www.dataport.cz%s" % action, post=inputs) check = self.checkDownload({"captcha": 'alert("\u0160patn\u011b opsan\u00fd k\u00f3d z obr\u00e1zu");', "slot" : 'alert("Je n\u00e1m l\u00edto, ale moment\u00e1ln\u011b nejsou'}) diff --git a/module/plugins/hoster/DateiTo.py b/module/plugins/hoster/DateiTo.py index 99b86e000..92a96b9ec 100644 --- a/module/plugins/hoster/DateiTo.py +++ b/module/plugins/hoster/DateiTo.py @@ -2,16 +2,17 @@ import re -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DateiTo(SimpleHoster): __name__ = "DateiTo" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.09" __pattern__ = r'http://(?:www\.)?datei\.to/datei/(?P<ID>\w+)\.html' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Datei.to hoster plugin""" __license__ = "GPLv3" @@ -23,7 +24,7 @@ class DateiTo(SimpleHoster): OFFLINE_PATTERN = r'>Datei wurde nicht gefunden<|>Bitte wÀhle deine Datei aus... <' WAIT_PATTERN = r'countdown\({seconds: (\d+)' - MULTIDL_PATTERN = r'>Du lÀdst bereits eine Datei herunter<' + DOWNLOAD_PATTERN = r'>Du lÀdst bereits eine Datei herunter' DATA_PATTERN = r'url: "(.*?)", data: "(.*?)",' @@ -52,23 +53,11 @@ class DateiTo(SimpleHoster): data = dict(x.split('=') for x in m.group(2).split('&')) if url.endswith('recaptcha.php'): - data['recaptcha_challenge_field'], data['recaptcha_response_field'] = recaptcha.challenge() + data['recaptcha_response_field'], data['recaptcha_challenge_field'] = recaptcha.challenge() else: self.fail(_("Too bad...")) - self.download(self.html) - - - def checkErrors(self): - m = re.search(self.MULTIDL_PATTERN, self.html) - if m: - m = re.search(self.WAIT_PATTERN, self.html) - wait_time = int(m.group(1)) if m else 30 - - errmsg = self.info['error'] = _("Parallel downloads") - self.retry(wait_time=wait_time, reason=errmsg) - - self.info.pop('error', None) + self.link = self.html def doWait(self): diff --git a/module/plugins/hoster/DdlstorageCom.py b/module/plugins/hoster/DdlstorageCom.py index a45ef27e9..976d9ccf9 100644 --- a/module/plugins/hoster/DdlstorageCom.py +++ b/module/plugins/hoster/DdlstorageCom.py @@ -9,6 +9,7 @@ class DdlstorageCom(DeadHoster): __version__ = "1.02" __pattern__ = r'https?://(?:www\.)?ddlstorage\.com/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """DDLStorage.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/DebridItaliaCom.py b/module/plugins/hoster/DebridItaliaCom.py index 8cda56c0c..4f64215fa 100644 --- a/module/plugins/hoster/DebridItaliaCom.py +++ b/module/plugins/hoster/DebridItaliaCom.py @@ -11,6 +11,7 @@ class DebridItaliaCom(MultiHoster): __version__ = "0.17" __pattern__ = r'https?://(?:www\.|s\d+\.)?debriditalia\.com/dl/\d+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Debriditalia.com multi-hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py index 9d060e75d..18f23f552 100644 --- a/module/plugins/hoster/DepositfilesCom.py +++ b/module/plugins/hoster/DepositfilesCom.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- import re +import urllib -from urllib import unquote - -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DepositfilesCom(SimpleHoster): __name__ = "DepositfilesCom" __type__ = "hoster" - __version__ = "0.52" + __version__ = "0.56" __pattern__ = r'https?://(?:www\.)?(depositfiles\.com|dfiles\.(eu|ru))(/\w{1,3})?/files/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Depositfiles.com hoster plugin""" __license__ = "GPLv3" @@ -27,37 +27,23 @@ class DepositfilesCom(SimpleHoster): OFFLINE_PATTERN = r'<span class="html_download_api-not_exists"></span>' NAME_REPLACEMENTS = [(r'\%u([0-9A-Fa-f]{4})', lambda m: unichr(int(m.group(1), 16))), - (r'.*<b title="(?P<N>[^"]+).*', "\g<N>")] + (r'.*<b title="(?P<N>.+?)".*', "\g<N>")] URL_REPLACEMENTS = [(__pattern__ + ".*", "https://dfiles.eu/files/\g<ID>")] COOKIES = [("dfiles.eu", "lang_current", "en")] + WAIT_PATTERN = r'(?:download_waiter_remain">|html_download_api-limit_interval">|>Please wait|>Try in).+' + ERROR_PATTER = r'File is checked, please try again in a minute' + LINK_FREE_PATTERN = r'<form id="downloader_file_form" action="(http://.+?\.(dfiles\.eu|depositfiles\.com)/.+?)" method="post"' LINK_PREMIUM_PATTERN = r'class="repeat"><a href="(.+?)"' LINK_MIRROR_PATTERN = r'class="repeat_mirror"><a href="(.+?)"' def handleFree(self, pyfile): - if re.search(r'File is checked, please try again in a minute.', self.html) is not None: - self.logInfo(_("The file is being checked. Waiting 1 minute")) - self.retry(wait_time=60) - - wait = re.search(r'html_download_api-limit_interval\">(\d+)</span>', self.html) - if wait: - wait_time = int(wait.group(1)) - self.logInfo(_("Traffic used up. Waiting %d seconds") % wait_time) - self.wait(wait_time, True) - self.retry() + self.html = self.load(pyfile.url, post={'gateway_result': "1"}) - wait = re.search(r'>Try in (\d+) minutes or use GOLD account', self.html) - if wait: - wait_time = int(wait.group(1)) - self.logInfo(_("All free slots occupied. Waiting %d minutes") % wait_time) - self.setWait(wait_time * 60, False) - - wait = re.search(r'Please wait (\d+) sec', self.html) - if wait: - self.setWait(int(wait.group(1))) + self.checkErrors() m = re.search(r"var fid = '(\w+)';", self.html) if m is None: @@ -65,57 +51,42 @@ class DepositfilesCom(SimpleHoster): params = {'fid': m.group(1)} self.logDebug("FID: %s" % params['fid']) - self.wait() + self.checkErrors() + recaptcha = ReCaptcha(self) captcha_key = recaptcha.detect_key() if captcha_key is None: - self.error(_("ReCaptcha key not found")) + return - for _i in xrange(5): - self.html = self.load("https://dfiles.eu/get_file.php", get=params) + self.html = self.load("https://dfiles.eu/get_file.php", get=params) - if '<input type=button value="Continue" onclick="check_recaptcha' in self.html: - if 'response' in params: - self.invalidCaptcha() - params['challenge'], params['response'] = recaptcha.challenge(captcha_key) - self.logDebug(params) - continue - - m = re.search(self.LINK_FREE_PATTERN, self.html) - if m: - if 'response' in params: - self.correctCaptcha() - link = unquote(m.group(1)) - self.logDebug("LINK: %s" % link) - break - else: - self.error(_("Download link")) - else: - self.fail(_("No valid captcha response received")) + if '<input type=button value="Continue" onclick="check_recaptcha' in self.html: + params['response'], params['challenge'] = recaptcha.challenge(captcha_key) + self.html = self.load("https://dfiles.eu/get_file.php", get=params) - try: - self.download(link, disposition=True) - except: - self.retry(wait_time=60) + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + self.link = urllib.unquote(m.group(1)) def handlePremium(self, pyfile): if '<span class="html_download_api-gold_traffic_limit">' in self.html: self.logWarning(_("Download limit reached")) self.retry(25, 60 * 60, "Download limit reached") + elif 'onClick="show_gold_offer' in self.html: self.account.relogin(self.user) self.retry() + else: - link = re.search(self.LINK_PREMIUM_PATTERN, self.html) + link = re.search(self.LINK_PREMIUM_PATTERN, self.html) mirror = re.search(self.LINK_MIRROR_PATTERN, self.html) + if link: - dlink = link.group(1) + self.link = link.group(1) + elif mirror: - dlink = mirror.group(1) - else: - self.error(_("No direct download link or mirror found")) - self.download(dlink, disposition=True) + self.link = mirror.group(1) getInfo = create_getInfo(DepositfilesCom) diff --git a/module/plugins/hoster/DevhostSt.py b/module/plugins/hoster/DevhostSt.py index 2a8734655..e7a23f6b1 100644 --- a/module/plugins/hoster/DevhostSt.py +++ b/module/plugins/hoster/DevhostSt.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Test links: -# http://d-h.st/mM8 +# http://d-h.st/mM8 import re @@ -11,24 +11,26 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DevhostSt(SimpleHoster): __name__ = "DevhostSt" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'http://(?:www\.)?d-h\.st/(?!users/)\w{3}' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """d-h.st hoster plugin""" __license__ = "GPLv3" __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] - NAME_PATTERN = r'>Filename:</span> <div title="(?P<N>.+?)"' - SIZE_PATTERN = r'>Size:</span> (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + NAME_PATTERN = r'<span title="(?P<N>.*?)"' + SIZE_PATTERN = r'</span> \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)<br' + HASHSUM_PATTERN = r'>(?P<T>.*?) Sum</span>: (?P<H>.*?)<br' - OFFLINE_PATTERN = r'>File Not Found<' - LINK_FREE_PATTERN = r'id="downloadfile" href="(.+?)"' + OFFLINE_PATTERN = r'>File Not Found' + LINK_FREE_PATTERN = r'var product_download_url= \'(.+?)\'' def setup(self): - self.multiDL = True + self.multiDL = True self.chunkLimit = 1 diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index ec40b64ce..ff5ee7b92 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -5,7 +5,6 @@ import re from module.network.Browser import Browser from module.network.CookieJar import CookieJar -from module.plugins.internal.CaptchaService import AdYouLike from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns @@ -36,19 +35,20 @@ class CustomBrowser(Browser): class DlFreeFr(SimpleHoster): __name__ = "DlFreeFr" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.30" __pattern__ = r'http://(?:www\.)?dl\.free\.fr/(\w+|getfile\.pl\?file=/\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Dl.free.fr hoster plugin""" __license__ = "GPLv3" __authors__ = [("the-razer", "daniel_ AT gmx DOT net"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("Toilal", "toilal.dev@gmail.com")] + ("zoidberg" , "zoidberg@mujmail.cz" ), + ("Toilal" , "toilal.dev@gmail.com" )] - NAME_PATTERN = r'Fichier:</td>\s*<td[^>]*>(?P<N>[^>]*)</td>' - SIZE_PATTERN = r'Taille:</td>\s*<td[^>]*>(?P<S>[\d.,]+\w)o' + NAME_PATTERN = r'Fichier:</td>\s*<td.*?>(?P<N>[^>]*)</td>' + SIZE_PATTERN = r'Taille:</td>\s*<td.*?>(?P<S>[\d.,]+\w)o' OFFLINE_PATTERN = r'Erreur 404 - Document non trouv|Fichier inexistant|Le fichier demandé n\'a pas été trouvé' @@ -81,31 +81,31 @@ class DlFreeFr(SimpleHoster): self.handleFree(pyfile) else: # Direct access to requested file for users using free.fr as Internet Service Provider. - self.download(valid_url, disposition=True) + self.link = valid_url + + self.download(self.link, disposition=True) + elif headers.get('code') == 404: self.offline() + else: self.fail(_("Invalid return code: ") + str(headers.get('code'))) def handleFree(self, pyfile): action, inputs = self.parseHtmlForm('action="getfile.pl"') - - adyoulike = AdYouLike(self) - inputs.update(adyoulike.challenge()) - self.load("http://dl.free.fr/getfile.pl", post=inputs) headers = self.getLastHeaders() if headers.get("code") == 302 and "set-cookie" in headers and "location" in headers: - m = re.search("(.*?)=(.*?); path=(.*?); domain=(.*?)", headers.get("set-cookie")) + m = re.search("(.*?)=(.*?); path=(.*?); domain=(.*)", headers.get("set-cookie")) cj = CookieJar(__name__) if m: cj.setCookie(m.group(4), m.group(1), m.group(2), m.group(3)) else: self.fail(_("Cookie error")) - location = headers.get("location") + + self.link = headers.get("location") self.req.setCookieJar(cj) - self.download(location, disposition=True) else: self.fail(_("Invalid response")) diff --git a/module/plugins/hoster/DodanePl.py b/module/plugins/hoster/DodanePl.py index 65d8452fa..46f748cc4 100644 --- a/module/plugins/hoster/DodanePl.py +++ b/module/plugins/hoster/DodanePl.py @@ -9,6 +9,7 @@ class DodanePl(DeadHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?dodane\.pl/file/\d+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Dodane.pl hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/DropboxCom.py b/module/plugins/hoster/DropboxCom.py index a8ef5b4bc..eec968f5a 100644 --- a/module/plugins/hoster/DropboxCom.py +++ b/module/plugins/hoster/DropboxCom.py @@ -11,6 +11,7 @@ class DropboxCom(SimpleHoster): __version__ = "0.04" __pattern__ = r'https?://(?:www\.)?dropbox\.com/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Dropbox.com hoster plugin""" __license__ = "GPLv3" @@ -26,8 +27,8 @@ class DropboxCom(SimpleHoster): def setup(self): - self.multiDL = True - self.chunkLimit = 1 + self.multiDL = True + self.chunkLimit = 1 self.resumeDownload = True diff --git a/module/plugins/hoster/DuploadOrg.py b/module/plugins/hoster/DuploadOrg.py index 73702eb67..076171544 100644 --- a/module/plugins/hoster/DuploadOrg.py +++ b/module/plugins/hoster/DuploadOrg.py @@ -9,6 +9,7 @@ class DuploadOrg(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?dupload\.org/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Dupload.grg hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py index cd54bdc70..693910c1b 100644 --- a/module/plugins/hoster/EasybytezCom.py +++ b/module/plugins/hoster/EasybytezCom.py @@ -16,8 +16,6 @@ class EasybytezCom(XFSHoster): ("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "easybytez.com" - OFFLINE_PATTERN = r'>File not available' LINK_PATTERN = r'(http://(\w+\.(easybytez|easyload|ezbytez|zingload)\.(com|to)|\d+\.\d+\.\d+\.\d+)/files/\d+/\w+/.+?)["\'<]' diff --git a/module/plugins/hoster/EdiskCz.py b/module/plugins/hoster/EdiskCz.py index f8ccc972e..b8485001e 100644 --- a/module/plugins/hoster/EdiskCz.py +++ b/module/plugins/hoster/EdiskCz.py @@ -11,13 +11,14 @@ class EdiskCz(SimpleHoster): __version__ = "0.23" __pattern__ = r'http://(?:www\.)?edisk\.(cz|sk|eu)/(stahni|sk/stahni|en/download)/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Edisk.cz hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - INFO_PATTERN = r'<span class="fl" title="(?P<N>[^"]+)">\s*.*?\((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)</h1></span>' + INFO_PATTERN = r'<span class="fl" title="(?P<N>.+?)">\s*.*?\((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)</h1></span>' OFFLINE_PATTERN = r'<h3>This file does not exist due to one of the following:</h3><ul><li>' ACTION_PATTERN = r'/en/download/(\d+/.*\.html)' @@ -50,7 +51,7 @@ class EdiskCz(SimpleHoster): if not re.match(self.LINK_FREE_PATTERN, url): self.fail(_("Unexpected server response")) - self.download(url) + self.link = url getInfo = create_getInfo(EdiskCz) diff --git a/module/plugins/hoster/EgoFilesCom.py b/module/plugins/hoster/EgoFilesCom.py index 9a2f50ed4..78108962f 100644 --- a/module/plugins/hoster/EgoFilesCom.py +++ b/module/plugins/hoster/EgoFilesCom.py @@ -9,6 +9,7 @@ class EgoFilesCom(DeadHoster): __version__ = "0.16" __pattern__ = r'https?://(?:www\.)?egofiles\.com/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Egofiles.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/EnteruploadCom.py b/module/plugins/hoster/EnteruploadCom.py index bbd613f57..8ef994e1e 100644 --- a/module/plugins/hoster/EnteruploadCom.py +++ b/module/plugins/hoster/EnteruploadCom.py @@ -9,6 +9,7 @@ class EnteruploadCom(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?enterupload\.com/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """EnterUpload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/EpicShareNet.py b/module/plugins/hoster/EpicShareNet.py index 8ac8cdaf2..7e3c8ba0e 100644 --- a/module/plugins/hoster/EpicShareNet.py +++ b/module/plugins/hoster/EpicShareNet.py @@ -9,6 +9,7 @@ class EpicShareNet(DeadHoster): __version__ = "0.02" __pattern__ = r'https?://(?:www\.)?epicshare\.net/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """EpicShare.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py index 08d8a2e3e..842e0cb87 100644 --- a/module/plugins/hoster/EuroshareEu.py +++ b/module/plugins/hoster/EuroshareEu.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class EuroshareEu(SimpleHoster): __name__ = "EuroshareEu" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.28" __pattern__ = r'http://(?:www\.)?euroshare\.(eu|sk|cz|hu|pl)/file/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Euroshare.eu hoster plugin""" __license__ = "GPLv3" @@ -33,7 +34,7 @@ class EuroshareEu(SimpleHoster): self.account.relogin(self.user) self.retry(reason=_("User not logged in")) - self.download(pyfile.url.rstrip('/') + "/download/") + self.link = pyfile.url.rstrip('/') + "/download/" check = self.checkDownload({"login": re.compile(self.ERR_NOT_LOGGED_IN_PATTERN), "json" : re.compile(r'\{"status":"error".*?"message":"(.*?)"')}) @@ -47,22 +48,21 @@ class EuroshareEu(SimpleHoster): def handleFree(self, pyfile): - if re.search(self.ERR_PARDL_PATTERN, self.html) is not None: + if re.search(self.ERR_PARDL_PATTERN, self.html): self.longWait(5 * 60, 12) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: self.error(_("LINK_FREE_PATTERN not found")) - parsed_url = "http://euroshare.eu%s" % m.group(1) - self.logDebug("URL", parsed_url) - self.download(parsed_url, disposition=True) + self.link = "http://euroshare.eu%s" % m.group(1) - def checkFile(): + + def checkFile(self, rules={}): if self.checkDownload({"multi-dl": re.compile(self.ERR_PARDL_PATTERN)}) self.longWait(5 * 60, 12) - return super(EuroshareEu, self).checkFile() + return super(EuroshareEu, self).checkFile(rules) getInfo = create_getInfo(EuroshareEu) diff --git a/module/plugins/hoster/ExashareCom.py b/module/plugins/hoster/ExashareCom.py new file mode 100644 index 000000000..d70ca6d3a --- /dev/null +++ b/module/plugins/hoster/ExashareCom.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo + + +class ExashareCom(XFSHoster): + __name__ = "ExashareCom" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?exashare\.com/\w{12}' + + __description__ = """Exashare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + INFO_PATTERN = r'>(?P<NAME>.+?)<small>\( (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + self.resumeDownload = self.premium + + + def handleFree(self, pyfile): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("Free download link not found")) + else: + self.link = m.group(1) + + +getInfo = create_getInfo(ExashareCom) diff --git a/module/plugins/hoster/ExtabitCom.py b/module/plugins/hoster/ExtabitCom.py index 5c2976a29..ead2e1e53 100644 --- a/module/plugins/hoster/ExtabitCom.py +++ b/module/plugins/hoster/ExtabitCom.py @@ -4,23 +4,24 @@ import re from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight class ExtabitCom(SimpleHoster): __name__ = "ExtabitCom" __type__ = "hoster" - __version__ = "0.64" + __version__ = "0.66" __pattern__ = r'http://(?:www\.)?extabit\.com/(file|go|fid)/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Extabit.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<th>File:</th>\s*<td class="col-fileinfo">\s*<div title="(?P<N>[^"]+)">' + NAME_PATTERN = r'<th>File:</th>\s*<td class="col-fileinfo">\s*<div title="(?P<N>.+?)">' SIZE_PATTERN = r'<th>Size:</th>\s*<td class="col-fileinfo">(?P<S>[^<]+)</td>' OFFLINE_PATTERN = r'>File not found<' TEMP_OFFLINE_PATTERN = r'>(File is temporary unavailable|No download mirror)<' @@ -50,7 +51,7 @@ class ExtabitCom(SimpleHoster): for _i in xrange(5): get_data = {"type": "recaptcha"} - get_data['challenge'], get_data['capture'] = recaptcha.challenge(captcha_key) + get_data['capture'], get_data['challenge'] = recaptcha.challenge(captcha_key) res = json_loads(self.load("http://extabit.com/file/%s/" % fileID, get=get_data)) if "ok" in res: self.correctCaptcha() @@ -71,8 +72,7 @@ class ExtabitCom(SimpleHoster): if m is None: self.error(_("LINK_FREE_PATTERN not found")) - url = m.group(1) - self.download(url) + self.link = m.group(1) getInfo = create_getInfo(ExtabitCom) diff --git a/module/plugins/hoster/FastixRu.py b/module/plugins/hoster/FastixRu.py index 9a38f5a72..cc50f4229 100644 --- a/module/plugins/hoster/FastixRu.py +++ b/module/plugins/hoster/FastixRu.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- import re - -from random import randrange -from urllib import unquote +import urllib from module.common.json_layer import json_loads from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -12,25 +10,16 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class FastixRu(MultiHoster): __name__ = "FastixRu" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.11" __pattern__ = r'http://(?:www\.)?fastix\.(ru|it)/file/\w{24}' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Fastix multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("Massimo Rosamilia", "max@spiritix.eu")] - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - if name.endswith("..."): # incomplete filename, append random stuff - name += "%s.tmp" % randrange(100, 999) - return name - - def setup(self): self.chunkLimit = 3 @@ -39,30 +28,17 @@ class FastixRu(MultiHoster): api_key = self.account.getAccountData(self.user) api_key = api_key['api'] - page = self.load("http://fastix.ru/api_v2/", + self.html = self.load("http://fastix.ru/api_v2/", get={'apikey': api_key, 'sub': "getdirectlink", 'link': pyfile.url}) - data = json_loads(page) + + data = json_loads(self.html) self.logDebug("Json data", data) - if "error\":true" in page: + if "error\":true" in self.html: self.offline() else: self.link = data['downloadlink'] - if self.link != pyfile.url: - self.logDebug("New URL: %s" % self.link) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown"): - #only use when name wasnt already set - pyfile.name = self.getFilename(self.link) - - - def checkFile(self): - if self.checkDownload({"error": "<title>An error occurred while processing your request</title>"}): - self.retry(wait_time=60, reason=_("An error occurred while generating link")) - - return super(FastixRu, self).checkFile() - getInfo = create_getInfo(FastixRu) diff --git a/module/plugins/hoster/FastshareCz.py b/module/plugins/hoster/FastshareCz.py index 6fe871ad4..330a6e3b9 100644 --- a/module/plugins/hoster/FastshareCz.py +++ b/module/plugins/hoster/FastshareCz.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -10,9 +9,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FastshareCz(SimpleHoster): __name__ = "FastshareCz" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.29" __pattern__ = r'http://(?:www\.)?fastshare\.cz/\d+/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FastShare.cz hoster plugin""" __license__ = "GPLv3" @@ -22,11 +22,12 @@ class FastshareCz(SimpleHoster): COOKIES = [("fastshare.cz", "lang", "en")] - INFO_PATTERN = r'<h1 class="dwp">(?P<N>[^<]+)</h1>\s*<div class="fileinfo">\s*Size\s*: (?P<S>\d+) (?P<U>[\w^_]+),' + NAME_PATTERN = r'<h3 class="section_title">(?P<N>.+?)<' + SIZE_PATTERN = r'>Size\s*:</strong> (?P<S>[\d.,]+) (?P<U>[\w^_]+)' OFFLINE_PATTERN = r'>(The file has been deleted|Requested page not found)' - LINK_FREE_PATTERN = r'action=(/free/.*?)>\s*<img src="([^"]*)"><br' - LINK_PREMIUM_PATTERN = r'(http://data\d+\.fastshare\.cz/download\.php\?id=\d+&)' + LINK_FREE_PATTERN = r'>Enter the code\s*:</em>\s*<span><img src="(.+?)"' + LINK_PREMIUM_PATTERN = r'(http://\w+\.fastshare\.cz/download\.php\?id=\d+&)' SLOT_ERROR = "> 100% of FREE slots are full" CREDIT_ERROR = " credit for " @@ -53,11 +54,11 @@ class FastshareCz(SimpleHoster): self.error(_("FREE_URL_PATTERN not found")) baseurl = "http://www.fastshare.cz" - captcha = self.decryptCaptcha(urljoin(baseurl, captcha_src)) - self.download(urljoin(baseurl, action), post={'code': captcha, 'btn.x': 77, 'btn.y': 18}) + captcha = self.decryptCaptcha(urlparse.urljoin(baseurl, captcha_src)) + self.download(urlparse.urljoin(baseurl, action), post={'code': captcha, 'btn.x': 77, 'btn.y': 18}) - def checkFile(self): + def checkFile(self, rules={}): check = self.checkDownload({ 'paralell-dl' : re.compile(r"<title>FastShare.cz</title>|<script>alert\('Pres FREE muzete stahovat jen jeden soubor najednou.'\)"), 'wrong captcha': re.compile(r'Download for FREE'), @@ -73,7 +74,7 @@ class FastshareCz(SimpleHoster): elif check == "credit": self.resetAccount() - return super(FastshareCz, self).checkFile() + return super(FastshareCz, self).checkFile(rules) getInfo = create_getInfo(FastshareCz) diff --git a/module/plugins/hoster/FileApeCom.py b/module/plugins/hoster/FileApeCom.py index db843586b..79157539b 100644 --- a/module/plugins/hoster/FileApeCom.py +++ b/module/plugins/hoster/FileApeCom.py @@ -9,6 +9,7 @@ class FileApeCom(DeadHoster): __version__ = "0.12" __pattern__ = r'http://(?:www\.)?fileape\.com/(index\.php\?act=download\&id=|dl/)\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """FileApe.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FileSharkPl.py b/module/plugins/hoster/FileSharkPl.py index d686be480..5be339896 100644 --- a/module/plugins/hoster/FileSharkPl.py +++ b/module/plugins/hoster/FileSharkPl.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -10,9 +9,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FileSharkPl(SimpleHoster): __name__ = "FileSharkPl" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.11" - __pattern__ = r'http://(?:www\.)?fileshark\.pl/pobierz/\d{6}/\w{5}' + __pattern__ = r'http://(?:www\.)?fileshark\.pl/pobierz/\d+/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FileShark.pl hoster plugin""" __license__ = "GPLv3" @@ -20,25 +20,25 @@ class FileSharkPl(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<h2 class="name-file">(?P<N>.+)</h2>' - SIZE_PATTERN = r'<p class="size-file">(.*?)<strong>(?P<S>\d+\.?\d*)\s(?P<U>\w+)</strong></p>' - - OFFLINE_PATTERN = '(P|p)lik zosta. (usuni.ty|przeniesiony)' + NAME_PATTERN = r'<h2 class="name-file">(?P<N>.+?)</h2>' + SIZE_PATTERN = r'<p class="size-file">(.*?)<strong>(?P<S>\d+\.?\d*)\s(?P<U>\w+)</strong></p>' + OFFLINE_PATTERN = r'(P|p)lik zosta. (usuni.ty|przeniesiony)' - LINK_FREE_PATTERN = r'<a href="(.*?)" class="btn-upload-free">' - LINK_PREMIUM_PATTERN = r'<a href="(.*?)" class="btn-upload-premium">' + LINK_FREE_PATTERN = r'<a rel="nofollow" href="(.*?)" class="btn-upload-free">' + LINK_PREMIUM_PATTERN = r'<a rel="nofollow" href="(.*?)" class="btn-upload-premium">' WAIT_PATTERN = r'var timeToDownload = (\d+);' ERROR_PATTERN = r'<p class="lead text-center alert alert-warning">(.*?)</p>' IP_ERROR_PATTERN = r'Strona jest dost.pna wy..cznie dla u.ytkownik.w znajduj.cych si. na terenie Polski' SLOT_ERROR_PATTERN = r'Osi.gni.to maksymaln. liczb. .ci.ganych jednocze.nie plik.w\.' - CAPTCHA_PATTERN = '<img src="data:image/jpeg;base64,(.*?)" title="captcha"' + CAPTCHA_PATTERN = r'<img src="data:image/jpeg;base64,(.*?)" title="captcha"' TOKEN_PATTERN = r'name="form\[_token\]" value="(.*?)" />' def setup(self): self.resumeDownload = True + if self.premium: self.multiDL = True self.limitDL = 20 @@ -53,7 +53,7 @@ class FileSharkPl(SimpleHoster): errmsg = self.info['error'] = _("Another download already run") self.retry(15, int(m.group(1)), errmsg) - m = re.search(self.ERROR_PATTERN, self.html): + m = re.search(self.ERROR_PATTERN, self.html) if m: alert = m.group(1) @@ -72,19 +72,14 @@ class FileSharkPl(SimpleHoster): self.info.pop('error', None) - #@NOTE: handlePremium method was never been tested - def handlePremium(self, pyfile): - super(FilerNet, self).handlePremium(pyfile) - if self.link: - self.link = urljoin("http://fileshark.pl/", self.link) - - def handleFree(self, pyfile): m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: self.error(_("Download url not found")) - link = urljoin("http://fileshark.pl", m.group(1)) + link = urlparse.urljoin("http://fileshark.pl", m.group(1)) + + self.html = self.load(link) m = re.search(self.WAIT_PATTERN, self.html) if m: @@ -112,29 +107,11 @@ class FileSharkPl(SimpleHoster): self.load = tmp_load - self.download(link, post=inputs, cookies=True, disposition=True) - - - def checkFile(self): - check = self.checkDownload({'wrong_captcha': re.compile(r'<label for="form_captcha" generated="true" class="error">(.*?)</label>'), - 'wait_pattern' : re.compile(self.SECONDS_PATTERN), - 'DL-found' : re.compile('<a href="(.*)">')}) - - if check == "DL-found": - self.correctCaptcha() - - elif check == "wrong_captcha": - self.invalidCaptcha() - self.retry(10, 1, _("Wrong captcha solution")) - - elif check == "wait_pattern": - self.retry() - - return super(FileSharkPl, self).checkFile() + self.download(link, post=inputs, disposition=True) def _decode64(self, data, *args, **kwargs): - return data.decode("base64") + return data.decode('base64') getInfo = create_getInfo(FileSharkPl) diff --git a/module/plugins/hoster/FileStoreTo.py b/module/plugins/hoster/FileStoreTo.py index 7b93c03f1..3262b73ad 100644 --- a/module/plugins/hoster/FileStoreTo.py +++ b/module/plugins/hoster/FileStoreTo.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FileStoreTo(SimpleHoster): __name__ = "FileStoreTo" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.05" __pattern__ = r'http://(?:www\.)?filestore\.to/\?d=(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FileStore.to hoster plugin""" __license__ = "GPLv3" @@ -18,8 +19,9 @@ class FileStoreTo(SimpleHoster): ("stickell", "l.stickell@yahoo.it")] - INFO_PATTERN = r'File: <span[^>]*>(?P<N>.+)</span><br />Size: (?P<S>[\d.,]+) (?P<U>[\w^_]+)' - OFFLINE_PATTERN = r'>Download-Datei wurde nicht gefunden<' + INFO_PATTERN = r'File: <span.*?>(?P<N>.+?)<.*>Size: (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'>Download-Datei wurde nicht gefunden<' + TEMP_OFFLINE_PATTERN = r'>Der Download ist nicht bereit !<' def setup(self): @@ -29,9 +31,8 @@ class FileStoreTo(SimpleHoster): def handleFree(self, pyfile): self.wait(10) - ldc = re.search(r'wert="(\w+)"', self.html).group(1) - link = self.load("http://filestore.to/ajax/download.php", get={"LDC": ldc}) - self.download(link) + self.link = self.load("http://filestore.to/ajax/download.php", + get={'D': re.search(r'"D=(\w+)', self.html).group(1)}) getInfo = create_getInfo(FileStoreTo) diff --git a/module/plugins/hoster/FilebeerInfo.py b/module/plugins/hoster/FilebeerInfo.py index 885010a2c..244131dc7 100644 --- a/module/plugins/hoster/FilebeerInfo.py +++ b/module/plugins/hoster/FilebeerInfo.py @@ -9,6 +9,7 @@ class FilebeerInfo(DeadHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?filebeer\.info/(?!\d*~f)(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Filebeer.info plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FileboomMe.py b/module/plugins/hoster/FileboomMe.py new file mode 100644 index 000000000..9cdeebe3d --- /dev/null +++ b/module/plugins/hoster/FileboomMe.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class FileboomMe(SimpleHoster): + __name__ = "FileboomMe" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'https?://f(?:ile)?boom\.me/file/(?P<ID>\w+)' + + __description__ = """Fileboom.me hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("GammaC0de", None)] + + + NAME_PATTERN = r'<i class="icon-download"></i>\s*(?P<N>.+?)\s*<' + SIZE_PATTERN = r'File size: (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'>This file is no longer available' + + WAIT_PATTERN = r'<div class="tik-tak">([\d:]+)' + LINK_PATTERN = r'/file/url\.html\?file=\w+' + + CAPTCHA_PATTERN = r'<img .* src="(/file/captcha.html\?v=\w+)"' + + + def setup(self): + self.resumeDownload = True + self.multiDL = False + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + post_url = urljoin(pyfile.url, "/file/" + self.info['pattern']['ID']) + + m = re.search(r'data-slow-id="(\w+)"', self.html) + if m: + self.html = self.load(post_url, + post={'slow_id': m.group(1)}) + + m = re.search(self.LINK_PATTERN, self.html) + if m: + self.link = urljoin(pyfile.url, m.group(0)) + + else: + for _i in xrange(5): + m = re.search(r'<input type="hidden" name="uniqueId" value="(\w+)">', self.html) + if m: + uniqueId = m.group(1) + + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + captcha = self.decryptCaptcha(urljoin(pyfile.url, m.group(1))) + + self.html = self.load(post_url, + post={'CaptchaForm[code]' : captcha, + 'free' : 1, + 'freeDownloadRequest': 1, + 'uniqueId' : uniqueId}) + + if 'The verification code is incorrect' in self.html: + self.invalidCaptcha() + + else: + self.checkErrors() + + self.html = self.load(post_url, + post={'free' : 1, + 'uniqueId': uniqueId}) + + m = re.search(self.LINK_PATTERN, self.html) + if m: + self.link = urljoin(pyfile.url, m.group(0)) + + else: + self.invalidCaptcha() + + break + + else: + self.fail(_("Captcha not found")) + + else: + m = re.search(r'>\s*Please wait ([\d:]+)', self.html) + if m: + wait_time = 0 + for v in re.findall(r'(\d+)', m.group(1), re.I): + wait_time = 60 * wait_time + int(v) + self.wait(wait_time) + self.retry() + break + + else: + self.fail(_("Invalid captcha")) + + +getInfo = create_getInfo(FileboomMe) diff --git a/module/plugins/hoster/FilecloudIo.py b/module/plugins/hoster/FilecloudIo.py index bc328ddbd..346e9c444 100644 --- a/module/plugins/hoster/FilecloudIo.py +++ b/module/plugins/hoster/FilecloudIo.py @@ -3,16 +3,17 @@ import re from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FilecloudIo(SimpleHoster): __name__ = "FilecloudIo" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.09" __pattern__ = r'http://(?:www\.)?(?:filecloud\.io|ifile\.it|mihd\.net)/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Filecloud.io hoster plugin""" __license__ = "GPLv3" @@ -59,7 +60,7 @@ class FilecloudIo(SimpleHoster): if captcha_key is None: self.error(_("ReCaptcha key not found")) - challenge, response = recaptcha.challenge(captcha_key) + response, challenge = recaptcha.challenge(captcha_key) self.account.form_data = {"recaptcha_challenge_field": challenge, "recaptcha_response_field" : response} self.account.relogin(self.user) @@ -78,7 +79,7 @@ class FilecloudIo(SimpleHoster): data['ctype'] = "recaptcha" for _i in xrange(5): - data['recaptcha_challenge'], data['recaptcha_response'] = recaptcha.challenge(captcha_key) + data['recaptcha_response'], data['recaptcha_challenge'] = recaptcha.challenge(captcha_key) json_url = "http://filecloud.io/download-request.json" res = self.load(json_url, post=data) @@ -103,8 +104,7 @@ class FilecloudIo(SimpleHoster): if "size" in self.info and self.info['size']: self.check_data = {"size": int(self.info['size'])} - download_url = m.group(1) - self.download(download_url) + self.link = m.group(1) else: self.fail(_("Unexpected server response")) @@ -118,7 +118,7 @@ class FilecloudIo(SimpleHoster): self.logDebug("FetchDownloadUrl: " + rep) rep = json_loads(rep) if rep['status'] == 'ok': - self.download(rep['download_url'], disposition=True) + self.link = rep['download_url'] else: self.fail(rep['message']) diff --git a/module/plugins/hoster/FilefactoryCom.py b/module/plugins/hoster/FilefactoryCom.py index 2c8ec58f4..ea1a38b7a 100644 --- a/module/plugins/hoster/FilefactoryCom.py +++ b/module/plugins/hoster/FilefactoryCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.network.RequestFactory import getURL from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo @@ -21,9 +20,10 @@ def getInfo(urls): class FilefactoryCom(SimpleHoster): __name__ = "FilefactoryCom" __type__ = "hoster" - __version__ = "0.53" + __version__ = "0.55" __pattern__ = r'https?://(?:www\.)?filefactory\.com/(file|trafficshare/\w+)/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Filefactory.com hoster plugin""" __license__ = "GPLv3" @@ -32,7 +32,7 @@ class FilefactoryCom(SimpleHoster): INFO_PATTERN = r'<div id="file_name"[^>]*>\s*<h2>(?P<N>[^<]+)</h2>\s*<div id="file_info">\s*(?P<S>[\d.,]+) (?P<U>[\w^_]+) uploaded' - OFFLINE_PATTERN = r'<h2>File Removed</h2>|This file is no longer available' + OFFLINE_PATTERN = r'<h2>File Removed</h2>|This file is no longer available|Invalid Download Link' LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'"([^"]+filefactory\.com/get.+?)"' @@ -52,14 +52,14 @@ class FilefactoryCom(SimpleHoster): if m is None: self.error(_("Free download link not found")) - dl_link = m.group(1) + self.link = m.group(1) m = re.search(self.WAIT_PATTERN, self.html) if m: - self.wait(int(m.group(1))) + self.wait(m.group(1)) - self.download(dl_link, disposition=True) + def checkFile(self, rules={}): check = self.checkDownload({'multiple': "You are currently downloading too many files at once.", 'error' : '<div id="errorMessage">'}) @@ -70,6 +70,8 @@ class FilefactoryCom(SimpleHoster): elif check == "error": self.error(_("Unknown error")) + return super(FilefactoryCom, self).checkFile(rules) + def handlePremium(self, pyfile): self.link = self.directLink(self.load(pyfile.url, just_header=True)) diff --git a/module/plugins/hoster/FilejungleCom.py b/module/plugins/hoster/FilejungleCom.py index 8a8aee9e2..236603c6e 100644 --- a/module/plugins/hoster/FilejungleCom.py +++ b/module/plugins/hoster/FilejungleCom.py @@ -19,7 +19,7 @@ class FilejungleCom(FileserveCom): URLS = ["http://www.filejungle.com/f/", "http://www.filejungle.com/check_links.php", "http://www.filejungle.com/checkReCaptcha.php"] LINKCHECK_TR = r'<li>\s*(<div class="col1">.*?)</li>' - LINKCHECK_TD = r'<div class="(?:col )?col\d">(?:<[^>]*>| )*([^<]*)' + LINKCHECK_TD = r'<div class="(?:col )?col\d">(?:<.*?>| )*([^<]*)' LONG_WAIT_PATTERN = r'<h1>Please wait for (\d+) (\w+)\s*to download the next file\.</h1>' diff --git a/module/plugins/hoster/FileomCom.py b/module/plugins/hoster/FileomCom.py index 2b6fd34db..306953b84 100644 --- a/module/plugins/hoster/FileomCom.py +++ b/module/plugins/hoster/FileomCom.py @@ -18,8 +18,6 @@ class FileomCom(XFSHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = "fileom.com" - NAME_PATTERN = r'Filename: <span>(?P<N>.+?)<' SIZE_PATTERN = r'File Size: <span class="size">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' diff --git a/module/plugins/hoster/FilepostCom.py b/module/plugins/hoster/FilepostCom.py index 4dd36c6e4..1f3de6717 100644 --- a/module/plugins/hoster/FilepostCom.py +++ b/module/plugins/hoster/FilepostCom.py @@ -1,27 +1,27 @@ # -*- coding: utf-8 -*- import re - -from time import time +import time from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FilepostCom(SimpleHoster): __name__ = "FilepostCom" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.34" __pattern__ = r'https?://(?:www\.)?(?:filepost\.com/files|fp\.io)/(?P<ID>[^/]+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Filepost.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - INFO_PATTERN = r'<input type="text" id="url" value=\'<a href[^>]*>(?P<N>[^>]+?) - (?P<S>[\d.,]+) (?P<U>[\w^_]+)</a>\' class="inp_text"/>' + INFO_PATTERN = r'<input type="text" id="url" value=\'<a href.*?>(?P<N>[^>]+?) - (?P<S>[\d.,]+) (?P<U>[\w^_]+)</a>\' class="inp_text"/>' OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>|<div class="file_info file_info_deleted">' PREMIUM_ONLY_PATTERN = r'members only. Please upgrade to premium|a premium membership is required to download this file' @@ -41,7 +41,7 @@ class FilepostCom(SimpleHoster): captcha_key = m.group(1) # Get wait time - get_dict = {'SID': self.req.cj.getCookie('SID'), 'JsHttpRequest': str(int(time() * 10000)) + '-xml'} + get_dict = {'SID': self.req.cj.getCookie('SID'), 'JsHttpRequest': str(int(time.time() * 10000)) + '-xml'} post_dict = {'action': 'set_download', 'token': flp_token, 'code': self.info['pattern']['ID']} wait_time = int(self.getJsonResponse(get_dict, post_dict, 'wait_time')) @@ -57,7 +57,7 @@ class FilepostCom(SimpleHoster): if password: self.logInfo(_("Password protected link, trying ") + file_pass) - get_dict['JsHttpRequest'] = str(int(time() * 10000)) + '-xml' + get_dict['JsHttpRequest'] = str(int(time.time() * 10000)) + '-xml' post_dict['file_pass'] = file_pass self.link = self.getJsonResponse(get_dict, post_dict, 'link') @@ -72,27 +72,18 @@ class FilepostCom(SimpleHoster): recaptcha = ReCaptcha(self) for i in xrange(5): - get_dict['JsHttpRequest'] = str(int(time() * 10000)) + '-xml' + get_dict['JsHttpRequest'] = str(int(time.time() * 10000)) + '-xml' if i: - post_dict['recaptcha_challenge_field'], post_dict['recaptcha_response_field'] = recaptcha.challenge( + post_dict['recaptcha_response_field'], post_dict['recaptcha_challenge_field'] = recaptcha.challenge( captcha_key) self.logDebug(u"RECAPTCHA: %s : %s : %s" % ( captcha_key, post_dict['recaptcha_challenge_field'], post_dict['recaptcha_response_field'])) - download_url = self.getJsonResponse(get_dict, post_dict, 'link') - if download_url: - if i: - self.correctCaptcha() - break - elif i: - self.invalidCaptcha() + self.link = self.getJsonResponse(get_dict, post_dict, 'link') else: self.fail(_("Invalid captcha")) - # Download - self.download(download_url) - def getJsonResponse(self, get_dict, post_dict, field): res = json_loads(self.load('https://filepost.com/files/get/', get=get_dict, post=post_dict)) @@ -112,9 +103,9 @@ class FilepostCom(SimpleHoster): self.retry(wait_time=res['js']['params']['next_download']) # ~? self.retry(wait_time=js_answer['params']['next_download']) - elif ('Wrong file password' in res['js']['error'] - or 'You entered a wrong CAPTCHA code' in res['js']['error'] - or 'CAPTCHA Code nicht korrekt' in res['js']['error']): + elif 'Wrong file password' in res['js']['error'] \ + or 'You entered a wrong CAPTCHA code' in res['js']['error'] \ + or 'CAPTCHA Code nicht korrekt' in res['js']['error']: return None elif 'CAPTCHA' in res['js']['error']: diff --git a/module/plugins/hoster/FilepupNet.py b/module/plugins/hoster/FilepupNet.py index a53773f81..874fde3c8 100644 --- a/module/plugins/hoster/FilepupNet.py +++ b/module/plugins/hoster/FilepupNet.py @@ -15,6 +15,7 @@ class FilepupNet(SimpleHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?filepup\.net/files/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Filepup.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FilerNet.py b/module/plugins/hoster/FilerNet.py index 159aab964..3b876ed48 100644 --- a/module/plugins/hoster/FilerNet.py +++ b/module/plugins/hoster/FilerNet.py @@ -6,19 +6,19 @@ import pycurl import re +import urlparse -from urlparse import urljoin - -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class FilerNet(SimpleHoster): __name__ = "FilerNet" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.20" __pattern__ = r'https?://(?:www\.)?filer\.net/get/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Filer.net hoster plugin""" __license__ = "GPLv3" @@ -29,17 +29,9 @@ class FilerNet(SimpleHoster): INFO_PATTERN = r'<h1 class="page-header">Free Download (?P<N>\S+) <small>(?P<S>[\w.]+) (?P<U>[\w^_]+)</small></h1>' OFFLINE_PATTERN = r'Nicht gefunden' - LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'href="([^"]+)">Get download</a>' - + WAIT_PATTERN = r'musst du <span id="time">(\d+)' - def checkErrors(self): - # Wait between downloads - m = re.search(r'musst du <span id="time">(\d+)</span> Sekunden warten', self.html) - if m: - errmsg = self.info['error'] = _("Wait between free downloads") - self.retry(wait_time=int(m.group(1)), reason=errmsg) - - self.info.pop('error', None) + LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'href="([^"]+)">Get download</a>' def handleFree(self, pyfile): @@ -53,30 +45,22 @@ class FilerNet(SimpleHoster): if 'hash' not in inputs: self.error(_("Unable to detect hash")) - recaptcha = ReCaptcha(self) - - for _i in xrange(5): - challenge, response = recaptcha.challenge() - - #@NOTE: Work-around for v0.4.9 just_header issue - #@TODO: Check for v0.4.10 - self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.load(pyfile.url, post={'recaptcha_challenge_field': challenge, - 'recaptcha_response_field' : response, - 'hash' : inputs['hash']}) - self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) - - if 'location' in self.req.http.header.lower(): - self.link = re.search(r'location: (\S+)', self.req.http.header, re.I).group(1) - self.correctCaptcha() - break - else: - self.invalidCaptcha() - - - def downloadLink(self, link): - if link and isinstance(link, basestring): - self.download(urljoin("http://filer.net/", link), disposition=True) + recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge() + + #@NOTE: Work-around for v0.4.9 just_header issue + #@TODO: Check for v0.4.10 + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) + self.load(pyfile.url, post={'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response, + 'hash' : inputs['hash']}) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) + + if 'location' in self.req.http.header.lower(): + self.link = re.search(r'location: (\S+)', self.req.http.header, re.I).group(1) + self.correctCaptcha() + else: + self.invalidCaptcha() getInfo = create_getInfo(FilerNet) diff --git a/module/plugins/hoster/FilerioCom.py b/module/plugins/hoster/FilerioCom.py index db81f5b16..c6ebbd444 100644 --- a/module/plugins/hoster/FilerioCom.py +++ b/module/plugins/hoster/FilerioCom.py @@ -15,8 +15,6 @@ class FilerioCom(XFSHoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - HOSTER_DOMAIN = "filerio.in" - URL_REPLACEMENTS = [(r'filekeen\.com', "filerio.in")] OFFLINE_PATTERN = r'>"File Not Found|File has been removed' diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py index 18fca7176..7bd099282 100644 --- a/module/plugins/hoster/FilesMailRu.py +++ b/module/plugins/hoster/FilesMailRu.py @@ -21,7 +21,7 @@ def getInfo(urls): url_pattern = '<a href="(.+?)" onclick="return Act\(this\, \'dlink\'\, event\)">(.+?)</a>' file_name = re.search(url_pattern, html).group(0).split(', event)">')[1].split('</a>')[0] result.append((file_name, 0, 2, url)) - except: + except Exception: pass # status 1=OFFLINE, 2=OK, 3=UNKNOWN diff --git a/module/plugins/hoster/FileserveCom.py b/module/plugins/hoster/FileserveCom.py index 515ec9029..f8cf652b9 100644 --- a/module/plugins/hoster/FileserveCom.py +++ b/module/plugins/hoster/FileserveCom.py @@ -6,7 +6,7 @@ from module.common.json_layer import json_loads from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import chunks -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import secondsToMidnight from module.utils import parseFileSize @@ -33,7 +33,7 @@ def checkFile(plugin, urls): class FileserveCom(Hoster): __name__ = "FileserveCom" __type__ = "hoster" - __version__ = "0.53" + __version__ = "0.55" __pattern__ = r'http://(?:www\.)?fileserve\.com/file/(?P<ID>[^/]+)' @@ -48,7 +48,7 @@ class FileserveCom(Hoster): URLS = ["http://www.fileserve.com/file/", "http://www.fileserve.com/link-checker.php", "http://www.fileserve.com/checkReCaptcha.php"] LINKCHECK_TR = r'<tr>\s*(<td>http://www\.fileserve\.com/file/.*?)</tr>' - LINKCHECK_TD = r'<td>(?:<[^>]*>| )*([^<]*)' + LINKCHECK_TD = r'<td>(?:<.*?>| )*([^<]*)' CAPTCHA_KEY_PATTERN = r'var reCAPTCHA_publickey=\'(.+?)\'' LONG_WAIT_PATTERN = r'<li class="title">You need to wait (\d+) (\w+) to start another download\.</li>' @@ -161,7 +161,7 @@ class FileserveCom(Hoster): recaptcha = ReCaptcha(self) for _i in xrange(5): - challenge, response = recaptcha.challenge(captcha_key) + response, challenge = recaptcha.challenge(captcha_key) res = json_loads(self.load(self.URLS[2], post={'recaptcha_challenge_field' : challenge, 'recaptcha_response_field' : response, diff --git a/module/plugins/hoster/FileshareInUa.py b/module/plugins/hoster/FileshareInUa.py index 08e10dccb..aa1785f19 100644 --- a/module/plugins/hoster/FileshareInUa.py +++ b/module/plugins/hoster/FileshareInUa.py @@ -9,6 +9,7 @@ class FileshareInUa(DeadHoster): __version__ = "0.02" __pattern__ = r'https?://(?:www\.)?fileshare\.in\.ua/\w{7}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Fileshare.in.ua hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FilesonicCom.py b/module/plugins/hoster/FilesonicCom.py index 8bfa0fa2e..5254a6f9b 100644 --- a/module/plugins/hoster/FilesonicCom.py +++ b/module/plugins/hoster/FilesonicCom.py @@ -9,6 +9,7 @@ class FilesonicCom(DeadHoster): __version__ = "0.35" __pattern__ = r'http://(?:www\.)?filesonic\.com/file/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Filesonic.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FilezyNet.py b/module/plugins/hoster/FilezyNet.py index 4197a2858..9f442c846 100644 --- a/module/plugins/hoster/FilezyNet.py +++ b/module/plugins/hoster/FilezyNet.py @@ -9,6 +9,7 @@ class FilezyNet(DeadHoster): __version__ = "0.20" __pattern__ = r'http://(?:www\.)?filezy\.net/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Filezy.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FiredriveCom.py b/module/plugins/hoster/FiredriveCom.py index 0e3a4e847..8faee5b52 100644 --- a/module/plugins/hoster/FiredriveCom.py +++ b/module/plugins/hoster/FiredriveCom.py @@ -9,6 +9,7 @@ class FiredriveCom(DeadHoster): __version__ = "0.05" __pattern__ = r'https?://(?:www\.)?(firedrive|putlocker)\.com/(mobile/)?(file|embed)/(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Firedrive.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FlyFilesNet.py b/module/plugins/hoster/FlyFilesNet.py index 49705958d..689eb3c66 100644 --- a/module/plugins/hoster/FlyFilesNet.py +++ b/module/plugins/hoster/FlyFilesNet.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote +import urllib from module.network.RequestFactory import getURL from module.plugins.internal.SimpleHoster import SimpleHoster @@ -14,6 +13,7 @@ class FlyFilesNet(SimpleHoster): __version__ = "0.10" __pattern__ = r'http://(?:www\.)?flyfiles\.net/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FlyFiles.net hoster plugin""" __license__ = "GPLv3" @@ -25,14 +25,14 @@ class FlyFilesNet(SimpleHoster): def process(self, pyfile): name = re.search(self.NAME_PATTERN, pyfile.url).group(1) - pyfile.name = unquote_plus(name) + pyfile.name = urllib.unquote_plus(name) session = re.search(self.SESSION_PATTERN, pyfile.url).group(1) url = "http://flyfiles.net" # get download URL - parsed_url = getURL(url, post={"getDownLink": session}, cookies=True) + parsed_url = getURL(url, post={"getDownLink": session}) self.logDebug("Parsed URL: %s" % parsed_url) if parsed_url == '#downlink|' or parsed_url == "#downlink|#": @@ -40,6 +40,4 @@ class FlyFilesNet(SimpleHoster): self.wait(10 * 60, True) self.retry() - download_url = parsed_url.replace('#downlink|', '') - - self.download(download_url) + self.link = parsed_url.replace('#downlink|', '') diff --git a/module/plugins/hoster/FourSharedCom.py b/module/plugins/hoster/FourSharedCom.py index 78aeece44..79eb1fb83 100644 --- a/module/plugins/hoster/FourSharedCom.py +++ b/module/plugins/hoster/FourSharedCom.py @@ -11,6 +11,7 @@ class FourSharedCom(SimpleHoster): __version__ = "0.31" __pattern__ = r'https?://(?:www\.)?4shared(\-china)?\.com/(account/)?(download|get|file|document|photo|video|audio|mp3|office|rar|zip|archive|music)/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """4Shared.com hoster plugin""" __license__ = "GPLv3" @@ -53,7 +54,7 @@ class FourSharedCom(SimpleHoster): m = re.search(self.ID_PATTERN, self.html) res = self.load('http://www.4shared.com/web/d2/getFreeDownloadLimitInfo?fileId=%s' % m.group(1)) self.logDebug(res) - except: + except Exception: pass self.wait(20) diff --git a/module/plugins/hoster/FreakshareCom.py b/module/plugins/hoster/FreakshareCom.py index 9a11f933d..d005443d9 100644 --- a/module/plugins/hoster/FreakshareCom.py +++ b/module/plugins/hoster/FreakshareCom.py @@ -3,14 +3,14 @@ import re from module.plugins.Hoster import Hoster -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import secondsToMidnight class FreakshareCom(Hoster): __name__ = "FreakshareCom" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" __pattern__ = r'http://(?:www\.)?freakshare\.(net|com)/files/\S*?/' @@ -107,12 +107,14 @@ class FreakshareCom(Hoster): def get_file_name(self): if not self.html: self.download_html() + if not self.wantReconnect: - file_name = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">([^ ]+)", self.html) - if file_name is not None: - file_name = file_name.group(1) + m = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">([^ ]+)", self.html) + if m: + file_name = m.group(1) else: file_name = self.pyfile.url + return file_name else: return self.pyfile.url @@ -122,12 +124,12 @@ class FreakshareCom(Hoster): size = 0 if not self.html: self.download_html() + if not self.wantReconnect: - file_size_check = re.search( - r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">[^ ]+ - ([^ ]+) (\w\w)yte", self.html) - if file_size_check is not None: - units = float(file_size_check.group(1).replace(",", "")) - pow = {'KB': 1, 'MB': 2, 'GB': 3}[file_size_check.group(2)] + m = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center;\">[^ ]+ - ([^ ]+) (\w\w)yte", self.html) + if m: + units = float(m.group(1).replace(",", "")) + pow = {'KB': 1, 'MB': 2, 'GB': 3}[m.group(2)] size = int(units * 1024 ** pow) return size @@ -153,7 +155,7 @@ class FreakshareCom(Hoster): """ if not self.html: self.download_html() - if re.search(r"This file does not exist!", self.html) is not None: + if re.search(r"This file does not exist!", self.html): return False else: return True diff --git a/module/plugins/hoster/FreeWayMe.py b/module/plugins/hoster/FreeWayMe.py index ffc8115d3..1cca24c3b 100644 --- a/module/plugins/hoster/FreeWayMe.py +++ b/module/plugins/hoster/FreeWayMe.py @@ -6,9 +6,10 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class FreeWayMe(MultiHoster): __name__ = "FreeWayMe" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __pattern__ = r'https://(?:www\.)?free-way\.me/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FreeWayMe multi-hoster plugin""" __license__ = "GPLv3" @@ -24,13 +25,30 @@ class FreeWayMe(MultiHoster): def handlePremium(self, pyfile): user, data = self.account.selectAccount() - self.download("https://www.free-way.me/load.php", - get={'multiget': 7, - 'url' : pyfile.url, - 'user' : user, - 'pw' : self.account.getAccountData(user)['password'], - 'json' : ""}, - disposition=True) + for _i in xrange(5): + # try it five times + header = self.load("https://www.free-way.me/load.php", + get={'multiget': 7, + 'url' : pyfile.url, + 'user' : user, + 'pw' : self.account.getAccountData(user)['password'], + 'json' : ""}, + just_header=True) + + if 'location' in header: + headers = self.load(header['location'], just_header=True) + if headers['code'] == 500: + # error on 2nd stage + self.logError(_("Error [stage2]")) + else: + # seems to work.. + self.download(header['location']) + break + else: + # error page first stage + self.logError(_("Error [stage1]")) + + #@TODO: handle errors getInfo = create_getInfo(FreeWayMe) diff --git a/module/plugins/hoster/FreevideoCz.py b/module/plugins/hoster/FreevideoCz.py index e56d1a299..49f02a28b 100644 --- a/module/plugins/hoster/FreevideoCz.py +++ b/module/plugins/hoster/FreevideoCz.py @@ -9,6 +9,7 @@ class FreevideoCz(DeadHoster): __version__ = "0.30" __pattern__ = r'http://(?:www\.)?freevideo\.cz/vase-videa/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Freevideo.cz hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/FshareVn.py b/module/plugins/hoster/FshareVn.py index 872511afc..50128db10 100644 --- a/module/plugins/hoster/FshareVn.py +++ b/module/plugins/hoster/FshareVn.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- import re - -from time import strptime, mktime, gmtime -from urlparse import urljoin +import time +import urlparse from module.network.RequestFactory import getURL from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo @@ -25,9 +24,10 @@ def doubleDecode(m): class FshareVn(SimpleHoster): __name__ = "FshareVn" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __pattern__ = r'http://(?:www\.)?fshare\.vn/file/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """FshareVn hoster plugin""" __license__ = "GPLv3" @@ -58,7 +58,7 @@ class FshareVn(SimpleHoster): self.checkErrors() action, inputs = self.parseHtmlForm('frm_download') - self.url = urljoin(pyfile.url, action) + url = urlparse.urljoin(pyfile.url, action) if not inputs: self.error(_("No FORM")) @@ -69,7 +69,7 @@ class FshareVn(SimpleHoster): if password: self.logInfo(_("Password protected link, trying ") + password) inputs['link_file_pwd_dl'] = password - self.html = self.load(self.url, post=inputs, decode=True) + self.html = self.load(url, post=inputs, decode=True) if 'name="link_file_pwd_dl"' in self.html: self.fail(_("Incorrect password")) @@ -77,7 +77,7 @@ class FshareVn(SimpleHoster): self.fail(_("No password found")) else: - self.html = self.load(self.url, post=inputs, decode=True) + self.html = self.load(url, post=inputs, decode=True) self.checkErrors() @@ -87,11 +87,9 @@ class FshareVn(SimpleHoster): m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: self.error(_("LINK_FREE_PATTERN not found")) - self.url = m.group(1) - self.logDebug("FREE DL URL: %s" % self.url) + self.link = m.group(1) self.wait() - self.download(self.url) def checkErrors(self): @@ -101,8 +99,8 @@ class FshareVn(SimpleHoster): m = re.search(self.WAIT_PATTERN, self.html) if m: self.logInfo(_("Wait until %s ICT") % m.group(1)) - wait_until = mktime(strptime(m.group(1), "%d/%m/%Y %H:%M")) - self.wait(wait_until - mktime(gmtime()) - 7 * 60 * 60, True) + wait_until = time.mktime.time(time.strptime.time(m.group(1), "%d/%m/%Y %H:%M")) + self.wait(wait_until - time.mktime.time(time.gmtime.time()) - 7 * 60 * 60, True) self.retry() elif '<ul class="message-error">' in self.html: msg = "Unknown error occured or wait time not parsed" diff --git a/module/plugins/hoster/Ftp.py b/module/plugins/hoster/Ftp.py index 8562778c4..295955cbe 100644 --- a/module/plugins/hoster/Ftp.py +++ b/module/plugins/hoster/Ftp.py @@ -2,9 +2,8 @@ import pycurl import re - -from urllib import quote, unquote -from urlparse import urlparse +import urllib +import urlparse from module.plugins.Hoster import Hoster @@ -12,7 +11,7 @@ from module.plugins.Hoster import Hoster class Ftp(Hoster): __name__ = "Ftp" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.52" __pattern__ = r'(?:ftps?|sftp)://([\w.-]+(:[\w.-]+)?@)?[\w.-]+(:\d+)?/.+' @@ -29,13 +28,13 @@ class Ftp(Hoster): def process(self, pyfile): - parsed_url = urlparse(pyfile.url) + parsed_url = urlparse.urlparse(pyfile.url) netloc = parsed_url.netloc pyfile.name = parsed_url.path.rpartition('/')[2] try: - pyfile.name = unquote(str(pyfile.name)).decode('utf8') - except: + pyfile.name = urllib.unquote(str(pyfile.name)).decode('utf8') + except Exception: pass if not "@" in netloc: @@ -67,11 +66,11 @@ class Ftp(Hoster): #Naive ftp directory listing if re.search(r'^25\d.*?"', self.req.http.header, re.M): pyfile.url = pyfile.url.rstrip('/') - pkgname = "/".join(pyfile.package().name, urlparse(pyfile.url).path.rpartition('/')[2]) + pkgname = "/".join([pyfile.package().name, urlparse.urlparse(pyfile.url).path.rpartition('/')[2]]) pyfile.url += '/' self.req.http.c.setopt(48, 1) # CURLOPT_DIRLISTONLY res = self.load(pyfile.url, decode=False) - links = [pyfile.url + quote(x) for x in res.splitlines()] + links = [pyfile.url + urllib.quote(x) for x in res.splitlines()] self.logDebug("LINKS", links) self.core.api.addPackage(pkgname, links) else: diff --git a/module/plugins/hoster/GigapetaCom.py b/module/plugins/hoster/GigapetaCom.py index 9d3ace3b2..c2feeb6f8 100644 --- a/module/plugins/hoster/GigapetaCom.py +++ b/module/plugins/hoster/GigapetaCom.py @@ -1,37 +1,39 @@ # -*- coding: utf-8 -*- +import pycurl +import random import re -from pycurl import FOLLOWLOCATION -from random import randint - from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class GigapetaCom(SimpleHoster): __name__ = "GigapetaCom" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'http://(?:www\.)?gigapeta\.com/dl/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """GigaPeta.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<img src=".*" alt="file" />-->\s*(?P<N>.*?)\s*</td>' - SIZE_PATTERN = r'<th>\s*Size\s*</th>\s*<td>\s*(?P<S>.*?)\s*</td>' + NAME_PATTERN = r'<img src=".*" alt="file" />-->\s*(?P<N>.*?)\s*</td>' + SIZE_PATTERN = r'<th>\s*Size\s*</th>\s*<td>\s*(?P<S>.*?)\s*</td>' OFFLINE_PATTERN = r'<div id="page_error">' + DOWNLOAD_PATTERN = r'"All threads for IP' + COOKIES = [("gigapeta.com", "lang", "us")] def handleFree(self, pyfile): - captcha_key = str(randint(1, 100000000)) + captcha_key = str(random.randint(1, 100000000)) captcha_url = "http://gigapeta.com/img/captcha.gif?x=%s" % captcha_key - self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) for _i in xrange(5): self.checkErrors() @@ -44,24 +46,14 @@ class GigapetaCom(SimpleHoster): m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) if m: - download_url = m.group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 + self.link = m.group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 break elif "Entered figures don`t coincide with the picture" in self.html: self.invalidCaptcha() else: self.fail(_("No valid captcha code entered")) - self.req.http.c.setopt(FOLLOWLOCATION, 1) - self.download(download_url) - - - def checkErrors(self): - if "All threads for IP" in self.html: - self.logDebug("Your IP is already downloading a file") - self.wait(5 * 60, True) - self.retry() - - self.info.pop('error', None) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) getInfo = create_getInfo(GigapetaCom) diff --git a/module/plugins/hoster/GooIm.py b/module/plugins/hoster/GooIm.py index 2cb012e50..4b27e6cc8 100644 --- a/module/plugins/hoster/GooIm.py +++ b/module/plugins/hoster/GooIm.py @@ -14,6 +14,7 @@ class GooIm(SimpleHoster): __version__ = "0.04" __pattern__ = r'https?://(?:www\.)?goo\.im/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Goo.im hoster plugin""" __license__ = "GPLv3" @@ -31,7 +32,7 @@ class GooIm(SimpleHoster): def handleFree(self, pyfile): self.wait(10) - self.download(pyfile.url, cookies=True) + self.link = pyfile.url getInfo = create_getInfo(GooIm) diff --git a/module/plugins/hoster/GoogledriveCom.py b/module/plugins/hoster/GoogledriveCom.py new file mode 100644 index 000000000..1c33a1e4e --- /dev/null +++ b/module/plugins/hoster/GoogledriveCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -* +# +# Test links: +# https://drive.google.com/file/d/0B6RNTe4ygItBQm15RnJiTmMyckU/view?pli=1 + +import re +import urlparse + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.utils import html_unescape + + +class GoogledriveCom(SimpleHoster): + __name__ = "GoogledriveCom" + __type__ = "hoster" + __version__ = "0.12" + + __pattern__ = r'https?://(?:www\.)?(drive|docs)\.google\.com/(file/d/\w+|uc\?.*id=)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """Drive.google.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + DISPOSITION = False #: Remove in 0.4.10 + + NAME_PATTERN = r'(?:<title>|class="uc-name-size".*>)(?P<N>.+?)(?: - Google Drive</title>|</a> \()' + OFFLINE_PATTERN = r'align="center"><p class="errorMessage"' + + LINK_FREE_PATTERN = r'"([^"]+uc\?.*?)"' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + for _i in xrange(2): + m = re.search(self.LINK_FREE_PATTERN, self.html) + + if m is None: + self.error(_("Free download link not found")) + + else: + link = html_unescape(m.group(1).decode('unicode-escape')) + if not urlparse.urlparse(link).scheme: + link = urlparse.urljoin("https://docs.google.com/", link) + + direct_link = self.directLink(link, False) + if not direct_link: + self.html = self.load(link, decode=True) + else: + self.link = direct_link + break + + +getInfo = create_getInfo(GoogledriveCom) diff --git a/module/plugins/hoster/HellshareCz.py b/module/plugins/hoster/HellshareCz.py index 47b981dd7..ac0043b37 100644 --- a/module/plugins/hoster/HellshareCz.py +++ b/module/plugins/hoster/HellshareCz.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class HellshareCz(SimpleHoster): __name__ = "HellshareCz" __type__ = "hoster" - __version__ = "0.84" + __version__ = "0.85" __pattern__ = r'http://(?:www\.)?hellshare\.(?:cz|com|sk|hu|pl)/[^?]*/\d+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Hellshare.cz hoster plugin""" __license__ = "GPLv3" @@ -32,10 +33,4 @@ class HellshareCz(SimpleHoster): self.chunkLimit = 1 - def downloadLink(self, link): - if link and isinstance(link, basestring): - self.correctCaptcha() - self.download(urljoin("http://www.hellshare.com/", link), disposition=True) - - getInfo = create_getInfo(HellshareCz) diff --git a/module/plugins/hoster/HellspyCz.py b/module/plugins/hoster/HellspyCz.py index 2b9b76b8a..9b10760bd 100644 --- a/module/plugins/hoster/HellspyCz.py +++ b/module/plugins/hoster/HellspyCz.py @@ -9,6 +9,7 @@ class HellspyCz(DeadHoster): __version__ = "0.28" __pattern__ = r'http://(?:www\.)?(?:hellspy\.(?:cz|com|sk|hu|pl)|sciagaj\.pl)(/\S+/\d+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """HellSpy.cz hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/HostujeNet.py b/module/plugins/hoster/HostujeNet.py new file mode 100644 index 000000000..ec91e50b9 --- /dev/null +++ b/module/plugins/hoster/HostujeNet.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class HostujeNet(SimpleHoster): + __name__ = "HostujeNet" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?hostuje\.net/\w+' + + __description__ = """Hostuje.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("GammaC0de", None)] + + + NAME_PATTERN = r'<input type="hidden" name="name" value="(?P<N>.+?)">' + SIZE_PATTERN = r'<b>Rozmiar:</b> (?P<S>[\d.,]+) (?P<U>[\w^_]+)<br>' + OFFLINE_PATTERN = ur'Podany plik nie zostaÅ odnaleziony\.\.\.' + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + m = re.search(r'<script src="([\w^_]+.php)"></script>', self.html) + if m: + jscript = self.load("http://hostuje.net/" + m.group(1)) + m = re.search(r"\('(\w+\.php\?i=\w+)'\);", jscript) + if m: + self.load("http://hostuje.net/" + m.group(1)) + else: + self.error(_("unexpected javascript format")) + else: + self.error(_("script not found")) + + action, inputs = self.parseHtmlForm(pyfile.url.replace(".", "\.").replace( "?", "\?")) + if not action: + self.error(_("form not found")) + + self.download(action, post=inputs) + + +getInfo = create_getInfo(HostujeNet) diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py index f7724faf2..082415c6b 100644 --- a/module/plugins/hoster/HotfileCom.py +++ b/module/plugins/hoster/HotfileCom.py @@ -9,6 +9,7 @@ class HotfileCom(DeadHoster): __version__ = "0.37" __pattern__ = r'https?://(?:www\.)?hotfile\.com/dl/\d+/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Hotfile.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/HugefilesNet.py b/module/plugins/hoster/HugefilesNet.py index f7221f8c5..3fdcca1ba 100644 --- a/module/plugins/hoster/HugefilesNet.py +++ b/module/plugins/hoster/HugefilesNet.py @@ -17,9 +17,7 @@ class HugefilesNet(XFSHoster): __authors__ = [("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "hugefiles.net" - - SIZE_PATTERN = r'File Size:</span>\s*<span[^>]*>(?P<S>[^<]+)</span></div>' + SIZE_PATTERN = r'File Size:</span>\s*<span.*?>(?P<S>[^<]+)</span></div>' FORM_INPUTS_MAP = {'ctype': re.compile(r'\d+')} diff --git a/module/plugins/hoster/HundredEightyUploadCom.py b/module/plugins/hoster/HundredEightyUploadCom.py index 4fc96a2b1..2a35a008f 100644 --- a/module/plugins/hoster/HundredEightyUploadCom.py +++ b/module/plugins/hoster/HundredEightyUploadCom.py @@ -6,7 +6,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class HundredEightyUploadCom(XFSHoster): __name__ = "HundredEightyUploadCom" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'http://(?:www\.)?180upload\.com/\w{12}' @@ -15,7 +15,7 @@ class HundredEightyUploadCom(XFSHoster): __authors__ = [("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "180upload.com" + OFFLINE_PATTERN = r'>File Not Found' getInfo = create_getInfo(HundredEightyUploadCom) diff --git a/module/plugins/hoster/IFileWs.py b/module/plugins/hoster/IFileWs.py index b4f225c4b..ff263d43a 100644 --- a/module/plugins/hoster/IFileWs.py +++ b/module/plugins/hoster/IFileWs.py @@ -9,6 +9,7 @@ class IFileWs(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?ifile\.ws/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Ifile.ws hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/IcyFilesCom.py b/module/plugins/hoster/IcyFilesCom.py index 921b64207..f81381c66 100644 --- a/module/plugins/hoster/IcyFilesCom.py +++ b/module/plugins/hoster/IcyFilesCom.py @@ -9,6 +9,7 @@ class IcyFilesCom(DeadHoster): __version__ = "0.06" __pattern__ = r'http://(?:www\.)?icyfiles\.com/(.+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """IcyFiles.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/IfileIt.py b/module/plugins/hoster/IfileIt.py index 056b05a65..ef267b505 100644 --- a/module/plugins/hoster/IfileIt.py +++ b/module/plugins/hoster/IfileIt.py @@ -9,6 +9,7 @@ class IfileIt(DeadHoster): __version__ = "0.29" __pattern__ = r'^unmatchable$' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Ifile.it""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/IfolderRu.py b/module/plugins/hoster/IfolderRu.py index 249c2feb1..0f09731e4 100644 --- a/module/plugins/hoster/IfolderRu.py +++ b/module/plugins/hoster/IfolderRu.py @@ -10,7 +10,8 @@ class IfolderRu(SimpleHoster): __type__ = "hoster" __version__ = "0.39" - __pattern__ = r'http://(?:www\.)?(?:ifolder\.ru|rusfolder\.(?:com|net|ru))/(?:files/)?(?P<ID>\d+)' + __pattern__ = r'http://(?:www)?(files\.)?(ifolder\.ru|metalarea\.org|rusfolder\.(com|net|ru))/(files/)?(?P<ID>\d+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Ifolder.ru hoster plugin""" __license__ = "GPLv3" @@ -21,13 +22,13 @@ class IfolderRu(SimpleHoster): NAME_PATTERN = ur'(?:<div><span>)?ÐазваМОе:(?:</span>)? <b>(?P<N>[^<]+)</b><(?:/div|br)>' SIZE_PATTERN = ur'(?:<div><span>)?РазЌеÑ:(?:</span>)? <b>(?P<S>[^<]+)</b><(?:/div|br)>' - OFFLINE_PATTERN = ur'<p>Ѐайл ÐœÐŸÐŒÐµÑ <b>[^<]*</b> (Ме МайЎеМ|ÑЎалеМ) !!!</p>' + OFFLINE_PATTERN = ur'<p>Ѐайл ÐœÐŸÐŒÐµÑ <b>.*?</b> (Ме МайЎеМ|ÑЎалеМ) !!!</p>' - SESSION_ID_PATTERN = r'<a href=(http://ints\.(?:rusfolder\.com|ifolder\.ru)/ints/sponsor/\?bi=\d*&session=([^&]+)&u=[^>]+)>' - INTS_SESSION_PATTERN = r'\(\'ints_session\'\);\s*if\(tag\)\{tag\.value = "([^"]+)";\}' + SESSION_ID_PATTERN = r'<input type="hidden" name="session" value="(.+?)"' + INTS_SESSION_PATTERN = r'\(\'ints_session\'\);\s*if\(tag\)\{tag\.value = "(.+?)";\}' HIDDEN_INPUT_PATTERN = r'var v = .*?name=\'(.+?)\' value=\'1\'' - LINK_FREE_PATTERN = r'<a id="download_file_href" href="([^"]+)"' + LINK_FREE_PATTERN = r'<a href="(.+?)" class="downloadbutton_files"' WRONG_CAPTCHA_PATTERN = ur'<font color=Red>МевеÑÐœÑй кПЎ,<br>ввеЎОÑе еÑе Ñаз</font><br>' @@ -38,31 +39,20 @@ class IfolderRu(SimpleHoster): def handleFree(self, pyfile): - self.html = self.load("http://rusfolder.com/%s" % self.info['pattern']['ID'], cookies=True, decode=True) + url = "http://rusfolder.com/%s" % self.info['pattern']['ID'] + self.html = self.load("http://rusfolder.com/%s" % self.info['pattern']['ID'], decode=True) self.getFileInfo() - url = re.search(r"location\.href = '(http://ints\..*?=)'", self.html).group(1) - self.html = self.load(url, cookies=True, decode=True) - - url, session_id = re.search(self.SESSION_ID_PATTERN, self.html).groups() - self.html = self.load(url, cookies=True, decode=True) - - url = "http://ints.rusfolder.com/ints/frame/?session=%s" % session_id - self.html = self.load(url, cookies=True) - - self.wait(31, False) + session_id = re.search(self.SESSION_ID_PATTERN, self.html).groups() captcha_url = "http://ints.rusfolder.com/random/images/?session=%s" % session_id for _i in xrange(5): - self.html = self.load(url, cookies=True) - action, inputs = self.parseHtmlForm('ID="Form1"') - inputs['ints_session'] = re.search(self.INTS_SESSION_PATTERN, self.html).group(1) - inputs[re.search(self.HIDDEN_INPUT_PATTERN, self.html).group(1)] = '1' + action, inputs = self.parseHtmlForm('id="download-step-one-form"') inputs['confirmed_number'] = self.decryptCaptcha(captcha_url, cookies=True) inputs['action'] = '1' self.logDebug(inputs) - self.html = self.load(url, decode=True, cookies=True, post=inputs) + self.html = self.load(url, decode=True, post=inputs) if self.WRONG_CAPTCHA_PATTERN in self.html: self.invalidCaptcha() else: @@ -70,7 +60,8 @@ class IfolderRu(SimpleHoster): else: self.fail(_("Invalid captcha")) - self.link = re.search(self.LINK_PATTERN, self.html).group(1) + self.link = re.search(self.LINK_FREE_PATTERN, self.html).group(1) getInfo = create_getInfo(IfolderRu) + diff --git a/module/plugins/hoster/JumbofilesCom.py b/module/plugins/hoster/JumbofilesCom.py index 7adc1a029..a9bf14c6d 100644 --- a/module/plugins/hoster/JumbofilesCom.py +++ b/module/plugins/hoster/JumbofilesCom.py @@ -11,6 +11,7 @@ class JumbofilesCom(SimpleHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?jumbofiles\.com/(?P<ID>\w{12})' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """JumboFiles.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/JunocloudMe.py b/module/plugins/hoster/JunocloudMe.py index 56d6588fa..415d5e2d0 100644 --- a/module/plugins/hoster/JunocloudMe.py +++ b/module/plugins/hoster/JunocloudMe.py @@ -15,8 +15,6 @@ class JunocloudMe(XFSHoster): __authors__ = [("guidobelix", "guidobelix@hotmail.it")] - HOSTER_DOMAIN = "junocloud.me" - URL_REPLACEMENTS = [(r'//(www\.)?junocloud', "//dl3.junocloud")] OFFLINE_PATTERN = r'>No such file with this filename<' diff --git a/module/plugins/hoster/Keep2shareCc.py b/module/plugins/hoster/Keep2ShareCc.py index 16bda27e0..19a8f2ba6 100644 --- a/module/plugins/hoster/Keep2shareCc.py +++ b/module/plugins/hoster/Keep2ShareCc.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- import re +import urlparse -from urlparse import urljoin, urlparse - -from module.plugins.internal.CaptchaService import ReCaptcha -from module.plugins.internal.SimpleHoster import _isDirectLink, SimpleHoster, create_getInfo +from module.plugins.internal.ReCaptcha import ReCaptcha +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class Keep2ShareCc(SimpleHoster): __name__ = "Keep2ShareCc" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.23" __pattern__ = r'https?://(?:www\.)?(keep2share|k2s|keep2s)\.cc/file/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Keep2Share.cc hoster plugin""" __license__ = "GPLv3" @@ -21,15 +21,16 @@ class Keep2ShareCc(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - URL_REPLACEMENTS = [(__pattern__ + ".*", "http://k2s.cc/file/\g<ID>")] + URL_REPLACEMENTS = [(__pattern__ + ".*", "http://keep2s.cc/file/\g<ID>")] - NAME_PATTERN = r'File: <span>(?P<N>.+)</span>' + NAME_PATTERN = r'File: <span>(?P<N>.+?)</span>' SIZE_PATTERN = r'Size: (?P<S>[^<]+)</div>' OFFLINE_PATTERN = r'File not found or deleted|Sorry, this file is blocked or deleted|Error 404' TEMP_OFFLINE_PATTERN = r'Downloading blocked due to' - LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'"([^"]+url.html?file=.+?)"|window\.location\.href = \'(.+?)\';' + LINK_FREE_PATTERN = r'"(.+?url.html\?file=.+?)"|window\.location\.href = \'(.+?)\';' + LINK_PREMIUM_PATTERN = r'window\.location\.href = \'(.+?)\';' CAPTCHA_PATTERN = r'src="(/file/captcha\.html.+?)"' @@ -56,7 +57,7 @@ class Keep2ShareCc(SimpleHoster): # string to time convert courtesy of https://stackoverflow.com/questions/10663720 ftr = [3600, 60, 1] - wait_time = sum([a * b for a, b in zip(ftr, map(int, m.group(1).split(':')))]) + wait_time = sum(a * b for a, b in zip(ftr, map(int, m.group(1).split(':')))) self.wantReconnect = True self.retry(wait_time=wait_time, reason="Please wait to download this file") @@ -64,67 +65,54 @@ class Keep2ShareCc(SimpleHoster): self.info.pop('error', None) - def handleFree(self): - self.fid = re.search(r'<input type="hidden" name="slow_id" value="([^"]+)">', self.html).group(1) - self.html = self.load(self.pyfile.url, post={'yt0': '', 'slow_id': self.fid}) + def handleFree(self, pyfile): + self.fid = re.search(r'<input type="hidden" name="slow_id" value="(.+?)">', self.html).group(1) + self.html = self.load(pyfile.url, post={'yt0': '', 'slow_id': self.fid}) + + # self.logDebug(self.fid) + # self.logDebug(pyfile.url) self.checkErrors() m = re.search(self.LINK_FREE_PATTERN, self.html) - if m is None: self.handleCaptcha() - - self.wait(30) - - self.html = self.load(self.pyfile.url, post={'uniqueId': self.fid, 'free': 1}) - - self.checkErrors() + self.wait(31) + self.html = self.load(pyfile.url) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.error(_("LINK_FREE_PATTERN not found")) + self.error(_("Free download link not found")) self.link = m.group(1) def handleCaptcha(self): - recaptcha = ReCaptcha(self) - - for _i in xrange(5): - post_data = {'free' : 1, - 'freeDownloadRequest': 1, - 'uniqueId' : self.fid, - 'yt0' : ''} - - m = re.search(self.CAPTCHA_PATTERN, self.html) - if m: - captcha_url = urljoin("http://k2s.cc/", m.group(1)) - post_data['CaptchaForm[code]'] = self.decryptCaptcha(captcha_url) - else: - challenge, response = recaptcha.challenge() - post_data.update({'recaptcha_challenge_field': challenge, - 'recaptcha_response_field' : response}) - - self.html = self.load(self.pyfile.url, post=post_data) - - if 'recaptcha' not in self.html: - self.correctCaptcha() - break - else: - self.invalidCaptcha() - else: - self.fail(_("All captcha attempts failed")) + post_data = {'free' : 1, + 'freeDownloadRequest': 1, + 'uniqueId' : self.fid, + 'yt0' : ''} + m = re.search(r'id="(captcha\-form)"', self.html) + self.logDebug("captcha-form found %s" % m) - def downloadLink(self, link): - if not link or not isinstance(link, basestring): - return + m = re.search(self.CAPTCHA_PATTERN, self.html) + self.logDebug("CAPTCHA_PATTERN found %s" % m) + if m: + captcha_url = urlparse.urljoin("http://keep2s.cc/", m.group(1)) + post_data['CaptchaForm[code]'] = self.decryptCaptcha(captcha_url) + else: + recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge() + post_data.update({'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) - link = _isDirectLink(self, link, self.premium) + self.html = self.load(self.pyfile.url, post=post_data) - if link: - self.download(urljoin("http://k2s.cc/", link), disposition=True) + if 'verification code is incorrect' not in self.html: + self.correctCaptcha() + else: + self.invalidCaptcha() getInfo = create_getInfo(Keep2ShareCc) diff --git a/module/plugins/hoster/KickloadCom.py b/module/plugins/hoster/KickloadCom.py index 1c39db46c..7690c85c8 100644 --- a/module/plugins/hoster/KickloadCom.py +++ b/module/plugins/hoster/KickloadCom.py @@ -9,6 +9,7 @@ class KickloadCom(DeadHoster): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?kickload\.com/get/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Kickload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/KingfilesNet.py b/module/plugins/hoster/KingfilesNet.py index a75ef5f3e..42547d658 100644 --- a/module/plugins/hoster/KingfilesNet.py +++ b/module/plugins/hoster/KingfilesNet.py @@ -2,16 +2,17 @@ import re -from module.plugins.internal.CaptchaService import SolveMedia +from module.plugins.internal.SolveMedia import SolveMedia from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class KingfilesNet(SimpleHoster): __name__ = "KingfilesNet" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.08" __pattern__ = r'http://(?:www\.)?kingfiles\.net/(?P<ID>\w{12})' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Kingfiles.net hoster plugin""" __license__ = "GPLv3" @@ -43,10 +44,10 @@ class KingfilesNet(SimpleHoster): 'referer' : "", 'method_free': "+"} - self.html = self.load(pyfile.url, post=post_data, cookies=True, decode=True) + self.html = self.load(pyfile.url, post=post_data, decode=True) solvemedia = SolveMedia(self) - challenge, response = solvemedia.challenge() + response, challenge = solvemedia.challenge() # Make the downloadlink appear and load the file m = re.search(self.RAND_ID_PATTERN, self.html) @@ -66,13 +67,13 @@ class KingfilesNet(SimpleHoster): 'adcopy_challenge': challenge, 'down_direct' : "1"} - self.html = self.load(pyfile.url, post=post_data, cookies=True, decode=True) + self.html = self.load(pyfile.url, post=post_data, decode=True) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: self.error(_("Download url not found")) - self.download(m.group(1), cookies=True, disposition=True) + self.link = m.group(1) getInfo = create_getInfo(KingfilesNet) diff --git a/module/plugins/hoster/LemUploadsCom.py b/module/plugins/hoster/LemUploadsCom.py index 22fbd60dd..098867c8b 100644 --- a/module/plugins/hoster/LemUploadsCom.py +++ b/module/plugins/hoster/LemUploadsCom.py @@ -9,6 +9,7 @@ class LemUploadsCom(DeadHoster): __version__ = "0.02" __pattern__ = r'https?://(?:www\.)?lemuploads\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """LemUploads.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/LetitbitNet.py b/module/plugins/hoster/LetitbitNet.py index 6d5641778..62afb6c7c 100644 --- a/module/plugins/hoster/LetitbitNet.py +++ b/module/plugins/hoster/LetitbitNet.py @@ -7,12 +7,11 @@ # http://letitbit.net/download/07874.0b5709a7d3beee2408bb1f2eefce/random.bin.html import re - -from urlparse import urljoin +import urlparse from module.common.json_layer import json_loads, json_dumps from module.network.RequestFactory import getURL -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, secondsToMidnight @@ -36,9 +35,10 @@ def getInfo(urls): class LetitbitNet(SimpleHoster): __name__ = "LetitbitNet" __type__ = "hoster" - __version__ = "0.29" + __version__ = "0.31" __pattern__ = r'https?://(?:www\.)?(letitbit|shareflare)\.net/download/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Letitbit.net hoster plugin""" __license__ = "GPLv3" @@ -56,16 +56,6 @@ class LetitbitNet(SimpleHoster): self.resumeDownload = True - def getFileInfo(self): - api_rep = api_response(self.pyfile.url) - if api_rep['status'] == 'OK': - self.api_data = api_rep['data'][0] - self.pyfile.name = self.api_data['name'] - self.pyfile.size = self.api_data['size'] - else: - self.offline() - - def handleFree(self, pyfile): action, inputs = self.parseHtmlForm('id="ifree_form"') if not action: @@ -75,7 +65,7 @@ class LetitbitNet(SimpleHoster): self.logDebug(action, inputs) inputs['desc'] = "" - self.html = self.load(urljoin("http://letitbit.net/", action), post=inputs, cookies=True) + self.html = self.load(urlparse.urljoin("http://letitbit.net/", action), post=inputs) m = re.search(self.SECONDS_PATTERN, self.html) seconds = int(m.group(1)) if m else 60 @@ -89,14 +79,14 @@ class LetitbitNet(SimpleHoster): self.wait(seconds) - res = self.load("http://letitbit.net/ajax/download3.php", post=" ", cookies=True) + res = self.load("http://letitbit.net/ajax/download3.php", post=" ") if res != '1': self.error(_("Unknown response - ajax_check_url")) self.logDebug(res) recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge() + response, challenge = recaptcha.challenge() post_data = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": response, @@ -104,7 +94,7 @@ class LetitbitNet(SimpleHoster): self.logDebug("Post data to send", post_data) - res = self.load("http://letitbit.net/ajax/check_recaptcha.php", post=post_data, cookies=True) + res = self.load("http://letitbit.net/ajax/check_recaptcha.php", post=post_data) self.logDebug(res) @@ -128,16 +118,7 @@ class LetitbitNet(SimpleHoster): else: self.error(_("Unknown response - captcha check")) - self.correctCaptcha() - - for download_url in urls: - try: - self.download(download_url) - break - except Exception, e: - self.logError(e) - else: - self.fail(_("Download did not finish correctly")) + self.link = urls[0] def handlePremium(self, pyfile): @@ -152,4 +133,4 @@ class LetitbitNet(SimpleHoster): if api_rep['status'] == 'FAIL': self.fail(api_rep['data']) - self.download(api_rep['data'][0][0], disposition=True) + self.link = api_rep['data'][0][0] diff --git a/module/plugins/hoster/LinksnappyCom.py b/module/plugins/hoster/LinksnappyCom.py index aa3466036..9c3af4ae3 100644 --- a/module/plugins/hoster/LinksnappyCom.py +++ b/module/plugins/hoster/LinksnappyCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urlsplit +import urlparse from module.common.json_layer import json_loads, json_dumps from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -11,50 +10,48 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class LinksnappyCom(MultiHoster): __name__ = "LinksnappyCom" __type__ = "hoster" - __version__ = "0.07" + __version__ = "0.08" - __pattern__ = r'https?://(?:[^/]*\.)?linksnappy\.com' + __pattern__ = r'https?://(?:[^/]+\.)?linksnappy\.com' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Linksnappy.com multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - SINGLE_CHUNK_HOSTERS = ('easybytez.com') + SINGLE_CHUNK_HOSTERS = ["easybytez.com"] def handlePremium(self, pyfile): - host = self._get_host(pyfile.url) - json_params = json_dumps({'link': pyfile.url, - 'type': host, + host = self._get_host(pyfile.url) + json_params = json_dumps({'link' : pyfile.url, + 'type' : host, 'username': self.user, 'password': self.account.getAccountData(self.user)['password']}) - r = self.load('http://gen.linksnappy.com/genAPI.php', + + r = self.load("http://gen.linksnappy.com/genAPI.php", post={'genLinks': json_params}) + self.logDebug("JSON data: " + r) j = json_loads(r)['links'][0] if j['error']: - msg = _("Error converting the link") - self.logError(msg, j['error']) - self.fail(msg) + self.error(_("Error converting the link")) pyfile.name = j['filename'] - self.link = j['generated'] + self.link = j['generated'] if host in self.SINGLE_CHUNK_HOSTERS: self.chunkLimit = 1 else: self.setup() - if self.link != pyfile.url: - self.logDebug("New URL: " + self.link) - @staticmethod def _get_host(url): - host = urlsplit(url).netloc + host = urlparse.urlsplit(url).netloc return re.search(r'[\w-]+\.\w+$', host).group(0) diff --git a/module/plugins/hoster/LoadTo.py b/module/plugins/hoster/LoadTo.py index b21a69b5f..11c96de60 100644 --- a/module/plugins/hoster/LoadTo.py +++ b/module/plugins/hoster/LoadTo.py @@ -6,16 +6,17 @@ import re -from module.plugins.internal.CaptchaService import SolveMedia +from module.plugins.internal.SolveMedia import SolveMedia from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class LoadTo(SimpleHoster): __name__ = "LoadTo" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.24" __pattern__ = r'http://(?:www\.)?load\.to/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """ Load.to hoster plugin """ __license__ = "GPLv3" @@ -23,7 +24,7 @@ class LoadTo(SimpleHoster): ("stickell", "l.stickell@yahoo.it")] - NAME_PATTERN = r'<h1>(?P<N>.+)</h1>' + NAME_PATTERN = r'<h1>(?P<N>.+?)</h1>' SIZE_PATTERN = r'Size: (?P<S>[\d.,]+) (?P<U>[\w^_]+)' OFFLINE_PATTERN = r'>Can\'t find file' @@ -44,22 +45,23 @@ class LoadTo(SimpleHoster): if m is None: self.error(_("LINK_FREE_PATTERN not found")) - download_url = m.group(1) + self.link = m.group(1) # Set Timer - may be obsolete m = re.search(self.WAIT_PATTERN, self.html) if m: - self.wait(int(m.group(1))) + self.wait(m.group(1)) # Load.to is using solvemedia captchas since ~july 2014: - solvemedia = SolveMedia(self) + solvemedia = SolveMedia(self) captcha_key = solvemedia.detect_key() - if captcha_key is None: - self.download(download_url) - else: - challenge, response = solvemedia.challenge(captcha_key) - self.download(download_url, post={"adcopy_challenge": challenge, "adcopy_response": response}) + if captcha_key: + response, challenge = solvemedia.challenge(captcha_key) + self.download(self.link, + post={'adcopy_challenge': challenge, + 'adcopy_response' : response, + 'returnUrl' : pyfile.url}) getInfo = create_getInfo(LoadTo) diff --git a/module/plugins/hoster/LolabitsEs.py b/module/plugins/hoster/LolabitsEs.py new file mode 100644 index 000000000..61df5f0bb --- /dev/null +++ b/module/plugins/hoster/LolabitsEs.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -* + +import HTMLParser +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class LolabitsEs(SimpleHoster): + __name__ = "LolabitsEs" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'https?://(?:www\.)?lolabits\.es/.+' + + __description__ = """Lolabits.es hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'Descargar: <b>(?P<N>.+?)<' + SIZE_PATTERN = r'class="fileSize">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'Un usuario con este nombre no existe' + + FILEID_PATTERN = r'name="FileId" value="(\d+)"' + TOKEN_PATTERN = r'name="__RequestVerificationToken" type="hidden" value="(.+?)"' + LINK_PATTERN = r'"redirectUrl":"(.+?)"' + + + def setup(self): + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + fileid = re.search(self.FILEID_PATTERN, self.html).group(1) + self.logDebug("FileID: " + fileid) + + token = re.search(self.TOKEN_PATTERN, self.html).group(1) + self.logDebug("Token: " + token) + + self.html = self.load("http://lolabits.es/action/License/Download", + post={'fileId' : fileid, + '__RequestVerificationToken' : token}).decode('unicode-escape') + + self.link = HTMLParser.HTMLParser().unescape(re.search(self.LINK_PATTERN, self.html).group(1)) + + +getInfo = create_getInfo(LolabitsEs) diff --git a/module/plugins/hoster/LomafileCom.py b/module/plugins/hoster/LomafileCom.py index 475cdacaa..ef05cd1ea 100644 --- a/module/plugins/hoster/LomafileCom.py +++ b/module/plugins/hoster/LomafileCom.py @@ -9,6 +9,7 @@ class LomafileCom(DeadHoster): __version__ = "0.52" __pattern__ = r'http://lomafile\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Lomafile.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/LuckyShareNet.py b/module/plugins/hoster/LuckyShareNet.py index 09300a41a..f4932d93d 100644 --- a/module/plugins/hoster/LuckyShareNet.py +++ b/module/plugins/hoster/LuckyShareNet.py @@ -4,16 +4,17 @@ import re from bottle import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class LuckyShareNet(SimpleHoster): __name__ = "LuckyShareNet" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.07" __pattern__ = r'https?://(?:www\.)?luckyshare\.net/(?P<ID>\d{10,})' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """LuckyShare.net hoster plugin""" __license__ = "GPLv3" @@ -47,12 +48,12 @@ class LuckyShareNet(SimpleHoster): self.logDebug("JSON: " + rep) json = self.parseJson(rep) - self.wait(int(json['time'])) + self.wait(json['time']) recaptcha = ReCaptcha(self) for _i in xrange(5): - challenge, response = recaptcha.challenge() + response, challenge = recaptcha.challenge() rep = self.load(r"http://luckyshare.net/download/verify/challenge/%s/response/%s/hash/%s" % (challenge, response, json['hash']), decode=True) self.logDebug("JSON: " + rep) @@ -68,7 +69,7 @@ class LuckyShareNet(SimpleHoster): if not json['link']: self.fail(_("No Download url retrieved/all captcha attempts failed")) - self.download(json['link']) + self.link = json['link'] getInfo = create_getInfo(LuckyShareNet) diff --git a/module/plugins/hoster/MediafireCom.py b/module/plugins/hoster/MediafireCom.py index 81bd36f73..09eb4705d 100644 --- a/module/plugins/hoster/MediafireCom.py +++ b/module/plugins/hoster/MediafireCom.py @@ -1,128 +1,78 @@ # -*- coding: utf-8 -*- -import re - -from module.plugins.internal.CaptchaService import SolveMedia -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - - -def replace_eval(js_expr): - return js_expr.replace(r'eval("', '').replace(r"\'", r"'").replace(r'\"', r'"') - - -def checkHTMLHeader(url): - try: - for _i in xrange(3): - header = getURL(url, just_header=True) - - for line in header.splitlines(): - line = line.lower() - - if 'location' in line: - url = line.split(':', 1)[1].strip() - if 'error.php?errno=320' in url: - return url, 1 - - if not url.startswith('http://'): - url = 'http://www.mediafire.com' + url - - break - - elif 'content-disposition' in line: - return url, 2 - else: - break - except: - return url, 3 - else: - return url, 0 - - -def getInfo(urls): - for url in urls: - location, status = checkHTMLHeader(url) - - if status: - file_info = (url, 0, status, url) - else: - file_info = parseFileInfo(MediafireCom, url, getURL(url, decode=True)) - - yield file_info +from module.plugins.internal.ReCaptcha import ReCaptcha +from module.plugins.internal.SolveMedia import SolveMedia +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class MediafireCom(SimpleHoster): __name__ = "MediafireCom" __type__ = "hoster" - __version__ = "0.83" + __version__ = "0.89" - __pattern__ = r'http://(?:www\.)?mediafire\.com/(file/|(view/?|download\.php)?\?)(\w{11}|\w{15})($|/)' + __pattern__ = r'https?://(?:www\.)?mediafire\.com/(file/|view/\??|download(\.php\?|/)|\?)(?P<ID>\w{15})' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Mediafire.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it")] + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<META NAME="description" CONTENT="(?P<N>[^"]+)"/>' - INFO_PATTERN = r'oFileSharePopup\.ald\(\'(?P<ID>[^\']*)\',\'(?P<N>[^\']*)\',\'(?P<S>[^\']*)\',\'\',\'(?P<H>[^\']*)\'\)' - OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>' + NAME_PATTERN = r'<META NAME="description" CONTENT="(?P<N>.+?)"/>' + SIZE_PATTERN = r'<li>File size: <span>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + INFO_PATTERN = r'oFileSharePopup\.ald\(\'.*?\',\'(?P<N>.+?)\',\'(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)\',\'\',\'(?P<H>.+?)\'\)' + OFFLINE_PATTERN = r'class="error_msg_title"' + + LINK_FREE_PATTERN = r'kNO = "(.+?)"' PASSWORD_PATTERN = r'<form name="form_password"' def setup(self): - self.multiDL = False + self.resumeDownload = True + self.multiDL = True - def process(self, pyfile): - pyfile.url = re.sub(r'/view/?\?', '/?', pyfile.url) + def handleCaptcha(self): + solvemedia = SolveMedia(self) + captcha_key = solvemedia.detect_key() - self.link, result = checkHTMLHeader(pyfile.url) - self.logDebug("Location (%d): %s" % (result, self.link)) + if captcha_key: + response, challenge = solvemedia.challenge(captcha_key) + self.html = self.load("http://www.mediafire.com/?" + self.info['pattern']['ID'], + post={'adcopy_challenge': challenge, + 'adcopy_response' : response}, + decode=True) + return - if result == 0: - self.html = self.load(self.link, decode=True) - self.checkCaptcha() - self.multiDL = True - self.check_data = self.getFileInfo() + recaptcha = ReCaptcha(self) + captcha_key = recaptcha.detect_key() - if self.account: - self.handlePremium(pyfile) - else: - self.handleFree(pyfile) - elif result == 1: - self.offline() - else: - self.multiDL = True - self.download(self.link, disposition=True) + if captcha_key: + response, challenge = recaptcha.challenge(captcha_key) + self.html = self.load(self.pyfile.url, + post={'g-recaptcha-response': response}, + decode=True) def handleFree(self, pyfile): + self.handleCaptcha() + if self.PASSWORD_PATTERN in self.html: password = self.getPassword() - if password: - self.logInfo(_("Password protected link, trying ") + password) - self.html = self.load(self.link, post={"downloadp": password}) + if not password: + self.fail(_("No password found")) + else: + self.logInfo(_("Password protected link, trying: ") + password) + self.html = self.load(self.link, post={'downloadp': password}) if self.PASSWORD_PATTERN in self.html: self.fail(_("Incorrect password")) - else: - self.fail(_("No password found")) - - m = re.search(r'kNO = r"(http://.*?)";', self.html) - if m is None: - self.error(_("No download URL")) - download_url = m.group(1) - self.download(download_url) + return super(MediafireCom, self).handleFree(pyfile) - def checkCaptcha(self): - solvemedia = SolveMedia(self) - challenge, response = solvemedia.challenge() - self.html = self.load(self.link, - post={'adcopy_challenge': challenge, - 'adcopy_response' : response}, - decode=True) +getInfo = create_getInfo(MediafireCom) diff --git a/module/plugins/hoster/MegaCoNz.py b/module/plugins/hoster/MegaCoNz.py index fbd34c563..5280b42ec 100644 --- a/module/plugins/hoster/MegaCoNz.py +++ b/module/plugins/hoster/MegaCoNz.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- +import array import os +# import pycurl import random import re -from array import array from base64 import standard_b64decode from Crypto.Cipher import AES from Crypto.Util import Counter -# from pycurl import SSL_CIPHER_LIST from module.common.json_layer import json_loads, json_dumps from module.plugins.Hoster import Hoster @@ -48,9 +48,9 @@ from module.utils import decode, fs_decode, fs_encode class MegaCoNz(Hoster): __name__ = "MegaCoNz" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.29" - __pattern__ = r'(?:https?://(?:www\.)?mega\.co\.nz/|mega:|chrome:.+?)#(?P<TYPE>N|)!(?P<ID>[\w^_]+)!(?P<KEY>[\w,\\-]+)' + __pattern__ = r'(?:https?://(?:www\.)?mega(?:\.co)?\.nz/|mega:|chrome:.+?)#(?P<TYPE>N|)!(?P<ID>[\w^_]+)!(?P<KEY>[\w,-]+)' __description__ = """Mega.co.nz hoster plugin""" __license__ = "GPLv3" @@ -69,10 +69,10 @@ class MegaCoNz(Hoster): def getCipherKey(self, key): """ Construct the cipher key from the given data """ - a = array("I", self.b64_decode(key)) + a = array.array("I", self.b64_decode(key)) - k = array("I", (a[0] ^ a[4], a[1] ^ a[5], a[2] ^ a[6], a[3] ^ a[7])) - iv = a[4:6] + array("I", (0, 0)) + k = array.array("I", (a[0] ^ a[4], a[1] ^ a[5], a[2] ^ a[6], a[3] ^ a[7])) + iv = a[4:6] + array.array("I", (0, 0)) meta_mac = a[6:8] return k, iv, meta_mac @@ -124,7 +124,7 @@ class MegaCoNz(Hoster): df = open(file_decrypted, "wb") except IOError, e: - self.fail(str(e)) + self.fail(e) chunk_size = 2 ** 15 # buffer size, 32k # file_mac = [0, 0, 0, 0] # calculate CBC-MAC for checksum @@ -145,7 +145,7 @@ class MegaCoNz(Hoster): # block = chunk[i:i+16] # if len(block) % 16: # block += '=' * (16 - (len(block) % 16)) - # block = array("I", block) + # block = array.array("I", block) # chunk_mac = [chunk_mac[0] ^ a_[0], chunk_mac[1] ^ block[1], chunk_mac[2] ^ block[2], chunk_mac[3] ^ block[3]] # chunk_mac = aes_cbc_encrypt_a32(chunk_mac, k) @@ -160,7 +160,7 @@ class MegaCoNz(Hoster): # if file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3] != meta_mac: # os.remove(file_decrypted) - # self.fail("Checksum mismatch") + # self.fail(_("Checksum mismatch")) os.remove(file_crypted) self.lastDownload = fs_decode(file_decrypted) @@ -207,7 +207,7 @@ class MegaCoNz(Hoster): pyfile.name = attr['n'] + self.FILE_SUFFIX pyfile.size = mega['s'] - # self.req.http.c.setopt(SSL_CIPHER_LIST, "RC4-MD5:DEFAULT") + # self.req.http.c.setopt(pycurl.SSL_CIPHER_LIST, "RC4-MD5:DEFAULT") self.download(mega['g']) diff --git a/module/plugins/hoster/MegaDebridEu.py b/module/plugins/hoster/MegaDebridEu.py index 8bbc733e0..9c8cc2a90 100644 --- a/module/plugins/hoster/MegaDebridEu.py +++ b/module/plugins/hoster/MegaDebridEu.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote_plus +import urllib from module.common.json_layer import json_loads from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -11,9 +10,10 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class MegaDebridEu(MultiHoster): __name__ = "MegaDebridEu" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.47" __pattern__ = r'http://((?:www\d+\.|s\d+\.)?mega-debrid\.eu|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/download/file/[\w^_]+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """mega-debrid.eu multi-hoster plugin""" __license__ = "GPLv3" @@ -23,25 +23,6 @@ class MegaDebridEu(MultiHoster): API_URL = "https://www.mega-debrid.eu/api.php" - def getFilename(self, url): - try: - return unquote_plus(url.rsplit("/", 1)[1]) - except IndexError: - return "" - - - def handlePremium(self, pyfile): - if not self.api_load(): - self.exitOnFail("Unable to connect to Mega-debrid.eu") - - self.link = self.debridLink(pyfile.url) - self.logDebug("New URL: " + self.link) - - filename = self.getFilename(self.link) - if filename != "": - pyfile.name = filename - - def api_load(self): """ Connexion to the mega-debrid API @@ -59,32 +40,21 @@ class MegaDebridEu(MultiHoster): return False - def debridLink(self, linkToDebrid): + def handlePremium(self, pyfile): """ Debrid a link Return The debrided link if succeed or original link if fail """ - jsonResponse = self.load(self.API_URL, get={'action': 'getLink', 'token': self.token}, - post={"link": linkToDebrid}) - res = json_loads(jsonResponse) - - if res['response_code'] == "ok": - debridedLink = res['debridLink'][1:-1] - return debridedLink - else: - self.exitOnFail("Unable to debrid %s" % linkToDebrid) + if not self.api_load(): + self.error("Unable to connect to remote API") + jsonResponse = self.load(self.API_URL, + get={'action': 'getLink', 'token': self.token}, + post={'link': pyfile.url}) - def exitOnFail(self, msg): - """ - exit the plugin on fail case - And display the reason of this failure - """ - if self.getConfig("unloadFailing"): - self.logError(_(msg)) - self.resetAccount() - else: - self.fail(_(msg)) + res = json_loads(jsonResponse) + if res['response_code'] == "ok": + self.link = res['debridLink'][1:-1] getInfo = create_getInfo(MegaDebridEu) diff --git a/module/plugins/hoster/MegaFilesSe.py b/module/plugins/hoster/MegaFilesSe.py index c3de57914..249eff952 100644 --- a/module/plugins/hoster/MegaFilesSe.py +++ b/module/plugins/hoster/MegaFilesSe.py @@ -9,6 +9,7 @@ class MegaFilesSe(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?megafiles\.se/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """MegaFiles.se hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/MegaRapidCz.py b/module/plugins/hoster/MegaRapidCz.py index 048561ac5..e17dd4730 100644 --- a/module/plugins/hoster/MegaRapidCz.py +++ b/module/plugins/hoster/MegaRapidCz.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re -from pycurl import HTTPHEADER - from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getRequest from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo @@ -11,7 +10,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo def getInfo(urls): h = getRequest() - h.c.setopt(HTTPHEADER, + h.c.setopt(pycurl.HTTPHEADER, ["Accept: text/html", "User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"]) @@ -26,6 +25,7 @@ class MegaRapidCz(SimpleHoster): __version__ = "0.56" __pattern__ = r'http://(?:www\.)?(share|mega)rapid\.cz/soubor/\d+/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """MegaRapid.cz hoster plugin""" __license__ = "GPLv3" @@ -35,13 +35,13 @@ class MegaRapidCz(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<h1[^>]*><span[^>]*>(?:<a[^>]*>)?(?P<N>[^<]+)' + NAME_PATTERN = r'<h1.*?><span.*?>(?:<a.*?>)?(?P<N>[^<]+)' SIZE_PATTERN = r'<td class="i">Velikost:</td>\s*<td class="h"><strong>\s*(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong></td>' OFFLINE_PATTERN = ur'Nastala chyba 404|Soubor byl smazán' CHECK_TRAFFIC = True - LINK_PREMIUM_PATTERN = r'<a href="([^"]+)" title="Stahnout">([^<]+)</a>' + LINK_PREMIUM_PATTERN = r'<a href="(.+?)" title="Stahnout">([^<]+)</a>' ERR_LOGIN_PATTERN = ur'<div class="error_div"><strong>Stahovánà je pÅÃstupné pouze pÅihlášenÃœm uÅŸivatelům' ERR_CREDIT_PATTERN = ur'<div class="error_div"><strong>Stahovánà zdarma je moÅŸné jen pÅes náš' @@ -54,14 +54,14 @@ class MegaRapidCz(SimpleHoster): def handlePremium(self, pyfile): m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m: - link = m.group(1) - self.logDebug("Premium link: %s" % link) - self.download(link, disposition=True) + self.link = m.group(1) else: if re.search(self.ERR_LOGIN_PATTERN, self.html): self.relogin(self.user) self.retry(wait_time=60, reason=_("User login failed")) + elif re.search(self.ERR_CREDIT_PATTERN, self.html): self.fail(_("Not enough credit left")) + else: self.fail(_("Download link not found")) diff --git a/module/plugins/hoster/MegaRapidoNet.py b/module/plugins/hoster/MegaRapidoNet.py new file mode 100644 index 000000000..a3b4c72ba --- /dev/null +++ b/module/plugins/hoster/MegaRapidoNet.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +import random + +from module.plugins.internal.MultiHoster import MultiHoster + + +def random_with_N_digits(n): + rand = "0." + not_zero = 0 + for i in range(1, n + 1): + r = random.randint(0, 9) + if(r > 0): + not_zero += 1 + rand += str(r) + + if not_zero > 0: + return rand + else: + return random_with_N_digits(n) + + +class MegaRapidoNet(MultiHoster): + __name__ = "MegaRapidoNet" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?\w+\.megarapido\.net/\?file=\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """MegaRapido.net multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + LINK_PREMIUM_PATTERN = r'<\s*?a[^>]*?title\s*?=\s*?["\'].*?download["\'][^>]*?href=["\']([^"\']+)' + + ERROR_PATTERN = r'<\s*?div[^>]*?class\s*?=\s*?["\']?alert-message error.*?>([^<]*)' + + + def handlePremium(self, pyfile): + self.html = self.load("http://megarapido.net/gerar.php", + post={'rand' :random_with_N_digits(16), + 'urllist' : pyfile.url, + 'links' : pyfile.url, + 'exibir' : "normal", + 'usar' : "premium", + 'user' : self.account.getAccountInfo(self.user).get('sid', None), + 'autoreset': ""}) + + if "desloga e loga novamente para gerar seus links" in self.html.lower(): + self.error("You have logged in at another place") + + return super(MegaRapidoNet, self).handlePremium(pyfile) diff --git a/module/plugins/hoster/MegareleaseOrg.py b/module/plugins/hoster/MegareleaseOrg.py index 60796c1ee..6349144ec 100644 --- a/module/plugins/hoster/MegareleaseOrg.py +++ b/module/plugins/hoster/MegareleaseOrg.py @@ -9,6 +9,7 @@ class MegareleaseOrg(DeadHoster): __version__ = "0.02" __pattern__ = r'https?://(?:www\.)?megarelease\.org/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Megarelease.org hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/MegasharesCom.py b/module/plugins/hoster/MegasharesCom.py index bdb428143..ed2363fe3 100644 --- a/module/plugins/hoster/MegasharesCom.py +++ b/module/plugins/hoster/MegasharesCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import time +import time from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,6 +12,7 @@ class MegasharesCom(SimpleHoster): __version__ = "0.28" __pattern__ = r'http://(?:www\.)?(d\d{2}\.)?megashares\.com/((index\.php)?\?d\d{2}=|dl/)\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Megashares.com hoster plugin""" __license__ = "GPLv3" @@ -20,17 +20,17 @@ class MegasharesCom(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<h1 class="black xxl"[^>]*title="(?P<N>[^"]+)">' + NAME_PATTERN = r'<h1 class="black xxl"[^>]*title="(?P<N>.+?)">' SIZE_PATTERN = r'<strong><span class="black">Filesize:</span></strong> (?P<S>[\d.,]+) (?P<U>[\w^_]+)' OFFLINE_PATTERN = r'<dd class="red">(Invalid Link Request|Link has been deleted|Invalid link)' - LINK_PATTERN = r'<div id="show_download_button_%d"[^>]*>\s*<a href="([^"]+)">' + LINK_PATTERN = r'<div id="show_download_button_%d".*?>\s*<a href="(.+?)">' - PASSPORT_LEFT_PATTERN = r'Your Download Passport is: <[^>]*>(\w+).*?You have.*?<[^>]*>.*?([\d.]+) (\w+)' + PASSPORT_LEFT_PATTERN = r'Your Download Passport is: <.*?>(\w+).*?You have.*?<.*?>.*?([\d.]+) (\w+)' PASSPORT_RENEW_PATTERN = r'(\d+):<strong>(\d+)</strong>:<strong>(\d+)</strong>' REACTIVATE_NUM_PATTERN = r'<input[^>]*id="random_num" value="(\d+)" />' REACTIVATE_PASSPORT_PATTERN = r'<input[^>]*id="passport_num" value="(\w+)" />' - REQUEST_URI_PATTERN = r'var request_uri = "([^"]+)";' + REQUEST_URI_PATTERN = r'var request_uri = "(.+?)";' NO_SLOTS_PATTERN = r'<dd class="red">All download slots for this link are currently filled' @@ -66,7 +66,7 @@ class MegasharesCom(SimpleHoster): 'rsargs[]': random_num, 'rsargs[]': passport_num, 'rsargs[]': "replace_sec_pprenewal", - 'rsrnd[]' : str(int(time() * 1000))}) + 'rsrnd[]' : str(int(time.time() * 1000))}) if 'Thank you for reactivating your passport.' in res: self.correctCaptcha() @@ -105,9 +105,8 @@ class MegasharesCom(SimpleHoster): if m is None: self.error(msg) - download_url = m.group(1) - self.logDebug("%s: %s" % (msg, download_url)) - self.download(download_url) + self.link = m.group(1) + self.logDebug("%s: %s" % (msg, self.link)) getInfo = create_getInfo(MegasharesCom) diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py index 7f51a8a46..590819f7f 100644 --- a/module/plugins/hoster/MegauploadCom.py +++ b/module/plugins/hoster/MegauploadCom.py @@ -9,6 +9,7 @@ class MegauploadCom(DeadHoster): __version__ = "0.31" __pattern__ = r'http://(?:www\.)?megaupload\.com/\?.*&?(d|v)=\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Megaupload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/MegavideoCom.py b/module/plugins/hoster/MegavideoCom.py index 24905ce62..accd079fc 100644 --- a/module/plugins/hoster/MegavideoCom.py +++ b/module/plugins/hoster/MegavideoCom.py @@ -9,6 +9,7 @@ class MegavideoCom(DeadHoster): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?megavideo\.com/\?.*&?(d|v)=\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Megavideo.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/MovReelCom.py b/module/plugins/hoster/MovReelCom.py index 9bb63701c..2fe5184ae 100644 --- a/module/plugins/hoster/MovReelCom.py +++ b/module/plugins/hoster/MovReelCom.py @@ -15,9 +15,7 @@ class MovReelCom(XFSHoster): __authors__ = [("JorisV83", "jorisv83-pyload@yahoo.com")] - HOSTER_DOMAIN = "movreel.com" - - LINK_PATTERN = r'<a href="([^"]+)">Download Link' + LINK_PATTERN = r'<a href="(.+?)">Download Link' getInfo = create_getInfo(MovReelCom) diff --git a/module/plugins/hoster/MultihostersCom.py b/module/plugins/hoster/MultihostersCom.py index c5a48310e..bcd7c6237 100644 --- a/module/plugins/hoster/MultihostersCom.py +++ b/module/plugins/hoster/MultihostersCom.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.hosters.ZeveraCom import ZeveraCom +from module.plugins.hoster.ZeveraCom import ZeveraCom class MultihostersCom(ZeveraCom): __name__ = "MultihostersCom" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.03" __pattern__ = r'https?://(?:www\.)multihosters\.com/(getFiles\.ashx|Members/download\.ashx)\?.*ourl=.+' diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index 7a6e73252..bbb77f525 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import random import re -from random import random - from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,6 +12,7 @@ class MultishareCz(SimpleHoster): __version__ = "0.40" __pattern__ = r'http://(?:www\.)?multishare\.cz/stahnout/(?P<ID>\d+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """MultiShare.cz hoster plugin""" __license__ = "GPLv3" @@ -44,7 +44,7 @@ class MultishareCz(SimpleHoster): if not self.checkTrafficLeft(): self.fail(_("Not enough credit left to download file")) - self.download("http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random() * 10000 * random()), + self.download("http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random.random() * 10000 * random.random()), get={'u_ID' : self.acc_info['u_ID'], 'u_hash': self.acc_info['u_hash'], 'link' : pyfile.url}, diff --git a/module/plugins/hoster/MyfastfileCom.py b/module/plugins/hoster/MyfastfileCom.py index 5ea2639f1..662638843 100644 --- a/module/plugins/hoster/MyfastfileCom.py +++ b/module/plugins/hoster/MyfastfileCom.py @@ -12,6 +12,7 @@ class MyfastfileCom(MultiHoster): __version__ = "0.08" __pattern__ = r'http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/dl/' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Myfastfile.com multi-hoster plugin""" __license__ = "GPLv3" @@ -23,19 +24,16 @@ class MyfastfileCom(MultiHoster): def handlePremium(self, pyfile): - self.logDebug("Original URL: %s" % pyfile.url) - - page = self.load('http://myfastfile.com/api.php', + self.html = self.load('http://myfastfile.com/api.php', get={'user': self.user, 'pass': self.account.getAccountData(self.user)['password'], 'link': pyfile.url}) - self.logDebug("JSON data: " + page) - page = json_loads(page) - if page['status'] != 'ok': + self.logDebug("JSON data: " + self.html) + + self.html = json_loads(self.html) + if self.html['status'] != 'ok': self.fail(_("Unable to unrestrict link")) - self.link = page['link'] - if self.link != pyfile.url: - self.logDebug("Unrestricted URL: " + self.link) + self.link = self.html['link'] getInfo = create_getInfo(MyfastfileCom) diff --git a/module/plugins/hoster/MystoreTo.py b/module/plugins/hoster/MystoreTo.py new file mode 100644 index 000000000..531368134 --- /dev/null +++ b/module/plugins/hoster/MystoreTo.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# Test link: +# http://mystore.to/dl/mxcA50jKfP + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class MystoreTo(SimpleHoster): + __name__ = "MystoreTo" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?mystore\.to/dl/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """Mystore.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "")] + + + NAME_PATTERN = r'<h1>(?P<N>.+?)<' + SIZE_PATTERN = r'FILESIZE: (?P<S>[\d\.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'>file not found<' + + + def setup(self): + self.chunkLimit = 1 + self.resumeDownload = True + self.multiDL = True + + + def handleFree(self, pyfile): + try: + fid = re.search(r'wert="(.+?)"', self.html).group(1) + + except AttributeError: + self.error(_("File-ID not found")) + + self.link = self.load("http://mystore.to/api/download", + post={'FID': fid}) + + +getInfo = create_getInfo(MystoreTo) diff --git a/module/plugins/hoster/NahrajCz.py b/module/plugins/hoster/NahrajCz.py index 6b5699408..cf708be09 100644 --- a/module/plugins/hoster/NahrajCz.py +++ b/module/plugins/hoster/NahrajCz.py @@ -9,6 +9,7 @@ class NahrajCz(DeadHoster): __version__ = "0.21" __pattern__ = r'http://(?:www\.)?nahraj\.cz/content/download/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Nahraj.cz hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/NarodRu.py b/module/plugins/hoster/NarodRu.py index 8b56c0908..c201ac250 100644 --- a/module/plugins/hoster/NarodRu.py +++ b/module/plugins/hoster/NarodRu.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +import random import re - -from random import random +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,6 +13,7 @@ class NarodRu(SimpleHoster): __version__ = "0.12" __pattern__ = r'http://(?:www\.)?narod(\.yandex)?\.ru/(disk|start/\d+\.\w+-narod\.yandex\.ru)/(?P<ID>\d+)/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Narod.ru hoster plugin""" __license__ = "GPLv3" @@ -33,7 +34,7 @@ class NarodRu(SimpleHoster): def handleFree(self, pyfile): for _i in xrange(5): - self.html = self.load('http://narod.ru/disk/getcapchaxml/?rnd=%d' % int(random() * 777)) + self.html = self.load('http://narod.ru/disk/getcapchaxml/?rnd=%d' % int(random.random() * 777)) m = re.search(self.CAPTCHA_PATTERN, self.html) if m is None: @@ -47,7 +48,7 @@ class NarodRu(SimpleHoster): m = re.search(self.LINK_FREE_PATTERN, self.html) if m: - url = 'http://narod.ru' + m.group(1) + self.link = urlparse.urljoin("http://narod.ru", m.group(1)) self.correctCaptcha() break @@ -60,7 +61,5 @@ class NarodRu(SimpleHoster): else: self.fail(_("No valid captcha code entered")) - self.download(url) - getInfo = create_getInfo(NarodRu) diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index 35c814656..82f5c22fb 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -1,295 +1,21 @@ # -*- coding: utf-8 -*- -import re +from module.plugins.internal.DeadHoster import DeadHoster, create_getInfo -from urlparse import urljoin -from time import time -from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster -from module.plugins.Plugin import chunks -from module.plugins.internal.CaptchaService import ReCaptcha - - -def getInfo(urls): - ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) - - apiurl = "http://api.netload.in/info.php" - id_regex = re.compile(NetloadIn.__pattern__) - urls_per_query = 80 - - for chunk in chunks(urls, urls_per_query): - ids = "" - for url in chunk: - match = id_regex.search(url) - if match: - ids = ids + match.group(1) + ";" - - api = getURL(apiurl, - get={'auth' : "Zf9SnQh9WiReEsb18akjvQGqT0I830e8", - 'bz' : 1, - 'md5' : 1, - 'file_id': ids}, - decode=True) - - if api is None or len(api) < 10: - self.logDebug("Prefetch failed") - return - - if api.find("unknown_auth") >= 0: - self.logDebug("Outdated auth code") - return - - result = [] - - for i, r in enumerate(api.splitlines()): - try: - tmp = r.split(";") - - try: - size = int(tmp[2]) - except Exception: - size = 0 - - result.append((tmp[1], size, 2 if tmp[3] == "online" else 1, chunk[i] )) - - except Exception: - self.logDebug("Error while processing response: %s" % r) - - yield result - - -class NetloadIn(Hoster): +class NetloadIn(DeadHoster): __name__ = "NetloadIn" __type__ = "hoster" - __version__ = "0.47" + __version__ = "0.50" - __pattern__ = r'https?://(?:[^/]*\.)?netload\.in/(?:datei(.*?)(?:\.htm|/)|index\.php?id=10&file_id=)' + __pattern__ = r'https?://(?:www\.)?netload\.(in|me)/(?P<PATH>datei|index\.php\?id=10&file_id=)(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Netload.in hoster plugin""" __license__ = "GPLv3" __authors__ = [("spoob", "spoob@pyload.org"), ("RaNaN", "ranan@pyload.org"), - ("Gregy", "gregy@gregy.cz")] - - - def setup(self): - self.multiDL = self.resumeDownload = self.premium - - - def process(self, pyfile): - self.url = pyfile.url - - self.prepare() - - pyfile.setStatus("downloading") - - self.proceed(self.url) - - - def prepare(self): - self.api_load() - - if self.api_data and self.api_data['filename']: - self.pyfile.name = self.api_data['filename'] - - if self.premium: - self.logDebug("Use Premium Account") - - settings = self.load("http://www.netload.in/index.php", get={'id': 2, 'lang': "en"}) - - if '<option value="2" selected="selected">Direkter Download' in settings: - self.logDebug("Using direct download") - return True - else: - self.logDebug("Direct downloads not enabled. Parsing html for a download URL") - - if self.download_html(): - return True - else: - self.fail(_("Failed")) - return False - - - def api_load(self, n=0): - url = self.url - id_regex = re.compile(self.__pattern__) - match = id_regex.search(url) - - if match: - #normalize url - self.url = 'http://www.netload.in/datei%s.htm' % match.group(1) - self.logDebug("URL: %s" % self.url) - else: - self.api_data = False - return - - apiurl = "http://api.netload.in/info.php" - html = self.load(apiurl, cookies=False, - get={"file_id": match.group(1), "auth": "Zf9SnQh9WiReEsb18akjvQGqT0I830e8", "bz": "1", - "md5": "1"}, decode=True).strip() - if not html and n <= 3: - self.setWait(2) - self.wait() - self.api_load(n + 1) - return - - self.logDebug("APIDATA: " + html) - - self.api_data = {} - - if html and ";" in html and html not in ("unknown file_data", "unknown_server_data", "No input file specified."): - lines = html.split(";") - self.api_data['exists'] = True - self.api_data['fileid'] = lines[0] - self.api_data['filename'] = lines[1] - self.api_data['size'] = lines[2] - self.api_data['status'] = lines[3] - - if self.api_data['status'] == "online": - self.api_data['checksum'] = lines[4].strip() - else: - self.api_data = False # check manually since api data is useless sometimes - - if lines[0] == lines[1] and lines[2] == "0": # useless api data - self.api_data = False - else: - self.api_data = False - - - def final_wait(self, page): - wait_time = self.get_wait_time(page) - - self.setWait(wait_time) - - self.logDebug("Final wait %d seconds" % wait_time) - - self.wait() - - self.url = self.get_file_url(page) - - - def check_free_wait(self,page): - if ">An access request has been made from IP address <" in page: - self.wantReconnect = True - self.setWait(self.get_wait_time(page) or 30) - self.wait() - return True - else: - return False - - - def download_html(self): - page = self.load(self.url, decode=True) - - if "/share/templates/download_hddcrash.tpl" in page: - self.logError(_("Netload HDD Crash")) - self.fail(_("File temporarily not available")) - - if not self.api_data: - self.logDebug("API Data may be useless, get details from html page") - - if "* The file was deleted" in page: - self.offline() - - name = re.search(r'class="dl_first_filename">([^<]+)', page, re.M) - # the found filename is not truncated - if name: - name = name.group(1).strip() - if not name.endswith(".."): - self.pyfile.name = name - - captchawaited = False - - for i in xrange(5): - if not page: - page = self.load(self.url) - t = time() + 30 - - if "/share/templates/download_hddcrash.tpl" in page: - self.logError(_("Netload HDD Crash")) - self.fail(_("File temporarily not available")) - - self.logDebug("Try number %d " % i) - - if ">Your download is being prepared.<" in page: - self.logDebug("We will prepare your download") - self.final_wait(page) - return True - - self.logDebug("Trying to find captcha") - - try: - url_captcha_html = re.search(r'(index.php\?id=10&.*&captcha=1)', page).group(1).replace("amp;", "") - - except Exception, e: - self.logDebug("Exception during Captcha regex: %s" % e.message) - page = None - - else: - url_captcha_html = urljoin("http://netload.in/", url_captcha_html) - break - - self.html = self.load(url_captcha_html) - - recaptcha = ReCaptcha(self) - - for _i in xrange(5): - challenge, response = recaptcha.challenge() - - response_page = self.load("http://www.netload.in/index.php?id=10", - post={'captcha_check' : '1', - 'recaptcha_challenge_field': challenge, - 'recaptcha_response_field' : response, - 'file_id' : self.api_data['fileid'], - 'Download_Next' : ''}) - if "Orange_Link" in response_page: - break - - if self.check_free_wait(response_page): - self.logDebug("Had to wait for next free slot, trying again") - return self.download_html() - - else: - download_url = self.get_file_url(response_page) - self.logDebug("Download URL after get_file: " + download_url) - if not download_url.startswith("http://"): - self.error("download url: %s" % download_url) - self.wait() - - self.url = download_url - return True - - - def get_file_url(self, page): - try: - file_url_pattern = r'<a class="Orange_Link" href="(http://.+)".?>Or click here' - attempt = re.search(file_url_pattern, page) - if attempt is not None: - return attempt.group(1) - else: - self.logDebug("Backup try for final link") - file_url_pattern = r'<a href="(.+)" class="Orange_Link">Click here' - attempt = re.search(file_url_pattern, page) - return "http://netload.in/" + attempt.group(1) - - except Exception, e: - self.logDebug("Getting final link failed", e.message) - return None - - - def get_wait_time(self, page): - return int(re.search(r"countdown\((.+),'change\(\)'\)", page).group(1)) / 100 - - - def proceed(self, url): - self.download(url, disposition=True) + ("Gregy", "gregy@gregy.cz" )] - check = self.checkDownload({'empty' : re.compile(r'^$'), - 'offline': re.compile("The file was deleted")}) - if check == "empty": - self.logInfo(_("Downloaded File was empty")) - self.retry() - elif check == "offline": - self.offline() +getInfo = create_getInfo(NetloadIn) diff --git a/module/plugins/hoster/NitroflareCom.py b/module/plugins/hoster/NitroflareCom.py index 2367493e5..a0734d9b1 100644 --- a/module/plugins/hoster/NitroflareCom.py +++ b/module/plugins/hoster/NitroflareCom.py @@ -1,79 +1,62 @@ # -*- coding: utf-8 -*- -# -# Note: -# Right now premium support is not added -# Thus, any file that require premium support -# cannot be downloaded. Only the file that is free to -# download can be downloaded. import re -from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster class NitroflareCom(SimpleHoster): __name__ = "NitroflareCom" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.14" __pattern__ = r'https?://(?:www\.)?nitroflare\.com/view/(?P<ID>[\w^_]+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Nitroflare.com hoster plugin""" __license__ = "GPLv3" - __authors__ = [("sahil", ""), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("sahil" , "sahilshekhawat01@gmail.com"), + ("Walter Purcaro", "vuolter@gmail.com" ), + ("Stickell" , "l.stickell@yahoo.it" )] + INFO_PATTERN = r'title="(?P<N>.+?)".+>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'>File doesn\'t exist' - # URL_REPLACEMENTS = [("http://", "https://")] + LINK_PREMIUM_PATTERN = LINK_FREE_PATTERN = r'(https?://[\w\-]+\.nitroflare\.com/.+?)"' - LINK_FREE_PATTERN = r'(https?://[\w\\-]+\\.nitroflare\\.com/[^<>\"]*?)"' + RECAPTCHA_KEY = "6Lenx_USAAAAAF5L1pmTWvWcH73dipAEzNnmNLgy" + PREMIUM_ONLY_PATTERN = r'This file is available with Premium only' + WAIT_PATTERN = r'You have to wait (\d+ minutes)' + # ERROR_PATTERN = r'downloading is not possible' def handleFree(self, pyfile): - file_info = self.load("https://nitroflare.com/api/getDownloadLink", - get={'file': self.info['pattern']['ID']}) - - self.logWarning(file_info[3:]) - file_info = json_loads(file_info[3:]) # removing non ascii characters - if file_info['type'] == "success": - result = file_info['result'] # already a dict - if result['linkType'] == "free": - delay = result['delay'] # Don't need the delay for free downloads - captch_key = result['recaptchaPublic'] - filename = result['name'] - recaptcha = ReCaptcha(self) - # try upto 3 times to solve reCaptcha - for i in xrange(3): - challenge, response = recaptcha.challenge(key=captch_key) - res = self.load("https://nitroflare.com/ajax/freeDownload.php", - post={"method": "fetchDownload", - "recaptcha_challenge_field": challenge, - "recaptcha_response_field": response}) - if self.handleCaptchaErrors(res): - break - if "The captcha wasn't entered correctly" or "You have to fill the captcha" in res: - continue - else: - break - - if "The captcha wasn't entered correctly" or "You have to fill the captcha" in res: - self.logError("Captcha Failed") - self.offline() - # Captcha failed - else: - self.logInfo("result of the captcha is") - self.logInfo(res) - # self.offline() - download_link = re.search(self.DOWNLOAD_PATTERN, res) - if download_link is None: - print "downloasd link failed" - # Download link failed - else: - self.download(download_link) - else: - print "link is invalid" - self.offline() - # Link is invalid - # self.download() + # used here to load the cookies which will be required later + self.load(pyfile.url, post={'goToFreePage': ""}) + + self.load("http://nitroflare.com/ajax/setCookie.php", post={'fileId': self.info['pattern']['ID']}) + self.html = self.load("http://nitroflare.com/ajax/freeDownload.php", + post={'method': "startTimer", 'fileId': self.info['pattern']['ID']}) + + self.checkErrors() + + try: + js_file = self.load("http://nitroflare.com/js/downloadFree.js?v=1.0.1") + var_time = re.search("var time = (\\d+);", js_file) + wait_time = int(var_time.groups()[0]) + + except Exception: + wait_time = 60 + + self.wait(wait_time) + + recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) + + self.html = self.load("http://nitroflare.com/ajax/freeDownload.php", + post={'method' : "fetchDownload", + 'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) + + return super(NitroflareCom, self).handleFree(pyfile) diff --git a/module/plugins/hoster/NoPremiumPl.py b/module/plugins/hoster/NoPremiumPl.py index 361d48c2d..c3313c525 100644 --- a/module/plugins/hoster/NoPremiumPl.py +++ b/module/plugins/hoster/NoPremiumPl.py @@ -10,6 +10,7 @@ class NoPremiumPl(MultiHoster): __version__ = "0.02" __pattern__ = r'https?://direct\.nopremium\.pl.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """NoPremium.pl multi-hoster plugin""" __license__ = "GPLv3" @@ -83,14 +84,14 @@ class NoPremiumPl(MultiHoster): # error code isn't yet added to plugin self.fail( parsed["errstring"] - or "Unknown error (code: %s)" % parsed["errno"] + or _("Unknown error (code: %s)") % parsed["errno"] ) if "sdownload" in parsed: if parsed["sdownload"] == "1": self.fail( - "Download from %s is possible only using NoPremium.pl webiste \ - directly. Update this plugin." % parsed["hosting"]) + _("Download from %s is possible only using NoPremium.pl website \ + directly") % parsed["hosting"]) pyfile.name = parsed["filename"] pyfile.size = parsed["filesize"] diff --git a/module/plugins/hoster/NosuploadCom.py b/module/plugins/hoster/NosuploadCom.py index 8a03d7090..241ee3a29 100644 --- a/module/plugins/hoster/NosuploadCom.py +++ b/module/plugins/hoster/NosuploadCom.py @@ -17,8 +17,6 @@ class NosuploadCom(XFSHoster): __authors__ = [("igel", "igelkun@myopera.com")] - HOSTER_DOMAIN = "nosupload.com" - SIZE_PATTERN = r'<p><strong>Size:</strong> (?P<S>[\d.,]+) (?P<U>[\w^_]+)</p>' LINK_PATTERN = r'<a class="select" href="(http://.+?)">Download</a>' @@ -28,14 +26,14 @@ class NosuploadCom(XFSHoster): def getDownloadLink(self): # stage1: press the "Free Download" button data = self.getPostParameters() - self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) + self.html = self.load(self.pyfile.url, post=data, decode=True) # stage2: wait some time and press the "Download File" button data = self.getPostParameters() wait_time = re.search(self.WAIT_PATTERN, self.html, re.M | re.S).group(1) self.logDebug("Hoster told us to wait %s seconds" % wait_time) self.wait(wait_time) - self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) + self.html = self.load(self.pyfile.url, post=data, decode=True) # stage3: get the download link return re.search(self.LINK_PATTERN, self.html, re.S).group(1) diff --git a/module/plugins/hoster/NovafileCom.py b/module/plugins/hoster/NovafileCom.py index 9754ceed1..b00f71635 100644 --- a/module/plugins/hoster/NovafileCom.py +++ b/module/plugins/hoster/NovafileCom.py @@ -20,10 +20,8 @@ class NovafileCom(XFSHoster): ("stickell", "l.stickell@yahoo.it")] - HOSTER_DOMAIN = "novafile.com" - - ERROR_PATTERN = r'class="alert[^"]*alert-separate"[^>]*>\s*(?:<p>)?(.*?)\s*</' - WAIT_PATTERN = r'<p>Please wait <span id="count"[^>]*>(\d+)</span> seconds</p>' + ERROR_PATTERN = r'class="alert.+?alert-separate".*?>\s*(?:<p>)?(.*?)\s*</' + WAIT_PATTERN = r'<p>Please wait <span id="count".*?>(\d+)</span> seconds</p>' LINK_PATTERN = r'<a href="(http://s\d+\.novafile\.com/.*?)" class="btn btn-green">Download File</a>' diff --git a/module/plugins/hoster/NowDownloadSx.py b/module/plugins/hoster/NowDownloadSx.py index b69242a86..a1cf9baf7 100644 --- a/module/plugins/hoster/NowDownloadSx.py +++ b/module/plugins/hoster/NowDownloadSx.py @@ -9,9 +9,10 @@ from module.utils import fixup class NowDownloadSx(SimpleHoster): __name__ = "NowDownloadSx" __type__ = "hoster" - __version__ = "0.07" + __version__ = "0.09" - __pattern__ = r'http://(?:www\.)?(nowdownload\.(at|ch|co|eu|sx)/(dl/|download\.php\?id=)|likeupload\.org/)\w+' + __pattern__ = r'http://(?:www\.)?(nowdownload\.[a-zA-Z]{2,}/(dl/|download\.php.+?id=|mobile/(#/files/|.+?id=))|likeupload\.org/)\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """NowDownload.sx hoster plugin""" __license__ = "GPLv3" @@ -27,7 +28,7 @@ class NowDownloadSx(SimpleHoster): WAIT_PATTERN = r'\.countdown\(\{until: \+(\d+),' LINK_FREE_PATTERN = r'(http://s\d+\.coolcdn\.info/nowdownload/.+?)["\']' - NAME_REPLACEMENTS = [("&#?\w+;", fixup), (r'<[^>]*>', '')] + NAME_REPLACEMENTS = [("&#?\w+;", fixup), (r'<.*?>', '')] def setup(self): @@ -54,11 +55,11 @@ class NowDownloadSx(SimpleHoster): self.html = self.load(baseurl + str(continuelink.group(1))) - url = re.search(self.LINK_FREE_PATTERN, self.html) - if url is None: + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: self.error(_("Download link not found")) - self.download(str(url.group(1))) + self.link = m.group(1) getInfo = create_getInfo(NowDownloadSx) diff --git a/module/plugins/hoster/NowVideoSx.py b/module/plugins/hoster/NowVideoSx.py index 3f75a33f7..477379597 100644 --- a/module/plugins/hoster/NowVideoSx.py +++ b/module/plugins/hoster/NowVideoSx.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class NowVideoSx(SimpleHoster): __name__ = "NowVideoSx" __type__ = "hoster" - __version__ = "0.10" + __version__ = "0.12" - __pattern__ = r'http://(?:www\.)?nowvideo\.(at|ch|co|eu|li|sx)/(video|mobile/#/videos)/(?P<ID>\w+)' + __pattern__ = r'http://(?:www\.)?nowvideo\.[a-zA-Z]{2,}/(video/|mobile/(#/videos/|.+?id=))(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """NowVideo.sx hoster plugin""" __license__ = "GPLv3" @@ -38,7 +39,7 @@ class NowVideoSx(SimpleHoster): if m is None: self.error(_("Free download link not found")) - self.download(m.group(1)) + self.link = m.group(1) getInfo = create_getInfo(NowVideoSx) diff --git a/module/plugins/hoster/OboomCom.py b/module/plugins/hoster/OboomCom.py index 588d8f64a..b50e2fb42 100644 --- a/module/plugins/hoster/OboomCom.py +++ b/module/plugins/hoster/OboomCom.py @@ -7,15 +7,15 @@ import re from module.common.json_layer import json_loads from module.plugins.Hoster import Hoster -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha class OboomCom(Hoster): __name__ = "OboomCom" __type__ = "hoster" - __version__ = "0.30" + __version__ = "0.33" - __pattern__ = r'https?://(?:www\.)?oboom\.com/(#(id=|/)?)?(?P<ID>\w{8})' + __pattern__ = r'https?://(?:www\.)?oboom\.com/(?:#(?:id=|/)?)?(?P<ID>\w{8})' __description__ = """oboom.com hoster plugin""" __license__ = "GPLv3" @@ -33,6 +33,7 @@ class OboomCom(Hoster): def process(self, pyfile): self.pyfile.url.replace(".com/#id=", ".com/#") self.pyfile.url.replace(".com/#/", ".com/#") + self.html = self.load(pyfile.url) self.getFileId(self.pyfile.url) self.getSessionToken() self.getFileInfo(self.sessionToken, self.fileId) @@ -74,7 +75,7 @@ class OboomCom(Hoster): recaptcha = ReCaptcha(self) for _i in xrange(5): - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) apiUrl = "https://www.oboom.com/1.0/download/ticket" params = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": response, diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py index f9c0dbe85..4b947c554 100644 --- a/module/plugins/hoster/OneFichierCom.py +++ b/module/plugins/hoster/OneFichierCom.py @@ -8,9 +8,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class OneFichierCom(SimpleHoster): __name__ = "OneFichierCom" __type__ = "hoster" - __version__ = "0.76" + __version__ = "0.84" __pattern__ = r'https?://(?:www\.)?(?:(?P<ID1>\w+)\.)?(?P<HOST>1fichier\.com|alterupload\.com|cjoint\.net|d(es)?fichiers\.com|dl4free\.com|megadl\.fr|mesfichiers\.org|piecejointe\.net|pjointe\.com|tenvoi\.com)(?:/\?(?P<ID2>\w+))?' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """1fichier.com hoster plugin""" __license__ = "GPLv3" @@ -20,17 +21,18 @@ class OneFichierCom(SimpleHoster): ("imclem", None), ("stickell", "l.stickell@yahoo.it"), ("Elrick69", "elrick69[AT]rocketmail[DOT]com"), - ("Walter Purcaro", "vuolter@gmail.com")] + ("Walter Purcaro", "vuolter@gmail.com"), + ("Ludovic Lehmann", "ludo.lehmann@gmail.com")] - NAME_PATTERN = r'>FileName :</td>\s*<td.*>(?P<N>.+?)<' - SIZE_PATTERN = r'>Size :</td>\s*<td.*>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + COOKIES = [("1fichier.com", "LG", "en")] + DISPOSITION = False #: Remove in 0.4.10 + NAME_PATTERN = r'>FileName :</td>\s*<td.*>(?P<N>.+?)<' + SIZE_PATTERN = r'>Size :</td>\s*<td.*>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' OFFLINE_PATTERN = r'File not found !\s*<' - COOKIES = [("1fichier.com", "LG", "en")] - - WAIT_PATTERN = r'>You must wait (\d+) minutes' + WAIT_PATTERN = r'>You must wait \d+ minutes' def setup(self): @@ -54,7 +56,7 @@ class OneFichierCom(SimpleHoster): def handlePremium(self, pyfile): - return self.handleFree(pyfile) + self.download(pyfile.url, post={'dl': "Download", 'did': 0}) getInfo = create_getInfo(OneFichierCom) diff --git a/module/plugins/hoster/OpenloadIo.py b/module/plugins/hoster/OpenloadIo.py new file mode 100644 index 000000000..5a60b6787 --- /dev/null +++ b/module/plugins/hoster/OpenloadIo.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class OpenloadIo(SimpleHoster): + __name__ = "OpenloadIo" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'https?://(?:www\.)?openload\.io/f/\w{11}' + + __description__ = """Openload.io hoster plugin""" + __license__ = "GPLv3" + + NAME_PATTERN = r'<span id="filename">(?P<N>.+?)</' + SIZE_PATTERN = r'<span class="count">(?P<S>[\d.,]+) (?P<U>[\w^_]+)<' + OFFLINE_PATTERN = r">(We can't find the file you are looking for)" + + LINK_FREE_PATTERN = r'id="realdownload"><a href="(https?://[\w\.]+\.openload\.io/dl/.*?)"' + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + +getInfo = create_getInfo(OpenloadIo) diff --git a/module/plugins/hoster/OronCom.py b/module/plugins/hoster/OronCom.py index 7e8423ec9..2e4961704 100644 --- a/module/plugins/hoster/OronCom.py +++ b/module/plugins/hoster/OronCom.py @@ -9,6 +9,7 @@ class OronCom(DeadHoster): __version__ = "0.14" __pattern__ = r'https?://(?:www\.)?oron\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Oron.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/OverLoadMe.py b/module/plugins/hoster/OverLoadMe.py index d4f4c0135..d06baa0f5 100644 --- a/module/plugins/hoster/OverLoadMe.py +++ b/module/plugins/hoster/OverLoadMe.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- import re - -from random import randrange -from urllib import unquote +import urllib from module.common.json_layer import json_loads from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -13,66 +11,41 @@ from module.utils import parseFileSize class OverLoadMe(MultiHoster): __name__ = "OverLoadMe" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.11" __pattern__ = r'https?://.*overload\.me/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Over-Load.me multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("marley", "marley@over-load.me")] - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - - if name.endswith("..."): #: incomplete filename, append random stuff - name += "%s.tmp" % randrange(100, 999) - - return name - - def setup(self): self.chunkLimit = 5 def handlePremium(self, pyfile): - https = "https" if self.getConfig("ssl") else "http" + https = "https" if self.getConfig('ssl') else "http" data = self.account.getAccountData(self.user) page = self.load(https + "://api.over-load.me/getdownload.php", get={'auth': data['password'], 'link': pyfile.url}) data = json_loads(page) + self.logDebug(data) if data['error'] == 1: self.logWarning(data['msg']) self.tempOffline() else: - if pyfile.name is not None and pyfile.name.endswith('.tmp') and data['filename']: + if pyfile.name and pyfile.name.endswith('.tmp') and data['filename']: pyfile.name = data['filename'] pyfile.size = parseFileSize(data['filesize']) http_repl = ["http://", "https://"] - self.link = data['downloadlink'].replace(*http_repl if self.getConfig("ssl") else *http_repl[::-1]) - - if self.link != pyfile.url: - self.logDebug("New URL: %s" % self.link) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): - # only use when name wasn't already set - pyfile.name = self.getFilename(self.link) - - - def checkFile(self): - if self.checkDownload({"error": "<title>An error occured while processing your request</title>"}) - # usual this download can safely be retried - self.retry(wait_time=60, reason=_("An error occured while generating link.")) - - return super(OverLoadMe, self).checkFile() + self.link = data['downloadlink'].replace(*http_repl if self.getConfig('ssl') else *http_repl[::-1]) getInfo = create_getInfo(OverLoadMe) diff --git a/module/plugins/hoster/PandaplaNet.py b/module/plugins/hoster/PandaplaNet.py index 78a1ed177..ae8406d82 100644 --- a/module/plugins/hoster/PandaplaNet.py +++ b/module/plugins/hoster/PandaplaNet.py @@ -9,6 +9,7 @@ class PandaplaNet(DeadHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?pandapla\.net/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Pandapla.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/PornhostCom.py b/module/plugins/hoster/PornhostCom.py index 71342f3e0..0c3b84a9d 100644 --- a/module/plugins/hoster/PornhostCom.py +++ b/module/plugins/hoster/PornhostCom.py @@ -73,8 +73,7 @@ class PornhostCom(Hoster): if not self.html: self.download_html() - if (re.search(r'gallery not found', self.html) is not None or - re.search(r'You will be redirected to', self.html) is not None): + if re.search(r'gallery not found|You will be redirected to', self.html): return False else: return True diff --git a/module/plugins/hoster/PornhubCom.py b/module/plugins/hoster/PornhubCom.py index 1bb787f09..16ce36ea9 100644 --- a/module/plugins/hoster/PornhubCom.py +++ b/module/plugins/hoster/PornhubCom.py @@ -64,7 +64,7 @@ class PornhubCom(Hoster): if not self.html: self.download_html() - m = re.search(r'<title[^>]+>([^<]+) - ', self.html) + m = re.search(r'<title.+?>([^<]+) - ', self.html) if m: name = m.group(1) else: @@ -83,7 +83,7 @@ class PornhubCom(Hoster): if not self.html: self.download_html() - if re.search(r'This video is no longer in our database or is in conversion', self.html) is not None: + if re.search(r'This video is no longer in our database or is in conversion', self.html): return False else: return True diff --git a/module/plugins/hoster/PotloadCom.py b/module/plugins/hoster/PotloadCom.py index d6261af3a..5a41f7d3c 100644 --- a/module/plugins/hoster/PotloadCom.py +++ b/module/plugins/hoster/PotloadCom.py @@ -9,6 +9,7 @@ class PotloadCom(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?potload\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Potload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/PremiumTo.py b/module/plugins/hoster/PremiumTo.py index 665005ec3..ab5016673 100644 --- a/module/plugins/hoster/PremiumTo.py +++ b/module/plugins/hoster/PremiumTo.py @@ -2,7 +2,7 @@ from __future__ import with_statement -from os import remove +import os from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo from module.utils import fs_encode @@ -11,9 +11,10 @@ from module.utils import fs_encode class PremiumTo(MultiHoster): __name__ = "PremiumTo" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.22" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Premium.to multi-hoster plugin""" __license__ = "GPLv3" @@ -22,6 +23,9 @@ class PremiumTo(MultiHoster): ("stickell", "l.stickell@yahoo.it")] + CHECK_TRAFFIC = True + + def handlePremium(self, pyfile): #raise timeout to 2min self.download("http://premium.to/api/getfile.php", @@ -31,22 +35,22 @@ class PremiumTo(MultiHoster): disposition=True) - def checkFile(self): + def checkFile(self, rules={}): if self.checkDownload({'nopremium': "No premium account available"}): self.retry(60, 5 * 60, "No premium account available") err = '' if self.req.http.code == '420': # Custom error code send - fail - lastDownload = fs_encode(self.lastDownload) - with open(lastDownload, "rb") as f: + file = fs_encode(self.lastDownload) + with open(file, "rb") as f: err = f.read(256).strip() - remove(lastDownload) + os.remove(file) if err: self.fail(err) - return super(PremiumTo, self).checkFile() + return super(PremiumTo, self).checkFile(rules) getInfo = create_getInfo(PremiumTo) diff --git a/module/plugins/hoster/PremiumizeMe.py b/module/plugins/hoster/PremiumizeMe.py index 3d19c88ac..181c08b7a 100644 --- a/module/plugins/hoster/PremiumizeMe.py +++ b/module/plugins/hoster/PremiumizeMe.py @@ -7,9 +7,10 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class PremiumizeMe(MultiHoster): __name__ = "PremiumizeMe" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __pattern__ = r'^unmatchable$' #: Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Premiumize.me multi-hoster plugin""" __license__ = "GPLv3" @@ -31,7 +32,7 @@ class PremiumizeMe(MultiHoster): user, data = self.account.selectAccount() # Get rewritten link using the premiumize.me api v1 (see https://secure.premiumize.me/?show=api) - data = json_loads(self.load("https://api.premiumize.me/pm-api/v1.php", + data = json_loads(self.load("http://api.premiumize.me/pm-api/v1.php", get={'method' : "directdownloadlink", 'params[login]': user, 'params[pass]' : data['password'], diff --git a/module/plugins/hoster/PromptfileCom.py b/module/plugins/hoster/PromptfileCom.py index ce0a384e3..3815a1a24 100644 --- a/module/plugins/hoster/PromptfileCom.py +++ b/module/plugins/hoster/PromptfileCom.py @@ -11,16 +11,17 @@ class PromptfileCom(SimpleHoster): __version__ = "0.13" __pattern__ = r'https?://(?:www\.)?promptfile\.com/' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Promptfile.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("igel", "igelkun@myopera.com")] - INFO_PATTERN = r'<span style="[^"]*" title="[^"]*">(?P<N>.*?) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)</span>' - OFFLINE_PATTERN = r'<span style="[^"]*" title="File Not Found">File Not Found</span>' + INFO_PATTERN = r'<span style=".+?" title=".+?">(?P<N>.*?) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)</span>' + OFFLINE_PATTERN = r'<span style=".+?" title="File Not Found">File Not Found</span>' - CHASH_PATTERN = r'<input type="hidden" name="chash" value="([^"]*)" />' + CHASH_PATTERN = r'<input type="hidden" name="chash" value="(.+?)" />' LINK_FREE_PATTERN = r'<a href=\"(.+)\" target=\"_blank\" class=\"view_dl_link\">Download File</a>' @@ -39,7 +40,7 @@ class PromptfileCom(SimpleHoster): if m is None: self.error(_("LINK_FREE_PATTERN not found")) - self.download(m.group(1), disposition=True) + self.link = m.group(1) getInfo = create_getInfo(PromptfileCom) diff --git a/module/plugins/hoster/PrzeklejPl.py b/module/plugins/hoster/PrzeklejPl.py index 3a59a2c9e..e984d0033 100644 --- a/module/plugins/hoster/PrzeklejPl.py +++ b/module/plugins/hoster/PrzeklejPl.py @@ -9,6 +9,7 @@ class PrzeklejPl(DeadHoster): __version__ = "0.11" __pattern__ = r'http://(?:www\.)?przeklej\.pl/plik/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Przeklej.pl hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/PutdriveCom.py b/module/plugins/hoster/PutdriveCom.py index 67732a4cd..7f4b7b6cc 100644 --- a/module/plugins/hoster/PutdriveCom.py +++ b/module/plugins/hoster/PutdriveCom.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.hosters.ZeveraCom import ZeveraCom +from module.plugins.hoster.ZeveraCom import ZeveraCom class PutdriveCom(ZeveraCom): __name__ = "PutdriveCom" __type__ = "hoster" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = r'https?://(?:www\.)putdrive\.com/(getFiles\.ashx|Members/download\.ashx)\?.*ourl=.+' diff --git a/module/plugins/hoster/QuickshareCz.py b/module/plugins/hoster/QuickshareCz.py index 210bf4cb2..1e0750b88 100644 --- a/module/plugins/hoster/QuickshareCz.py +++ b/module/plugins/hoster/QuickshareCz.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re -from pycurl import FOLLOWLOCATION - from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,6 +12,7 @@ class QuickshareCz(SimpleHoster): __version__ = "0.56" __pattern__ = r'http://(?:[^/]*\.)?quickshare\.cz/stahnout-soubor/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Quickshare.cz hoster plugin""" __license__ = "GPLv3" @@ -29,7 +29,7 @@ class QuickshareCz(SimpleHoster): self.getFileInfo() # parse js variables - self.jsvars = dict((x, y.strip("'")) for x, y in re.findall(r"var (\w+) = ([\d.]+|'[^']*')", self.html)) + self.jsvars = dict((x, y.strip("'")) for x, y in re.findall(r"var (\w+) = ([\d.]+|'.+?')", self.html)) self.logDebug(self.jsvars) pyfile.name = self.jsvars['ID3'] @@ -59,19 +59,20 @@ class QuickshareCz(SimpleHoster): data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ("ID1", "ID2", "ID3", "ID4")) self.logDebug("FREE URL1:" + download_url, data) - self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) self.load(download_url, post=data) self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) if m is None: self.fail(_("File not found")) - download_url = m.group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 - self.logDebug("FREE URL2:" + download_url) + + self.link = m.group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 + self.logDebug("FREE URL2:" + self.link) # check errors - m = re.search(r'/chyba/(\d+)', download_url) + m = re.search(r'/chyba/(\d+)', self.link) if m: if m.group(1) == '1': self.retry(60, 2 * 60, "This IP is already downloading") @@ -80,9 +81,6 @@ class QuickshareCz(SimpleHoster): else: self.fail(_("Error %d") % m.group(1)) - # download file - self.download(download_url) - def handlePremium(self, pyfile): download_url = '%s/download_premium.php' % self.jsvars['server'] diff --git a/module/plugins/hoster/RPNetBiz.py b/module/plugins/hoster/RPNetBiz.py index 13c2f8776..710faf25c 100644 --- a/module/plugins/hoster/RPNetBiz.py +++ b/module/plugins/hoster/RPNetBiz.py @@ -11,10 +11,11 @@ class RPNetBiz(MultiHoster): __type__ = "hoster" __version__ = "0.14" + __pattern__ = r'https?://.+rpnet\.biz' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + __description__ = """RPNet.biz multi-hoster plugin""" __license__ = "GPLv3" - - __pattern__ = r'https?://.+rpnet\.biz' __authors__ = [("Dman", "dmanugm@gmail.com")] @@ -25,7 +26,6 @@ class RPNetBiz(MultiHoster): def handlePremium(self, pyfile): user, data = self.account.selectAccount() - self.logDebug("Original URL: %s" % pyfile.url) # Get the download link res = self.load("https://premium.rpnet.biz/client_api.php", get={"username": user, diff --git a/module/plugins/hoster/RapideoPl.py b/module/plugins/hoster/RapideoPl.py index 2f7365e3b..50804e8cd 100644 --- a/module/plugins/hoster/RapideoPl.py +++ b/module/plugins/hoster/RapideoPl.py @@ -10,6 +10,7 @@ class RapideoPl(MultiHoster): __version__ = "0.02" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Rapideo.pl multi-hoster plugin""" __license__ = "GPLv3" @@ -83,14 +84,14 @@ class RapideoPl(MultiHoster): # error code isn't yet added to plugin self.fail( parsed["errstring"] - or "Unknown error (code: %s)" % parsed["errno"] + or _("Unknown error (code: %s)") % parsed["errno"] ) if "sdownload" in parsed: if parsed["sdownload"] == "1": self.fail( - "Download from %s is possible only using Rapideo.pl webiste \ - directly. Update this plugin." % parsed["hosting"]) + _("Download from %s is possible only using Rapideo.pl website \ + directly") % parsed["hosting"]) pyfile.name = parsed["filename"] pyfile.size = parsed["filesize"] diff --git a/module/plugins/hoster/RapidfileshareNet.py b/module/plugins/hoster/RapidfileshareNet.py index 3473b1c7c..0bbaed57f 100644 --- a/module/plugins/hoster/RapidfileshareNet.py +++ b/module/plugins/hoster/RapidfileshareNet.py @@ -15,8 +15,6 @@ class RapidfileshareNet(XFSHoster): __authors__ = [("guidobelix", "guidobelix@hotmail.it")] - HOSTER_DOMAIN = "rapidfileshare.net" - NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>.+?)">' SIZE_PATTERN = r'>http://www.rapidfileshare.net/\w+?</font> \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)</font>' diff --git a/module/plugins/hoster/RapidgatorNet.py b/module/plugins/hoster/RapidgatorNet.py index 84c3b20d6..bd6bb8582 100644 --- a/module/plugins/hoster/RapidgatorNet.py +++ b/module/plugins/hoster/RapidgatorNet.py @@ -1,21 +1,23 @@ # -*- coding: utf-8 -*- +import pycurl import re -from pycurl import HTTPHEADER - from module.common.json_layer import json_loads from module.network.HTTPRequest import BadHeader -from module.plugins.internal.CaptchaService import AdsCaptcha, ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight +from module.plugins.internal.AdsCaptcha import AdsCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha +from module.plugins.internal.SolveMedia import SolveMedia +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class RapidgatorNet(SimpleHoster): __name__ = "RapidgatorNet" __type__ = "hoster" - __version__ = "0.29" + __version__ = "0.34" __pattern__ = r'http://(?:www\.)?(rapidgator\.net|rg\.to)/file/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Rapidgator.net hoster plugin""" __license__ = "GPLv3" @@ -29,24 +31,26 @@ class RapidgatorNet(SimpleHoster): COOKIES = [("rapidgator.net", "lang", "en")] - NAME_PATTERN = r'<title>Download file (?P<N>.*)</title>' - SIZE_PATTERN = r'File size:\s*<strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong>' + NAME_PATTERN = r'<title>Download file (?P<N>.*)</title>' + SIZE_PATTERN = r'File size:\s*<strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong>' OFFLINE_PATTERN = r'>(File not found|Error 404)' JSVARS_PATTERN = r'\s+var\s*(startTimerUrl|getDownloadUrl|captchaUrl|fid|secs)\s*=\s*\'?(.*?)\'?;' - PREMIUM_ONLY_ERROR_PATTERN = r'You can download files up to|This file can be downloaded by premium only<' - DOWNLOAD_LIMIT_ERROR_PATTERN = r'You have reached your (daily|hourly) downloads limit' - WAIT_PATTERN = r'(?:Delay between downloads must be not less than|Try again in)\s*(\d+)\s*(hour|min)' + + PREMIUM_ONLY_PATTERN = r'You can download files up to|This file can be downloaded by premium only<' + ERROR_PATTERN = r'You have reached your (?:daily|hourly) downloads limit' + WAIT_PATTERN = r'(Delay between downloads must be not less than|Try again in).+' + LINK_FREE_PATTERN = r'return \'(http://\w+.rapidgator.net/.*)\';' - RECAPTCHA_PATTERN = r'"http://api\.recaptcha\.net/challenge\?k=(.*?)"' - ADSCAPTCHA_PATTERN = r'(http://api\.adscaptcha\.com/Get\.aspx[^"\']*)' + RECAPTCHA_PATTERN = r'"http://api\.recaptcha\.net/challenge\?k=(.*?)"' + ADSCAPTCHA_PATTERN = r'(http://api\.adscaptcha\.com/Get\.aspx[^"\']+)' SOLVEMEDIA_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.script\?k=(.*?)"' def setup(self): if self.account: - self.sid = self.account.getAccountInfo(self.user).get('SID', None) + self.sid = self.account.getAccountInfo(self.user).get('sid', None) else: self.sid = None @@ -95,26 +99,24 @@ class RapidgatorNet(SimpleHoster): def handleFree(self, pyfile): - self.checkFree() - jsvars = dict(re.findall(self.JSVARS_PATTERN, self.html)) self.logDebug(jsvars) self.req.http.lastURL = pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) url = "http://rapidgator.net%s?fid=%s" % ( jsvars.get('startTimerUrl', '/download/AjaxStartTimer'), jsvars['fid']) jsvars.update(self.getJsonResponse(url)) - self.wait(int(jsvars.get('secs', 45)), False) + self.wait(jsvars.get('secs', 45), False) url = "http://rapidgator.net%s?sid=%s" % ( jsvars.get('getDownloadUrl', '/download/AjaxGetDownload'), jsvars['sid']) jsvars.update(self.getJsonResponse(url)) self.req.http.lastURL = pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With:"]) url = "http://rapidgator.net%s" % jsvars.get('captchaUrl', '/download/captcha') self.html = self.load(url) @@ -122,13 +124,15 @@ class RapidgatorNet(SimpleHoster): for _i in xrange(5): m = re.search(self.LINK_FREE_PATTERN, self.html) if m: - link = m.group(1) - self.logDebug(link) - self.download(link, disposition=True) + self.link = m.group(1) break else: - captcha, captcha_key = self.getCaptcha() - challenge, response = captcha.challenge(captcha_key) + captcha = self.handleCaptcha() + + if not captcha: + self.error(_("Captcha pattern not found")) + + response, challenge = captcha.challenge() self.html = self.load(url, post={'DownloadCaptchaForm[captcha]': "", 'adcopy_challenge' : challenge, @@ -142,49 +146,11 @@ class RapidgatorNet(SimpleHoster): self.error(_("Download link")) - def getCaptcha(self): - m = re.search(self.ADSCAPTCHA_PATTERN, self.html) - if m: - captcha_key = m.group(1) - captcha = AdsCaptcha(self) - else: - m = re.search(self.RECAPTCHA_PATTERN, self.html) - if m: - captcha_key = m.group(1) - captcha = ReCaptcha(self) - else: - m = re.search(self.SOLVEMEDIA_PATTERN, self.html) - if m: - captcha_key = m.group(1) - captcha = SolveMedia(self) - else: - self.error(_("Captcha")) - - return captcha, captcha_key - - - def checkFree(self): - m = re.search(self.PREMIUM_ONLY_ERROR_PATTERN, self.html) - if m: - self.fail(_("Premium account needed for download")) - else: - m = re.search(self.WAIT_PATTERN, self.html) - - if m: - wait_time = int(m.group(1)) * {"hour": 60, "min": 1}[m.group(2)] - else: - m = re.search(self.DOWNLOAD_LIMIT_ERROR_PATTERN, self.html) - if m is None: - return - elif m.group(1) == "daily": - self.logWarning(_("You have reached your daily downloads limit for today")) - wait_time = secondsToMidnight(gmt=2) - else: - wait_time = 1 * 60 * 60 - - self.logDebug("Waiting %d minutes" % wait_time / 60) - self.wait(wait_time, True) - self.retry() + def handleCaptcha(self): + for klass in (AdsCaptcha, ReCaptcha, SolveMedia): + inst = klass(self) + if inst.detect_key(): + return inst def getJsonResponse(self, url): diff --git a/module/plugins/hoster/RapiduNet.py b/module/plugins/hoster/RapiduNet.py index 0c40d95b9..da353ec70 100644 --- a/module/plugins/hoster/RapiduNet.py +++ b/module/plugins/hoster/RapiduNet.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- +import pycurl import re - -from pycurl import HTTPHEADER -from time import time, altzone +import time from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class RapiduNet(SimpleHoster): __name__ = "RapiduNet" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.09" __pattern__ = r'https?://(?:www\.)?rapidu\.net/(?P<ID>\d{10})' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Rapidu.net hoster plugin""" __license__ = "GPLv3" @@ -24,8 +24,8 @@ class RapiduNet(SimpleHoster): COOKIES = [("rapidu.net", "rapidu_lang", "en")] - FILE_INFO_PATTERN = r'<h1 title="(?P<N>.*)">.*</h1>\s*<small>(?P<S>\d+(\.\d+)?)\s(?P<U>\w+)</small>' - OFFLINE_PATTERN = r'404 - File not found' + INFO_PATTERN = r'<h1 title="(?P<N>.*)">.*</h1>\s*<small>(?P<S>\d+(\.\d+)?)\s(?P<U>\w+)</small>' + OFFLINE_PATTERN = r'<h1>404' ERROR_PATTERN = r'<div class="error">' @@ -39,7 +39,7 @@ class RapiduNet(SimpleHoster): def handleFree(self, pyfile): self.req.http.lastURL = pyfile.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", get={'a': "getLoadTimeToDownload"}, @@ -47,30 +47,28 @@ class RapiduNet(SimpleHoster): decode=True) if str(jsvars['timeToDownload']) is "stop": - t = (24 * 60 * 60) - (int(time()) % (24 * 60 * 60)) + altzone + t = (24 * 60 * 60) - (int(time.time()) % (24 * 60 * 60)) + time.altzone self.logInfo("You've reach your daily download transfer") self.retry(10, 10 if t < 1 else None, _("Try tomorrow again")) #@NOTE: check t in case of not synchronised clock else: - self.wait(int(jsvars['timeToDownload']) - int(time())) + self.wait(int(jsvars['timeToDownload']) - int(time.time())) recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) + + jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", + get={'a': "getCheckCaptcha"}, + post={'_go' : "", + 'captcha1': challenge, + 'captcha2': response, + 'fileId' : self.info['pattern']['ID']}, + decode=True) - for _i in xrange(10): - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) - - jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", - get={'a': "getCheckCaptcha"}, - post={'_go' : "", - 'captcha1': challenge, - 'captcha2': response, - 'fileId' : self.info['pattern']['ID']}, - decode=True) - if jsvars['message'] == 'success': - self.download(jsvars['url']) - break + if jsvars['message'] == 'success': + self.link = jsvars['url'] def getJsonResponse(self, *args, **kwargs): diff --git a/module/plugins/hoster/RarefileNet.py b/module/plugins/hoster/RarefileNet.py index 2be952efe..a45e4ed4d 100644 --- a/module/plugins/hoster/RarefileNet.py +++ b/module/plugins/hoster/RarefileNet.py @@ -17,8 +17,6 @@ class RarefileNet(XFSHoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - HOSTER_DOMAIN = "rarefile.net" - LINK_PATTERN = r'<a href="(.+?)">\1</a>' diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index bf9fda293..4500fcefc 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- import re - -from random import randrange -from urllib import unquote -from time import time +import time +import urllib from module.common.json_layer import json_loads from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -14,25 +12,16 @@ from module.utils import parseFileSize class RealdebridCom(MultiHoster): __name__ = "RealdebridCom" __type__ = "hoster" - __version__ = "0.64" + __version__ = "0.67" __pattern__ = r'https?://((?:www\.|s\d+\.)?real-debrid\.com/dl/|[\w^_]\.rdb\.so/d/)[\w^_]+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Real-Debrid.com multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("Devirex Hazzard", "naibaf_11@yahoo.de")] - def getFilename(self, url): - try: - name = unquote(url.rsplit("/", 1)[1]) - except IndexError: - name = "Unknown_Filename..." - if not name or name.endswith(".."): #: incomplete filename, append random stuff - name += "%s.tmp" % randrange(100, 999) - return name - - def setup(self): self.chunkLimit = 3 @@ -42,7 +31,7 @@ class RealdebridCom(MultiHoster): get={'lang' : "en", 'link' : pyfile.url, 'password': self.getPassword(), - 'time' : int(time() * 1000)})) + 'time' : int(time.time() * 1000)})) self.logDebug("Returned Data: %s" % data) @@ -53,30 +42,15 @@ class RealdebridCom(MultiHoster): self.logWarning(data['message']) self.tempOffline() else: - if pyfile.name is not None and pyfile.name.endswith('.tmp') and data['file_name']: + if pyfile.name and pyfile.name.endswith('.tmp') and data['file_name']: pyfile.name = data['file_name'] pyfile.size = parseFileSize(data['file_size']) self.link = data['generated_links'][0][-1] - if self.getConfig("ssl"): + if self.getConfig('ssl'): self.link = self.link.replace("http://", "https://") else: self.link = self.link.replace("https://", "http://") - if self.link != pyfile.url: - self.logDebug("New URL: %s" % self.link) - - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): - #only use when name wasnt already set - pyfile.name = self.getFilename(self.link) - - - def checkFile(self): - if self.checkDownload({"error": "<title>An error occured while processing your request</title>"}): - #usual this download can safely be retried - self.retry(wait_time=60, reason=_("An error occured while generating link")) - - return super(RealdebridCom, self).checkFile() - getInfo = create_getInfo(RealdebridCom) diff --git a/module/plugins/hoster/RedtubeCom.py b/module/plugins/hoster/RedtubeCom.py index d68fbe262..9051c498a 100644 --- a/module/plugins/hoster/RedtubeCom.py +++ b/module/plugins/hoster/RedtubeCom.py @@ -56,7 +56,7 @@ class RedtubeCom(Hoster): if not self.html: self.download_html() - if re.search(r'This video has been removed.', self.html) is not None: + if re.search(r'This video has been removed.', self.html): return False else: return True diff --git a/module/plugins/hoster/RehostTo.py b/module/plugins/hoster/RehostTo.py index 28de35d08..9c07364ec 100644 --- a/module/plugins/hoster/RehostTo.py +++ b/module/plugins/hoster/RehostTo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from urllib import unquote +import urllib from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -11,6 +11,7 @@ class RehostTo(MultiHoster): __version__ = "0.21" __pattern__ = r'https?://.*rehost\.to\..+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Rehost.com multi-hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/RemixshareCom.py b/module/plugins/hoster/RemixshareCom.py index 11c9f3b0f..d60101aed 100644 --- a/module/plugins/hoster/RemixshareCom.py +++ b/module/plugins/hoster/RemixshareCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Test links: -# http://remixshare.com/download/p946u +# http://remixshare.com/download/z8uli # # Note: # The remixshare.com website is very very slow, so @@ -16,22 +16,26 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class RemixshareCom(SimpleHoster): __name__ = "RemixshareCom" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.05" __pattern__ = r'https?://remixshare\.com/(download|dl)/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Remixshare.com hoster plugin""" __license__ = "GPLv3" - __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de" ), + ("Walter Purcaro", "vuolter@gmail.com" ), + ("sraedler" , "simon.raedler@yahoo.de")] - INFO_PATTERN = r'title=\'.+?\'>(?P<N>.+?)</span><span class=\'light2\'> \((?P<S>\d+) (?P<U>[\w^_]+)\)<' - OFFLINE_PATTERN = r'<h1>Ooops!<' + INFO_PATTERN = r'title=\'.+?\'>(?P<N>.+?)</span><span class=\'light2\'> \((?P<S>\d+) (?P<U>[\w^_]+)\)<' + HASHSUM_PATTERN = r'>(?P<T>MD5): (?P<H>\w+)' + OFFLINE_PATTERN = r'<h1>Ooops!' - LINK_FREE_PATTERN = r'(http://remixshare\.com/downloadfinal/.+?)"' + LINK_PATTERN = r'var uri = "(.+?)"' TOKEN_PATTERN = r'var acc = (\d+)' - WAIT_PATTERN = r'var XYZ = r"(\d+)"' + + WAIT_PATTERN = r'var XYZ = "(\d+)"' def setup(self): @@ -40,24 +44,15 @@ class RemixshareCom(SimpleHoster): def handleFree(self, pyfile): - b = re.search(self.LINK_FREE_PATTERN, self.html) + b = re.search(self.LINK_PATTERN, self.html) if not b: - self.error(_("Cannot parse download url")) + self.error(_("File url")) c = re.search(self.TOKEN_PATTERN, self.html) if not c: - self.error(_("Cannot parse file token")) - - dl_url = b.group(1) + c.group(1) - - #Check if we have to wait - seconds = re.search(self.WAIT_PATTERN, self.html) - if seconds: - self.logDebug("Wait " + seconds.group(1)) - self.wait(int(seconds.group(1))) + self.error(_("File token")) - # Finally start downloading... - self.download(dl_url, disposition=True) + self.link = b.group(1) + "/zzz/" + c.group(1) getInfo = create_getInfo(RemixshareCom) diff --git a/module/plugins/hoster/RgHostNet.py b/module/plugins/hoster/RgHostNet.py index aa4830563..1a45def44 100644 --- a/module/plugins/hoster/RgHostNet.py +++ b/module/plugins/hoster/RgHostNet.py @@ -8,19 +8,21 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class RgHostNet(SimpleHoster): __name__ = "RgHostNet" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.04" - __pattern__ = r'http://(?:www\.)?rghost\.net/\d+(?:r=\d+)?' + __pattern__ = r'http://(?:www\.)?rghost\.(net|ru)/[\d-]+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """RgHost.net hoster plugin""" __license__ = "GPLv3" __authors__ = [("z00nx", "z00nx0@gmail.com")] - INFO_PATTERN = r'<h1>\s+(<a[^>]+>)?(?P<N>[^<]+)(</a>)?\s+<small[^>]+>\s+\((?P<S>[^)]+)\)\s+</small>\s+</h1>' - OFFLINE_PATTERN = r'File is deleted|this page is not found' + INFO_PATTERN = r'data-share42-text="(?P<N>.+?) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)' + HASHSUM_PATTERN = r'<dt>(?P<T>\w+)</dt>\s*<dd>(?P<H>\w+)' + OFFLINE_PATTERN = r'>(File is deleted|page not found)' - LINK_FREE_PATTERN = r'<a\s+href="([^"]+)"\s+class="btn\s+large\s+download"[^>]+>Download</a>' + LINK_FREE_PATTERN = r'<a href="(.+?)" class="btn large' getInfo = create_getInfo(RgHostNet) diff --git a/module/plugins/hoster/SafesharingEu.py b/module/plugins/hoster/SafesharingEu.py index bb6e0f646..08612e413 100644 --- a/module/plugins/hoster/SafesharingEu.py +++ b/module/plugins/hoster/SafesharingEu.py @@ -15,8 +15,6 @@ class SafesharingEu(XFSHoster): __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] - HOSTER_DOMAIN = "safesharing.eu" - ERROR_PATTERN = r'(?:<div class="alert alert-danger">)(.+?)(?:</div>)' diff --git a/module/plugins/hoster/SecureUploadEu.py b/module/plugins/hoster/SecureUploadEu.py index 64e6456a9..6bfbce328 100644 --- a/module/plugins/hoster/SecureUploadEu.py +++ b/module/plugins/hoster/SecureUploadEu.py @@ -15,8 +15,6 @@ class SecureUploadEu(XFSHoster): __authors__ = [("z00nx", "z00nx0@gmail.com")] - HOSTER_DOMAIN = "secureupload.eu" - INFO_PATTERN = r'<h3>Downloading (?P<N>[^<]+) \((?P<S>[^<]+)\)</h3>' diff --git a/module/plugins/hoster/SendspaceCom.py b/module/plugins/hoster/SendspaceCom.py index ffd2995b8..0ab20949d 100644 --- a/module/plugins/hoster/SendspaceCom.py +++ b/module/plugins/hoster/SendspaceCom.py @@ -11,6 +11,7 @@ class SendspaceCom(SimpleHoster): __version__ = "0.17" __pattern__ = r'https?://(?:www\.)?sendspace\.com/file/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Sendspace.com hoster plugin""" __license__ = "GPLv3" @@ -21,10 +22,10 @@ class SendspaceCom(SimpleHoster): SIZE_PATTERN = r'<div class="file_description reverse margin_center">\s*<b>File Size:</b>\s*(?P<S>[\d.,]+)(?P<U>[\w^_]+)\s*</div>' OFFLINE_PATTERN = r'<div class="msg error" style="cursor: default">Sorry, the file you requested is not available.</div>' - LINK_FREE_PATTERN = r'<a id="download_button" href="([^"]+)"' + LINK_FREE_PATTERN = r'<a id="download_button" href="(.+?)"' - CAPTCHA_PATTERN = r'<td><img src="(/captchas/captcha\.php?captcha=([^"]+))"></td>' - USER_CAPTCHA_PATTERN = r'<td><img src="/captchas/captcha\.php?user=([^"]+))"></td>' + CAPTCHA_PATTERN = r'<td><img src="(/captchas/captcha\.php?captcha=(.+?))"></td>' + USER_CAPTCHA_PATTERN = r'<td><img src="/captchas/captcha\.php?user=(.+?))"></td>' def handleFree(self, pyfile): @@ -34,7 +35,7 @@ class SendspaceCom(SimpleHoster): if m: if 'captcha_hash' in params: self.correctCaptcha() - download_url = m.group(1) + self.link = m.group(1) break m = re.search(self.CAPTCHA_PATTERN, self.html) @@ -55,7 +56,5 @@ class SendspaceCom(SimpleHoster): else: self.fail(_("Download link not found")) - self.download(download_url) - getInfo = create_getInfo(SendspaceCom) diff --git a/module/plugins/hoster/Share4webCom.py b/module/plugins/hoster/Share4WebCom.py index 7a276c1fe..7a276c1fe 100644 --- a/module/plugins/hoster/Share4webCom.py +++ b/module/plugins/hoster/Share4WebCom.py diff --git a/module/plugins/hoster/Share76Com.py b/module/plugins/hoster/Share76Com.py index 1ac8a64e7..4a4af1672 100644 --- a/module/plugins/hoster/Share76Com.py +++ b/module/plugins/hoster/Share76Com.py @@ -9,6 +9,7 @@ class Share76Com(DeadHoster): __version__ = "0.04" __pattern__ = r'http://(?:www\.)?share76\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Share76.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/ShareFilesCo.py b/module/plugins/hoster/ShareFilesCo.py index 4996014d8..d9a71d786 100644 --- a/module/plugins/hoster/ShareFilesCo.py +++ b/module/plugins/hoster/ShareFilesCo.py @@ -9,6 +9,7 @@ class ShareFilesCo(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?sharefiles\.co/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Sharefiles.co hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/SharebeesCom.py b/module/plugins/hoster/SharebeesCom.py index c0fd22c91..9387483cc 100644 --- a/module/plugins/hoster/SharebeesCom.py +++ b/module/plugins/hoster/SharebeesCom.py @@ -9,6 +9,7 @@ class SharebeesCom(DeadHoster): __version__ = "0.02" __pattern__ = r'http://(?:www\.)?sharebees\.com/\w{12}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """ShareBees hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index 636e5824d..64a490993 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -1,22 +1,22 @@ # -*- coding: utf-8 -*- import re - -from time import time -from urllib import unquote -from urlparse import urlparse +import time +import urllib +import urlparse from module.network.RequestFactory import getURL -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class ShareonlineBiz(SimpleHoster): __name__ = "ShareonlineBiz" __type__ = "hoster" - __version__ = "0.47" + __version__ = "0.51" __pattern__ = r'https?://(?:www\.)?(share-online\.biz|egoshare\.com)/(download\.php\?id=|dl/)(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Shareonline.biz hoster plugin""" __license__ = "GPLv3" @@ -36,17 +36,15 @@ class ShareonlineBiz(SimpleHoster): @classmethod - def getInfo(cls, url="", html=""): - info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url} - - if url: - info['pattern'] = re.match(cls.__pattern__, url).groupdict() + def apiInfo(cls, url): + info = super(ShareonlineBiz, cls).apiInfo(url) - field = getURL("http://api.share-online.biz/linkcheck.php", - get={'md5': "1"}, - post={'links': info['pattern']['ID']}, - decode=True).split(";") + field = getURL("http://api.share-online.biz/linkcheck.php", + get={'md5' : "1", + 'links': re.match(cls.__pattern__, url).group("ID")}, + decode=True).split(";") + try: if field[1] == "OK": info['fileid'] = field[0] info['status'] = 2 @@ -57,6 +55,9 @@ class ShareonlineBiz(SimpleHoster): elif field[1] in ("DELETED", "NOT FOUND"): info['status'] = 1 + except IndexError: + pass + return info @@ -69,12 +70,12 @@ class ShareonlineBiz(SimpleHoster): recaptcha = ReCaptcha(self) for _i in xrange(5): - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) m = re.search(r'var wait=(\d+);', self.html) self.setWait(int(m.group(1)) if m else 30) - res = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time() * 1000)), + res = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time.time() * 1000)), post={'dl_free' : "1", 'recaptcha_challenge_field': challenge, 'recaptcha_response_field' : response}) @@ -98,18 +99,15 @@ class ShareonlineBiz(SimpleHoster): self.checkErrors() res = self.handleCaptcha() + self.link = res.decode('base64') - download_url = res.decode("base64") - - if not download_url.startswith("http://"): + if not self.link.startswith("http://"): self.error(_("Wrong download url")) self.wait() - self.download(download_url) - - def checkFile(self): + def checkFile(self, rules={}): check = self.checkDownload({'cookie': re.compile(r'<div id="dl_failure"'), 'fail' : re.compile(r"<title>Share-Online")}) @@ -121,7 +119,7 @@ class ShareonlineBiz(SimpleHoster): self.invalidCaptcha() self.retry(5, 5 * 60, _("Download failed")) - return super(ShareonlineBiz, self).checkFile() + return super(ShareonlineBiz, self).checkFile(rules) def handlePremium(self, pyfile): #: should be working better loading (account) api internally @@ -145,13 +143,12 @@ class ShareonlineBiz(SimpleHoster): pyfile.name = dlinfo['name'] pyfile.size = int(dlinfo['size']) - dlLink = dlinfo['url'] + self.link = dlinfo['url'] - if dlLink == "server_under_maintenance": + if self.link == "server_under_maintenance": self.tempOffline() else: self.multiDL = True - self.download(dlLink) def checkErrors(self): @@ -164,7 +161,7 @@ class ShareonlineBiz(SimpleHoster): try: self.logError(errmsg, re.search(self.ERROR_PATTERN, self.html).group(1)) - except: + except Exception: self.logError("Unknown error occurred", errmsg) if errmsg is "invalid": diff --git a/module/plugins/hoster/ShareplaceCom.py b/module/plugins/hoster/ShareplaceCom.py index 07724a9d1..babd0d1d4 100644 --- a/module/plugins/hoster/ShareplaceCom.py +++ b/module/plugins/hoster/ShareplaceCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote +import urllib from module.plugins.Hoster import Hoster @@ -61,7 +60,7 @@ class ShareplaceCom(Hoster): url = re.search(r"var beer = '(.*?)';", self.html) if url: url = url.group(1) - url = unquote( + url = urllib.unquote( url.replace("http://http:/", "").replace("vvvvvvvvv", "").replace("lllllllll", "").replace( "teletubbies", "")) self.logDebug("URL: %s" % url) @@ -83,7 +82,7 @@ class ShareplaceCom(Hoster): if not self.html: self.download_html() - if re.search(r"HTTP Status 404", self.html) is not None: + if re.search(r"HTTP Status 404", self.html): return False else: return True diff --git a/module/plugins/hoster/SharingmatrixCom.py b/module/plugins/hoster/SharingmatrixCom.py index fa08a4a8f..3958ebbd6 100644 --- a/module/plugins/hoster/SharingmatrixCom.py +++ b/module/plugins/hoster/SharingmatrixCom.py @@ -9,6 +9,7 @@ class SharingmatrixCom(DeadHoster): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?sharingmatrix\.com/file/\w+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Sharingmatrix.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/ShragleCom.py b/module/plugins/hoster/ShragleCom.py index bec30f6f2..689223322 100644 --- a/module/plugins/hoster/ShragleCom.py +++ b/module/plugins/hoster/ShragleCom.py @@ -9,6 +9,7 @@ class ShragleCom(DeadHoster): __version__ = "0.22" __pattern__ = r'http://(?:www\.)?(cloudnator|shragle)\.com/files/(?P<ID>.+?)/' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Cloudnator.com (Shragle.com) hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/SimplyPremiumCom.py b/module/plugins/hoster/SimplyPremiumCom.py index 39e8a6aeb..b3f52bc8c 100644 --- a/module/plugins/hoster/SimplyPremiumCom.py +++ b/module/plugins/hoster/SimplyPremiumCom.py @@ -2,8 +2,6 @@ import re -from datetime import datetime, timedelta - from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo from module.plugins.internal.SimpleHoster import secondsToMidnight @@ -14,6 +12,7 @@ class SimplyPremiumCom(MultiHoster): __version__ = "0.08" __pattern__ = r'https?://.+simply-premium\.com' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Simply-Premium.com multi-hoster plugin""" __license__ = "GPLv3" @@ -24,53 +23,58 @@ class SimplyPremiumCom(MultiHoster): self.chunkLimit = 16 - def handlePremium(self, pyfile): - for i in xrange(5): - page = self.load("http://www.simply-premium.com/premium.php", get={'info': "", 'link': self.pyfile.url}) - self.logDebug("JSON data: " + page) - if page != '': - break - else: - self.logInfo(_("Unable to get API data, waiting 1 minute and retry")) - self.retry(5, 60, "Unable to get API data") - - if '<valid>0</valid>' in page or ( - "You are not allowed to download from this host" in page and self.premium): + def checkErrors(self): + if '<valid>0</valid>' in self.html or ( + "You are not allowed to download from this host" in self.html and self.premium): self.account.relogin(self.user) self.retry() - elif "NOTFOUND" in page: + elif "NOTFOUND" in self.html: self.offline() - elif "downloadlimit" in page: + elif "downloadlimit" in self.html: self.logWarning(_("Reached maximum connctions")) - self.retry(5, 60, "Reached maximum connctions") + self.retry(5, 60, _("Reached maximum connctions")) - elif "trafficlimit" in page: + elif "trafficlimit" in self.html: self.logWarning(_("Reached daily limit for this host")) self.retry(wait_time=secondsToMidnight(gmt=2), reason="Daily limit for this host reached") - elif "hostererror" in page: + elif "hostererror" in self.html: self.logWarning(_("Hoster temporarily unavailable, waiting 1 minute and retry")) - self.retry(5, 60, "Hoster is temporarily unavailable") + self.retry(5, 60, _("Hoster is temporarily unavailable")) + + + def handlePremium(self, pyfile): + for i in xrange(5): + self.html = self.load("http://www.simply-premium.com/premium.php", get={'info': "", 'link': self.pyfile.url}) + + if self.html: + self.logDebug("JSON data: " + self.html) + break + else: + self.logInfo(_("Unable to get API data, waiting 1 minute and retry")) + self.retry(5, 60, _("Unable to get API data")) + + self.checkErrors() try: - self.pyfile.name = re.search(r'<name>([^<]+)</name>', page).group(1) + self.pyfile.name = re.search(r'<name>([^<]+)</name>', self.html).group(1) + except AttributeError: self.pyfile.name = "" try: - self.pyfile.size = re.search(r'<size>(\d+)</size>', page).group(1) + self.pyfile.size = re.search(r'<size>(\d+)</size>', self.html).group(1) + except AttributeError: self.pyfile.size = 0 try: - self.link = re.search(r'<download>([^<]+)</download>', page).group(1) + self.link = re.search(r'<download>([^<]+)</download>', self.html).group(1) + except AttributeError: self.link = 'http://www.simply-premium.com/premium.php?link=' + self.pyfile.url - if self.link != self.pyfile.url: - self.logDebug("New URL: " + self.link) - getInfo = create_getInfo(SimplyPremiumCom) diff --git a/module/plugins/hoster/SimplydebridCom.py b/module/plugins/hoster/SimplydebridCom.py index a26bc5751..aae71d983 100644 --- a/module/plugins/hoster/SimplydebridCom.py +++ b/module/plugins/hoster/SimplydebridCom.py @@ -2,15 +2,16 @@ import re -from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo +from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo, replace_patterns class SimplydebridCom(MultiHoster): __name__ = "SimplydebridCom" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.17" __pattern__ = r'http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/sd\.php' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Simply-debrid.com multi-hoster plugin""" __license__ = "GPLv3" @@ -19,37 +20,30 @@ class SimplydebridCom(MultiHoster): def handlePremium(self, pyfile): #fix the links for simply-debrid.com! - self.link = pyfile.url - self.link = self.link.replace("clz.to", "cloudzer.net/file") - self.link = self.link.replace("http://share-online", "http://www.share-online") - self.link = self.link.replace("ul.to", "uploaded.net/file") - self.link = self.link.replace("uploaded.com", "uploaded.net") - self.link = self.link.replace("filerio.com", "filerio.in") - self.link = self.link.replace("lumfile.com", "lumfile.se") - - if('fileparadox' in self.link): + self.link = replace_patterns(pyfile.url, [("clz.to", "cloudzer.net/file") + ("http://share-online", "http://www.share-online") + ("ul.to", "uploaded.net/file") + ("uploaded.com", "uploaded.net") + ("filerio.com", "filerio.in") + ("lumfile.com", "lumfile.se")]) + + if 'fileparadox' in self.link: self.link = self.link.replace("http://", "https://") - if re.match(self.__pattern__, self.link): - self.link = self.link + self.html = self.load("http://simply-debrid.com/api.php", get={'dl': self.link}) + if 'tiger Link' in self.html or 'Invalid Link' in self.html or ('API' in self.html and 'ERROR' in self.html): + self.error(_("Unable to unrestrict link")) - self.logDebug("New URL: %s" % self.link) + self.link = self.html - if not re.match(self.__pattern__, self.link): - page = self.load("http://simply-debrid.com/api.php", get={'dl': self.link}) - if 'tiger Link' in page or 'Invalid Link' in page or ('API' in page and 'ERROR' in page): - self.fail(_("Unable to unrestrict link")) - self.link = page + self.wait(5) - self.setWait(5) - self.wait() - - def checkFile(self): + def checkFile(self, rules={}): if self.checkDownload({"error": "No address associated with hostname"}): - self.retry(24, 3 * 60, "Bad file downloaded") + self.retry(24, 3 * 60, _("Bad file downloaded")) - return super(SimplydebridCom, self).checkFile() + return super(SimplydebridCom, self).checkFile(rules) getInfo = create_getInfo(SimplydebridCom) diff --git a/module/plugins/hoster/SizedriveCom.py b/module/plugins/hoster/SizedriveCom.py new file mode 100644 index 000000000..4c5aedb6d --- /dev/null +++ b/module/plugins/hoster/SizedriveCom.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class SizedriveCom(SimpleHoster): + __name__ = "SizedriveCom" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?sizedrive\.com/[rd]/(?P<ID>\w+)' + + __description__ = """Sizedrive.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("GammaC0de", None)] + + + NAME_PATTERN = r'>Nome:</b> (?P<N>.+?)<' + SIZE_PATTERN = r'>Tamanho:</b>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'ARQUIVO DELATADO POR' + + + def setup(self): + self.resumeDownload = False + self.multiDL = False + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + self.wait(5) + self.html = self.load("http://www.sizedrive.com/getdownload.php", + post={'id': self.info['pattern']['ID']}) + + m = re.search(r'<span id="boton_download" ><a href="(.+?)"', self.html) + if m: + self.link = m.group(1) + + +getInfo = create_getInfo(SizedriveCom) diff --git a/module/plugins/hoster/SmoozedCom.py b/module/plugins/hoster/SmoozedCom.py index bd653ff59..9ae12f145 100644 --- a/module/plugins/hoster/SmoozedCom.py +++ b/module/plugins/hoster/SmoozedCom.py @@ -7,16 +7,17 @@ from module.plugins.internal.MultiHoster import MultiHoster class SmoozedCom(MultiHoster): __name__ = "SmoozedCom" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.05" __pattern__ = r'^unmatchable$' #: Since we want to allow the user to specify the list of hoster to use we let MultiHoster.coreReady + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Smoozed.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("", "")] - def handlePremium(self, pyfile): + def handleFree(self, pyfile): # In some cases hostsers do not supply us with a filename at download, so we # are going to set a fall back filename (e.g. for freakshare or xfileshare) pyfile.name = pyfile.name.split('/').pop() # Remove everthing before last slash @@ -55,9 +56,9 @@ class SmoozedCom(MultiHoster): self.link = header["location"][-1] if isinstance(header["location"], list) else header["location"] - def checkFile(self): + def checkFile(self, rules={}): if self.checkDownload({'error': '{"state":"error"}', 'retry': '{"state":"retry"}'}): self.fail(_("Error response received")) - return super(SmoozedCom, self).checkFile() + return super(SmoozedCom, self).checkFile(rules) diff --git a/module/plugins/hoster/SockshareCom.py b/module/plugins/hoster/SockshareCom.py index aabb8dcd1..44eedf9d4 100644 --- a/module/plugins/hoster/SockshareCom.py +++ b/module/plugins/hoster/SockshareCom.py @@ -9,6 +9,7 @@ class SockshareCom(DeadHoster): __version__ = "0.05" __pattern__ = r'http://(?:www\.)?sockshare\.com/(mobile/)?(file|embed)/(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Sockshare.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/SolidfilesCom.py b/module/plugins/hoster/SolidfilesCom.py new file mode 100644 index 000000000..d359577d6 --- /dev/null +++ b/module/plugins/hoster/SolidfilesCom.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://www.solidfiles.com/d/609cdb4b1b + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class SolidfilesCom(SimpleHoster): + __name__ = "SolidfilesCom" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?solidfiles\.com\/d/\w+' + + __description__ = """Solidfiles.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("sraedler", "simon.raedler@yahoo.de")] + + + NAME_PATTERN = r'<h1 title="(?P<N>.+?)"' + SIZE_PATTERN = r'<p class="meta">(?P<S>[\d.,]+) (?P<U>[\w_^]+)' + OFFLINE_PATTERN = r'<h1>404' + + LINK_FREE_PATTERN = r'id="ddl-text" href="(.+?)"' + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + + +getInfo = create_getInfo(SolidfilesCom) diff --git a/module/plugins/hoster/SoundcloudCom.py b/module/plugins/hoster/SoundcloudCom.py index 3eb546604..8dff4f42a 100644 --- a/module/plugins/hoster/SoundcloudCom.py +++ b/module/plugins/hoster/SoundcloudCom.py @@ -1,57 +1,56 @@ # -*- coding: utf-8 -*- -import pycurl import re -from module.plugins.Hoster import Hoster +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.common.json_layer import json_loads -class SoundcloudCom(Hoster): +class SoundcloudCom(SimpleHoster): __name__ = "SoundcloudCom" __type__ = "hoster" - __version__ = "0.10" + __version__ = "0.11" - __pattern__ = r'https?://(?:www\.)?soundcloud\.com/(?P<UID>.+?)/(?P<SID>.+)' + __pattern__ = r'https?://(?:www\.)?soundcloud\.com/[\w-]+/[\w-]+' + __config__ = [("use_premium", "bool" , "Use premium account if available", True ), + ("quality" , "Lower;Higher", "Quality" , "Higher")] __description__ = """SoundCloud.com hoster plugin""" __license__ = "GPLv3" - __authors__ = [("Peekayy", "peekayy.dev@gmail.com")] - - - def process(self, pyfile): - # default UserAgent of HTTPRequest fails for this hoster so we use this one - self.req.http.c.setopt(pycurl.USERAGENT, 'Mozilla/5.0') - page = self.load(pyfile.url) - m = re.search(r'<div class="haudio.*?large.*?" data-sc-track="(?P<ID>\d*)"', page) - songId = clientId = "" - if m: - songId = m.group('ID') - if len(songId) <= 0: - self.logError(_("Could not find song id")) - self.offline() - else: - m = re.search(r'"clientID":"(?P<CID>.*?)"', page) - if m: - clientId = m.group('CID') - - if len(clientId) <= 0: - clientId = "b45b1aa10f1ac2941910a7f0d10f8e28" - - m = re.search(r'<em itemprop="name">\s(?P<TITLE>.*?)\s</em>', page) - if m: - pyfile.name = m.group('TITLE') + ".mp3" - else: - pyfile.name = re.match(self.__pattern__, pyfile.url).group('SID') + ".mp3" - - # url to retrieve the actual song url - page = self.load("https://api.sndcdn.com/i1/tracks/%s/streams" % songId, get={"client_id": clientId}) - # getting streams - # for now we choose the first stream found in all cases - # it could be improved if relevant for this hoster - streams = [ - (result.group('QUALITY'), result.group('URL')) - for result in re.finditer(r'"(?P<QUALITY>.*?)":"(?P<URL>.*?)"', page) - ] - self.logDebug("Found Streams", streams) - self.logDebug("Downloading", streams[0][0], streams[0][1]) - self.download(streams[0][1]) + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'title" content="(?P<N>.+?)"' + OFFLINE_PATTERN = r'<title>"SoundCloud - Hear the worldâs sounds"</title>' + + + def handleFree(self, pyfile): + try: + song_id = re.search(r'sounds:(\d+)"', self.html).group(1) + + except Exception: + self.error(_("Could not find song id")) + + try: + client_id = re.search(r'"clientID":"(.+?)"', self.html).group(1) + + except Exception: + client_id = "b45b1aa10f1ac2941910a7f0d10f8e28" + + # url to retrieve the actual song url + streams = json_loads(self.load("https://api.soundcloud.com/tracks/%s/streams" % song_id, + get={'client_id': client_id})) + + regex = re.compile(r'[^\d]') + http_streams = sorted([(key, value) for key, value in streams.iteritems() if key.startswith('http_')], + key=lambda t: regex.sub(t[0], ''), + reverse=True) + + self.logDebug("Streams found: %s" % (http_streams or "None")) + + if http_streams: + stream_name, self.link = http_streams[0 if self.getConfig('quality') == "Higher" else -1] + pyfile.name += '.' + stream_name.split('_')[1].lower() + + +getInfo = create_getInfo(SoundcloudCom) diff --git a/module/plugins/hoster/SpeedLoadOrg.py b/module/plugins/hoster/SpeedLoadOrg.py index a13220eab..18f04b611 100644 --- a/module/plugins/hoster/SpeedLoadOrg.py +++ b/module/plugins/hoster/SpeedLoadOrg.py @@ -9,6 +9,7 @@ class SpeedLoadOrg(DeadHoster): __version__ = "1.02" __pattern__ = r'http://(?:www\.)?speedload\.org/(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Speedload.org hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/SpeedfileCz.py b/module/plugins/hoster/SpeedfileCz.py index f23c8d4c7..cd6d65082 100644 --- a/module/plugins/hoster/SpeedfileCz.py +++ b/module/plugins/hoster/SpeedfileCz.py @@ -9,6 +9,7 @@ class SpeedfileCz(DeadHoster): __version__ = "0.32" __pattern__ = r'http://(?:www\.)?speedfile\.cz/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Speedfile.cz hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/SpeedyshareCom.py b/module/plugins/hoster/SpeedyshareCom.py index 0acdb7647..91788b2c8 100644 --- a/module/plugins/hoster/SpeedyshareCom.py +++ b/module/plugins/hoster/SpeedyshareCom.py @@ -4,8 +4,7 @@ # http://speedy.sh/ep2qY/Zapp-Brannigan.jpg import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,9 +12,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SpeedyshareCom(SimpleHoster): __name__ = "SpeedyshareCom" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'https?://(?:www\.)?(speedyshare\.com|speedy\.sh)/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Speedyshare.com hoster plugin""" __license__ = "GPLv3" @@ -38,10 +38,7 @@ class SpeedyshareCom(SimpleHoster): def handleFree(self, pyfile): m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.error(_("Download link not found")) - - dl_link = urljoin("http://www.speedyshare.com", m.group(1)) - self.download(dl_link, disposition=True) + self.link = m.group(1) getInfo = create_getInfo(SpeedyshareCom) diff --git a/module/plugins/hoster/StorageTo.py b/module/plugins/hoster/StorageTo.py index bedc2934f..feb9f3300 100644 --- a/module/plugins/hoster/StorageTo.py +++ b/module/plugins/hoster/StorageTo.py @@ -9,6 +9,7 @@ class StorageTo(DeadHoster): __version__ = "0.01" __pattern__ = r'http://(?:www\.)?storage\.to/get/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Storage.to hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/StreamCz.py b/module/plugins/hoster/StreamCz.py index 11d4efcdb..97bed8109 100644 --- a/module/plugins/hoster/StreamCz.py +++ b/module/plugins/hoster/StreamCz.py @@ -32,10 +32,10 @@ class StreamCz(Hoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<link rel="video_src" href="http://www\.stream\.cz/\w+/(\d+)-([^"]+)" />' + NAME_PATTERN = r'<link rel="video_src" href="http://www\.stream\.cz/\w+/(\d+)-(.+?)" />' OFFLINE_PATTERN = r'<h1 class="commonTitle">Str.nku nebylo mo.n. nal.zt \(404\)</h1>' - CDN_PATTERN = r'<param name="flashvars" value="[^"]*&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' + CDN_PATTERN = r'<param name="flashvars" value=".+?&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' def setup(self): diff --git a/module/plugins/hoster/StreamcloudEu.py b/module/plugins/hoster/StreamcloudEu.py index b9d6d9ae6..0f8c08ad9 100644 --- a/module/plugins/hoster/StreamcloudEu.py +++ b/module/plugins/hoster/StreamcloudEu.py @@ -17,12 +17,8 @@ class StreamcloudEu(XFSHoster): __authors__ = [("seoester", "seoester@googlemail.com")] - HOSTER_DOMAIN = "streamcloud.eu" - WAIT_PATTERN = r'var count = (\d+)' - LINK_PATTERN = r'file: "(http://(stor|cdn)\d+\.streamcloud\.eu:?\d*/.*/video\.(mp4|flv))",' - def setup(self): self.multiDL = True diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py index ccca76018..254e06c2a 100644 --- a/module/plugins/hoster/TurbobitNet.py +++ b/module/plugins/hoster/TurbobitNet.py @@ -1,25 +1,26 @@ # -*- coding: utf-8 -*- +import binascii +import pycurl import random import re import time +import urllib from Crypto.Cipher import ARC4 -from binascii import hexlify, unhexlify -from pycurl import HTTPHEADER -from urllib import quote from module.network.RequestFactory import getURL -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp class TurbobitNet(SimpleHoster): __name__ = "TurbobitNet" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.20" __pattern__ = r'http://(?:www\.)?turbobit\.net/(?:download/free/)?(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """ Turbobit.net hoster plugin """ __license__ = "GPLv3" @@ -35,30 +36,29 @@ class TurbobitNet(SimpleHoster): SIZE_PATTERN = r'class="file-size">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File (?:was )?not found' - LINK_PATTERN = r'(/download/redirect/[^"\']+)' + LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'(/download/redirect/[^"\']+)' LIMIT_WAIT_PATTERN = r'<div id=\'timeout\'>(\d+)<' CAPTCHA_PATTERN = r'<img alt="Captcha" src="(.+?)"' def handleFree(self, pyfile): - self.url = "http://turbobit.net/download/free/%s" % self.info['pattern']['ID'] - self.html = self.load(self.url, ref=True, decode=True) + self.html = self.load("http://turbobit.net/download/free/%s" % self.info['pattern']['ID'], + decode=True) rtUpdate = self.getRtUpdate() self.solveCaptcha() - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - self.url = self.getDownloadUrl(rtUpdate) + self.html = self.load(self.getDownloadUrl(rtUpdate)) - self.wait() - self.html = self.load(self.url) - - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With:"]) - self.downloadFile() + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + self.link = m.group(1) def solveCaptcha(self): @@ -76,7 +76,7 @@ class TurbobitNet(SimpleHoster): if inputs['captcha_type'] == 'recaptcha': recaptcha = ReCaptcha(self) - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge() + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge() else: m = re.search(self.CAPTCHA_PATTERN, self.html) if m is None: @@ -134,25 +134,29 @@ class TurbobitNet(SimpleHoster): for b in [1, 3]: self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % ( - self.info['pattern']['ID'], b, quote(fun), rtUpdate) + self.info['pattern']['ID'], b, urllib.quote(fun), rtUpdate) try: out = self.js.eval(self.jscode) self.logDebug("URL", self.js.engine, out) if out.startswith('/download/'): return "http://turbobit.net%s" % out.strip() + except Exception, e: self.logError(e) else: if self.retries >= 2: # retry with updated js self.delStorage("rtUpdate") - self.retry() + else: + self.retry() + + self.wait() def decrypt(self, data): - cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) - return unhexlify(cipher.encrypt(unhexlify(data))) + cipher = ARC4.new(binascii.hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) + return binascii.unhexlify(cipher.encrypt(binascii.unhexlify(data))) def getLocalTimeString(self): @@ -161,17 +165,4 @@ class TurbobitNet(SimpleHoster): return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) - def handlePremium(self, pyfile): - self.logDebug("Premium download as user %s" % self.user) - self.downloadFile() - - - def downloadFile(self): - m = re.search(self.LINK_PATTERN, self.html) - if m is None: - self.error(_("Download link not found")) - self.url = "http://turbobit.net" + m.group(1) - self.download(self.url) - - getInfo = create_getInfo(TurbobitNet) diff --git a/module/plugins/hoster/TurbouploadCom.py b/module/plugins/hoster/TurbouploadCom.py index f0893cef6..e4b53b818 100644 --- a/module/plugins/hoster/TurbouploadCom.py +++ b/module/plugins/hoster/TurbouploadCom.py @@ -9,6 +9,7 @@ class TurbouploadCom(DeadHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?turboupload\.com/(\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Turboupload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/TusfilesNet.py b/module/plugins/hoster/TusfilesNet.py index 9d0a86fe9..a79d8c869 100644 --- a/module/plugins/hoster/TusfilesNet.py +++ b/module/plugins/hoster/TusfilesNet.py @@ -7,7 +7,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class TusfilesNet(XFSHoster): __name__ = "TusfilesNet" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.10" __pattern__ = r'https?://(?:www\.)?tusfiles\.net/\w{12}' @@ -17,10 +17,7 @@ class TusfilesNet(XFSHoster): ("guidobelix", "guidobelix@hotmail.it")] - HOSTER_DOMAIN = "tusfiles.net" - INFO_PATTERN = r'\](?P<N>.+) - (?P<S>[\d.,]+) (?P<U>[\w^_]+)\[' - OFFLINE_PATTERN = r'>File Not Found|<Title>TusFiles - Fast Sharing Files!|The file you are trying to download is no longer available' def setup(self): @@ -29,9 +26,9 @@ class TusfilesNet(XFSHoster): self.resumeDownload = True - def downloadLink(self, link): + def downloadLink(self, link, disposition=True): try: - return super(TusfilesNet, self).downloadLink(link) + return super(TusfilesNet, self).downloadLink(link, disposition) except BadHeader, e: if e.code is 503: diff --git a/module/plugins/hoster/TwoSharedCom.py b/module/plugins/hoster/TwoSharedCom.py index c6ca2ab29..086228637 100644 --- a/module/plugins/hoster/TwoSharedCom.py +++ b/module/plugins/hoster/TwoSharedCom.py @@ -11,6 +11,7 @@ class TwoSharedCom(SimpleHoster): __version__ = "0.13" __pattern__ = r'http://(?:www\.)?2shared\.com/(account/)?(download|get|file|document|photo|video|audio)/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """2Shared.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/UlozTo.py b/module/plugins/hoster/UlozTo.py index cadd51149..b43ff8d92 100644 --- a/module/plugins/hoster/UlozTo.py +++ b/module/plugins/hoster/UlozTo.py @@ -15,9 +15,10 @@ def convertDecimalPrefix(m): class UlozTo(SimpleHoster): __name__ = "UlozTo" __type__ = "hoster" - __version__ = "1.02" + __version__ = "1.09" __pattern__ = r'http://(?:www\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj\.cz|zachowajto\.pl)/(?:live/)?(?P<ID>\w+/[^/?]*)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Uloz.to hoster plugin""" __license__ = "GPLv3" @@ -29,61 +30,24 @@ class UlozTo(SimpleHoster): SIZE_PATTERN = r'<span id="fileSize">.*?(?P<S>[\d.,]+\s[kMG]?B)</span>' OFFLINE_PATTERN = r'<title>404 - Page not found</title>|<h1 class="h1">File (has been deleted|was banned)</h1>' - URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "www.ulozto.net")] - SIZE_REPLACEMENTS = [('([\d.]+)\s([kMG])B', convertDecimalPrefix)] + URL_REPLACEMENTS = [(r'(?<=http://)([^/]+)', "www.ulozto.net")] + SIZE_REPLACEMENTS = [(r'([\d.]+)\s([kMG])B', convertDecimalPrefix)] - ADULT_PATTERN = r'<form action="([^\"]*)" method="post" id="frm-askAgeForm">' + CHECK_TRAFFIC = True + DISPOSITION = False #: Remove in 0.4.10 + + ADULT_PATTERN = r'<form action="(.+?)" method="post" id="frm-askAgeForm">' PASSWD_PATTERN = r'<div class="passwordProtectedFile">' - VIPLINK_PATTERN = r'<a href="[^"]*\?disclaimer=1" class="linkVip">' + VIPLINK_PATTERN = r'<a href=".+?\?disclaimer=1" class="linkVip">' TOKEN_PATTERN = r'<input type="hidden" name="_token_" .*?value="(.+?)"' def setup(self): - self.multiDL = self.premium + self.chunkLimit = 16 if self.premium else 1 + self.multiDL = True self.resumeDownload = True - def process(self, pyfile): - pyfile.url = re.sub(r"(?<=http://)([^/]+)", "www.ulozto.net", pyfile.url) - self.html = self.load(pyfile.url, decode=True, cookies=True) - - if re.search(self.ADULT_PATTERN, self.html): - self.logInfo(_("Adult content confirmation needed")) - - m = re.search(self.TOKEN_PATTERN, self.html) - if m is None: - self.error(_("TOKEN_PATTERN not found")) - token = m.group(1) - - self.html = self.load(pyfile.url, get={"do": "askAgeForm-submit"}, - post={"agree": "Confirm", "_token_": token}, cookies=True) - - if self.PASSWD_PATTERN in self.html: - password = self.getPassword() - - if password: - self.logInfo(_("Password protected link, trying ") + password) - self.html = self.load(pyfile.url, get={"do": "passwordProtectedForm-submit"}, - post={"password": password, "password_send": 'Send'}, cookies=True) - - if self.PASSWD_PATTERN in self.html: - self.fail(_("Incorrect password")) - else: - self.fail(_("No password found")) - - if re.search(self.VIPLINK_PATTERN, self.html): - self.html = self.load(pyfile.url, get={"disclaimer": "1"}) - - self.getFileInfo() - - if self.premium and self.checkTrafficLeft(): - self.handlePremium(pyfile) - else: - self.handleFree(pyfile) - - self.doCheckDownload() - - def handleFree(self, pyfile): action, inputs = self.parseHtmlForm('id="frm-downloadDialog-freeDownloadForm"') if not action or not inputs: @@ -104,7 +68,7 @@ class UlozTo(SimpleHoster): # New version - better to get new parameters (like captcha reload) because of image url - since 6.12.2013 self.logDebug('Using "new" version') - xapca = self.load("http://www.ulozto.net/reloadXapca.php", get={"rnd": str(int(time.time()))}) + xapca = self.load("http://www.ulozto.net/reloadXapca.php", get={'rnd': str(int(time.time()))}) self.logDebug("xapca = " + str(xapca)) data = json_loads(xapca) @@ -112,18 +76,50 @@ class UlozTo(SimpleHoster): self.logDebug("CAPTCHA HASH: " + data['hash'], "CAPTCHA SALT: " + str(data['salt']), "CAPTCHA VALUE: " + captcha_value) inputs.update({'timestamp': data['timestamp'], 'salt': data['salt'], 'hash': data['hash'], 'captcha_value': captcha_value}) + else: self.error(_("CAPTCHA form changed")) - self.multiDL = True - self.download("http://www.ulozto.net" + action, post=inputs, cookies=True, disposition=True) + self.download("http://www.ulozto.net" + action, post=inputs) def handlePremium(self, pyfile): - self.download(pyfile.url, get={'do': directDownload}, disposition=True) + self.download(pyfile.url, get={'do': "directDownload"}) + + + def checkErrors(self): + if re.search(self.ADULT_PATTERN, self.html): + self.logInfo(_("Adult content confirmation needed")) + + m = re.search(self.TOKEN_PATTERN, self.html) + if m is None: + self.error(_("TOKEN_PATTERN not found")) + + self.html = self.load(pyfile.url, + get={'do': "askAgeForm-submit"}, + post={"agree": "Confirm", "_token_": m.group(1)}) + + if self.PASSWD_PATTERN in self.html: + password = self.getPassword() + if password: + self.logInfo(_("Password protected link, trying ") + password) + self.html = self.load(pyfile.url, + get={'do': "passwordProtectedForm-submit"}, + post={"password": password, "password_send": 'Send'}) - def doCheckDownload(self): + if self.PASSWD_PATTERN in self.html: + self.fail(_("Incorrect password")) + else: + self.fail(_("No password found")) + + if re.search(self.VIPLINK_PATTERN, self.html): + self.html = self.load(pyfile.url, get={'disclaimer': "1"}) + + return super(UlozTo, self).checkErrors() + + + def checkFile(self, rules={}): check = self.checkDownload({ "wrong_captcha": re.compile(r'<ul class="error">\s*<li>Error rewriting the text.</li>'), "offline" : re.compile(self.OFFLINE_PATTERN), @@ -133,8 +129,6 @@ class UlozTo(SimpleHoster): }) if check == "wrong_captcha": - #self.delStorage("captcha_id") - #self.delStorage("captcha_text") self.invalidCaptcha() self.retry(reason=_("Wrong captcha code")) @@ -151,7 +145,9 @@ class UlozTo(SimpleHoster): self.retry() elif check == "not_found": - self.fail(_("Server error - file not downloadable")) + self.fail(_("Server error, file not downloadable")) + + return super(UlozTo, self).checkFile(rules) getInfo = create_getInfo(UlozTo) diff --git a/module/plugins/hoster/UloziskoSk.py b/module/plugins/hoster/UloziskoSk.py index 0267d4b61..ad809b47c 100644 --- a/module/plugins/hoster/UloziskoSk.py +++ b/module/plugins/hoster/UloziskoSk.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -11,6 +12,7 @@ class UloziskoSk(SimpleHoster): __version__ = "0.25" __pattern__ = r'http://(?:www\.)?ulozisko\.sk/.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Ulozisko.sk hoster plugin""" __license__ = "GPLv3" @@ -21,10 +23,10 @@ class UloziskoSk(SimpleHoster): SIZE_PATTERN = ur'VeÄŸkosÅ¥ súboru: <strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong><br />' OFFLINE_PATTERN = ur'<span class = "red">ZadanÃœ súbor neexistuje z jedného z nasledujúcich dÃŽvodov:</span>' - LINK_FREE_PATTERN = r'<form name = "formular" action = "([^"]+)" method = "post">' - ID_PATTERN = r'<input type = "hidden" name = "id" value = "([^"]+)" />' - CAPTCHA_PATTERN = r'<img src="(/obrazky/obrazky\.php\?fid=[^"]+)" alt="" />' - IMG_PATTERN = ur'<strong>PRE ZVÃÄÅ ENIE KLIKNITE NA OBRÃZOK</strong><br /><a href = "([^"]+)">' + LINK_FREE_PATTERN = r'<form name = "formular" action = "(.+?)" method = "post">' + ID_PATTERN = r'<input type = "hidden" name = "id" value = "(.+?)" />' + CAPTCHA_PATTERN = r'<img src="(/obrazky/obrazky\.php\?fid=.+?)" alt="" />' + IMG_PATTERN = ur'<strong>PRE ZVÃÄÅ ENIE KLIKNITE NA OBRÃZOK</strong><br /><a href = "(.+?)">' def process(self, pyfile): @@ -33,8 +35,7 @@ class UloziskoSk(SimpleHoster): m = re.search(self.IMG_PATTERN, self.html) if m: - url = "http://ulozisko.sk" + m.group(1) - self.download(url) + self.link = "http://ulozisko.sk" + m.group(1) else: self.handleFree(pyfile) @@ -55,8 +56,8 @@ class UloziskoSk(SimpleHoster): m = re.search(self.CAPTCHA_PATTERN, self.html) if m is None: self.error(_("CAPTCHA_PATTERN not found")) - captcha_url = 'http://www.ulozisko.sk' + m.group(1) + captcha_url = urlparse.urljoin("http://www.ulozisko.sk", m.group(1)) captcha = self.decryptCaptcha(captcha_url, cookies=True) self.logDebug("CAPTCHA_URL:" + captcha_url + ' CAPTCHA:' + captcha) diff --git a/module/plugins/hoster/UnibytesCom.py b/module/plugins/hoster/UnibytesCom.py index cd4e506c4..d090c8e7d 100644 --- a/module/plugins/hoster/UnibytesCom.py +++ b/module/plugins/hoster/UnibytesCom.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re - -from urlparse import urljoin - -from pycurl import FOLLOWLOCATION +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -15,6 +13,7 @@ class UnibytesCom(SimpleHoster): __version__ = "0.12" __pattern__ = r'https?://(?:www\.)?unibytes\.com/[\w .-]{11}B' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """UniBytes.com hoster plugin""" __license__ = "GPLv3" @@ -23,25 +22,25 @@ class UnibytesCom(SimpleHoster): HOSTER_DOMAIN = "unibytes.com" - INFO_PATTERN = r'<span[^>]*?id="fileName"[^>]*>(?P<N>[^>]+)</span>\s*\((?P<S>\d.*?)\)' + INFO_PATTERN = r'<span[^>]*?id="fileName".*?>(?P<N>[^>]+)</span>\s*\((?P<S>\d.*?)\)' WAIT_PATTERN = r'Wait for <span id="slowRest">(\d+)</span> sec' - LINK_FREE_PATTERN = r'<a href="([^"]+)">Download</a>' + LINK_FREE_PATTERN = r'<a href="(.+?)">Download</a>' def handleFree(self, pyfile): domain = "http://www.%s/" % self.HOSTER_DOMAIN action, post_data = self.parseHtmlForm('id="startForm"') - self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) for _i in xrange(8): self.logDebug(action, post_data) - self.html = self.load(urljoin(domain, action), post=post_data) + self.html = self.load(urlparse.urljoin(domain, action), post=post_data) m = re.search(r'location:\s*(\S+)', self.req.http.header, re.I) if m: - url = m.group(1) + self.link = m.group(1) break if '>Somebody else is already downloading using your IP-address<' in self.html: @@ -51,7 +50,7 @@ class UnibytesCom(SimpleHoster): if post_data['step'] == 'last': m = re.search(self.LINK_FREE_PATTERN, self.html) if m: - url = m.group(1) + self.link = m.group(1) self.correctCaptcha() break else: @@ -62,17 +61,15 @@ class UnibytesCom(SimpleHoster): if last_step == 'timer': m = re.search(self.WAIT_PATTERN, self.html) - self.wait(int(m.group(1)) if m else 60, False) + self.wait(m.group(1) if m else 60, False) elif last_step in ("captcha", "last"): - post_data['captcha'] = self.decryptCaptcha(urljoin(domain, "/captcha.jpg")) + post_data['captcha'] = self.decryptCaptcha(urlparse.urljoin(domain, "/captcha.jpg")) else: self.fail(_("No valid captcha code entered")) - self.req.http.c.setopt(FOLLOWLOCATION, 1) - - self.download(url) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) getInfo = create_getInfo(UnibytesCom) diff --git a/module/plugins/hoster/UnrestrictLi.py b/module/plugins/hoster/UnrestrictLi.py index 52f12b5b8..b63796fe0 100644 --- a/module/plugins/hoster/UnrestrictLi.py +++ b/module/plugins/hoster/UnrestrictLi.py @@ -1,88 +1,19 @@ # -*- coding: utf-8 -*- -import re +from module.plugins.internal.DeadHoster import DeadHoster, create_getInfo -from datetime import datetime, timedelta -from module.common.json_layer import json_loads -from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo -from module.plugins.internal.SimpleHoster import secondsToMidnight - - -class UnrestrictLi(MultiHoster): +class UnrestrictLi(DeadHoster): __name__ = "UnrestrictLi" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.23" __pattern__ = r'https?://(?:www\.)?(unrestrict|unr)\.li/dl/[\w^_]+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Unrestrict.li multi-hoster plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] - LOGIN_ACCOUNT = False - - - def setup(self): - self.chunkLimit = 16 - self.resumeDownload = True - - - def handleFree(self, pyfile): - for _i in xrange(5): - page = self.load('https://unrestrict.li/unrestrict.php', - post={'link': pyfile.url, 'domain': 'long'}) - self.logDebug("JSON data: " + page) - if page != '': - break - else: - self.logInfo(_("Unable to get API data, waiting 1 minute and retry")) - self.retry(5, 60, "Unable to get API data") - - if 'Expired session' in page or ("You are not allowed to " - "download from this host" in page and self.premium): - self.account.relogin(self.user) - self.retry() - - elif "File offline" in page: - self.offline() - - elif "You are not allowed to download from this host" in page: - self.fail(_("You are not allowed to download from this host")) - - elif "You have reached your daily limit for this host" in page: - self.logWarning(_("Reached daily limit for this host")) - self.retry(5, secondsToMidnight(gmt=2), "Daily limit for this host reached") - - elif "ERROR_HOSTER_TEMPORARILY_UNAVAILABLE" in page: - self.logInfo(_("Hoster temporarily unavailable, waiting 1 minute and retry")) - self.retry(5, 60, "Hoster is temporarily unavailable") - - page = json_loads(page) - self.link = page.keys()[0] - self.api_data = page[self.link] - - if self.link != pyfile.url: - self.logDebug("New URL: " + self.link) - - if hasattr(self, 'api_data'): - self.setNameSize() - - - def checkFile(self): - super(UnrestrictLi, self).checkFile() - - if self.getConfig("history"): - self.load("https://unrestrict.li/history/", get={'delete': "all"}) - self.logInfo(_("Download history deleted")) - - - def setNameSize(self): - if 'name' in self.api_data: - self.pyfile.name = self.api_data['name'] - if 'size' in self.api_data: - self.pyfile.size = self.api_data['size'] - - getInfo = create_getInfo(UnrestrictLi) diff --git a/module/plugins/hoster/UpleaCom.py b/module/plugins/hoster/UpleaCom.py index 4ea4e345f..9d460ef98 100644 --- a/module/plugins/hoster/UpleaCom.py +++ b/module/plugins/hoster/UpleaCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo @@ -10,26 +9,31 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class UpleaCom(XFSHoster): __name__ = "UpleaCom" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.10" __pattern__ = r'https?://(?:www\.)?uplea\.com/dl/\w{15}' __description__ = """Uplea.com hoster plugin""" __license__ = "GPLv3" - __authors__ = [("Redleon", None)] + __authors__ = [("Redleon", None), + ("GammaC0de", None)] + + DISPOSITION = False #@TODO: Remove in 0.4.10 HOSTER_DOMAIN = "uplea.com" - NAME_PATTERN = r'class="agmd size18">(?P<N>.+?)<' - SIZE_PATTERN = r'size14">(?P<S>[\d.,]+) (?P<U>[\w^_])</span>' + SIZE_REPLACEMENTS = [('ko','KB'), ('mo','MB'), ('go','GB'), ('Ko','KB'), ('Mo','MB'), ('Go','GB')] + NAME_PATTERN = r'<span class="gold-text">(?P<N>.+?)</span>' + SIZE_PATTERN = r'<span class="label label-info agmd">(?P<S>[\d.,]+) (?P<U>[\w^_]+?)</span>' OFFLINE_PATTERN = r'>You followed an invalid or expired link' - LINK_PATTERN = r'"(http?://\w+\.uplea\.com/anonym/.*?)"' + LINK_PATTERN = r'"(https?://\w+\.uplea\.com/anonym/.*?)"' - WAIT_PATTERN = r'timeText:([\d.]+),' - STEP_PATTERN = r'<a href="(/step/.+)">' + PREMIUM_ONLY_PATTERN = r'You need to have a Premium subscription to download this file' + WAIT_PATTERN = r'timeText: ?([\d.]+),' + STEP_PATTERN = r'<a href="(/step/.+)">' def setup(self): @@ -41,21 +45,22 @@ class UpleaCom(XFSHoster): def handleFree(self, pyfile): m = re.search(self.STEP_PATTERN, self.html) if m is None: - self.error("STEP_PATTERN not found") + self.error(_("STEP_PATTERN not found")) - self.html = self.load(urljoin("http://uplea.com/", m.group(1))) + self.html = self.load(urlparse.urljoin("http://uplea.com/", m.group(1))) m = re.search(self.WAIT_PATTERN, self.html) if m: - self.wait(int(m.group(1)), True) + self.logDebug(_("Waiting %s seconds") % m.group(1)) + self.wait(m.group(1), True) self.retry() m = re.search(self.LINK_PATTERN, self.html) if m is None: self.error(_("LINK_PATTERN not found")) + self.link = m.group(1) self.wait(15) - self.download(m.group(1), disposition=True) getInfo = create_getInfo(UpleaCom) diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py index d987770fe..61226e3da 100644 --- a/module/plugins/hoster/UploadStationCom.py +++ b/module/plugins/hoster/UploadStationCom.py @@ -9,6 +9,7 @@ class UploadStationCom(DeadHoster): __version__ = "0.52" __pattern__ = r'http://(?:www\.)?uploadstation\.com/file/(?P<ID>\w+)' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """UploadStation.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/UploadableCh.py b/module/plugins/hoster/UploadableCh.py index e28944ab9..f323a3538 100644 --- a/module/plugins/hoster/UploadableCh.py +++ b/module/plugins/hoster/UploadableCh.py @@ -2,18 +2,17 @@ import re -from time import sleep - -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UploadableCh(SimpleHoster): __name__ = "UploadableCh" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.10" __pattern__ = r'http://(?:www\.)?uploadable\.ch/file/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Uploadable.ch hoster plugin""" __license__ = "GPLv3" @@ -21,9 +20,9 @@ class UploadableCh(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - FILE_URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://www.uploadable.ch/file/\g<ID>')] + URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://www.uploadable.ch/file/\g<ID>')] - FILE_INFO_PATTERN = r'div id=\"file_name\" title=.*>(?P<N>.+)<span class=\"filename_normal\">\((?P<S>[\d.]+) (?P<U>\w+)\)</span><' + INFO_PATTERN = r'div id=\"file_name\" title=.*>(?P<N>.+)<span class=\"filename_normal\">\((?P<S>[\d.]+) (?P<U>\w+)\)</span><' OFFLINE_PATTERN = r'>(File not available|This file is no longer available)' TEMP_OFFLINE_PATTERN = r'<div class="icon_err">' @@ -35,22 +34,21 @@ class UploadableCh(SimpleHoster): def handleFree(self, pyfile): # Click the "free user" button and wait - a = self.load(pyfile.url, cookies=True, post={'downloadLink': "wait"}, decode=True) + a = self.load(pyfile.url, post={'downloadLink': "wait"}, decode=True) self.logDebug(a) self.wait(30) # Make the recaptcha appear and show it the pyload interface - b = self.load(pyfile.url, cookies=True, post={'checkDownload': "check"}, decode=True) + b = self.load(pyfile.url, post={'checkDownload': "check"}, decode=True) self.logDebug(b) #: Expected output: {"success":"showCaptcha"} recaptcha = ReCaptcha(self) - challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY) + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) # Submit the captcha solution self.load("http://www.uploadable.ch/checkReCaptcha.php", - cookies=True, post={'recaptcha_challenge_field' : challenge, 'recaptcha_response_field' : response, 'recaptcha_shortencode_field': self.info['pattern']['ID']}, @@ -59,21 +57,21 @@ class UploadableCh(SimpleHoster): self.wait(3) # Get ready for downloading - self.load(pyfile.url, cookies=True, post={'downloadLink': "show"}, decode=True) + self.load(pyfile.url, post={'downloadLink': "show"}, decode=True) self.wait(3) # Download the file - self.download(pyfile.url, cookies=True, post={'download': "normal"}, disposition=True) + self.download(pyfile.url, post={'download': "normal"}, disposition=True) - def checkFile(self): + def checkFile(self, rules={}): if self.checkDownload({'wait': re.compile("Please wait for")}): self.logInfo("Downloadlimit reached, please wait or reconnect") self.wait(60 * 60, True) self.retry() - return super(UploadableCh, self).checkFile() + return super(UploadableCh, self).checkFile(rules) getInfo = create_getInfo(UploadableCh) diff --git a/module/plugins/hoster/UploadboxCom.py b/module/plugins/hoster/UploadboxCom.py index 031c5f761..cb4f50184 100644 --- a/module/plugins/hoster/UploadboxCom.py +++ b/module/plugins/hoster/UploadboxCom.py @@ -9,6 +9,7 @@ class UploadboxCom(DeadHoster): __version__ = "0.05" __pattern__ = r'http://(?:www\.)?uploadbox\.com/files/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """UploadBox.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index 833468a80..4db160ab4 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -1,245 +1,106 @@ # -*- coding: utf-8 -*- -# -# Test links: -# http://ul.to/044yug9o -# http://ul.to/gzfhd0xs import re - -from time import sleep +import time +import urlparse from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster -from module.plugins.Plugin import chunks -from module.plugins.internal.CaptchaService import ReCaptcha -from module.utils import html_unescape, parseFileSize - - -key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') - - -def getID(url): - """ returns id from file url""" - m = re.match(UploadedTo.__pattern__, url) - return m.group('ID') - - -def getAPIData(urls): - post = {"apikey": key} - - idMap = {} - - for i, url in enumerate(urls): - id = getID(url) - post['id_%s' % i] = id - idMap[id] = url - - for _i in xrange(5): - api = unicode(getURL("http://uploaded.net/api/filemultiple", post=post, decode=False), 'iso-8859-1') - if api != "can't find request": - break - else: - sleep(3) - - result = {} - - if api: - for line in api.splitlines(): - data = line.split(",", 4) - if data[1] in idMap: - result[data[1]] = (data[0], data[2], data[4], data[3], idMap[data[1]]) - - return result +from module.plugins.internal.ReCaptcha import ReCaptcha +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -def parseFileInfo(self, url='', html=''): - if not html and hasattr(self, "html"): - html = self.html - - name = url - size = 0 - fileid = None - - if re.search(self.OFFLINE_PATTERN, html): - # File offline - status = 1 - else: - m = re.search(self.INFO_PATTERN, html) - if m: - name, fileid = html_unescape(m.group('N')), m.group('ID') - size = parseFileSize(m.group('S')) - status = 2 - else: - status = 3 - - return name, size, status, fileid - - -def getInfo(urls): - for chunk in chunks(urls, 80): - result = [] - - api = getAPIData(chunk) - - for data in api.itervalues(): - if data[0] == "online": - result.append((html_unescape(data[2]), data[1], 2, data[4])) - - elif data[0] == "offline": - result.append((data[4], 0, 1, data[4])) - - yield result - - -class UploadedTo(Hoster): +class UploadedTo(SimpleHoster): __name__ = "UploadedTo" __type__ = "hoster" - __version__ = "0.75" + __version__ = "0.92" __pattern__ = r'https?://(?:www\.)?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Uploaded.net hoster plugin""" __license__ = "GPLv3" - __authors__ = [("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("netpok", "netpok@gmail.com"), - ("stickell", "l.stickell@yahoo.it")] - + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - INFO_PATTERN = r'<a href="file/(?P<ID>\w+)" id="filename">(?P<N>[^<]+)</a> \s*<small[^>]*>(?P<S>[^<]+)</small>' - OFFLINE_PATTERN = r'<small class="cL">Error: 404</small>' - DL_LIMIT_PATTERN = r'You have reached the max. number of possible free downloads for this hour' + CHECK_TRAFFIC = True + DISPOSITION = False - def setup(self): - self.multiDL = self.resumeDownload = self.premium - self.chunkLimit = 1 # critical problems with more chunks + API_KEY = "lhF2IeeprweDfu9ccWlxXVVypA5nA3EL" - self.fileID = getID(self.pyfile.url) - self.pyfile.url = "http://uploaded.net/file/%s" % self.fileID + URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://uploaded.net/file/\g<ID>')] + TEMP_OFFLINE_PATTERN = r'<title>uploaded\.net - Maintenance' - def process(self, pyfile): - self.load("http://uploaded.net/language/en", just_header=True) - - api = getAPIData([pyfile.url]) + WAIT_PATTERN = r'Current waiting period: <span>(\d+)' + DL_LIMIT_ERROR = r'You have reached the max. number of possible free downloads for this hour' - # TODO: fallback to parse from site, because api sometimes delivers wrong status codes - if not api: - self.logWarning(_("No response for API call")) + @classmethod + def apiInfo(cls, url): + info = super(UploadedTo, cls).apiInfo(url) - self.html = unicode(self.load(pyfile.url, decode=False), 'iso-8859-1') - name, size, status, self.fileID = parseFileInfo(self) - self.logDebug(name, size, status, self.fileID) - if status == 1: - self.offline() - elif status == 2: - pyfile.name, pyfile.size = name, size + for _i in xrange(5): + html = getURL("http://uploaded.net/api/filemultiple", + get={"apikey": cls.API_KEY, 'id_0': re.match(cls.__pattern__, url).group('ID')}, + decode=True) + + if html != "can't find request": + api = html.split(",", 4) + if api[0] == "online": + info.update({'name': api[4].strip(), 'size': api[2], 'status': 2}) + else: + info['status'] = 1 + break else: - self.error(_("file info")) + time.sleep(3) - elif api == 'Access denied': - self.fail(_("API key invalid")) + return info - else: - if self.fileID not in api: - self.offline() - self.data = api[self.fileID] - if self.data[0] != "online": - self.offline() + def setup(self): + self.multiDL = self.resumeDownload = self.premium + self.chunkLimit = 1 # critical problems with more chunks - pyfile.name = html_unescape(self.data[2]) - # pyfile.name = self.get_file_name() + def checkErrors(self): + if 'var free_enabled = false;' in self.html: + self.logError(_("Free-download capacities exhausted")) + self.retry(24, 5 * 60) - if self.premium: - self.handlePremium() - else: - self.handleFree() + return super(UploadedTo, self).checkErrors() - def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) - self.logDebug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name": self.__name__, - "left": info['trafficleft'] / 1024 / 1024}) - if int(self.data[1]) / 1024 > info['trafficleft']: - self.logInfo(_("Not enough traffic left")) - self.account.empty(self.user) - self.resetAccount() - self.fail(_("Traffic exceeded")) + def handlePremium(self, pyfile): + self.link = urlparse.urljoin(pyfile.url, "/ddl?pw=" + self.getPassword()) - header = self.load("http://uploaded.net/file/%s" % self.fileID, just_header=True) - if 'location' in header: - #Direct download - self.logDebug("Direct download link detected") - self.download(header['location']) - else: - #Indirect download - self.html = self.load("http://uploaded.net/file/%s" % self.fileID) - m = re.search(r'<div class="tfree".*\s*<form method="post" action="(.*?)"', self.html) - if m is None: - self.fail(_("Download URL not m. Try to enable direct downloads")) - url = m.group(1) - self.download(url, post={}) + def handleFree(self, pyfile): + self.load("http://uploaded.net/language/en", just_header=True) - def handleFree(self): - self.html = self.load(self.pyfile.url, decode=True) + self.html = self.load("http://uploaded.net/js/download.js", decode=True) - if 'var free_enabled = false;' in self.html: - self.logError(_("Free-download capacities exhausted")) - self.retry(24, 5 * 60) + recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge() - m = re.search(r"Current waiting period: <span>(\d+)</span> seconds", self.html) - if m is None: - self.fail(_("File not downloadable for free users")) - self.setWait(int(m.group(1))) + self.html = self.load("http://uploaded.net/io/ticket/captcha/%s" % self.info['pattern']['ID'], + post={'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) - self.html = self.load("http://uploaded.net/js/download.js", decode=True) + if "type:'download'" in self.html: + self.correctCaptcha() + try: + self.link = re.search("url:'(.+?)'", self.html).group(1) - url = "http://uploaded.net/io/ticket/captcha/%s" % self.fileID - downloadURL = "" + except Exception: + pass - recaptcha = ReCaptcha(self) - for _i in xrange(5): - challenge, response = recaptcha.challenge() - options = {"recaptcha_challenge_field": challenge, "recaptcha_response_field": response} - self.wait() - - result = self.load(url, post=options) - self.logDebug("Result: %s" % result) - - if "limit-size" in result: - self.fail(_("File too big for free download")) - elif "limit-slot" in result: # Temporary restriction so just wait a bit - self.setWait(30 * 60, True) - self.wait() - self.retry() - elif "limit-parallel" in result: - self.fail(_("Cannot download in parallel")) - elif "limit-dl" in result or self.DL_LIMIT_PATTERN in result: # limit-dl - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() - elif '"err":"captcha"' in result: - self.invalidCaptcha() - elif "type:'download'" in result: - self.correctCaptcha() - downloadURL = re.search("url:'([^']+)", result).group(1) - break - else: - self.error(_("Unknown error: %s") % result) + def checkFile(self, rules={}): + if self.checkDownload({'limit-dl': self.DL_LIMIT_ERROR}): + self.wait(3 * 60 * 60, True) + self.retry() - if not downloadURL: - self.fail(_("No Download url retrieved/all captcha attempts failed")) + return super(UploadedTo, self).checkFile(rules) - self.download(downloadURL, disposition=True) - check = self.checkDownload({"limit-dl": self.DL_LIMIT_PATTERN}) - if check == "limit-dl": - self.setWait(3 * 60 * 60, True) - self.wait() - self.retry() + +getInfo = create_getInfo(UploadedTo) diff --git a/module/plugins/hoster/UploadhereCom.py b/module/plugins/hoster/UploadhereCom.py index 8da30be46..bc94e644e 100644 --- a/module/plugins/hoster/UploadhereCom.py +++ b/module/plugins/hoster/UploadhereCom.py @@ -9,6 +9,7 @@ class UploadhereCom(DeadHoster): __version__ = "0.12" __pattern__ = r'http://(?:www\.)?uploadhere\.com/\w{10}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Uploadhere.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/UploadheroCom.py b/module/plugins/hoster/UploadheroCom.py index ac0b4b562..5c74f10eb 100644 --- a/module/plugins/hoster/UploadheroCom.py +++ b/module/plugins/hoster/UploadheroCom.py @@ -4,6 +4,7 @@ # http://uploadhero.co/dl/wQBRAVSM import re +import urlparse from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -11,9 +12,10 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UploadheroCom(SimpleHoster): __name__ = "UploadheroCom" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __pattern__ = r'http://(?:www\.)?uploadhero\.com?/dl/\w+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """UploadHero.co plugin""" __license__ = "GPLv3" @@ -21,56 +23,48 @@ class UploadheroCom(SimpleHoster): ("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'<div class="nom_de_fichier">(?P<N>.*?)</div>' - SIZE_PATTERN = r'Taille du fichier : </span><strong>(?P<S>.*?)</strong>' - OFFLINE_PATTERN = r'<p class="titre_dl_2">|<div class="raison"><strong>Le lien du fichier ci-dessus n\'existe plus.' + NAME_PATTERN = r'<div class="nom_de_fichier">(?P<N>.+?)<' + SIZE_PATTERN = r'>Filesize: </span><strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'<p class="titre_dl_2">' COOKIES = [("uploadhero.co", "lang", "en")] - IP_BLOCKED_PATTERN = r'href="(/lightbox_block_download\.php\?min=.*?)"' - IP_WAIT_PATTERN = r'<span id="minutes">(\d+)</span>.*\s*<span id="seconds">(\d+)</span>' + IP_BLOCKED_PATTERN = r'href="(/lightbox_block_download\.php\?min=.+?)"' + IP_WAIT_PATTERN = r'<span id="minutes">(\d+)</span>.*\s*<span id="seconds">(\d+)</span>' CAPTCHA_PATTERN = r'"(/captchadl\.php\?\w+)"' - LINK_FREE_PATTERN = r'var magicomfg = \'<a href="(http://[^<>"]*?)"|"(http://storage\d+\.uploadhero\.co/\?d=\w+/[^<>"/]+)"' - LINK_PREMIUM_PATTERN = r'<a href="([^"]+)" id="downloadnow"' + LINK_FREE_PATTERN = r'var magicomfg = \'<a href="(.+?)"|"(http://storage\d+\.uploadhero\.co.+?)"' + LINK_PREMIUM_PATTERN = r'<a href="(.+?)" id="downloadnow"' def handleFree(self, pyfile): - self.checkErrors() - m = re.search(self.CAPTCHA_PATTERN, self.html) if m is None: - self.error(_("CAPTCHA_PATTERN not found")) - captcha_url = "http://uploadhero.co" + m.group(1) + self.error(_("Captcha not found")) + + captcha = self.decryptCaptcha(urlparse.urljoin("http://uploadhero.co", m.group(1))) - for _i in xrange(5): - captcha = self.decryptCaptcha(captcha_url) - self.html = self.load(pyfile.url, get={"code": captcha}) - m = re.search(self.LINK_FREE_PATTERN, self.html) - if m: - self.correctCaptcha() - download_url = m.group(1) or m.group(2) - break - else: - self.invalidCaptcha() - else: - self.fail(_("No valid captcha code entered")) + self.html = self.load(pyfile.url, + get={"code": captcha}) - self.download(download_url) + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + self.link = m.group(1) or m.group(2) + self.wait(50) def checkErrors(self): m = re.search(self.IP_BLOCKED_PATTERN, self.html) if m: - self.html = self.load("http://uploadhero.co%s" % m.group(1)) + self.html = self.load(urlparse.urljoin("http://uploadhero.co", m.group(1))) m = re.search(self.IP_WAIT_PATTERN, self.html) wait_time = (int(m.group(1)) * 60 + int(m.group(2))) if m else 5 * 60 self.wait(wait_time, True) self.retry() - self.info.pop('error', None) + return super(UploadheroCom, self).checkErrors() getInfo = create_getInfo(UploadheroCom) diff --git a/module/plugins/hoster/UploadingCom.py b/module/plugins/hoster/UploadingCom.py index 164b7b243..d1cafcff4 100644 --- a/module/plugins/hoster/UploadingCom.py +++ b/module/plugins/hoster/UploadingCom.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re -from pycurl import HTTPHEADER - from module.common.json_layer import json_loads from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp @@ -11,7 +10,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, t class UploadingCom(SimpleHoster): __name__ = "UploadingCom" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" __pattern__ = r'http://(?:www\.)?uploading\.com/files/(?:get/)?(?P<ID>\w+)' @@ -22,7 +21,7 @@ class UploadingCom(SimpleHoster): ("zoidberg", "zoidberg@mujmail.cz")] - NAME_PATTERN = r'id="file_title">(?P<N>.+)</' + NAME_PATTERN = r'id="file_title">(?P<N>.+?)</' SIZE_PATTERN = r'size tip_container">(?P<S>[\d.,]+) (?P<U>[\w^_]+)<' OFFLINE_PATTERN = r'(Page|file) not found' @@ -53,8 +52,7 @@ class UploadingCom(SimpleHoster): self.html = self.load('http://uploading.com/files/get/?JsHttpRequest=%d-xml' % timestamp(), post=postData) url = re.search(r'"link"\s*:\s*"(.*?)"', self.html) if url: - url = url.group(1).replace("\\/", "/") - self.download(url) + self.link = url.group(1).replace("\\/", "/") raise Exception("Plugin defect") @@ -67,7 +65,7 @@ class UploadingCom(SimpleHoster): self.retry(6, (6 * 60 if m.group(2) else 15) * 60, pyfile.error) ajax_url = "http://uploading.com/files/get/?ajax" - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) self.req.http.lastURL = pyfile.url res = json_loads(self.load(ajax_url, post={'action': 'second_page', 'code': self.info['pattern']['ID']})) @@ -93,7 +91,7 @@ class UploadingCom(SimpleHoster): else: self.error(_("No URL")) - self.download(url) + self.link = url getInfo = create_getInfo(UploadingCom) diff --git a/module/plugins/hoster/UploadkingCom.py b/module/plugins/hoster/UploadkingCom.py index 743e700eb..0d60cef1f 100644 --- a/module/plugins/hoster/UploadkingCom.py +++ b/module/plugins/hoster/UploadkingCom.py @@ -9,6 +9,7 @@ class UploadkingCom(DeadHoster): __version__ = "0.14" __pattern__ = r'http://(?:www\.)?uploadking\.com/\w{10}' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """UploadKing.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/UpstoreNet.py b/module/plugins/hoster/UpstoreNet.py index f287a2b51..6410a2dce 100644 --- a/module/plugins/hoster/UpstoreNet.py +++ b/module/plugins/hoster/UpstoreNet.py @@ -2,16 +2,17 @@ import re -from module.plugins.internal.CaptchaService import ReCaptcha +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UpstoreNet(SimpleHoster): __name__ = "UpstoreNet" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.06" __pattern__ = r'https?://(?:www\.)?upstore\.net/' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Upstore.Net File Download Hoster""" __license__ = "GPLv3" @@ -22,7 +23,7 @@ class UpstoreNet(SimpleHoster): OFFLINE_PATTERN = r'<span class="error">File not found</span>' WAIT_PATTERN = r'var sec = (\d+)' - CHASH_PATTERN = r'<input type="hidden" name="hash" value="([^"]*)">' + CHASH_PATTERN = r'<input type="hidden" name="hash" value="(.+?)">' LINK_FREE_PATTERN = r'<a href="(https?://.*?)" target="_blank"><b>' @@ -52,7 +53,7 @@ class UpstoreNet(SimpleHoster): self.wait(wait_time) # then, handle the captcha - challenge, response = recaptcha.challenge() + response, challenge = recaptcha.challenge() post_data.update({'recaptcha_challenge_field': challenge, 'recaptcha_response_field' : response}) @@ -66,8 +67,7 @@ class UpstoreNet(SimpleHoster): if m is None: self.error(_("Download link not found")) - direct = m.group(1) - self.download(direct, disposition=True) + self.link = m.group(1) getInfo = create_getInfo(UpstoreNet) diff --git a/module/plugins/hoster/UptoboxCom.py b/module/plugins/hoster/UptoboxCom.py index 21d781f55..2c06558e4 100644 --- a/module/plugins/hoster/UptoboxCom.py +++ b/module/plugins/hoster/UptoboxCom.py @@ -6,24 +6,21 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class UptoboxCom(XFSHoster): __name__ = "UptoboxCom" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.20" - __pattern__ = r'https?://(?:www\.)?uptobox\.com/\w{12}' + __pattern__ = r'https?://(?:www\.)?(uptobox|uptostream)\.com/\w{12}' __description__ = """Uptobox.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = "uptobox.com" - - INFO_PATTERN = r'"para_title">(?P<N>.+) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)' - OFFLINE_PATTERN = r'>(File not found|Access Denied|404 Not Found)' + INFO_PATTERN = r'"para_title">(?P<N>.+) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)' + OFFLINE_PATTERN = r'>(File not found|Access Denied|404 Not Found)' + TEMP_OFFLINE_PATTERN = r'>Service Unavailable' LINK_PATTERN = r'"(https?://\w+\.uptobox\.com/d/.*?)"' - ERROR_PATTERN = r'>(You have to wait.+till next download.)<' #@TODO: Check XFSHoster ERROR_PATTERN - def setup(self): self.multiDL = True diff --git a/module/plugins/hoster/VeehdCom.py b/module/plugins/hoster/VeehdCom.py index d894dab24..78da91020 100644 --- a/module/plugins/hoster/VeehdCom.py +++ b/module/plugins/hoster/VeehdCom.py @@ -11,8 +11,8 @@ class VeehdCom(Hoster): __version__ = "0.23" __pattern__ = r'http://veehd\.com/video/\d+_\S+' - __config__ = [("filename_spaces", "bool", "Allow spaces in filename", False), - ("replacement_char", "str", "Filename replacement character", "_")] + __config__ = [("filename_spaces", "bool", "Allow spaces in filename", False), + ("replacement_char", "str", "Filename replacement character", "_")] __description__ = """Veehd.com hoster plugin""" __license__ = "GPLv3" @@ -52,7 +52,7 @@ class VeehdCom(Hoster): if not self.html: self.download_html() - m = re.search(r'<title[^>]*>([^<]+) on Veehd</title>', self.html) + m = re.search(r'<title.*?>([^<]+) on Veehd</title>', self.html) if m is None: self.error(_("Video title not found")) @@ -73,7 +73,7 @@ class VeehdCom(Hoster): if not self.html: self.download_html() - m = re.search(r'<embed type="video/divx" src="(http://([^/]*\.)?veehd\.com/dl/[^"]+)"', + m = re.search(r'<embed type="video/divx" src="(http://([^/]*\.)?veehd\.com/dl/.+?)"', self.html) if m is None: self.error(_("Embedded video url not found")) diff --git a/module/plugins/hoster/VeohCom.py b/module/plugins/hoster/VeohCom.py index 219efa8e1..57b24623b 100644 --- a/module/plugins/hoster/VeohCom.py +++ b/module/plugins/hoster/VeohCom.py @@ -11,7 +11,8 @@ class VeohCom(SimpleHoster): __version__ = "0.22" __pattern__ = r'http://(?:www\.)?veoh\.com/(tv/)?(watch|videos)/(?P<ID>v\w+)' - __config__ = [("quality", "Low;High;Auto", "Quality", "Auto")] + __config__ = [("use_premium", "bool" , "Use premium account if available", True ), + ("quality" , "Low;High;Auto", "Quality" , "Auto")] __description__ = """Veoh.com hoster plugin""" __license__ = "GPLv3" @@ -33,7 +34,7 @@ class VeohCom(SimpleHoster): def handleFree(self, pyfile): - quality = self.getConfig("quality") + quality = self.getConfig('quality') if quality == "Auto": quality = ("High", "Low") @@ -42,8 +43,7 @@ class VeohCom(SimpleHoster): m = re.search(pattern, self.html) if m: pyfile.name += ".mp4" - link = m.group(1).replace("\\", "") - self.download(link) + self.link = m.group(1).replace("\\", "") return else: self.logInfo(_("No %s quality video found") % q.upper()) diff --git a/module/plugins/hoster/VidPlayNet.py b/module/plugins/hoster/VidPlayNet.py index 76af05edd..f1a32a897 100644 --- a/module/plugins/hoster/VidPlayNet.py +++ b/module/plugins/hoster/VidPlayNet.py @@ -18,8 +18,6 @@ class VidPlayNet(XFSHoster): __authors__ = [("t4skforce", "t4skforce1337[AT]gmail[DOT]com")] - HOSTER_DOMAIN = "vidplay.net" - NAME_PATTERN = r'<b>Password:</b></div>\s*<h[1-6]>(?P<N>[^<]+)</h[1-6]>' diff --git a/module/plugins/hoster/VimeoCom.py b/module/plugins/hoster/VimeoCom.py index f7485f025..b263c445f 100644 --- a/module/plugins/hoster/VimeoCom.py +++ b/module/plugins/hoster/VimeoCom.py @@ -8,18 +8,19 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class VimeoCom(SimpleHoster): __name__ = "VimeoCom" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'https?://(?:www\.)?(player\.)?vimeo\.com/(video/)?(?P<ID>\d+)' - __config__ = [("quality", "Lowest;Mobile;SD;HD;Highest", "Quality", "Highest"), - ("original", "bool", "Try to download the original file first", True)] + __config__ = [("use_premium", "bool" , "Use premium account if available" , True ), + ("quality" , "Lowest;Mobile;SD;HD;Highest", "Quality" , "Highest"), + ("original" , "bool" , "Try to download the original file", True )] __description__ = """Vimeo.com hoster plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'<title>(?P<N>.+) on Vimeo<' + NAME_PATTERN = r'<title>(?P<N>.+?) on Vimeo<' OFFLINE_PATTERN = r'class="exception_header"' TEMP_OFFLINE_PATTERN = r'Please try again in a few minutes.<' @@ -44,16 +45,16 @@ class VimeoCom(SimpleHoster): html = self.load("https://player.vimeo.com/video/" + self.info['pattern']['ID'], get={'password': password}) pattern = r'"(?P<QL>\w+)":{"profile".*?"(?P<URL>http://pdl\.vimeocdn\.com.+?)"' - link = dict([(l.group('QL').lower(), l.group('URL')) for l in re.finditer(pattern, html)]) + link = dict((l.group('QL').lower(), l.group('URL')) for l in re.finditer(pattern, html)) - if self.getConfig("original"): + if self.getConfig('original'): if "original" in link: - self.download(link[q]) + self.link = link[q] return else: self.logInfo(_("Original file not downloadable")) - quality = self.getConfig("quality") + quality = self.getConfig('quality') if quality == "Highest": qlevel = ("hd", "sd", "mobile") elif quality == "Lowest": @@ -63,7 +64,7 @@ class VimeoCom(SimpleHoster): for q in qlevel: if q in link: - self.download(link[q]) + self.link = link[q] return else: self.logInfo(_("No %s quality video found") % q.upper()) diff --git a/module/plugins/hoster/Vipleech4uCom.py b/module/plugins/hoster/Vipleech4UCom.py index 2ae41b942..5668e0fad 100644 --- a/module/plugins/hoster/Vipleech4uCom.py +++ b/module/plugins/hoster/Vipleech4UCom.py @@ -9,6 +9,7 @@ class Vipleech4UCom(DeadHoster): __version__ = "0.20" __pattern__ = r'http://(?:www\.)?vipleech4u\.com/manager\.php' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Vipleech4u.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/WarserverCz.py b/module/plugins/hoster/WarserverCz.py index c83d5c03e..6cb94e57e 100644 --- a/module/plugins/hoster/WarserverCz.py +++ b/module/plugins/hoster/WarserverCz.py @@ -9,6 +9,7 @@ class WarserverCz(DeadHoster): __version__ = "0.13" __pattern__ = r'http://(?:www\.)?warserver\.cz/stahnout/\d+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Warserver.cz hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/WebshareCz.py b/module/plugins/hoster/WebshareCz.py index e3424fc21..c898d23ab 100644 --- a/module/plugins/hoster/WebshareCz.py +++ b/module/plugins/hoster/WebshareCz.py @@ -9,51 +9,49 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class WebshareCz(SimpleHoster): __name__ = "WebshareCz" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.18" - __pattern__ = r'https?://(?:www\.)?webshare\.cz/(?:#/)?file/(?P<ID>\w+)' + __pattern__ = r'https?://(?:www\.)?(en\.)?webshare\.cz/(?:#/)?file/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """WebShare.cz hoster plugin""" __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("rush", "radek.senfeld@gmail.com")] + __authors__ = [("stickell", "l.stickell@yahoo.it "), + ("rush" , "radek.senfeld@gmail.com")] @classmethod - def getInfo(cls, url="", html=""): - info = super(WebshareCz, self).getInfo(url, html) + def apiInfo(cls, url): + info = super(WebshareCz, cls).apiInfo(url) - if url: - info['pattern'] = re.match(cls.__pattern__, url).groupdict() + info['pattern'] = re.match(cls.__pattern__, url).groupdict() - api_data = getURL("https://webshare.cz/api/file_info/", - post={'ident': info['pattern']['ID']}, - decode=True) + api_data = getURL("https://webshare.cz/api/file_info/", + post={'ident': info['pattern']['ID'], 'wst': ""}, + decode=True) - if 'File not found' in api_data: - info['status'] = 1 - else: - info["status"] = 2 - info['name'] = re.search('<name>(.+)</name>', api_data).group(1) or info['name'] - info['size'] = re.search('<size>(.+)</size>', api_data).group(1) or info['size'] + if not re.search(r'<status>OK', api_data): + info['status'] = 1 + else: + info['status'] = 2 + info['name'] = re.search(r'<name>(.+?)<', api_data).group(1) + info['size'] = re.search(r'<size>(.+?)<', api_data).group(1) return info def handleFree(self, pyfile): - wst = self.account.infos['wst'] if self.account and 'wst' in self.account.infos else "" + wst = self.account.getAccountData(self.user).get('wst', None) if self.account else None - api_data = getURL('https://webshare.cz/api/file_link/', + api_data = getURL("https://webshare.cz/api/file_link/", post={'ident': self.info['pattern']['ID'], 'wst': wst}, decode=True) self.logDebug("API data: " + api_data) m = re.search('<link>(.+)</link>', api_data) - if m is None: - self.error(_("Unable to detect direct link")) - - self.link = m.group(1) + if m: + self.link = m.group(1) def handlePremium(self, pyfile): diff --git a/module/plugins/hoster/WrzucTo.py b/module/plugins/hoster/WrzucTo.py index 457e72865..f11d03ea8 100644 --- a/module/plugins/hoster/WrzucTo.py +++ b/module/plugins/hoster/WrzucTo.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- +import pycurl import re -from pycurl import HTTPHEADER - from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -13,6 +12,7 @@ class WrzucTo(SimpleHoster): __version__ = "0.03" __pattern__ = r'http://(?:www\.)?wrzuc\.to/(\w+(\.wt|\.html)|(\w+/?linki/\w+))' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Wrzuc.to hoster plugin""" __license__ = "GPLv3" @@ -34,7 +34,7 @@ class WrzucTo(SimpleHoster): if len(data) != 2: self.error(_("No file ID")) - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.c.setopt(pycurl.HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) self.req.http.lastURL = pyfile.url self.load("http://www.wrzuc.to/ajax/server/prepair", post={"md5": data['md5']}) @@ -45,8 +45,7 @@ class WrzucTo(SimpleHoster): if len(data) != 4: self.error(_("No download URL")) - download_url = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) - self.download(download_url) + self.link = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) getInfo = create_getInfo(WrzucTo) diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py index 75ce59353..73c008ae8 100644 --- a/module/plugins/hoster/WuploadCom.py +++ b/module/plugins/hoster/WuploadCom.py @@ -9,6 +9,7 @@ class WuploadCom(DeadHoster): __version__ = "0.23" __pattern__ = r'http://(?:www\.)?wupload\..+?/file/((\w+/)?\d+)(/.*)?' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """Wupload.com hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/X7To.py b/module/plugins/hoster/X7To.py index a4e4b04bd..8e2dd3a53 100644 --- a/module/plugins/hoster/X7To.py +++ b/module/plugins/hoster/X7To.py @@ -9,6 +9,7 @@ class X7To(DeadHoster): __version__ = "0.41" __pattern__ = r'http://(?:www\.)?x7\.to/' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """X7.to hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py index 1794ae513..169fbc8d3 100644 --- a/module/plugins/hoster/XFileSharingPro.py +++ b/module/plugins/hoster/XFileSharingPro.py @@ -8,9 +8,9 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class XFileSharingPro(XFSHoster): __name__ = "XFileSharingPro" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.46" - __pattern__ = r'^unmatchable$' + __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)' __description__ = """XFileSharingPro dummy hoster plugin for hook""" __license__ = "GPLv3" @@ -21,7 +21,7 @@ class XFileSharingPro(XFSHoster): def _log(self, type, args): - msg = " | ".join([str(a).strip() for a in args if a]) + msg = " | ".join(str(a).strip() for a in args if a) logger = getattr(self.log, type) logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) @@ -32,10 +32,7 @@ class XFileSharingPro(XFSHoster): self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] self.HOSTER_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() - self.HOSTER_NAME = "".join([part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.']) - - if self.HOSTER_NAME[0].isdigit(): - self.HOSTER_NAME = 'X' + self.HOSTER_NAME + self.HOSTER_NAME = "".join(part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.') account = self.core.accountManager.getAccountPlugin(self.HOSTER_NAME) diff --git a/module/plugins/hoster/XHamsterCom.py b/module/plugins/hoster/XHamsterCom.py index c6e789fa8..a1711cb0e 100644 --- a/module/plugins/hoster/XHamsterCom.py +++ b/module/plugins/hoster/XHamsterCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote +import urllib from module.common.json_layer import json_loads from module.plugins.Hoster import Hoster @@ -22,7 +21,7 @@ class XHamsterCom(Hoster): __version__ = "0.12" __pattern__ = r'http://(?:www\.)?xhamster\.com/movies/.+' - __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] + __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] __description__ = """XHamster.com hoster plugin""" __license__ = "GPLv3" @@ -35,8 +34,8 @@ class XHamsterCom(Hoster): if not self.file_exists(): self.offline() - if self.getConfig("type"): - self.desired_fmt = self.getConfig("type") + if self.getConfig('type'): + self.desired_fmt = self.getConfig('type') pyfile.name = self.get_file_name() + self.desired_fmt self.download(self.get_file_url()) @@ -83,7 +82,7 @@ class XHamsterCom(Hoster): self.logDebug("long_url = " + long_url) else: if flashvars['file']: - file_url = unquote(flashvars['file']) + file_url = urllib.unquote(flashvars['file']) else: self.error(_("file_url not found")) @@ -123,7 +122,7 @@ class XHamsterCom(Hoster): """ if not self.html: self.download_html() - if re.search(r"(.*Video not found.*)", self.html) is not None: + if re.search(r"(.*Video not found.*)", self.html): return False else: return True diff --git a/module/plugins/hoster/XVideosCom.py b/module/plugins/hoster/XVideosCom.py index 9bb2da424..a8f291824 100644 --- a/module/plugins/hoster/XVideosCom.py +++ b/module/plugins/hoster/XVideosCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote +import urllib from module.plugins.Hoster import Hoster @@ -25,4 +24,4 @@ class XVideosCom(Hoster): re.search(r"<h2>([^<]+)<span", site).group(1), re.match(self.__pattern__, pyfile.url).group(1), ) - self.download(unquote(re.search(r"flv_url=([^&]+)&", site).group(1))) + self.download(urllib.unquote(re.search(r"flv_url=([^&]+)&", site).group(1))) diff --git a/module/plugins/hoster/XdadevelopersCom.py b/module/plugins/hoster/XdadevelopersCom.py new file mode 100644 index 000000000..45d1e92cb --- /dev/null +++ b/module/plugins/hoster/XdadevelopersCom.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -* +# +# Test links: +# http://forum.xda-developers.com/devdb/project/dl/?id=10885 + +import re + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class XdadevelopersCom(SimpleHoster): + __name__ = "XdadevelopersCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?forum\.xda-developers\.com/devdb/project/dl/\?id=\d+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """Xda-developers.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'<label>Filename:</label>\s*<div>\s*(?P<N>.*?)\n' + SIZE_PATTERN = r'<label>Size:</label>\s*<div>\s*(?P<S>[\d.,]+)(?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'</i> Device Filter</h3>' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + self.link = pyfile.url + "&task=get" #@TODO: Revert to `get={'task': "get"}` in 0.4.10 + + +getInfo = create_getInfo(XdadevelopersCom) diff --git a/module/plugins/hoster/Xdcc.py b/module/plugins/hoster/Xdcc.py index bfa62b8db..d167e4cab 100644 --- a/module/plugins/hoster/Xdcc.py +++ b/module/plugins/hoster/Xdcc.py @@ -4,11 +4,9 @@ import re import socket import struct import sys +import time -from os import makedirs -from os.path import exists, join from select import select -from time import time from module.plugins.Hoster import Hoster from module.utils import save_join @@ -84,12 +82,12 @@ class Xdcc(Hoster): ####################### # CONNECT TO IRC AND IDLE FOR REAL LINK - dl_time = time() + dl_time = time.time() sock = socket.socket() sock.connect((host, int(port))) if nick == "pyload": - nick = "pyload-%d" % (time() % 1000) # last 3 digits + nick = "pyload-%d" % (time.time() % 1000) # last 3 digits sock.send("NICK %s\r\n" % nick) sock.send("USER %s %s bla :%s\r\n" % (ident, host, real)) @@ -111,13 +109,13 @@ class Xdcc(Hoster): break if retry: - if time() > retry: + if time.time() > retry: retry = None - dl_time = time() + dl_time = time.time() sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) else: - if (dl_time + self.timeout) < time(): # todo: add in config + if (dl_time + self.timeout) < time.time(): # todo: add in config sock.send("QUIT :byebye\r\n") sock.close() self.fail(_("XDCC Bot did not answer")) @@ -159,7 +157,7 @@ class Xdcc(Hoster): sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) elif msg['text'] == "\x01TIME\x01": self.logDebug("Sending CTCP TIME") - sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time())) + sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) elif msg['text'] == "\x01LAG\x01": pass # don't know how to answer @@ -172,7 +170,7 @@ class Xdcc(Hoster): print "%s: %s" % (msg['origin'], msg['text']) if "You already requested that pack" in msg['text']: - retry = time() + 300 + retry = time.time() + 300 if "you must be on a known channel to request a pack" in msg['text']: self.fail(_("Wrong channel")) diff --git a/module/plugins/hoster/YadiSk.py b/module/plugins/hoster/YadiSk.py new file mode 100644 index 000000000..3b4c9d985 --- /dev/null +++ b/module/plugins/hoster/YadiSk.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +import re +import random + +from module.common.json_layer import json_loads +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class YadiSk(SimpleHoster): + __name__ = "YadiSk" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'https?://yadi\.sk/d/[\w-]+' + + __description__ = """Yadi.sk hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("GammaC0de", None)] + + + OFFLINE_PATTERN = r'Nothing found' + + + @classmethod + def getInfo(cls, url="", html=""): + info = super(YadiSk, cls).getInfo(url, html) + + if html: + if 'idclient' not in info: + info['idclient'] = "" + for _i in xrange(32): + info ['idclient'] += random.choice('0123456abcdef') + + m = re.search(r'<script id="models-client" type="application/json">(.+?)</script>', html) + if m: + api_data = json_loads(m.group(1)) + try: + for sect in api_data: + if 'model' in sect: + if sect['model'] == "config": + info['version'] = sect['data']['version'] + info['sk'] = sect['data']['sk'] + + elif sect['model'] == "resource": + info['id'] = sect['data']['id'] + info['size'] = sect['data']['meta']['size'] + info['name'] = sect['data']['name'] + + except Exception, e: + info['status'] = 8 + info['error'] = _("Unexpected server response: %s") % e.message + + else: + info['status'] = 8 + info['error'] = _("could not find required json data") + + return info + + + def setup(self): + self.resumeDownload = False + self.multiDL = False + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + if any(True for _k in ['id', 'sk', 'version', 'idclient'] if _k not in self.info): + self.error(_("Missing JSON data")) + + + try: + self.html = self.load("https://yadi.sk/models/", + get={'_m': "do-get-resource-url"}, + post={'idClient': self.info['idclient'], + 'version' : self.info['version'], + '_model.0': "do-get-resource-url", + 'sk' : self.info['sk'], + 'id.0' : self.info['id']}) + + self.link = json_loads(self.html)['models'][0]['data']['file'] + + except Exception: + pass + + +getInfo = create_getInfo(YadiSk) diff --git a/module/plugins/hoster/YibaishiwuCom.py b/module/plugins/hoster/YibaishiwuCom.py index adc403de4..00185c05a 100644 --- a/module/plugins/hoster/YibaishiwuCom.py +++ b/module/plugins/hoster/YibaishiwuCom.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re +import urlparse from module.common.json_layer import json_loads from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo @@ -12,6 +13,7 @@ class YibaishiwuCom(SimpleHoster): __version__ = "0.14" __pattern__ = r'http://(?:www\.)?(?:u\.)?115\.com/file/(?P<ID>\w+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """115.com hoster plugin""" __license__ = "GPLv3" @@ -34,7 +36,7 @@ class YibaishiwuCom(SimpleHoster): self.logDebug(('FREEUSER' if m.group(2) == 'download' else 'GUEST') + ' URL', url) - res = json_loads(self.load("http://115.com" + url, decode=False)) + res = json_loads(self.load(urlparse.urljoin("http://115.com", url), decode=False)) if "urls" in res: mirrors = res['urls'] @@ -46,11 +48,10 @@ class YibaishiwuCom(SimpleHoster): for mr in mirrors: try: - url = mr['url'].replace("\\", "") - self.logDebug("Trying URL: " + url) - self.download(url) + self.link = mr['url'].replace("\\", "") + self.logDebug("Trying URL: " + self.link) break - except: + except Exception: continue else: self.fail(_("No working link found")) diff --git a/module/plugins/hoster/YoupornCom.py b/module/plugins/hoster/YoupornCom.py index 4bb2520e6..19d07fa36 100644 --- a/module/plugins/hoster/YoupornCom.py +++ b/module/plugins/hoster/YoupornCom.py @@ -54,7 +54,7 @@ class YoupornCom(Hoster): """ if not self.html: self.download_html() - if re.search(r"(.*invalid video_id.*)", self.html) is not None: + if re.search(r"(.*invalid video_id.*)", self.html): return False else: return True diff --git a/module/plugins/hoster/YourfilesTo.py b/module/plugins/hoster/YourfilesTo.py index e0def108e..d0316d3ac 100644 --- a/module/plugins/hoster/YourfilesTo.py +++ b/module/plugins/hoster/YourfilesTo.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- -import re - -from urllib import unquote +import reimport urllib from module.plugins.Hoster import Hoster @@ -62,7 +60,7 @@ class YourfilesTo(Hoster): url = re.search(r"var bla = '(.*?)';", self.html) if url: url = url.group(1) - url = unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", "")) + url = urllib.unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", "")) return url else: self.error(_("Absolute filepath not found")) @@ -81,7 +79,7 @@ class YourfilesTo(Hoster): if not self.html: self.download_html() - if re.search(r"HTTP Status 404", self.html) is not None: + if re.search(r"HTTP Status 404", self.html): return False else: return True diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index 90c6df3c2..e2ab146a9 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -2,9 +2,7 @@ import os import re -import subprocess - -from urllib import unquote +import subprocessimport urllib from module.plugins.Hoster import Hoster from module.plugins.internal.SimpleHoster import replace_patterns @@ -13,40 +11,36 @@ from module.utils import html_unescape def which(program): """Works exactly like the unix command which - Courtesy of http://stackoverflow.com/a/377028/675646""" - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) fpath, fname = os.path.split(program) if fpath: - if is_exe(program): + if isExe(program): return program else: for path in os.environ['PATH'].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, program) - if is_exe(exe_file): + if isExe(exe_file): return exe_file - return None - class YoutubeCom(Hoster): __name__ = "YoutubeCom" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" - __pattern__ = r'https?://(?:[^/]*\.)?(?:youtube\.com|youtu\.be)/watch.*?[?&]v=.+' - __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting", "hd"), - ("fmt", "int", "FMT/ITAG Number (5-102, 0 for auto)", 0), - (".mp4", "bool", "Allow .mp4", True), - (".flv", "bool", "Allow .flv", True), - (".webm", "bool", "Allow .webm", False), - (".3gp", "bool", "Allow .3gp", False), - ("3d", "bool", "Prefer 3D", False)] + __pattern__ = r'https?://(?:[^/]*\.)?(youtube\.com|youtu\.be)/watch\?(?:.*&)?v=.+' + __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting" , "hd" ), + ("fmt" , "int" , "FMT/ITAG Number (0 for auto)", 0 ), + (".mp4" , "bool" , "Allow .mp4" , True ), + (".flv" , "bool" , "Allow .flv" , True ), + (".webm" , "bool" , "Allow .webm" , False), + (".3gp" , "bool" , "Allow .3gp" , False), + ("3d" , "bool" , "Prefer 3D" , False)] __description__ = """Youtube.com hoster plugin""" __license__ = "GPLv3" @@ -90,7 +84,7 @@ class YoutubeCom(Hoster): def process(self, pyfile): pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - html = self.load(pyfile.url, decode=True) + html = self.load(pyfile.url, decode=True) if re.search(r'<div id="player-unavailable" class="\s*player-width player-height\s*">', html): self.offline() @@ -99,33 +93,41 @@ class YoutubeCom(Hoster): self.tempOffline() #get config - use3d = self.getConfig("3d") + use3d = self.getConfig('3d') + if use3d: quality = {"sd": 82, "hd": 84, "fullhd": 85, "240p": 83, "360p": 82, "480p": 82, "720p": 84, "1080p": 85, "3072p": 85} else: quality = {"sd": 18, "hd": 22, "fullhd": 37, "240p": 5, "360p": 18, "480p": 35, "720p": 22, "1080p": 37, "3072p": 38} - desired_fmt = self.getConfig("fmt") - if desired_fmt and desired_fmt not in self.formats: + + desired_fmt = self.getConfig('fmt') + + if not desired_fmt: + desired_fmt = quality.get(self.getConfig('quality'), 18) + + elif desired_fmt not in self.formats: self.logWarning(_("FMT %d unknown, using default") % desired_fmt) desired_fmt = 0 - if not desired_fmt: - desired_fmt = quality.get(self.getConfig("quality"), 18) #parse available streams - streams = re.search(r'"url_encoded_fmt_stream_map": "(.*?)",', html).group(1) + streams = re.search(r'"url_encoded_fmt_stream_map":"(.+?)",', html).group(1) streams = [x.split('\u0026') for x in streams.split(',')] streams = [dict((y.split('=', 1)) for y in x) for x in streams] - streams = [(int(x['itag']), unquote(x['url'])) for x in streams] - #self.logDebug("Found links: %s" % streams) + streams = [(int(x['itag']), urllib.unquote(x['url'])) for x in streams] + + # self.logDebug("Found links: %s" % streams) + self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams]) #build dictionary of supported itags (3D/2D) allowed = lambda x: self.getConfig(self.formats[x][0]) streams = [x for x in streams if x[0] in self.formats and allowed(x[0])] + if not streams: self.fail(_("No available stream meets your preferences")) + fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams) self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" % @@ -136,15 +138,18 @@ class YoutubeCom(Hoster): if desired_fmt in fmt_dict and allowed(desired_fmt): fmt = desired_fmt else: - sel = lambda x: self.formats[x][3] # select quality index + sel = lambda x: self.formats[x][3] # select quality index comp = lambda x, y: abs(sel(x) - sel(y)) self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()]) + fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and sel(x) > sel(y) else y, fmt_dict.keys()) self.logDebug("Chosen fmt: %s" % fmt) + url = fmt_dict[fmt] + self.logDebug("URL: %s" % url) #set file name @@ -167,9 +172,9 @@ class YoutubeCom(Hoster): m = "0" pyfile.name += " (starting at %s:%s)" % (m, s) - pyfile.name += file_suffix - filename = self.download(url) + pyfile.name += file_suffix + filename = self.download(url) if ffmpeg and time: inputfile = filename + "_" @@ -182,4 +187,5 @@ class YoutubeCom(Hoster): "-vcodec", "copy", "-acodec", "copy", filename]) + os.remove(inputfile) diff --git a/module/plugins/hoster/ZShareNet.py b/module/plugins/hoster/ZShareNet.py index dc96facbe..6e4508a18 100644 --- a/module/plugins/hoster/ZShareNet.py +++ b/module/plugins/hoster/ZShareNet.py @@ -9,6 +9,7 @@ class ZShareNet(DeadHoster): __version__ = "0.21" __pattern__ = r'https?://(?:ww[2w]\.)?zshares?\.net/.+' + __config__ = [] #@TODO: Remove in 0.4.10 __description__ = """ZShare.net hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/hoster/ZahikiNet.py b/module/plugins/hoster/ZahikiNet.py new file mode 100644 index 000000000..b2705662b --- /dev/null +++ b/module/plugins/hoster/ZahikiNet.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + + +class ZahikiNet(SimpleHoster): + __name__ = "ZahikiNet" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?zahiki\.net/\w+/.+' + + __description__ = """Zahiki.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + DIRECT_LINK = True + + NAME_PATTERN = r'/(?P<N>.+?) </title>' + OFFLINE_PATTERN = r'>(Not Found|Il file selezionato non esiste)' + + LINK_FREE_PATTERN = r'file: "(.+?)"' + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + self.limitDL = 6 + + +getInfo = create_getInfo(ZahikiNet) diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py index fb251142e..617e00e58 100644 --- a/module/plugins/hoster/ZeveraCom.py +++ b/module/plugins/hoster/ZeveraCom.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urljoin +import urlparse from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo @@ -10,9 +9,10 @@ from module.plugins.internal.MultiHoster import MultiHoster, create_getInfo class ZeveraCom(MultiHoster): __name__ = "ZeveraCom" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.29" __pattern__ = r'https?://(?:www\.)zevera\.com/(getFiles\.ashx|Members/download\.ashx)\?.*ourl=.+' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Zevera.com multi-hoster plugin""" __license__ = "GPLv3" @@ -21,14 +21,14 @@ class ZeveraCom(MultiHoster): def handlePremium(self, pyfile): - return "https://%s/getFiles.ashx?ourl=%s" % (self.account.HOSTER_DOMAIN, pyfile.url) + self.link = "https://%s/getFiles.ashx?ourl=%s" % (self.account.HOSTER_DOMAIN, pyfile.url) - def checkFile(self): + def checkFile(self, rules={}): if self.checkDownload({"error": 'action="ErrorDownload.aspx'}): self.fail(_("Error response received")) - return super(ZeveraCom, self).checkFile() + return super(ZeveraCom, self).checkFile(rules) getInfo = create_getInfo(ZeveraCom) diff --git a/module/plugins/hoster/ZippyshareCom.py b/module/plugins/hoster/ZippyshareCom.py index 2648f532f..d776e5928 100644 --- a/module/plugins/hoster/ZippyshareCom.py +++ b/module/plugins/hoster/ZippyshareCom.py @@ -1,61 +1,94 @@ # -*- coding: utf-8 -*- import re +import urllib -from urlparse import urljoin +from BeautifulSoup import BeautifulSoup +from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class ZippyshareCom(SimpleHoster): __name__ = "ZippyshareCom" __type__ = "hoster" - __version__ = "0.66" + __version__ = "0.79" - __pattern__ = r'(?P<HOST>http://www\d{0,2}\.zippyshare\.com)/v(?:/|iew\.jsp.*key=)(?P<KEY>\d+)' + __pattern__ = r'http://www\d{0,2}\.zippyshare\.com/v(/|iew\.jsp.*key=)(?P<KEY>[\w^_]+)' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Zippyshare.com hoster plugin""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("sebdelsol", "seb.morin@gmail.com")] - NAME_PATTERN = r'("\d{6,}/"[ ]*\+.+?"/|<title>Zippyshare.com - )(?P<N>.+?)("|</title>)' - SIZE_PATTERN = r'>Size:.+?">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + COOKIES = [("zippyshare.com", "ziplocale", "en")] - OFFLINE_PATTERN = r'>File does not exist on this server<' + NAME_PATTERN = r'(<title>Zippyshare.com - |"/)(?P<N>[^/]+)(</title>|";)' + SIZE_PATTERN = r'>Size:.+?">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'does not exist (anymore )?on this server<' - COOKIES = [("zippyshare.com", "ziplocale", "en")] + LINK_PREMIUM_PATTERN = r"document.location = '(.+?)'" def setup(self): - self.multiDL = True self.chunkLimit = -1 + self.multiDL = True self.resumeDownload = True def handleFree(self, pyfile): - checksum = self.get_checksum() - p_url = '/'.join(("d", self.info['pattern']['KEY'], str(checksum), self.pyfile.name)) + recaptcha = ReCaptcha(self) + captcha_key = recaptcha.detect_key() - self.link = urljoin(self.info['pattern']['HOST'], p_url) + if captcha_key: + try: + self.link = re.search(self.LINK_PREMIUM_PATTERN, self.html) + recaptcha.challenge() + except Exception, e: + self.error(e) - def get_checksum(self): - try: - m = re.search(r'\+[ ]*\((\d+)[ ]*\%[ ]*(\d+)[ ]*\+[ ]*(\d+)[ ]*\%[ ]*(\d+)\)[ ]*\+', self.html) - if m: - a1, a2, c1, c2 = map(int, m.groups()) - b = (a1 % a2) + (c1 % c2) - else: - a1, a2 = map(int, re.search(r'\(\'downloadB\'\).omg = (\d+)%(\d+)' , self.html).groups()) - c1, c2 = map(int, re.search(r'\(\'downloadB\'\).omg\) \* \((\d+)%(\d+)', self.html).groups()) - b = (a1 % a2) * (c1 % c2) + 18 + else: + self.link = self.get_link() - except Exception, e: - self.error(_("Unable to calculate checksum"), e) + if self.link and pyfile.name == 'file.html': + pyfile.name = urllib.unquote(self.link.split('/')[-1]) - else: - return b + + def get_link(self): + # get all the scripts inside the html body + soup = BeautifulSoup(self.html) + scripts = (s.getText().strip() for s in soup.body.findAll('script', type='text/javascript')) + + # meant to be populated with the initialization of all the DOM elements found in the scripts + initScripts = set() + + def replElementById(element): + id = element.group(1) # id might be either 'x' (a real id) or x (a variable) + attr = element.group(4) # attr might be None + + varName = re.sub(r'-', '', 'GVAR[%s+"_%s"]' %(id, attr)) + + realid = id.strip('"\'') + if id != realid: #id is not a variable, so look for realid.attr in the html + initValues = filter(None, [elt.get(attr, None) for elt in soup.findAll(id=realid)]) + initValue = '"%s"' % initValues[-1] if initValues else 'null' + initScripts.add('%s = %s;' % (varName, initValue)) + + return varName + + # handle all getElementById + reVar = r'document.getElementById\(([\'"\w-]+)\)(\.)?(getAttribute\([\'"])?(\w+)?([\'"]\))?' + scripts = [re.sub(reVar, replElementById, script) for script in scripts if script] + + # add try/catch in JS to handle deliberate errors + scripts = ['\n'.join(('try{', script, '} catch(err){}')) for script in scripts] + + # get the file's url by evaluating all the scripts + scripts = ['var GVAR = {}'] + list(initScripts) + scripts + ['GVAR["dlbutton_href"]'] + return self.js.eval('\n'.join(scripts)) getInfo = create_getInfo(ZippyshareCom) diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py new file mode 100644 index 000000000..a9c194dda --- /dev/null +++ b/module/plugins/internal/AdYouLike.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +import re + +from module.common.json_layer import json_loads +from module.plugins.internal.Captcha import Captcha + + +class AdYouLike(Captcha): + __name__ = "AdYouLike" + __type__ = "captcha" + __version__ = "0.06" + + __description__ = """AdYouLike captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' + CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' + + + def detect_key(self, html=None): + html = html or self.retrieve_html() + + m = re.search(self.AYL_PATTERN, html) + n = re.search(self.CALLBACK_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) + self.logDebug("Ayl: %s | Callback: %s" % self.key) + return self.key #: key is the tuple(ayl, callback) + else: + self.logWarning("Ayl or callback pattern not found") + return None + + + def challenge(self, key=None, html=None): + ayl, callback = key or self.retrieve_key(html) + + # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + ayl = json_loads(ayl) + + html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + get={'key' : ayl['adyoulike']['key'], + 'env' : ayl['all']['env'], + 'callback': callback}) + try: + challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) + + except AttributeError: + self.fail(_("AdYouLike challenge pattern not found")) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(ayl, challenge), challenge + + + def result(self, server, challenge): + # Adyoulike.g._jsonp_5579316662423138 + # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, + # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], + # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", + # "iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\" + # height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256, + # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, + # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, + # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + + if isinstance(server, basestring): + server = json_loads(server) + + if isinstance(challenge, basestring): + challenge = json_loads(challenge) + + try: + instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] + result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() + + except AttributeError: + self.fail(_("AdYouLike result not found")) + + result = {'_ayl_captcha_engine' : "adyoulike", + '_ayl_env' : server['all']['env'], + '_ayl_tid' : challenge['tid'], + '_ayl_token_challenge': challenge['token'], + '_ayl_response' : response} + + self.logDebug("Result: %s" % result) + + return result diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py new file mode 100644 index 000000000..9cab99151 --- /dev/null +++ b/module/plugins/internal/AdsCaptcha.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import random +import re + +from module.plugins.internal.Captcha import Captcha + + +class AdsCaptcha(Captcha): + __name__ = "AdsCaptcha" + __type__ = "captcha" + __version__ = "0.09" + + __description__ = """AdsCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?.*?CaptchaId=(\d+)' + PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?.*?PublicKey=([\w-]+)' + + + def detect_key(self, html=None): + html = html or self.retrieve_html() + + m = re.search(self.PUBLICKEY_PATTERN, html) + n = re.search(self.CAPTCHAID_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) + self.logDebug("Key: %s | ID: %s" % self.key) + return self.key + else: + self.logWarning("Key or id pattern not found") + return None + + + def challenge(self, key=None, html=None): + PublicKey, CaptchaId = key or self.retrieve_key(html) + + html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", + get={'CaptchaId': CaptchaId, + 'PublicKey': PublicKey}) + try: + challenge = re.search("challenge: '(.+?)',", html).group(1) + server = re.search("server: '(.+?)',", html).group(1) + + except AttributeError: + self.fail(_("AdsCaptcha challenge pattern not found")) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(server, challenge), challenge + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, + get={'cid': challenge, 'dummy': random.random()}, + cookies=True, + imgtype="jpg") + + self.logDebug("Result: %s" % result) + + return result diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py new file mode 100644 index 000000000..b4af46493 --- /dev/null +++ b/module/plugins/internal/Captcha.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +from module.plugins.Plugin import Base + + +#@TODO: Extend (new) Plugin class; remove all `html` args +class Captcha(Base): + __name__ = "Captcha" + __type__ = "captcha" + __version__ = "0.29" + + __description__ = """Base captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + key = None #: last key detected + + + def __init__(self, plugin): + self.plugin = plugin + super(Captcha, self).__init__(plugin.core) + + + #@TODO: Recheck in 0.4.10 + def fail(self, reason): + self.plugin.fail(reason) + raise AttributeError(reason) + + + #@TODO: Recheck in 0.4.10 + def retrieve_key(self, html): + if self.detect_key(html): + return self.key + else: + self.fail(_("%s key not found") % self.__name__) + + + #@TODO: Recheck in 0.4.10 + def retrieve_html(self): + if hasattr(self.plugin, "html") and self.plugin.html: + return self.plugin.html + else: + self.fail(_("%s html not found") % self.__name__) + + + def detect_key(self, html=None): + raise NotImplementedError + + + def challenge(self, key=None, html=None): + raise NotImplementedError + + + def result(self, server, challenge): + raise NotImplementedError diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py deleted file mode 100644 index b429fd6e0..000000000 --- a/module/plugins/internal/CaptchaService.py +++ /dev/null @@ -1,353 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from base64 import urlsafe_b64encode -from random import random - -from module.common.json_layer import json_loads - - -class CaptchaService: - __name__ = "CaptchaService" - __version__ = "0.16" - - __description__ = """Base captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - KEY_PATTERN = None - - key = None #: last key detected - - - def __init__(self, plugin): - self.plugin = plugin - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("%s html not found") % self.__name__ - self.plugin.fail(errmsg) #@TODO: replace all plugin.fail(errmsg) with plugin.error(errmsg) in 0.4.10 - raise TypeError(errmsg) - - m = re.search(self.KEY_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.plugin.logDebug("%s key: %s" % (self.__name__, self.key)) - return self.key - else: - self.plugin.logDebug("%s key not found" % self.__name__) - return None - - - def challenge(self, key=None): - raise NotImplementedError - - - def result(self, server, challenge): - raise NotImplementedError - - -class ReCaptcha(CaptchaService): - __name__ = "ReCaptcha" - __version__ = "0.09" - - __description__ = """ReCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - KEY_PATTERN = r'(?:class="g-recaptcha"\s+data-sitekey="|recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=)([\w-]+)' - KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\']([\w-]+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("ReCaptcha html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.KEY_PATTERN, html) or re.search(self.KEY_AJAX_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.plugin.logDebug("ReCaptcha key: %s" % self.key) - return self.key - else: - self.plugin.logDebug("ReCaptcha key not found") - return None - - - def challenge(self, key=None, userverify=False): - if not key: - if self.detect_key(): - key = self.key - else: - errmsg = _("ReCaptcha key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) - try: - challenge = re.search("challenge : '(.+?)',", html).group(1) - server = re.search("server : '(.+?)',", html).group(1) - - except AttributeError: - errmsg = _("ReCaptcha challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.plugin.logDebug("ReCaptcha challenge: %s" % challenge) - - response = challenge, self.result(server, challenge) - - return self.userverify(*response) if userverify else response - - - def userverify(self, challenge, result): - response = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", - post={'c' : challenge, - 'response': urlsafe_b64encode('{"response":"%s"}' % result)}) - try: - return re.search(r'"uvresp","(.+?)"', response).group(1) - - except AttributeError: - errmsg = _("ReCaptcha userverify response not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha("%simage" % server, - get={'c': challenge}, - cookies=True, - forceUser=True, - imgtype="jpg") - - self.plugin.logDebug("ReCaptcha result: %s" % result) - - return result - - -class AdsCaptcha(CaptchaService): - __name__ = "AdsCaptcha" - __version__ = "0.06" - - __description__ = """AdsCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)' - PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("AdsCaptcha html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.PUBLICKEY_PATTERN, html) - n = re.search(self.CAPTCHAID_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) - self.plugin.logDebug("AdsCaptcha key|id: %s | %s" % self.key) - return self.key - else: - self.plugin.logDebug("AdsCaptcha key or id not found") - return None - - - def challenge(self, key=None): - if not key: - if self.detect_key(): - key = self.key - else: - errmsg = _("AdsCaptcha key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - PublicKey, CaptchaId = key - - html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) - try: - challenge = re.search("challenge: '(.+?)',", html).group(1) - server = re.search("server: '(.+?)',", html).group(1) - - except AttributeError: - errmsg = _("AdsCaptcha challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.plugin.logDebug("AdsCaptcha challenge: %s" % challenge) - - return challenge, self.result(server, challenge) - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, - get={'cid': challenge, 'dummy': random()}, - cookies=True, - imgtype="jpg") - - self.plugin.logDebug("AdsCaptcha result: %s" % result) - - return result - - -class SolveMedia(CaptchaService): - __name__ = "SolveMedia" - __version__ = "0.06" - - __description__ = """SolveMedia captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' - - - def challenge(self, key=None): - if not key: - if self.detect_key(): - key = self.key - else: - errmsg = _("SolveMedia key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}) - try: - challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="([^"]+)">', - html).group(1) - server = "http://api.solvemedia.com/papi/media" - - except AttributeError: - errmsg = _("SolveMedia challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.plugin.logDebug("SolveMedia challenge: %s" % challenge) - - return challenge, self.result(server, challenge) - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha(server, get={'c': challenge}, imgtype="gif") - - self.plugin.logDebug("SolveMedia result: %s" % result) - - return result - - -class AdYouLike(CaptchaService): - __name__ = "AdYouLike" - __version__ = "0.02" - - __description__ = """AdYouLike captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' - CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("AdYouLike html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.AYL_PATTERN, html) - n = re.search(self.CALLBACK_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) - self.plugin.logDebug("AdYouLike ayl|callback: %s | %s" % self.key) - return self.key #: key is the tuple(ayl, callback) - else: - self.plugin.logDebug("AdYouLike ayl or callback not found") - return None - - - def challenge(self, key=None): - if not key: - if self.detect_key(): - key = self.key - else: - errmsg = _("AdYouLike key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - ayl, callback = key - - # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, - # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} - ayl = json_loads(ayl) - - html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", - get={'key' : ayl['adyoulike']['key'], - 'env' : ayl['all']['env'], - 'callback': callback}) - try: - challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) - - except AttributeError: - errmsg = _("AdYouLike challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.plugin.logDebug("AdYouLike challenge: %s" % challenge) - - return self.result(ayl, challenge) - - - def result(self, server, challenge): - # Adyoulike.g._jsonp_5579316662423138 - # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, - # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], - # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", - # "iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\" - # height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256, - # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, - # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, - # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) - - if isinstance(server, basestring): - server = json_loads(server) - - if isinstance(challenge, basestring): - challenge = json_loads(challenge) - - try: - instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] - result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() - - except AttributeError: - errmsg = _("AdYouLike result not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - result = {'_ayl_captcha_engine' : "adyoulike", - '_ayl_env' : server['all']['env'], - '_ayl_tid' : challenge['tid'], - '_ayl_token_challenge': challenge['token'], - '_ayl_response' : response} - - self.plugin.logDebug("AdYouLike result: %s" % result) - - return result diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 0fa23eef3..a12399501 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from urllib import unquote -from urlparse import urlparse - from module.plugins.internal.SimpleCrypter import create_getInfo -from module.plugins.Crypter import Crypter as _Crypter +from module.plugins.Crypter import Crypter -class DeadCrypter(_Crypter): +class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' @@ -20,8 +17,8 @@ class DeadCrypter(_Crypter): @classmethod - def apiInfo(cls, url="", get={}, post={}): - api = super(DeadCrypter, self).apiInfo(url, get, post) + def apiInfo(cls, *args, **kwargs): + api = super(DeadCrypter, cls).apiInfo(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index cc7adf4df..22825e87b 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from urllib import unquote -from urlparse import urlparse - from module.plugins.internal.SimpleHoster import create_getInfo -from module.plugins.Hoster import Hoster as _Hoster +from module.plugins.Hoster import Hoster -class DeadHoster(_Hoster): +class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.14" + __version__ = "0.15" __pattern__ = r'^unmatchable$' @@ -20,8 +17,8 @@ class DeadHoster(_Hoster): @classmethod - def apiInfo(cls, url="", get={}, post={}): - api = super(DeadHoster, self).apiInfo(url, get, post) + def apiInfo(cls, *args, **kwargs): + api = super(DeadHoster, cls).apiInfo(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 0b2462dac..1a98060d9 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- +import os +import re + +from module.PyFile import PyFile + + class ArchiveError(Exception): pass @@ -14,28 +20,36 @@ class PasswordError(Exception): class Extractor: __name__ = "Extractor" - __version__ = "0.13" + __version__ = "0.24" __description__ = """Base extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "ranan@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz" , "immenz@gmx.net" )] EXTENSIONS = [] + REPAIR = False + VERSION = "" @classmethod - def checkDeps(cls): - """ Check if system statisfy dependencies - :return: boolean - """ - return True + def isArchive(cls, filename): + name = os.path.basename(filename).lower() + return any(name.endswith(ext) for ext in cls.EXTENSIONS) @classmethod - def isArchive(cls, file): - raise NotImplementedError + def isMultipart(cls, filename): + return False + + + @classmethod + def isUsable(cls): + """ Check if system statisfy dependencies + :return: boolean + """ + return None @classmethod @@ -45,28 +59,29 @@ class Extractor: :return: List of targets, id tuple list """ targets = [] - - for file, id in files_ids: - if cls.isArchive(file): - targets.append((file, id)) - + processed = [] + + for fname, id, fout in files_ids: + if cls.isArchive(fname): + pname = re.sub(cls.re_multipart, '', fname) if cls.isMultipart(fname) else os.path.splitext(fname)[0] + if pname not in processed: + processed.append(pname) + targets.append((fname, id, fout)) return targets - def __init__(self, m, file, out, password, fullpath, overwrite, excludefiles, renice, delete, keepbroken): - """Initialize extractor for specific file - - :param m: ExtractArchive Hook plugin - :param file: Absolute filepath - :param out: Absolute path to destination directory - :param fullpath: extract to fullpath - :param overwrite: Overwrite existing archives - :param renice: Renice value - """ - self.m = m - self.file = file + def __init__(self, manager, filename, out, + fullpath=True, + overwrite=False, + excludefiles=[], + renice=0, + delete='No', + keepbroken=False, + fid=None): + """ Initialize extractor for specific file """ + self.manager = manager + self.filename = filename self.out = out - self.password = password self.fullpath = fullpath self.overwrite = overwrite self.excludefiles = excludefiles @@ -75,50 +90,45 @@ class Extractor: self.keepbroken = keepbroken self.files = [] #: Store extracted files here + pyfile = self.manager.core.files.getFile(fid) if fid else None + self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None + def init(self): """ Initialize additional data structures """ pass - def verify(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. + def check(self): + """Quick Check by listing content of archive. + Raises error if password is needed, integrity is questionable or else. + :raises PasswordError + :raises CRCError :raises ArchiveError """ - pass - + raise NotImplementedError - def isPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. + def verify(self): + """Testing with Extractors buildt-in method + Raises error if password is needed, integrity is questionable or else. - :param password: - :return: boolean + :raises PasswordError + :raises CRCError + :raises ArchiveError """ - if isinstance(password, basestring): - return True - else: - return False - - - def setPassword(self, password): - if self.isPassword(password): - self.password = password - return True - else: - return False + raise NotImplementedError def repair(self): - return False + return None - def extract(self, progress=lambda x: None): + def extract(self, password=None): """Extract the archive. Raise specific errors in case of failure. :param progress: Progress function, call this to update status + :param password password to use :raises PasswordError :raises CRCError :raises ArchiveError @@ -132,9 +142,9 @@ class Extractor: :return: List with paths of files to delete """ - raise NotImplementedError + return [self.filename] - def getExtractedFiles(self): + def list(self, password=None): """Populate self.files at some point while extracting""" return self.files diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index a3b266679..01ff4b07d 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import re - -from time import sleep +import time +import traceback from module.plugins.Hook import Hook from module.utils import decode, remove_chars @@ -11,28 +11,22 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.35" + __version__ = "0.45" - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), - ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] __description__ = """Hook plugin for multi hoster/crypter""" __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org"), + __authors__ = [("pyLoad Team" , "admin@pyload.org" ), ("Walter Purcaro", "vuolter@gmail.com")] - MIN_INTERVAL = 1 * 60 * 60 + MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), - (r'1fichier\.com' , "onefichier.com" ), - (r'2shared\.com' , "twoshared.com" ), - (r'4shared\.com' , "fourshared.com" ), (r'bayfiles\.net' , "bayfiles.com" ), (r'cloudnator\.com' , "shragle.com" ), (r'dfiles\.eu' , "depositfiles.com" ), @@ -48,10 +42,21 @@ class MultiHook(Hook): (r'uploaded\.net' , "uploaded.to" ), (r'uploadhero\.co' , "uploadhero.com" ), (r'zshares\.net' , "zshare.net" ), - (r'\d+.+' , "X\0" )] + (r'^1' , "one" ), + (r'^2' , "two" ), + (r'^3' , "three" ), + (r'^4' , "four" ), + (r'^5' , "five" ), + (r'^6' , "six" ), + (r'^7' , "seven" ), + (r'^8' , "eight" ), + (r'^9' , "nine" ), + (r'^0' , "zero" )] def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.plugins = [] self.supported = [] self.new_supported = [] @@ -62,23 +67,22 @@ class MultiHook(Hook): self.pluginname = None self.plugintype = None - self._initPlugin() + self.initPlugin() - def _initPlugin(self): - plugin, type = self.core.pluginManager.findPlugin(self.__name__) + def initPlugin(self): + self.pluginname = self.__name__.rsplit("Hook", 1)[0] + plugin, self.plugintype = self.core.pluginManager.findPlugin(self.pluginname) - if not plugin: + if plugin: + self.pluginmodule = self.core.pluginManager.loadModule(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.pluginname) + else: self.logWarning("Hook plugin will be deactivated due missing plugin reference") self.setConfig('activated', False) - else: - self.pluginname = self.__name__ - self.plugintype = type - self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__) - self.pluginclass = getattr(self.pluginmodule, self.__name__) - def _loadAccount(self): + def loadAccount(self): self.account = self.core.accountManager.getAccountPlugin(self.pluginname) if self.account and not self.account.canUse(): @@ -89,10 +93,6 @@ class MultiHook(Hook): self.setConfig('activated', False) - def coreReady(self): - self._loadAccount() - - def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 """ see HTTPRequest for argument list """ h = pyreq.getHTTPRequest(timeout=120) @@ -106,7 +106,7 @@ class MultiHook(Hook): return rep - def getConfig(self, option, default=''): + def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10 """getConfig with default value - sublass may not implements all config options""" try: return self.getConf(option) @@ -118,18 +118,18 @@ class MultiHook(Hook): def pluginsCached(self): if self.plugins: return self.plugins - - for _i in xrange(3): + + for _i in xrange(2): try: - pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) - - except Exception, e: - self.logError(e, "Waiting 1 minute and retry") - sleep(60) - - else: + pluginset = self._pluginSet(self.getHosters()) break + + except Exception, e: + self.logDebug(e, "Waiting 1 minute and retry") + time.sleep(60) else: + self.logWarning(_("Fallback to default reload interval due plugin parse error")) + self.interval = self.MIN_RELOAD_INTERVAL return list() try: @@ -152,17 +152,15 @@ class MultiHook(Hook): def _pluginSet(self, plugins): - plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) + regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) + plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] - for rf, rt in self.DOMAIN_REPLACEMENTS: - regex = re.compile(rf) - for p in filter(lambda x: regex.match(x), plugins): - plugins.remove(p) - plugins.add(re.sub(rf, rt, p)) + for r in self.DOMAIN_REPLACEMENTS: + rf, rt = r + repr = re.compile(rf, re.I|re.U) + plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] - plugins.discard('') - - return plugins + return set(plugins) def getHosters(self): @@ -173,16 +171,30 @@ class MultiHook(Hook): raise NotImplementedError - def getCrypters(self): - """Load list of supported crypters + #: Threaded _periodical, remove in 0.4.10 and use built-in flag for that + def _periodical(self): + try: + if self.isActivated(): + self.periodical() - :return: List of domain names - """ - raise NotImplementedError + except Exception, e: + self.core.log.error(_("Error executing hooks: %s") % str(e)) + if self.core.debug: + traceback.print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical) def periodical(self): """reload plugin list periodically""" + self.loadAccount() + + if self.getConfig("reload", True): + self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) + else: + self.core.scheduler.removeJob(self.cb) + self.cb = None + self.logInfo(_("Reloading supported %s list") % self.plugintype) old_supported = self.supported @@ -200,12 +212,6 @@ class MultiHook(Hook): for plugin in old_supported: self.unloadPlugin(plugin) - if self.getConfig("reload", True): - self.interval = max(self.getConfig("reloadinterval", 12), self.MIN_INTERVAL) - else: - self.core.scheduler.removeJob(self.cb) - self.cb = None - def overridePlugins(self): excludedList = [] @@ -249,7 +255,7 @@ class MultiHook(Hook): self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) # create new regexp - regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) + regexp = r'.*(?P<DOMAIN>%s).*' % "|".join(x.replace('.', '\.') for x in plugins) if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) @@ -263,11 +269,11 @@ class MultiHook(Hook): def unloadPlugin(self, plugin): hdict = self.core.pluginManager.plugins[self.plugintype][plugin] if "module" in hdict: - del hdict['module'] + hdict.pop('module', None) if "new_module" in hdict: - del hdict['new_module'] - del hdict['new_name'] + hdict.pop('new_module', None) + hdict.pop('new_name', None) def unload(self): @@ -280,25 +286,3 @@ class MultiHook(Hook): hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') hdict['re'] = re.compile(hdict['pattern']) - - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.status != 8 or not self.getConfig("revertfailed", True): - return - - hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.pluginname: - if pyfile.error == "MultiHook": - self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) - self.unloadPlugin(pyfile.pluginname) - pyfile.setStatus("queued") - else: - retries = max(self.getConfig("retry", 10), 0) - wait_time = max(self.getConfig("retryinterval", 1), 0) - - if 0 < retries > pyfile.plugin.retries: - pyfile.setCustomStatus("MultiHook", "queued") - pyfile.plugin.retries += 1 - pyfile.plugin.setWait(wait_time) - pyfile.plugin.wait() diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 8ca4d427f..ff4414034 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,22 +2,24 @@ import re +from module.plugins.Plugin import Fail, Retry from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.40" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("revertfailed", "bool", "Revert to standard download if fails", True)] __description__ = """Multi hoster plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - CHECK_TRAFFIC = True LOGIN_ACCOUNT = True @@ -28,9 +30,13 @@ class MultiHoster(SimpleHoster): def prepare(self): - self.info = {} - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.directDL = False #@TODO: Move to hoster class in 0.4.10 + self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + + if not self.getConfig('use_premium', True): + self.retryFree() if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -41,47 +47,64 @@ class MultiHoster(SimpleHoster): set_cookies(self.req.cj, self.COOKIES) if self.DIRECT_LINK is None: - self.directDL = self.__pattern__ != r'^unmatchable$' + self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) else: self.directDL = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, - self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) def process(self, pyfile): - self.prepare() + try: + self.prepare() + + if self.directDL: + self.checkInfo() + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) + + if not self.link and not self.lastDownload: + self.preload() + + self.checkErrors() + self.checkStatus(getinfo=False) + + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) + + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) - if self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, pyfile.url): - self.checkInfo() + self.downloadLink(self.link, True) + self.checkFile() - if self.directDL: - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + err = str(e) #@TODO: Recheck in 0.4.10 - if not self.link and not self.lastDownload: - self.preload() + if self.premium: + self.logWarning(_("Premium download failed")) + self.retryFree() - self.checkErrors() - self.checkStatus(getinfo=False) + elif self.getConfig("revertfailed", True) \ + and "new_module" in self.core.pluginManager.hosterPlugins[self.__name__]: + hdict = self.core.pluginManager.hosterPlugins[self.__name__] - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) + tmp_module = hdict['new_module'] + tmp_name = hdict['new_name'] + hdict.pop('new_module', None) + hdict.pop('new_name', None) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) + pyfile.initPlugin() - self.downloadLink(self.link) - self.checkFile() + hdict['new_module'] = tmp_module + hdict['new_name'] = tmp_name + raise Retry(_("Revert to original hoster plugin")) - #@TODO: Remove in 0.4.10 - def downloadLink(self, link): - if link and isinstance(link, basestring): - self.correctCaptcha() - self.download(link, disposition=True) + else: + raise Fail(err) def handlePremium(self, pyfile): diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py new file mode 100644 index 000000000..a9d0f3752 --- /dev/null +++ b/module/plugins/internal/ReCaptcha.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- + +import random +import re +import time +import urlparse + +from base64 import b64encode + +from module.plugins.internal.Captcha import Captcha + + +class ReCaptcha(Captcha): + __name__ = "ReCaptcha" + __type__ = "captcha" + __version__ = "0.17" + + __description__ = """ReCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("zapp-brannigan", "fuerst.reinje@web.de")] + + + KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' + KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' + + + def detect_key(self, html=None): + html = html or self.retrieve_html() + + m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) + if m: + self.key = m.group(1).strip() + self.logDebug("Key: %s" % self.key) + return self.key + else: + self.logWarning("Key pattern not found") + return None + + + def challenge(self, key=None, html=None, version=None): + key = key or self.retrieve_key(html) + + if version in (1, 2): + return getattr(self, "_challenge_v%s" % version)(key) + + else: + return self.challenge(key, + version=2 if re.search(self.KEY_V2_PATTERN, html or self.retrieve_html()) else 1) + + + def _challenge_v1(self, key): + html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", + get={'k': key}) + try: + challenge = re.search("challenge : '(.+?)',", html).group(1) + server = re.search("server : '(.+?)',", html).group(1) + + except AttributeError: + self.fail(_("ReCaptcha challenge pattern not found")) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(server, challenge, key) + + + def result(self, server, challenge, key): + self.plugin.req.load("http://www.google.com/recaptcha/api/js/recaptcha.js") + html = self.plugin.req.load("http://www.google.com/recaptcha/api/reload", + get={'c' : challenge, + 'k' : key, + 'reason': "i", + 'type' : "image"}) + + try: + challenge = re.search('\(\'(.+?)\',',html).group(1) + + except AttributeError: + self.fail(_("ReCaptcha second challenge pattern not found")) + + self.logDebug("Second challenge: %s" % challenge) + result = self.plugin.decryptCaptcha("%simage" % server, + get={'c': challenge}, + cookies=True, + forceUser=True, + imgtype="jpg") + + self.logDebug("Result: %s" % result) + + return result, challenge + + + def _collectApiInfo(self): + html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") + a = re.search(r'po.src = \'(.*?)\';', html).group(1) + vers = a.split("/")[5] + + self.logDebug("API version: %s" % vers) + + language = a.split("__")[1].split(".")[0] + + self.logDebug("API language: %s" % language) + + html = self.plugin.req.load("https://apis.google.com/js/api.js") + b = re.search(r'"h":"(.*?)","', html).group(1) + jsh = b.decode('unicode-escape') + + self.logDebug("API jsh-string: %s" % jsh) + + return vers, language, jsh + + + def _prepareTimeAndRpc(self): + self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") + + millis = int(round(time.time() * 1000)) + + self.logDebug("Time: %s" % millis) + + rand = random.randint(1, 99999999) + a = "0.%s" % str(rand * 2147483647) + rpc = int(100000000 * float(a)) + + self.logDebug("Rpc-token: %s" % rpc) + + return millis, rpc + + + def _challenge_v2(self, key, parent=None): + if parent is None: + try: + parent = urlparse.urljoin("http://", urlparse.urlparse(self.plugin.pyfile.url).netloc) + + except Exception: + parent = "" + + botguardstring = "!A" + vers, language, jsh = self._collectApiInfo() + millis, rpc = self._prepareTimeAndRpc() + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", + get={'k' : key, + 'hl' : language, + 'v' : vers, + 'usegapi' : "1", + 'jsh' : "%s#id=IO_%s" % (jsh, millis), + 'parent' : parent, + 'pfname' : "", + 'rpctoken': rpc}) + + token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) + self.logDebug("Token #1: %s" % token1.group(1)) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", + get={'c' : token1.group(1), + 'hl' : language, + 'v' : vers, + 'bg' : botguardstring, + 'k' : key, + 'usegapi': "1", + 'jsh' : jsh}).decode('unicode-escape') + + token2 = re.search(r'"finput","(.*?)",', html) + self.logDebug("Token #2: %s" % token2.group(1)) + + token3 = re.search(r'"rresp","(.*?)",', html) + self.logDebug("Token #3: %s" % token3.group(1)) + + millis_captcha_loading = int(round(time.time() * 1000)) + captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", + get={'c':token3.group(1), 'k':key}, + cookies=True, + forceUser=True) + response = b64encode('{"response":"%s"}' % captcha_response) + + self.logDebug("Result: %s" % response) + + timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading + timeToSolveMore = timeToSolve + int(float("0." + str(random.randint(1, 99999999))) * 500) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", + post={'k' : key, + 'c' : token3.group(1), + 'response': response, + 't' : timeToSolve, + 'ct' : timeToSolveMore, + 'bg' : botguardstring}) + + token4 = re.search(r'"uvresp","(.*?)",', html) + self.logDebug("Token #4: %s" % token4.group(1)) + + result = token4.group(1) + + return result, None diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py new file mode 100644 index 000000000..624f6c939 --- /dev/null +++ b/module/plugins/internal/SevenZip.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +import os +import re +import subprocess + +from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice +from module.utils import fs_encode, save_join + + +class SevenZip(UnRar): + __name__ = "SevenZip" + __version__ = "0.11" + + __description__ = """7-Zip extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("Michael Nowak" , "" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + CMD = "7z" + VERSION = "" + + EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2", + ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab", + ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha", + ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt", + ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr", + ".squashfs", ".cramfs", ".scap"] + + + #@NOTE: there are some more uncovered 7z formats + re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') + re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I) + re_wrongcrc = re.compile(r'CRC Failed|Can not open file', re.I) + re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) + + + @classmethod + def isUsable(cls): + if os.name == "nt": + cls.CMD = os.path.join(pypath, "7z.exe") + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + else: + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + m = cls.re_version.search(out) + cls.VERSION = m.group(1) if m else '(version unknown)' + + return True + + + def verify(self, password): + # 7z can't distinguish crc and pw error in test + p = self.call_cmd("l", "-slt", fs_encode(self.filename)) + out, err = p.communicate() + + if self.re_wrongpwd.search(out): + raise PasswordError + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): + raise CRCError(err) + + + + def check(self, password): + p = self.call_cmd("l", "-slt", fs_encode(self.filename)) + out, err = p.communicate() + + # check if output or error macthes the 'wrong password'-Regexp + if self.re_wrongpwd.search(out): + raise PasswordError + + if self.re_wrongcrc.search(out): + raise CRCError(_("Header protected")) + + + def repair(self): + return False + + + def extract(self, password=None): + command = "x" if self.fullpath else "e" + + p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password) + + renice(p.pid, self.renice) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError(err) + + else: #: raise error if anything is on stderr + raise ArchiveError(err) + + if p.returncode > 1: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + self.files = self.list(password) + + + def list(self, password=None): + command = "l" if self.fullpath else "l" + + p = self.call_cmd(command, fs_encode(self.filename), password=password) + out, err = p.communicate() + + if "Can not open" in err: + raise ArchiveError(_("Cannot open file")) + + if p.returncode > 1: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + result = set() + for groups in self.re_filelist.findall(out): + f = groups[-1].strip() + result.add(save_join(self.out, f)) + + return list(result) + + + def call_cmd(self, command, *xargs, **kwargs): + args = [] + + #overwrite flag + if self.overwrite: + args.append("-y") + + #set a password + if "password" in kwargs and kwargs["password"]: + args.append("-p%s" % kwargs["password"]) + else: + args.append("-p-") + + #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue + call = [self.CMD, command] + args + list(xargs) + + self.manager.logDebug(" ".join(call)) + + p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return p diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 43b1347fd..81f977e57 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -1,28 +1,25 @@ # -*- coding: utf-8 -*- import re - -from urlparse import urlparse +import urlparse from module.plugins.Crypter import Crypter from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies -from module.utils import fixup +from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.38" + __version__ = "0.49" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] """ @@ -58,25 +55,16 @@ class SimpleCrypter(Crypter, SimpleHoster): LINK_PATTERN = None - NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - URL_REPLACEMENTS = [] - - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] - - LOGIN_ACCOUNT = False - LOGIN_PREMIUM = False - #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.core.accountManager.getAccountPlugin(account_name) + account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) if account and account.canUse(): self.user, data = account.selectAccount() - self.req = account.getAccountRequest(self.user) - self.premium = account.isPremium(self.user) + self.req = account.getAccountRequest(self.user) + self.premium = account.isPremium(self.user) self.account = account @@ -85,6 +73,8 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.error = "" #@TODO: Remove in 0.4.10 self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 self.links = [] #@TODO: Move to hoster class in 0.4.10 if self.LOGIN_PREMIUM and not self.premium: @@ -101,24 +91,40 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + def handleDirect(self, pyfile): + while True: + header = self.load(self.link or pyfile.url, just_header=True, decode=True) + if 'location' in header and header['location']: + self.link = header['location'] + else: + break + + def decrypt(self, pyfile): self.prepare() - self.preload() - self.checkInfo() + self.logDebug("Looking for link redirect...") + self.handleDirect(pyfile) - self.links = self.getLinks() + if self.link: + self.urls = [self.link] - if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): - self.handlePages(pyfile) + else: + self.preload() + self.checkInfo() - self.logDebug("Package has %d links" % len(self.links)) + self.links = self.getLinks() + + if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): + self.handlePages(pyfile) + + self.logDebug("Package has %d links" % len(self.links)) if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 - self.fail("No link grabbed") + self.fail(_("No link grabbed")) def checkNameSize(self, getinfo=True): @@ -128,8 +134,8 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("File info (AFTER): %s" % self.info) try: - url = self.info['url'] - name = self.info['name'] + url = self.info['url'].strip() + name = self.info['name'].strip() if name and name != url: self.pyfile.name = name @@ -151,13 +157,19 @@ class SimpleCrypter(Crypter, SimpleHoster): Returns the links extracted from self.html You should override this only if it's impossible to extract links using only the LINK_PATTERN. """ - return re.findall(self.LINK_PATTERN, self.html) + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + links = [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ + for link in re.findall(self.LINK_PATTERN, self.html)] + + return [html_unescape(l.decode('unicode-escape').strip()) for l in links] def handlePages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except: + except Exception: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py deleted file mode 100644 index 53b80f827..000000000 --- a/module/plugins/internal/SimpleDereferer.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urllib import unquote - -from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import directLink, set_cookies - - -class SimpleDereferer(Crypter): - __name__ = "SimpleDereferer" - __type__ = "crypter" - __version__ = "0.03" - - __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Simple dereferer plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - """ - Following patterns should be defined by each crypter: - - LINK_PATTERN: Regex to catch the redirect url in group(1) - example: LINK_PATTERN = r'<div class="link"><a href="(.+?)"' - - OFFLINE_PATTERN: (optional) Checks if the page is unreachable - example: OFFLINE_PATTERN = r'File (deleted|not found)' - - TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable - example: TEMP_OFFLINE_PATTERN = r'Server maintainance' - - - You can override the getLinks method if you need a more sophisticated way to extract the redirect url. - """ - - LINK_PATTERN = None - - TEXT_ENCODING = False - COOKIES = True - - - def decrypt(self, pyfile): - link = directLink(self, pyfile.url) - - if not link: - try: - link = unquote(re.match(self.__pattern__, pyfile.url).group('LINK')) - - except Exception: - self.prepare() - self.preload() - - if self.html is None: - self.fail(_("No html retrieved")) - - self.checkStatus() - - link = self.getLink() - - if link.strip(): - self.urls = [link.strip()] #@TODO: Remove `.strip()` in 0.4.10 - - elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 - self.fail("No link grabbed") - - - def prepare(self): - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - - def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) - - - def checkStatus(self): - if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, self.html): - self.offline() - - elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() - - - def getLink(self): - try: - return re.search(self.LINK_PATTERN, self.html).group(1) - - except Exception: - pass diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index fa1ea45f2..4c1441e4b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- +import datetime +import mimetypes +import os import re - -from inspect import isclass -from os.path import exists -from time import time -from urllib import unquote -from urlparse import urljoin, urlparse +import time +import urllib +import urlparse from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster -from module.plugins.Plugin import Fail -from module.utils import fixup, fs_encode, parseFileSize +from module.plugins.Plugin import Fail, Retry +from module.utils import fixup, fs_encode, html_unescape, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -27,7 +27,7 @@ def _error(self, reason, type): type = "unknown" msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += ": %s" % reason.strip() if reason else "" + msg += (": %s" % reason.strip()) if reason else "" msg += _(" | Plugin may be out of date") raise Fail(msg) @@ -74,7 +74,7 @@ def parseHtmlForm(attr_str, html, input_names={}): if name: value = parseHtmlTagAttrValue("value", inputtag.group(1)) if not value: - inputs[name] = inputtag.group(3) or '' + inputs[name] = inputtag.group(3) or "" else: inputs[name] = value @@ -100,43 +100,77 @@ def parseHtmlForm(attr_str, html, input_names={}): return {}, None #: no matching form found -#: Deprecated +#@TODO: Remove in 0.4.10 def parseFileInfo(plugin, url="", html=""): if hasattr(plugin, "getInfo"): info = plugin.getInfo(url, html) res = info['name'], info['size'], info['status'], info['url'] else: - res = urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 0, 3, url + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + res = ((url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 0, + 3 if url else 8, + url) return res #@TODO: Remove in 0.4.10 -#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 def create_getInfo(plugin): + def getInfo(urls): + for url in urls: + if hasattr(plugin, "URL_REPLACEMENTS"): + url = replace_patterns(url, plugin.URL_REPLACEMENTS) + yield parseFileInfo(plugin, url) + + return getInfo - def generator(list): - for x in list: - yield x - if hasattr(plugin, "parseInfos"): - fn = lambda urls: generator((info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)) +def timestamp(): + return int(time.time() * 1000) + + +#@TODO: Move to hoster class in 0.4.10 +def getFileURL(self, url, follow_location=None): + link = "" + redirect = 1 + + if type(follow_location) is int: + redirect = max(follow_location, 1) else: - fn = lambda urls: generator(parseFileInfo(url) for url in urls) + redirect = 10 - return fn + for i in xrange(redirect): + try: + self.logDebug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, just_header=True, decode=True) + except Exception: #: Bad bad bad... rewrite this part in 0.4.10 + req = pyreq.getHTTPRequest() + res = req.load(url, just_header=True, decode=True) -def timestamp(): - return int(time() * 1000) + req.close() + header = {"code": req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: + continue -#@TODO: Move to hoster class in 0.4.10 -def directLink(self, url, resumable=False): - link = "" + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() - for i in xrange(5 if resumable else 1): - header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value if 'content-disposition' in header: link = url @@ -144,42 +178,59 @@ def directLink(self, url, resumable=False): elif 'location' in header and header['location']: location = header['location'] - if not urlparse(location).scheme: - p = urlparse(url) - base = "%s://%s" % (p.scheme, p.netloc) - location = urljoin(base, location) + if not urlparse.urlparse(location).scheme: + url_p = urlparse.urlparse(url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + location = urlparse.urljoin(baseurl, location) - if resumable: + if 'code' in header and header['code'] == 302: + link = location + + if follow_location: url = location - self.logDebug("Redirect #%d to: %s" % (++i, location)) continue - elif 'code' in header and header['code'] == 302: - link = location + else: + extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] - elif 'content-type' in header and header['content-type'] and "html" not in header['content-type']: - link = url + if 'content-type' in header and header['content-type']: + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" break + else: - self.logError(_("Too many redirects")) + try: + self.logError(_("Too many redirects")) + except Exception: + pass return link def secondsToMidnight(gmt=0): - now = datetime.utcnow() + timedelta(hours=gmt) + now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) if now.hour is 0 and now.minute < 10: midnight = now else: - midnight = now + timedelta(days=1) + midnight = now + datetime.timedelta(days=1) td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now if hasattr(td, 'total_seconds'): res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds + else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 return int(res) @@ -188,39 +239,39 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.96" + __version__ = "1.54" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium", "bool", "Use premium account if available" , True), + ("fallback" , "bool", "Fallback to free download if premium fails", True)] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] """ - Info patterns should be defined by each hoster: + Info patterns: - INFO_PATTERN: (optional) Name and Size of the file + INFO_PATTERN: (mandatory) Name and Size of the file example: INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)' or - NAME_PATTERN: (optional) Name that will be set for the file + NAME_PATTERN: (mandatory) Name that will be set for the file example: NAME_PATTERN = r'(?P<N>file_name)' - SIZE_PATTERN: (optional) Size that will be checked for the file + SIZE_PATTERN: (mandatory) Size that will be checked for the file example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)' HASHSUM_PATTERN: (optional) Hash code and type of the file example: HASHSUM_PATTERN = r'(?P<H>hash_code) (?P<T>MD5)' - OFFLINE_PATTERN: (optional) Check if the page is unreachable + OFFLINE_PATTERN: (mandatory) Check if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - Error handling patterns are all optional: + Error patterns: WAIT_PATTERN: (optional) Detect waiting time example: WAIT_PATTERN = r'' @@ -228,11 +279,23 @@ class SimpleHoster(Hoster): PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + HAPPY_HOUR_PATTERN: (optional) + example: HAPPY_HOUR_PATTERN = r'Happy hour' + + IP_BLOCKED_PATTERN: (optional) + example: IP_BLOCKED_PATTERN = r'in your country' + + DOWNLOAD_LIMIT_PATTERN: (optional) + example: DOWNLOAD_LIMIT_PATTERN = r'download limit' + + SIZE_LIMIT_PATTERN: (optional) + example: SIZE_LIMIT_PATTERN = r'up to' + ERROR_PATTERN: (optional) Detect any error preventing download example: ERROR_PATTERN = r'' - Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: + Instead overriding handleFree and handlePremium methods you may define the following patterns for basic link handling: LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download example: LINK_FREE_PATTERN = r'<div class="link"><a href="(.+?)"' @@ -245,29 +308,26 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + COOKIES = True #: or False or list of tuples [(domain, name, value)] DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + DISPOSITION = True #: Set to True to use any content-disposition value in http header as file name LOGIN_ACCOUNT = False #: Set to True to require account login - - directLink = directLink #@TODO: Remove in 0.4.10 + LOGIN_PREMIUM = False #: Set to True to require premium account login + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct - @classmethod - def parseInfos(cls, urls): #@TODO: Built-in in 0.4.10 core, then remove from plugins - for url in urls: - url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - yield cls.getInfo(url) + directLink = getFileURL #@TODO: Remove in 0.4.10 @classmethod - def apiInfo(cls, url="", get={}, post={}): - url = unquote(url) - return {'name' : (urlparse(url).path.split('/')[-1] - or urlparse(url).query.split('=', 1)[::-1][0].split('&', 1)[0] - or _("Unknown")), + def apiInfo(cls, url): + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), 'size' : 0, 'status': 3 if url else 8, 'url' : url} @@ -276,7 +336,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): info = cls.apiInfo(url) - online = False + online = True if info['status'] is 2 else False try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here @@ -284,7 +344,7 @@ class SimpleHoster(Hoster): except Exception: info['pattern'] = {} - if not html: + if not html and not online: if not url: info['error'] = "missing url" info['status'] = 1 @@ -305,21 +365,18 @@ class SimpleHoster(Hoster): elif e.code is 503: info['status'] = 6 + except Exception: + pass + if html: if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): info['status'] = 1 - elif hasattr(cls, "FILE_OFFLINE_PATTERN") and re.search(cls.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 - info['status'] = 1 - elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html): info['status'] = 6 else: - for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", - "FILE_NAME_PATTERN", "NAME_PATTERN", - "FILE_SIZE_PATTERN", "SIZE_PATTERN", - "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 + for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", "HASHSUM_PATTERN"): try: attr = getattr(cls, pattern) pdict = re.search(attr, html).groupdict() @@ -333,19 +390,16 @@ class SimpleHoster(Hoster): else: online = True - if not info['pattern']: - info.pop('pattern', None) - if online: info['status'] = 2 if 'N' in info['pattern']: - info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()), - cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 + info['name'] = replace_patterns(urllib.unquote(info['pattern']['N'].strip()), + cls.NAME_REPLACEMENTS) if 'S' in info['pattern']: size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], - cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 + cls.SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) elif isinstance(info['size'], basestring): @@ -356,6 +410,9 @@ class SimpleHoster(Hoster): hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash" info[hashtype] = info['pattern']['H'] + if not info['pattern']: + info.pop('pattern', None) + return info @@ -367,10 +424,17 @@ class SimpleHoster(Hoster): self.pyfile.error = "" #@TODO: Remove in 0.4.10 self.info = {} + self.html = "" self.link = "" #@TODO: Move to hoster class in 0.4.10 self.directDL = False #@TODO: Move to hoster class in 0.4.10 self.multihost = False #@TODO: Move to hoster class in 0.4.10 + if not self.getConfig('use_premium', True): + self.retryFree() + + if self.LOGIN_PREMIUM and not self.premium: + self.fail(_("Required premium account not found")) + if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -390,8 +454,7 @@ class SimpleHoster(Hoster): else: self.directDL = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, - self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) def preload(self): @@ -402,68 +465,106 @@ class SimpleHoster(Hoster): def process(self, pyfile): - self.prepare() - self.checkInfo() + try: + self.prepare() + self.checkInfo() - if self.directDL: - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) - if self.multihost and not self.link and not self.lastDownload: - self.logDebug("Looking for leeched download link...") - self.handleMulti(pyfile) + if self.multihost and not self.link and not self.lastDownload: + self.logDebug("Looking for leeched download link...") + self.handleMulti(pyfile) + + if not self.link and not self.lastDownload: + self.MULTI_HOSTER = False + self.retry(1, reason=_("Multi hoster fails")) if not self.link and not self.lastDownload: - self.MULTI_HOSTER = False - self.retry(1, reason="Multi hoster fails") + self.preload() + self.checkInfo() - if not self.link and not self.lastDownload: - self.preload() - self.checkInfo() + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) + self.downloadLink(self.link, self.DISPOSITION) + self.checkFile() - self.downloadLink(self.link) - self.checkFile() + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + err = str(e) #@TODO: Recheck in 0.4.10 + if err == _("No captcha result obtained in appropiate time by any of the plugins."): #@TODO: Fix in 0.4.10 + self.checkFile() - def downloadLink(self, link): - if link and isinstance(link, basestring): - self.correctCaptcha() - self.download(link, disposition=False) #@TODO: Set `disposition=True` in 0.4.10 + elif self.getConfig('fallback', True) and self.premium: + self.logWarning(_("Premium download failed"), e) + self.retryFree() + else: + raise Fail(err) - def checkFile(self): + + def downloadLink(self, link, disposition=True): + if not link or not isinstance(link, basestring): + return + + self.correctCaptcha() + + link = html_unescape(link.decode('unicode-escape').strip()) #@TODO: Move this check to plugin `load` method in 0.4.10 + + if not urlparse.urlparse(link).scheme: + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + link = urlparse.urljoin(baseurl, link) + + self.download(link, ref=False, disposition=disposition) + + + def checkFile(self, rules={}): if self.cTask and not self.lastDownload: self.invalidCaptcha() self.retry(10, reason=_("Wrong captcha")) - elif not self.lastDownload or not exists(fs_encode(self.lastDownload)): + elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)): self.lastDownload = "" - self.fail(self.pyfile.error or _("No file downloaded")) + self.error(self.pyfile.error or _("No file downloaded")) else: - rules = {'empty file': re.compile(r'\A\Z'), - 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), - 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')} + errmsg = self.checkDownload({'Empty file': re.compile(r'\A\s*\Z'), + 'Html error': re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')}) - if hasattr(self, 'ERROR_PATTERN'): - rules['error'] = re.compile(self.ERROR_PATTERN) + if not errmsg: + for r, p in [('Html file' , re.compile(r'\A\s*<!DOCTYPE html') ), + ('Request error', re.compile(r'([Aa]n error occured while processing your request)'))]: + if r not in rules: + rules[r] = p - check = self.checkDownload(rules) - if check: #@TODO: Move to hoster in 0.4.10 - errmsg = check.strip().capitalize() - if self.lastCheck: - errmsg += " | " + self.lastCheck.group(0).strip() + for r, a in [('Error' , "ERROR_PATTERN" ), + ('Premium only', "PREMIUM_ONLY_PATTERN"), + ('Wait error' , "WAIT_PATTERN" )]: + if r not in rules and hasattr(self, a): + rules[r] = getattr(self, a) - self.lastDownload = "" - self.retry(10, 60, errmsg) + errmsg = self.checkDownload(rules) + + if not errmsg: + return + + errmsg = errmsg.strip().capitalize() + + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass + + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.retry(3, 60, errmsg) def checkErrors(self): @@ -471,30 +572,111 @@ class SimpleHoster(Hoster): self.logWarning(_("No html code to check")) return - if hasattr(self, 'PREMIUM_ONLY_PATTERN') and self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): - self.fail(_("Link require a premium account to be handled")) + if hasattr(self, 'IP_BLOCKED_PATTERN') and re.search(self.IP_BLOCKED_PATTERN, self.html): + self.fail(_("Connection from your current IP address is not allowed")) + + elif not self.premium: + if hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html): + self.fail(_("File can be downloaded by premium users only")) + + elif hasattr(self, 'SIZE_LIMIT_PATTERN') and re.search(self.SIZE_LIMIT_PATTERN, self.html): + self.fail(_("File too large for free download")) - elif hasattr(self, 'ERROR_PATTERN'): + elif hasattr(self, 'DOWNLOAD_LIMIT_PATTERN') and re.search(self.DOWNLOAD_LIMIT_PATTERN, self.html): + m = re.search(self.DOWNLOAD_LIMIT_PATTERN, self.html) + try: + errmsg = m.group(1).strip() + except Exception: + errmsg = m.group(0).strip() + + self.info['error'] = re.sub(r'<.*?>', " ", errmsg) + self.logWarning(self.info['error']) + + if re.search('da(il)?y|today', errmsg, re.I): + wait_time = secondsToMidnight(gmt=2) + else: + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) + + self.wantReconnect = wait_time > 300 + self.retry(1, wait_time, _("Download limit exceeded")) + + if hasattr(self, 'HAPPY_HOUR_PATTERN') and re.search(self.HAPPY_HOUR_PATTERN, self.html): + self.multiDL = True + + if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: - errmsg = self.info['error'] = m.group(1) - self.error(errmsg) + try: + errmsg = m.group(1).strip() + except Exception: + errmsg = m.group(0).strip() + + self.info['error'] = re.sub(r'<.*?>', " ", errmsg) + self.logWarning(self.info['error']) + + if re.search('limit|wait|slot', errmsg, re.I): + if re.search("da(il)?y|today", errmsg): + wait_time = secondsToMidnight(gmt=2) + else: + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) + + self.wantReconnect = wait_time > 300 + self.retry(1, wait_time, _("Download limit exceeded")) + + elif re.search('country|ip|region|nation', errmsg, re.I): + self.fail(_("Connection from your current IP address is not allowed")) + + elif re.search('captcha|code', errmsg, re.I): + self.invalidCaptcha() + self.retry(10, reason=_("Wrong captcha")) + + elif re.search('countdown|expired', errmsg, re.I): + self.retry(10, wait_time=60, reason=_("Link expired")) + + elif re.search('maintenance|maintainance|temp', errmsg, re.I): + self.tempOffline() + + elif re.search('up to|size', errmsg, re.I): + self.fail(_("File too large for free download")) + + elif re.search('offline|delet|remov|not? (found|(longer)? available)', errmsg, re.I): + self.offline() + + elif re.search('filename', errmsg, re.I): + url_p = urlparse.urlparse(self.pyfile.url) + self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.strip('/').split('/')[0]) + self.retry(1, reason=_("Wrong url")) + + elif re.search('premium', errmsg, re.I): + self.fail(_("File can be downloaded by premium users only")) + + else: + self.wantReconnect = True + self.retry(wait_time=60, reason=errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) if m: - wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) + try: + waitmsg = m.group(1).strip() + except Exception: + waitmsg = m.group(0).strip() + + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec|)', waitmsg, re.I)) self.wait(wait_time, wait_time > 300) - return self.info.pop('error', None) def checkStatus(self, getinfo=True): if not self.info or getinfo: - self.logDebug("File info (BEFORE): %s" % self.info) + self.logDebug("Update file info...") + self.logDebug("Previous file info: %s" % self.info) self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("Current file info: %s" % self.info) try: status = self.info['status'] @@ -506,22 +688,22 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is 8: - self.fail() + self.fail(self.info['error'] if 'error' in self.info else _("Failed")) finally: - self.logDebug("File status: %s" % statusMap[status], - "File info: %s" % self.info) + self.logDebug("File status: %s" % statusMap[status]) def checkNameSize(self, getinfo=True): if not self.info or getinfo: - self.logDebug("File info (BEFORE): %s" % self.info) + self.logDebug("Update file info...") + self.logDebug("Previous file info: %s" % self.info) self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("File info (AFTER): %s" % self.info) + self.logDebug("Current file info: %s" % self.info) try: - url = self.info['url'] - name = self.info['name'] + url = self.info['url'].strip() + name = self.info['name'].strip() if name and name != url: self.pyfile.name = name @@ -537,7 +719,7 @@ class SimpleHoster(Hoster): pass self.logDebug("File name: %s" % self.pyfile.name, - "File size: %s" % self.pyfile.size if self.pyfile.size > 0 else "Unknown") + "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") def checkInfo(self): @@ -562,7 +744,6 @@ class SimpleHoster(Hoster): if link: self.logInfo(_("Direct download link detected")) - self.link = link else: self.logDebug("Direct download link not found") @@ -576,16 +757,12 @@ class SimpleHoster(Hoster): if not hasattr(self, 'LINK_FREE_PATTERN'): self.logError(_("Free download not implemented")) - try: - m = re.search(self.LINK_FREE_PATTERN, self.html) - if m is None: - self.error(_("Free download link not found")) - + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("Free download link not found")) + else: self.link = m.group(1) - except Exception, e: - self.fail(e) - def handlePremium(self, pyfile): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): @@ -593,16 +770,12 @@ class SimpleHoster(Hoster): self.logDebug("Handled as free download") self.handleFree(pyfile) - try: - m = re.search(self.LINK_PREMIUM_PATTERN, self.html) - if m is None: - self.error(_("Premium download link not found")) - + m = re.search(self.LINK_PREMIUM_PATTERN, self.html) + if m is None: + self.error(_("Premium download link not found")) + else: self.link = m.group(1) - except Exception, e: - self.fail(e) - def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): @@ -614,8 +787,7 @@ class SimpleHoster(Hoster): self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) - self.setWait(wait_time, True) - self.wait() + self.wait(wait_time, True) self.retry(max_tries=max_tries, reason=_("Download limit reached")) @@ -640,6 +812,26 @@ class SimpleHoster(Hoster): #@TODO: Remove in 0.4.10 + def getConfig(self, option, default=''): + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + + except KeyError: + return default + + + def retryFree(self): + if not self.premium: + return + self.premium = False + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retries = -1 + raise Retry(_("Fallback to free download")) + + + #@TODO: Remove in 0.4.10 def wait(self, seconds=0, reconnect=None): return _wait(self, seconds, reconnect) diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py new file mode 100644 index 000000000..7f5de51e1 --- /dev/null +++ b/module/plugins/internal/SolveMedia.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Plugin import Fail +from module.plugins.internal.Captcha import Captcha + + +class SolveMedia(Captcha): + __name__ = "SolveMedia" + __type__ = "captcha" + __version__ = "0.13" + + __description__ = """SolveMedia captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' + + + def detect_key(self, html=None): + html = html or self.retrieve_html() + + m = re.search(self.KEY_PATTERN, html) + if m: + self.key = m.group(1).strip() + self.logDebug("Key: %s" % self.key) + return self.key + else: + self.logWarning("Key pattern not found") + return None + + + def challenge(self, key=None, html=None): + key = key or self.retrieve_key(html) + + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", + get={'k': key}) + + for i in xrange(1, 11): + try: + magic = re.search(r'name="magic" value="(.+?)"', html).group(1) + + except AttributeError: + self.logWarning("Magic pattern not found") + magic = None + + try: + challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="(.+?)">', + html).group(1) + + except AttributeError: + self.fail(_("SolveMedia challenge pattern not found")) + + else: + self.logDebug("Challenge: %s" % challenge) + + try: + result = self.result("http://api.solvemedia.com/papi/media", challenge) + + except Fail, e: + self.logWarning(e) + self.plugin.invalidCaptcha() + result = None + + html = self.plugin.req.load("http://api.solvemedia.com/papi/verify.noscript", + post={'adcopy_response' : result, + 'k' : key, + 'l' : "en", + 't' : "img", + 's' : "standard", + 'magic' : magic, + 'adcopy_challenge': challenge, + 'ref' : self.plugin.pyfile.url}) + try: + redirect = re.search(r'URL=(.+?)">', html).group(1) + + except AttributeError: + self.fail(_("SolveMedia verify pattern not found")) + + else: + if "error" in html: + self.logWarning("Captcha code was invalid") + self.logDebug("Retry #%d" % i) + html = self.plugin.req.load(redirect) + else: + break + + else: + self.fail(_("SolveMedia max retries exceeded")) + + return result, challenge + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha(server, + get={'c': challenge}, + cookies=True, + imgtype="gif") + + self.logDebug("Result: %s" % result) + + return result diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 572fe95b9..5b9f2e1c3 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -2,105 +2,108 @@ import os import re +import subprocess from glob import glob -from os.path import basename, dirname, join from string import digits -from subprocess import Popen, PIPE from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import save_join, decode +from module.utils import fs_decode, fs_encode, save_join def renice(pid, value): - if os.name != "nt" and value: + if value and os.name != "nt": try: - Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - except: - print "Renice failed" + subprocess.Popen(["renice", str(value), str(pid)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1) + + except Exception: + pass class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.01" + __version__ = "1.20" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), + ("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz" , "immenz@gmx.net" )] CMD = "unrar" + VERSION = "" + EXTENSIONS = [".rar"] - EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "bz2", "ace", "uue", "jar", "iso", "7z", "xz", "z"] + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?',re.I) - #@NOTE: there are some more uncovered rar formats - re_rarpart = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) - re_rarfile = re.compile(r'.*\.(rar|r\d+)$', re.I) + re_filefixed = re.compile(r'Building (.+)') + re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') re_wrongpwd = re.compile(r'password', re.I) - re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I) + + re_version = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I) @classmethod - def checkDeps(cls): + def isUsable(cls): if os.name == "nt": - cls.CMD = join(pypath, "UnRAR.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() - else: try: - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() + cls.CMD = os.path.join(pypath, "RAR.exe") + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + cls.__name__ = "RAR" + cls.REPAIR = True except OSError: - # fallback to rar - cls.CMD = "rar" - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - p.communicate() + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + else: + try: + p = subprocess.Popen(["rar"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + cls.__name__ = "RAR" + cls.REPAIR = True + + except OSError: #: fallback to unrar + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + m = cls.re_version.search(out) + cls.VERSION = m.group(1) if m else '(version unknown)' return True @classmethod - def isArchive(cls, file): - f = basename(file).lower() - return any(f.endswith('.%s' % ext) for ext in cls.EXTENSIONS) + def isMultipart(cls, filename): + return True if cls.re_multipart.search(filename) else False - @classmethod - def getTargets(cls, files_ids): - targets = [] - - for file, id in files_ids: - if not cls.isArchive(file): - continue - - m = cls.re_rarpart.findall(file) - if m: - # only add first parts - if int(m[0][1]) == 1: - targets.append((file, id)) - else: - targets.append((file, id)) + def verify(self, password): + p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password) + self._progress(p) + err = p.stderr.read().strip() - return targets + if self.re_wrongpwd.search(err): + raise PasswordError + if self.re_wrongcrc.search(err): + raise CRCError(err) - def check(self, out="", err=""): - if not out or not err: - return - if err.strip(): - if self.re_wrongpwd.search(err): - raise PasswordError + def check(self, password): + p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) + out, err = p.communicate() - elif self.re_wrongcrc.search(err): - raise CRCError + if self.re_wrongpwd.search(err): + raise PasswordError - else: #: raise error if anything is on stderr - raise ArchiveError(err.strip()) + if self.re_wrongcrc.search(err): + raise CRCError(err) # output only used to check if passworded files are present for attr in self.re_filelist.findall(out): @@ -108,114 +111,99 @@ class UnRar(Extractor): raise PasswordError - def verify(self): - p = self.call_cmd("l", "-v", self.file, password=self.password) - - self.check(*p.communicate()) - - if p and p.returncode: - raise ArchiveError("Process terminated") - - if not self.list(): - raise ArchiveError("Empty archive") - - - def isPassword(self, password): - if isinstance(password, basestring): - p = self.call_cmd("l", "-v", self.file, password=password) - out, err = p.communicate() - - if not self.re_wrongpwd.search(err): - return True - - return False - - def repair(self): - p = self.call_cmd("rc", self.file) - out, err = p.communicate() - - if p.returncode or err.strip(): - p = self.call_cmd("r", self.file) - out, err = p.communicate() - - if p.returncode or err.strip(): - return False - else: - self.file = join(dirname(self.file), re.search(r'(fixed|rebuild)\.%s' % basename(self.file), out).group(0)) + p = self.call_cmd("rc", fs_encode(self.filename)) + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + if err or p.returncode: + return False return True - def extract(self, progress=lambda x: None): - self.verify() - - progress(0) - - command = "x" if self.fullpath else "e" - - p = self.call_cmd(command, self.file, self.out, password=self.password) - - renice(p.pid, self.renice) - - progressstring = "" + def _progress(self, process): + s = "" while True: - c = p.stdout.read(1) + c = process.stdout.read(1) # quit loop on eof if not c: break # reading a percentage sign -> set progress and restart - if c is '%': - progress(int(progressstring)) - progressstring = "" + if c == '%': + self.notifyProgress(int(s)) + s = "" # not reading a digit -> therefore restart elif c not in digits: - progressstring = "" + s = "" # add digit to progressstring else: - progressstring += c + s += c + + + def extract(self, password=None): + command = "x" if self.fullpath else "e" - progress(100) + p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) - self.files = self.list() + renice(p.pid, self.renice) - # retrieve stderr - self.check(err=p.stderr.read()) + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError(err) + + else: #: raise error if anything is on stderr + raise ArchiveError(err) if p.returncode: - raise ArchiveError("Process terminated") + raise ArchiveError(_("Process return code: %d") % p.returncode) + + self.files = self.list(password) def getDeleteFiles(self): - if ".part" in basename(self.file): - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) + dir, name = os.path.split(self.filename) + + # actually extracted file + files = [self.filename] - # get files which matches .r* and filter unsuited files out - parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) + # eventually Multipart Files + files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) - return filter(lambda x: self.re_rarfile.match(x), parts) + return files - def list(self): + def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_cmd(command, "-v", self.file, password=self.password) + p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) out, err = p.communicate() - if err.strip(): - self.m.logError(err) - if "Cannot open" in err: - return list() + if "Cannot open" in err: + raise ArchiveError(_("Cannot open file")) - if p.returncode: - self.m.logError("Process terminated") - return list() + if err.strip(): #: only log error at this point + self.manager.logError(err.strip()) result = set() - - for f in decode(out).splitlines(): - f = f.strip() - result.add(save_join(self.out, f)) + if not self.fullpath and self.VERSION.startswith('5'): + # NOTE: Unrar 5 always list full path + for f in fs_decode(out).splitlines(): + f = save_join(self.out, os.path.basename(f.strip())) + if os.path.isfile(f): + result.add(save_join(self.out, os.path.basename(f))) + else: + for f in fs_decode(out).splitlines(): + f = f.strip() + result.add(save_join(self.out, f)) return list(result) @@ -228,11 +216,11 @@ class UnRar(Extractor): args.append("-o+") else: args.append("-o-") - if self.delete: + if self.delete != 'No': args.append("-or") for word in self.excludefiles: - args.append("-x%s" % word.strip()) + args.append("-x'%s'" % word.strip()) # assume yes on all queries args.append("-y") @@ -248,6 +236,8 @@ class UnRar(Extractor): # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) - return Popen(call, stdout=PIPE, stderr=PIPE) + self.manager.logDebug(" ".join(call)) + + p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return p diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 5ec56cbdf..4c18a0e35 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -2,85 +2,71 @@ from __future__ import with_statement +import os import sys import zipfile from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError +from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.01" + __version__ = "1.12" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - EXTENSIONS = ["zip", "zip64"] + VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) + EXTENSIONS = [".zip", ".zip64"] @classmethod - def checkDeps(cls): + def isUsable(cls): return sys.version_info[:2] >= (2, 6) - @classmethod - def isArchive(cls, file): - return zipfile.is_zipfile(file) + def list(self, password=None): + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + z.setpassword(password) + return z.namelist() - def verify(self): - try: - with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: - z.setpassword(self.password) - badcrc = z.testzip() - - except (BadZipfile, LargeZipFile), e: - raise ArchiveError(e) + def check(self, password): + pass - except RuntimeError, e: - if 'encrypted' in e: - raise PasswordError - else: - raise ArchiveError(e) - else: - if badcrc: - raise CRCError + def verify(self): + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + badfile = z.testzip() - if not self.list(): - raise ArchiveError("Empty archive") + if badfile: + raise CRCError(badfile) + else: + raise PasswordError - def list(self): + def extract(self, password=None): try: - with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: - z.setpassword(self.password) - return z.namelist() - except Exception: - return list() + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + z.setpassword(password) + badfile = z.testzip() - def extract(self, progress=lambda x: None): - try: - with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: - progress(0) - z.extractall(self.out, pwd=self.password) - progress(100) + if badfile: + raise CRCError(badfile) + else: + z.extractall(self.out) - except (BadZipfile, LargeZipFile), e: + except (zipfile.BadZipfile, zipfile.LargeZipFile), e: raise ArchiveError(e) except RuntimeError, e: - if e is "Bad password for file": + if "encrypted" in e: raise PasswordError else: raise ArchiveError(e) - - finally: - self.files = self.list() - - - def getDeleteFiles(self): - return [self.file] + else: + self.files = z.namelist() diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index c350729ac..e619cb038 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- import re - -from time import gmtime, mktime, strptime -from urlparse import urljoin +import time +import urlparse from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies @@ -12,18 +11,19 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.35" + __version__ = "0.37" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com" )] HOSTER_DOMAIN = None HOSTER_URL = None + LOGIN_URL = None - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + COOKIES = True PREMIUM_PATTERN = r'\(Premium only\)' @@ -46,9 +46,15 @@ class XFSAccount(Account): def init(self): if not self.HOSTER_DOMAIN: self.logError(_("Missing HOSTER_DOMAIN")) + self.COOKIES = False + + else: + if not self.HOSTER_URL: + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN - if not self.HOSTER_URL: - self.HOSTER_URL = "http://www.%s/" % (self.HOSTER_DOMAIN or "") + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + set_cookies(req.cj, self.COOKIES) def loadAccountInfo(self, user, req): @@ -57,6 +63,12 @@ class XFSAccount(Account): leechtraffic = None premium = None + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -67,7 +79,7 @@ class XFSAccount(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) + validuntil = time.mktime(time.strptime(expiredate, "%d %B %Y")) except Exception, e: self.logError(e) @@ -75,7 +87,7 @@ class XFSAccount(Account): else: self.logDebug("Valid until: %s" % validuntil) - if validuntil > mktime(gmtime()): + if validuntil > time.mktime(time.gmtime()): premium = True trafficleft = -1 else: @@ -136,25 +148,34 @@ class XFSAccount(Account): else: self.logDebug("LEECH_TRAFFIC_PATTERN not found") - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'leechtraffic': leechtraffic, 'premium': premium} + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} def login(self, user, data, req): - if isinstance(self.COOKIES, list): - set_cookies(req.cj, self.COOKIES) + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + raise Exception(_("Missing HOSTER_DOMAIN")) - url = urljoin(self.HOSTER_URL, "login.html") - html = req.load(url, decode=True) + if not self.LOGIN_URL: + self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") + html = req.load(self.LOGIN_URL, decode=True) action, inputs = parseHtmlForm('name="FL"', html) if not inputs: - inputs = {'op': "login", + inputs = {'op' : "login", 'redirect': self.HOSTER_URL} - inputs.update({'login': user, + inputs.update({'login' : user, 'password': data['password']}) - html = req.load(self.HOSTER_URL, post=inputs, decode=True) + if action: + url = urlparse.urljoin("http://", action) + else: + url = self.HOSTER_URL + + html = req.load(url, post=inputs, decode=True) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrongPassword() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 3cb837aac..8b333b45c 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.05" + __version__ = "0.09" __pattern__ = r'^unmatchable$' @@ -19,10 +19,27 @@ class XFSCrypter(SimpleCrypter): URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] - - LINK_PATTERN = r'<(?:td|TD).*?>\s*<a href="(.+?)".*?>.+?(?:</a>)?\s*</(?:td|TD)>' - NAME_PATTERN = r'<[tT]itle>.*?\: (?P<N>.+) folder</[tT]itle>' + NAME_PATTERN = r'<[Tt]itle>.*?\: (?P<N>.+) folder</[Tt]itle>' + LINK_PATTERN = r'<(?:td|TD).*?>\s*(?:<.+>\s*)?<a href="(.+?)".*?>.+?(?:</a>)?\s*(?:<.+>\s*)?</(?:td|TD)>' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + + def prepare(self): + if not self.HOSTER_DOMAIN: + if self.account: + account = self.account + else: + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + + return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index b32f5978f..e0fd8fa8d 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- +import pycurl +import random import re +import urlparse -from random import random -from time import sleep - -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME - -from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia +from module.plugins.internal.ReCaptcha import ReCaptcha +from module.plugins.internal.SolveMedia import SolveMedia from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight from module.utils import html_unescape @@ -15,32 +14,30 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.35" + __version__ = "0.53" __pattern__ = r'^unmatchable$' __description__ = """XFileSharing hoster plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("stickell" , "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com" )] HOSTER_DOMAIN = None - TEXT_ENCODING = False - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] - DIRECT_LINK = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*</b>(</td><td nowrap>)?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P<N>.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*</b>(</td><td>)?|File:.*>|</font>\s*\(|<[\w^_]+ class="size">)\s*(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)' - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>|id="countdown" value=".*?(\d+).*?"' + WAIT_PATTERN = r'<span id="countdown_str".*>(\d+)</span>|id="countdown" value=".*?(\d+).*?"' PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + HAPPY_HOUR_PATTERN = r'>[Hh]appy hour' ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter><b>|>Error</td>|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' LINK_LEECH_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' @@ -56,48 +53,39 @@ class XFSHoster(SimpleHoster): def setup(self): - self.chunkLimit = 1 + self.chunkLimit = -1 if self.premium else 1 self.resumeDownload = self.multiDL = self.premium def prepare(self): """ Initialize important variables """ if not self.HOSTER_DOMAIN: - self.fail(_("Missing HOSTER_DOMAIN")) + if self.account: + account = self.account + else: + account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) if not self.LINK_PATTERN: - pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' + pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - self.captcha = None - self.errmsg = None - super(XFSHoster, self).prepare() if self.DIRECT_LINK is None: - self.directDL = bool(self.premium) - - - def downloadLink(self, link): - if link and isinstance(link, basestring): - if self.captcha: - self.correctCaptcha() - - self.download(link, ref=True, cookies=True, disposition=False) #@TODO: Set `disposition=True` in 0.4.10 - - elif self.errmsg: - if 'captcha' in self.errmsg: - self.fail(_("No valid captcha code entered")) - else: - self.fail(self.errmsg) - - else: - self.fail(_("Download link not found")) + self.directDL = self.premium def handleFree(self, pyfile): for i in xrange(1, 6): - self.logDebug("Getting download link: #%d" % i) + self.logDebug("Getting download link #%d" % i) self.checkErrors() @@ -107,11 +95,11 @@ class XFSHoster(SimpleHoster): data = self.getPostParameters() - self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.html = self.load(pyfile.url, post=data, ref=True, decode=True) + self.html = self.load(pyfile.url, post=data, decode=True) - self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) if m and not "op=" in m.group(1): @@ -124,8 +112,6 @@ class XFSHoster(SimpleHoster): self.logError(data['op'] if 'op' in data else _("UNKNOWN")) return "" - self.errmsg = None - self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 @@ -142,7 +128,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parseHtmlForm() - upload_id = "%012d" % int(random() * 10 ** 12) + upload_id = "%012d" % int(random.random() * 10 ** 12) action += upload_id + "&js_on=1&utype=prem&upload_type=url" inputs['tos'] = '1' @@ -159,10 +145,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parseHtmlForm('F1') if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) + self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.logDebug(inputs) @@ -189,61 +172,6 @@ class XFSHoster(SimpleHoster): if 'location' in header: #: Direct download link self.link = header['location'] - else: - self.fail(_("Download link not found")) - - - def checkErrors(self): - m = re.search(self.ERROR_PATTERN, self.html) - if m is None: - self.errmsg = None - else: - self.errmsg = m.group(1).strip() - - self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) - - if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)]) - self.wait(wait_time, True) - - elif 'country' in self.errmsg: - self.fail(_("Downloads are disabled for your country")) - - elif 'captcha' in self.errmsg: - self.invalidCaptcha() - - elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail(_("File can be downloaded by premium users only")) - - elif 'limit' in self.errmsg: - if 'days' in self.errmsg: - delay = secondsToMidnight(gmt=2) - retries = 3 - else: - delay = 1 * 60 * 60 - retries = 24 - - self.wantReconnect = True - self.retry(retries, delay, _("Download limit exceeded")) - - elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: - self.retry(reason=_("Link expired")) - - elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: - self.tempOffline() - - elif 'up to' in self.errmsg: - self.fail(_("File too large for free download")) - - else: - self.wantReconnect = True - self.retry(wait_time=60, reason=self.errmsg) - - if self.errmsg: - self.info['error'] = self.errmsg - else: - self.info.pop('error', None) def getPostParameters(self): @@ -255,10 +183,7 @@ class XFSHoster(SimpleHoster): if not inputs: action, inputs = self.parseHtmlForm('F1') if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) + self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.logDebug(inputs) @@ -276,8 +201,7 @@ class XFSHoster(SimpleHoster): wait_time = int(m.group(1)) self.setWait(wait_time, False) - self.captcha = self.handleCaptcha(inputs) - + self.handleCaptcha(inputs) self.wait() else: inputs['referer'] = self.pyfile.url @@ -297,39 +221,43 @@ class XFSHoster(SimpleHoster): if m: captcha_url = m.group(1) inputs['code'] = self.decryptCaptcha(captcha_url) - return 1 + return m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) if m: captcha_div = m.group(1) numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) + self.logDebug(captcha_div) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) + + inputs['code'] = "".join(a[1] for a in sorted(numerals, key=lambda num: int(num[0]))) + self.logDebug("Captcha code: %s" % inputs['code'], numerals) - return 2 + return recaptcha = ReCaptcha(self) try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except: + + except Exception: captcha_key = recaptcha.detect_key() + else: self.logDebug("ReCaptcha key: %s" % captcha_key) if captcha_key: - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - return 3 + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key) + return solvemedia = SolveMedia(self) try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except: + + except Exception: captcha_key = solvemedia.detect_key() + else: self.logDebug("SolveMedia key: %s" % captcha_key) if captcha_key: - inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) - return 4 - - return 0 + inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key) |