From 536669858a949c5b34b902b7e05f336e33b2df28 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Tue, 31 Jan 2012 21:27:45 +0100 Subject: re-enable & refactor fileserve/filejungle/uploadstation --- module/plugins/accounts/FilejungleCom.py | 60 +++++ module/plugins/accounts/UploadstationCom.py | 13 + module/plugins/hoster/FilejungleCom.py | 75 ++---- module/plugins/hoster/FileserveCom.py | 378 +++++++++++++++------------- module/plugins/hoster/UploadStationCom.py | 146 +---------- 5 files changed, 311 insertions(+), 361 deletions(-) create mode 100644 module/plugins/accounts/FilejungleCom.py create mode 100644 module/plugins/accounts/UploadstationCom.py (limited to 'module/plugins') diff --git a/module/plugins/accounts/FilejungleCom.py b/module/plugins/accounts/FilejungleCom.py new file mode 100644 index 000000000..4fa4a06d8 --- /dev/null +++ b/module/plugins/accounts/FilejungleCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +from module.plugins.Account import Account +import re +from time import mktime, strptime + +class FilejungleCom(Account): + __name__ = "FilejungleCom" + __version__ = "0.1" + __type__ = "account" + __description__ = """filejungle.com account plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + login_timeout = 60 + + URL = "http://filejungle.com/" + TRAFFIC_LEFT_PATTERN = r'"/extend_premium\.php">Until (\d+ [A-Za-z]+ \d+)' + + def loadAccountInfo(self, user, req): + html = req.load(self.URL + "dashboard.php") + found = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if found: + premium = True + validuntil = mktime(strptime(found.group(1), "%d %B %Y")) + else: + premium = False + validuntil = -1 + + return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} + + def login(self, user, data, req): + html = req.load(self.URL + "login.php", post={ + "loginUserName": user, + "loginUserPassword": data["password"], + "loginFormSubmit": "Login", + "recaptcha_challenge_field": "", + "recaptcha_response_field": "", + "recaptcha_shortencode_field": ""}) + + if re.search(self.LOGIN_FAILED_PATTERN, html): + self.wrongPassword() \ No newline at end of file diff --git a/module/plugins/accounts/UploadstationCom.py b/module/plugins/accounts/UploadstationCom.py new file mode 100644 index 000000000..e86cec7ce --- /dev/null +++ b/module/plugins/accounts/UploadstationCom.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from module.plugins.accounts.FilejungleCom import FilejungleCom + +class UploadstationCom(FilejungleCom): + __name__ = "UploadstationCom" + __version__ = "0.1" + __type__ = "account" + __description__ = """uploadstation.com account plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + URL = "http://uploadstation.com/" diff --git a/module/plugins/hoster/FilejungleCom.py b/module/plugins/hoster/FilejungleCom.py index c49cc8506..c75ac1a98 100644 --- a/module/plugins/hoster/FilejungleCom.py +++ b/module/plugins/hoster/FilejungleCom.py @@ -17,69 +17,24 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -from module.network.RequestFactory import getURL -from module.plugins.ReCaptcha import ReCaptcha +from module.plugins.hoster.FileserveCom import FileserveCom, checkFile -class FilejungleCom(SimpleHoster): +# shares code with UploadstationCom + +class FilejungleCom(FileserveCom): __name__ = "FilejungleCom" __type__ = "hoster" - __pattern__ = r"http://(?:www\.)?filejungle\.com/f/([^/]+).*" - __version__ = "0.25" + __pattern__ = r"http://(?:www\.)?filejungle\.com/f/(?P[^/]+).*" + __version__ = "0.5" __description__ = """Filejungle.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - - FILE_INFO_PATTERN = r'
(?P[^<]+) \((?P[0-9.]+) (?P[kKMG])i?B\)
' - FILE_OFFLINE_PATTERN = r'(This file is no longer available.|class="error_msg_title"> Invalid or Deleted File. )' - RECAPTCHA_KEY_PATTERN = r"var reCAPTCHA_publickey='([^']+)'" - WAIT_TIME_PATTERN = r'

Please wait for (\d+) seconds to download the next file\.

' - - def handleFree(self): - file_id = re.search(self.__pattern__, self.pyfile.url).group(1) - url = "http://www.filejungle.com/f/%s" % file_id - self.logDebug("File ID: %s" % file_id) - - # Get captcha - found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - if not found: self.fail("Captcha key not found") - captcha_key = found.group(1) - - json_response = self.load(self.pyfile.url, post = {"checkDownload" : "check"}, decode = True) - self.logDebug(json_response) - if r'"success":"showCaptcha"' in json_response: - recaptcha = ReCaptcha(self) - for i in range(5): - captcha_challenge, captcha_response = recaptcha.challenge(captcha_key) - self.logDebug("RECAPTCHA: %s : %s : %s" % (captcha_key, captcha_challenge, captcha_response)) - - json_response = self.load("http://www.filejungle.com/checkReCaptcha.php", post = { - "recaptcha_challenge_field" : captcha_challenge, - "recaptcha_response_field" : captcha_response, - "recaptcha_shortencode_field" : file_id - }, decode = True) - self.logDebug(json_response) - if r'{"success":1}' in json_response: - self.correctCaptcha() - break - else: - self.invalidCaptcha() - else: self.fail("Invalid captcha") - elif r'"fail":"timeLimit"' in json_response: - self.html = self.load(url, post = {"checkDownload" : "showError", "errorType" : "timeLimit"}) - found = re.search(self.WAIT_TIME_PATTERN, self.html) - self.retry(5, int(found.group(1)) if found else 1200, "Time limit reached") - else: - self.fail("Unknown server response") - - json_response = self.load(url, post = {"downloadLink" : "wait"}, decode = True) - self.logDebug(json_response[:30]) - found = re.search(r'"waitTime":(\d+)', json_response) - if not found: self.fail("Cannot get wait time") - self.setWait(int(found.group(1))) - self.wait() - - response = self.load(url, post = {"downloadLink" : "show"}) - self.download(url, post = {"download" : "normal"}) - -getInfo = create_getInfo(FilejungleCom) + + URLS = ['http://www.filejungle.com/f/', 'http://www.filejungle.com/check_links.php', 'http://www.filejungle.com/checkReCaptcha.php'] + LINKCHECK_TR = r'
  • \s*(
    .*?)
    • ' + LINKCHECK_TD = r'
    (?:<[^>]*>| )*([^<]*)' + + LONG_WAIT_PATTERN = r'

    Please wait for (\d+) (\w+)\s*to download the next file\.

    ' + +def getInfo(urls): + yield checkFile(FilejungleCom, urls) \ No newline at end of file diff --git a/module/plugins/hoster/FileserveCom.py b/module/plugins/hoster/FileserveCom.py index ce3836a48..759fef96f 100644 --- a/module/plugins/hoster/FileserveCom.py +++ b/module/plugins/hoster/FileserveCom.py @@ -1,168 +1,210 @@ -# -*- coding: utf-8 -*- -from __future__ import with_statement - -import re - -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha - -from module.common.json_layer import json_loads -from module.network.RequestFactory import getURL -from module.utils import parseFileSize - - -def getInfo(urls): - yield [(url, 0, 1, url) for url in urls] - -class FileserveCom(Hoster): - __name__ = "FileserveCom" - __type__ = "hoster" - __pattern__ = r"http://(www\.)?fileserve\.com/file/[a-zA-Z0-9]+" - __version__ = "0.44" - __description__ = """Fileserve.Com File Download Hoster""" - __author_name__ = ("jeix", "mkaay", "paul king") - __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "") - - FILE_ID_KEY = r"fileserve\.com/file/(?P\w+)" - FILE_CHECK_KEY = r"http://www.fileserve\.com/file/(?P\w+).*?(?P.*?).*?(?P.*?) (?P.B).*?(?P.*?)" - CAPTCHA_KEY_PATTERN = r"var reCAPTCHA_publickey='(?P.*?)';" - LONG_WAIT_PATTERN = r"You need to wait (\d+) seconds to start another download" - - def init(self): - if not self.premium: - self.multiDL = False - self.resumeDownload = False - self.chunkLimit = 1 - - def process(self, pyfile): - self.fail("Hoster not longer available") - - def checkFile(self): - self.file_id = re.search(self.FILE_ID_KEY, self.pyfile.url).group("id") - self.logDebug("file id is %s" % self.file_id) - - self.pyfile.url = "http://www.fileserve.com/file/" + self.file_id - - linkCheck = self.load("http://www.fileserve.com/link-checker.php", - post={"urls": self.pyfile.url}, - ref=False, cookies=False if self.account else True, decode=True) - - linkMatch = re.search(self.FILE_CHECK_KEY, linkCheck.replace("\r\n", "")) - if not linkMatch: - self.logDebug("couldn't extract file status") - self.offline() - - if linkMatch.group("online").find("Available"): - self.logDebug("file is not available : %s" % linkMatch.group("online")) - self.offline() - - self.pyfile.name = linkMatch.group("name") - - - def handlePremium(self): - # TODO: handle login timeouts - self.download(self.pyfile.url) - - check = self.checkDownload({"login": '
    '}) - - if check == "login": - self.account.relogin(self.user) - self.retry(reason=_("Not logged in.")) - - - def handleFree(self): - self.html = self.load(self.pyfile.url) - action = self.load(self.pyfile.url, post={"checkDownload": "check"}, decode=True) - action = json_loads(action.replace(u"\ufeff", "")) - self.logDebug("action is : %s" % action) - - if "fail" in action: - if action["fail"] == "timeLimit": - html = self.load(self.pyfile.url, - post={"checkDownload": "showError", - "errorType": "timeLimit"}, - decode=True) - wait = re.search(self.LONG_WAIT_PATTERN, html) - if wait: - wait = int(wait.group(1)) - else: - wait = 720 - self.setWait(wait, True) - self.wait() - self.retry() - - elif action["fail"] == "parallelDownload": - self.logWarning(_("Parallel download error, now waiting 60s.")) - self.retry(wait_time=60, reason="parallelDownload") - - else: - self.fail("Download check returned %s" % action["fail"]) - - if action["success"] == "showCaptcha": - self.doCaptcha() - self.doTimmer() - elif action["success"] == "showTimmer": - self.doTimmer() - - # show download link - response = self.load(self.pyfile.url, post={"downloadLink": "show"}, decode=True) - self.logDebug("show downloadLink response : %s" % response) - if not response.find("fail"): - self.fail("Couldn't retrieve download url") - - # this may either download our file or forward us to an error page - self.download(self.pyfile.url, post={"download": "normal"}) - - check = self.checkDownload({"expired": "Your download link has expired", - "wait": re.compile(self.LONG_WAIT_PATTERN), - "limit": "Your daily download limit has been reached"}) - - if check == "expired": - self.logDebug("Download link was expired") - self.retry() - elif check == "wait": - wait_time = 720 - if self.lastCheck is not None: - wait_time = int(self.lastCheck.group(1)) - self.setWait(wait_time + 3, True) - self.wait() - self.retry() - elif check == "limit": - #download limited reached for today (not a exact time known) - - self.setWait(180 * 60, True) # wait 3 hours - self.wait() - self.retry(max_tries=0) - - self.thread.m.reconnecting.wait(3) # Ease issue with later downloads appearing to be in parallel - - def doTimmer(self): - wait = self.load(self.pyfile.url, - post={"downloadLink": "wait"}, - decode=True).replace(u"\ufeff", "") # remove UTF8 BOM - self.logDebug("wait response : %s" % wait) - - if not wait.find("fail"): - self.fail("Failed getting wait time") - - self.setWait(int(wait)) # remove UTF8 BOM - self.wait() - - def doCaptcha(self): - captcha_key = re.search(self.CAPTCHA_KEY_PATTERN, self.html).group("key") - recaptcha = ReCaptcha(self) - - for i in range(5): - challenge, code = recaptcha.challenge(captcha_key) - - response = json_loads(self.load("http://www.fileserve.com/checkReCaptcha.php", - post={'recaptcha_challenge_field': challenge, - 'recaptcha_response_field': code, - 'recaptcha_shortencode_field': self.file_id}).replace(u"\ufeff", "")) - self.logDebug("reCaptcha response : %s" % response) - if not response["success"]: - self.invalidCaptcha() - else: - self.correctCaptcha() - break - +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +""" + +import re +from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL +from module.plugins.ReCaptcha import ReCaptcha +from module.common.json_layer import json_loads +from module.utils import parseFileSize + +def checkFile(plugin, urls): + html = getURL(plugin.URLS[1], post = {"urls": "\n".join(urls)}, decode=True) + + file_info = [] + for li in re.finditer(plugin.LINKCHECK_TR, html, re.DOTALL): + try: + cols = re.findall(plugin.LINKCHECK_TD, li.group(1)) + if cols: + file_info.append(( + cols[1] if cols[1] != '--' else cols[0], + parseFileSize(cols[2]) if cols[2] != '--' else 0, + 2 if cols[3].startswith('Available') else 1, + cols[0])) + except Exception, e: + continue + + return file_info + +class FileserveCom(Hoster): + __name__ = "FileserveCom" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?fileserve\.com/file/(?P[^/]+).*" + __version__ = "0.5" + __description__ = """Fileserve.Com File Download Hoster""" + __author_name__ = ("jeix", "mkaay", "paul king", "zoidberg") + __author_mail__ = ("jeix@hasnomail.de", "mkaay@mkaay.de", "", "zoidberg@mujmail.cz") + + URLS = ['http://www.fileserve.com/file/', 'http://www.fileserve.com/link-checker.php', 'http://www.fileserve.com/checkReCaptcha.php'] + LINKCHECK_TR = r'\s*(http://www.fileserve\.com/file/.*?)' + LINKCHECK_TD = r'(?:<[^>]*>| )*([^<]*)' + + CAPTCHA_KEY_PATTERN = r"var reCAPTCHA_publickey='(?P[^']+)'" + LONG_WAIT_PATTERN = r'
  • You need to wait (\d+) (\w+) to start another download\.
    • ' + LINK_EXPIRED_PATTERN = "Your download link has expired" + DAILY_LIMIT_PATTERN = "Your daily download limit has been reached" + NOT_LOGGED_IN_PATTERN = 'Login' + + # shares code with FilejungleCom and UploadstationCom + + def setup(self): + self.resumeDownload = self.multiDL = True if self.premium else False + + self.file_id = re.search(self.__pattern__, self.pyfile.url).group('id') + self.url = "%s%s" % (self.URLS[0], self.file_id) + self.logDebug("File ID: %s URL: %s" % (self.file_id, self.url)) + + def process(self, pyfile): + pyfile.name, pyfile.size, status, self.url = checkFile(self, [self.url])[0] + if status != 2: self.offline() + self.logDebug("File Name: %s Size: %d" % (pyfile.name, pyfile.size)) + + if self.premium: + self.handlePremium() + else: + self.handleFree() + + def handleFree(self): + self.html = self.load(self.url) + action = self.load(self.url, post={"checkDownload": "check"}, decode=True) + action = json_loads(action) + self.logDebug(action) + + if "fail" in action: + if action["fail"] == "timeLimit": + self.html = self.load(self.url, + post={"checkDownload": "showError", + "errorType": "timeLimit"}, + decode=True) + + self.doLongWait(re.search(self.LONG_WAIT_PATTERN, self.html)) + + elif action["fail"] == "parallelDownload": + self.logWarning(_("Parallel download error, now waiting 60s.")) + self.retry(wait_time=60, reason="parallelDownload") + + else: + self.fail("Download check returned %s" % action["fail"]) + + elif "success" in action: + if action["success"] == "showCaptcha": + self.doCaptcha() + self.doTimmer() + elif action["success"] == "showTimmer": + self.doTimmer() + + else: + self.fail("Unknown server response") + + # show download link + response = self.load(self.url, post={"downloadLink": "show"}, decode=True) + self.logDebug("show downloadLink response : %s" % response) + if "fail" in response: + self.fail("Couldn't retrieve download url") + + # this may either download our file or forward us to an error page + self.download(self.url, post = {"download": "normal"}) + self.logDebug(self.req.http.lastEffectiveURL) + + check = self.checkDownload({"expired": self.LINK_EXPIRED_PATTERN, + "wait": re.compile(self.LONG_WAIT_PATTERN), + "limit": self.DAILY_LIMIT_PATTERN}) + + if check == "expired": + self.logDebug("Download link was expired") + self.retry() + elif check == "wait": + self.doLongWait(self.lastCheck) + elif check == "limit": + #download limited reached for today (not a exact time known) + self.setWait(180 * 60, True) # wait 3 hours + self.wait() + self.retry(max_tries=0) + + self.thread.m.reconnecting.wait(3) # Ease issue with later downloads appearing to be in parallel + + def doTimmer(self): + response = self.load(self.url, + post={"downloadLink": "wait"}, + decode=True) + self.logDebug("wait response : %s" % response[:80]) + + if "fail" in response: + self.fail("Failed getting wait time") + + if self.__name__ == "FilejungleCom": + found = re.search(r'"waitTime":(\d+)', response) + if not found: self.fail("Cannot get wait time") + wait_time = int(found.group(1)) + else: + wait_time = int(response) + 3 + + self.setWait(wait_time) + self.wait() + + def doCaptcha(self): + captcha_key = re.search(self.CAPTCHA_KEY_PATTERN, self.html).group("key") + recaptcha = ReCaptcha(self) + + for i in range(5): + challenge, code = recaptcha.challenge(captcha_key) + + response = json_loads(self.load(self.URLS[2], + post={'recaptcha_challenge_field': challenge, + 'recaptcha_response_field': code, + 'recaptcha_shortencode_field': self.file_id})) + self.logDebug("reCaptcha response : %s" % response) + if not response["success"]: + self.invalidCaptcha() + else: + self.correctCaptcha() + break + else: self.fail("Invalid captcha") + + def doLongWait(self, m): + wait_time = (int(m.group(1)) * {'seconds':1, 'minutes':60, 'hours':3600}[m.group(2)]) if m else 720 + self.setWait(wait_time, True) + self.wait() + self.retry() + + def handlePremium(self): + premium_url = None + if self.__name__ == "FileserveCom": + #try api download + response = self.load("http://app.fileserve.com/api/download/premium/", + post = {"username": self.user, + "password": self.account.getAccountData(self.user)["password"], + "shorten": self.file_id}, + decode = True) + if response: + response = json_loads(response) + if response['error_code'] == "302": premium_url = response['next'] + elif response['error_code'] in ["305", "500"]: self.tempOffline() + elif response['error_code'] in ["403", "605"]: self.resetAccount() + elif response['error_code'] in ["606", "607", "608"]: self.offline() + else: self.logError(response['error_code'], response['error_message']) + + self.download(premium_url or self.pyfile.url) + + if not premium_url: + check = self.checkDownload({"login": re.compile(self.NOT_LOGGED_IN_PATTERN)}) + + if check == "login": + self.account.relogin(self.user) + self.retry(reason=_("Not logged in.")) + +def getInfo(urls): + yield checkFile(FileserveCom, urls) \ No newline at end of file diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py index fea5f4245..ede176e1f 100644 --- a/module/plugins/hoster/UploadStationCom.py +++ b/module/plugins/hoster/UploadStationCom.py @@ -1,141 +1,21 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement - -from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster -from module.plugins.ReCaptcha import ReCaptcha - import re +from module.plugins.hoster.FileserveCom import FileserveCom, checkFile -def getInfo(urls): - yield [(url, 0, 1, url) for url in urls] - -class UploadStationCom(Hoster): +class UploadStationCom(FileserveCom): __name__ = "UploadStationCom" __type__ = "hoster" - __pattern__ = r"http://(www\.)?uploadstation\.com/file/(?P[A-Za-z0-9]+)" - __version__ = "0.33" + __pattern__ = r"http://(?:www\.)?uploadstation\.com/file/(?P[A-Za-z0-9]+)" + __version__ = "0.5" __description__ = """UploadStation.Com File Download Hoster""" - __author_name__ = ("fragonib") - __author_mail__ = ("fragonib[AT]yahoo[DOT]es") - - FILE_OFFLINE_PATTERN = r'''

    File not available

    |

    File is not available

    ''' - FILE_TITLE_PATTERN = r'''
    (.*?)
    ''' - FILE_SIZE_PATTERN = r'''
    File size: (.*?) (KB|MB|GB)''' - CAPTCHA_PRESENT_TOKEN = '