From 25ab9093d84cbeb86f7b626d7251f4a0180b32bf Mon Sep 17 00:00:00 2001 From: fragonib Date: Sun, 24 Apr 2011 17:40:11 +0200 Subject: UploadStation & BitshareCom: Improved debugging --- module/plugins/hoster/BitshareCom.py | 103 ++++++++++++++++---------- module/plugins/hoster/UploadStationCom.py | 117 ++++++++++++++++-------------- 2 files changed, 127 insertions(+), 93 deletions(-) (limited to 'module/plugins/hoster') diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py index f44a89e23..496071fa3 100644 --- a/module/plugins/hoster/BitshareCom.py +++ b/module/plugins/hoster/BitshareCom.py @@ -10,6 +10,7 @@ from module.plugins.Hoster import Hoster from module.plugins.ReCaptcha import ReCaptcha from module.network.RequestFactory import getURL +from wx.lib.analogclock.helpers import Hand def unicode2str(unitext): return unicodedata.normalize('NFKD', unitext).encode('ascii', 'ignore') @@ -19,20 +20,22 @@ def getInfo(urls): for url in urls: - # Get html + # Get file info html + # TODO: Force responses in english language html = getURL(url) - if re.search(BitshareCom.OFFLINE_PATTERN, html): + if re.search(BitshareCom.FILE_OFFLINE_PATTERN, html): result.append((url, 0, 1, url)) # Name name1 = re.search(BitshareCom.__pattern__, url).group('name') m = re.search(BitshareCom.FILE_INFO_PATTERN, html) name2 = m.group('name') - name = unicode2str(max(name1, name2)) + name = unicode2str(max(name1, name2)) # Unicode BUG workaround # Size value = float(m.group('size')) - pow = {'KB' : 1, 'MB' : 2, 'GB' : 3}[m.group('units')] + units = m.group('units') + pow = {'KB' : 1, 'MB' : 2, 'GB' : 3}[units] size = int(value*1024**pow) # Return info @@ -44,12 +47,14 @@ class BitshareCom(Hoster): __name__ = "BitshareCom" __type__ = "hoster" __pattern__ = r"http://(www\.)?bitshare\.com/(files/(?P[a-zA-Z0-9]+)(/(?P.*?)\.html)?|\?f=(?P[a-zA-Z0-9]+))" - __version__ = "0.2" + __version__ = "0.3" __description__ = """Bitshare.Com File Download Hoster""" - __author_name__ = ("paul", "king") + __author_name__ = ("paul", "king", "fragonib") - OFFLINE_PATTERN = r'''(>We are sorry, but the requested file was not found in our database|>Error - File not available<|The file was deleted either by the uploader, inactivity or due to copyright claim)''' + FILE_OFFLINE_PATTERN = r'''(>We are sorry, but the requested file was not found in our database|>Error - File not available<|The file was deleted either by the uploader, inactivity or due to copyright claim)''' FILE_INFO_PATTERN = r'

.*\s(?P.+?)\s-\s(?P\d+)\s(?P..)yte

' + FILE_AJAXID_PATTERN = r'var ajaxdl = "(.*?)";' + CAPTCHA_KEY_PATTERN = r"http://api\.recaptcha\.net/challenge\?k=(.*?) " def setup(self): self.multiDL = False @@ -57,74 +62,92 @@ class BitshareCom(Hoster): def process(self, pyfile): self.pyfile = pyfile + + # Force responses language self.req.cj.setCookie("bitshare.com", "language_selection", "EN") # File id m = re.match(self.__pattern__, self.pyfile.url) self.file_id = max(m.group('id1'), m.group('id2')) - - # File url - self.log.debug("%s: File_id is %s" % (self.__name__, self.file_id)) + self.log.debug("%s: File id is [%s]" % (self.__name__, self.file_id)) # Load main page self.html = self.load(self.pyfile.url, ref=False, utf8=True, cookies=True) # Check offline - if re.search(self.OFFLINE_PATTERN, self.html) is not None: + if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: self.offline() # File name name1 = re.search(BitshareCom.__pattern__, self.pyfile.url).group('name') name2 = re.search(BitshareCom.FILE_INFO_PATTERN, self.html).group('name') - self.pyfile.name = unicode2str(max(name1, name2)) + self.pyfile.name = unicode2str(max(name1, name2)) # Unicode BUG workaround - self.ajaxid = re.search("var ajaxdl = \"(.*?)\";",self.html).group(1) - - self.log.debug("%s: AjaxId %s" % (self.__name__, self.ajaxid)) + # Ajax file id + self.ajaxid = re.search(BitshareCom.FILE_AJAXID_PATTERN, self.html).group(1) + self.log.debug("%s: File ajax id is [%s]" % (self.__name__, self.ajaxid)) + # Handle free downloading self.handleFree() def handleFree(self): - action = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + # Get download info + self.log.debug("%s: Getting download info" % (self.__name__)) + response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", post={"request" : "generateID", "ajaxid" : self.ajaxid}) - self.log.debug("%s: Result of generateID %s" % (self.__name__, action)) - parts = action.split(":") - - if parts[0] == "ERROR": - self.fail(parts[1]) - + self.handleErrors(response, ':') + parts = response.split(":") filetype = parts[0] wait = int(parts[1]) captcha = int(parts[2]) + self.log.debug("%s: Download info [type: '%s', waiting: %d, captcha: %d]" % + (self.__name__, filetype, wait, captcha)) + # Waiting if wait > 0: - self.log.info("%s: Waiting %d seconds." % (self.__name__, wait)) + self.log.debug("%s: Waiting %d seconds." % (self.__name__, wait)) self.setWait(wait, True) self.wait() + # Resolve captcha if captcha == 1: - id = re.search(r"http://api\.recaptcha\.net/challenge\?k=(.*?) ", self.html).group(1) - self.log.debug("%s: ReCaptcha key %s" % (self.__name__, id)) - for i in range(3): # Try upto 3 times + self.log.debug("%s: File is captcha protected" % (self.__name__)) + id = re.search(BitshareCom.CAPTCHA_KEY_PATTERN, self.html).group(1) + # Try up to 3 times + for i in range(3): + self.log.debug("%s: Resolving ReCaptcha with key [%s], round %d" % (self.__name__, id, i+1)) recaptcha = ReCaptcha(self) challenge, code = recaptcha.challenge(id) - action = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", post={"request" : "validateCaptcha", "ajaxid" : self.ajaxid, "recaptcha_challenge_field" : challenge, "recaptcha_response_field" : code}) - parts = action.split(":") - if parts[0] != "SUCCESS": - self.invalidCaptcha() - else: + if self.handleCaptchaErrors(response): break - action = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", - post={"request" : "getDownloadURL", "ajaxid" : self.ajaxid}) - parts = action.split("#") - - if parts[0] == "ERROR": - self.fail(parts[1]) + # Get download URL + self.log.debug("%s: Getting download url" % (self.__name__)) + response = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + post={"request" : "getDownloadURL", "ajaxid" : self.ajaxid}) + self.handleErrors(response, '#') + url = response.split("#")[-1] - # this may either download our file or forward us to an error page - self.log.debug("%s: Download url %s" % (self.__name__, parts[1])) - dl = self.download(parts[1]) + # Request download URL + # This may either download our file or forward us to an error page + self.log.debug("%s: Downloading file with url [%s]" % (self.__name__, url)) + dl = self.download(url) + + def handleErrors(self, response, separator): + self.log.debug("%s: Checking response [%s]" % (self.__name__, response)) + if "ERROR" in response: + msg = response.split(separator)[-1] + self.fail(msg) + + def handleCaptchaErrors(self, response): + self.log.debug("%s: Result of captcha resolving [%s]" % (self.__name__, response)) + if "SUCCESS" in response: + return True + + self.log.debug("%s: Wrong captcha" % (self.__name__)) + self.invalidCaptcha() + diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py index 2723ae2ef..19c2d078d 100644 --- a/module/plugins/hoster/UploadStationCom.py +++ b/module/plugins/hoster/UploadStationCom.py @@ -12,29 +12,28 @@ from module.plugins.ReCaptcha import ReCaptcha from module.network.RequestFactory import getURL def unicode2str(unitext): - return unicodedata.normalize('NFKD', unitext).encode('ascii', 'ignore') - + return unicodedata.normalize('NFKD', unitext).encode('ascii', 'ignore') + def getInfo(urls): result = [] for url in urls: - # Get html - html = getURL(url) - pattern = r'''

File not available

|The file could not be found\. Please check the download link''' - if re.search(pattern, html): + # Get file info html + html = getURL(url) + if re.search(UploadStationCom.FILE_OFFLINE_PATTERN, html): result.append((url, 0, 1, url)) continue # Name - pattern = r'''
(.*?)
''' - name = re.search(pattern, html).group(1) + name = re.search(UploadStationCom.FILE_TITLE_PATTERN, html).group(1) + name = unicode2str(name) # Unicode BUG workaround # Size - pattern = r'''
File size: (.*?) (KB|MB|GB)''' - m = re.search(pattern, html) + m = re.search(UploadStationCom.FILE_SIZE_PATTERN, html) value = float(m.group(1)) - pow = {'KB' : 1, 'MB' : 2, 'GB' : 3}[m.group(2)] + units = m.group(2) + pow = {'KB' : 1, 'MB' : 2, 'GB' : 3}[units] size = int(value*1024**pow) # Return info @@ -46,16 +45,27 @@ def getInfo(urls): class UploadStationCom(Hoster): __name__ = "UploadStationCom" __type__ = "hoster" - __pattern__ = r"http://(www\.)?uploadstation\.com/file/[A-Za-z0-9]+" - __version__ = "0.2" + __pattern__ = r"http://(www\.)?uploadstation\.com/file/(?P[A-Za-z0-9]+)" + __version__ = "0.3" __description__ = """UploadStation.Com File Download Hoster""" __author_name__ = ("fragonib") __author_mail__ = ("fragonib[AT]yahoo[DOT]es") + + FILE_OFFLINE_PATTERN = r'''

File not available

|The file could not be found\. Please check the download link''' + FILE_TITLE_PATTERN = r'''
(.*?)
''' + FILE_SIZE_PATTERN = r'''
File size: (.*?) (KB|MB|GB)''' + CAPTCHA_PRESENT_TOKEN = '