From 35f5b61e7c49d72c13c5220940c3584edf79a4f3 Mon Sep 17 00:00:00 2001
From: Christian Plesche <chris-19@users.noreply.github.com>
Date: Sun, 23 Nov 2014 20:45:56 +0100
Subject: Update NetloadIn.py

changed to recaptcha, initial commit, needs rework
---
 module/plugins/hoster/NetloadIn.py | 203 +++++++++++++++++++------------------
 1 file changed, 103 insertions(+), 100 deletions(-)

(limited to 'module/plugins')

diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py
index 549ecb667..4efd3839b 100644
--- a/module/plugins/hoster/NetloadIn.py
+++ b/module/plugins/hoster/NetloadIn.py
@@ -1,13 +1,13 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import re
-
 from time import sleep, time
 
-from module.network.RequestFactory import getURL
 from module.plugins.Hoster import Hoster
+from module.network.RequestFactory import getURL
 from module.plugins.Plugin import chunks
-
+from module.plugins.internal.CaptchaService import ReCaptcha
 
 def getInfo(urls):
     ##  returns list of tupels (name, size (in bytes), status (see FileDatabase), url)
@@ -26,11 +26,10 @@ def getInfo(urls):
         api = getURL(apiurl + ids, decode=True)
 
         if api is None or len(api) < 10:
-            self.logDebug("Prefetch failed")
+            print "Netload prefetch: failed "
             return
         if api.find("unknown_auth") >= 0:
-            print
-            self.logDebug("Outdated auth code")
+            print "Netload prefetch: Outdated auth code "
             return
 
         result = []
@@ -42,46 +41,45 @@ def getInfo(urls):
                     size = int(tmp[2])
                 except:
                     size = 0
-                result.append((tmp[1], size, 2 if tmp[3] == "online" else 1, chunk[i]))
+                result.append((tmp[1], size, 2 if tmp[3] == "online" else 1, chunk[i] ))
             except:
-                self.logDebug("Error while processing response: %s" % r)
+                print "Netload prefetch: Error while processing response: "
+                print r
 
         yield result
 
 
 class NetloadIn(Hoster):
-    __name__    = "NetloadIn"
-    __type__    = "hoster"
-    __version__ = "0.45"
-
-    __pattern__ = r'https?://(?:[^/]*\.)?netload\.in/(?:datei(.*?)(?:\.htm|/)|index\.php?id=10&file_id=)'
-
-    __description__ = """Netload.in hoster plugin"""
-    __license__     = "GPLv3"
-    __authors__     = [("spoob", "spoob@pyload.org"),
-                       ("RaNaN", "ranan@pyload.org"),
-                       ("Gregy", "gregy@gregy.cz")]
-
+    __name__ = "NetloadIn"
+    __type__ = "hoster"
+    __pattern__ = r"https?://.*netload\.in/(?:datei(.*?)(?:\.htm|/)|index.php?id=10&file_id=)"
+    __version__ = "0.451"
+    __description__ = """Netload.in Download Hoster"""
+    __author_name__ = ("spoob", "RaNaN", "Gregy")
+    __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "gregy@gregy.cz")
+    RECAPTCHA_KEY="6LcLJMQSAAAAAJzquPUPKNovIhbK6LpSqCjYrsR1"
 
     def setup(self):
-        self.multiDL = self.resumeDownload = self.premium
-
+        self.multiDL = False
+        if self.premium:
+            self.multiDL = True
+            self.chunkLimit = -1
+            self.resumeDownload = True
 
     def process(self, pyfile):
         self.url = pyfile.url
         self.prepare()
-        pyfile.setStatus("downloading")
+        self.pyfile.setStatus("downloading")
         self.proceed(self.url)
 
-
     def prepare(self):
         self.download_api_data()
 
-        if self.api_data and self.api_data['filename']:
-            self.pyfile.name = self.api_data['filename']
+        if self.api_data and self.api_data["filename"]:
+            self.pyfile.name = self.api_data["filename"]
 
         if self.premium:
-            self.logDebug("Use Premium Account")
+            self.logDebug("Netload: Use Premium Account")
             settings = self.load("http://www.netload.in/index.php?id=2&lang=en")
             if '<option value="2" selected="selected">Direkter Download' in settings:
                 self.logDebug("Using direct download")
@@ -92,10 +90,9 @@ class NetloadIn(Hoster):
         if self.download_html():
             return True
         else:
-            self.fail(_("Failed"))
+            self.fail("Failed")
             return False
 
-
     def download_api_data(self, n=0):
         url = self.url
         id_regex = re.compile(self.__pattern__)
@@ -104,31 +101,31 @@ class NetloadIn(Hoster):
         if match:
             #normalize url
             self.url = 'http://www.netload.in/datei%s.htm' % match.group(1)
-            self.logDebug("URL: %s" % self.url)
+            self.logDebug("URL after norm: %s" % self.url)
         else:
             self.api_data = False
             return
 
         apiurl = "http://api.netload.in/info.php"
-        html = self.load(apiurl, cookies=False,
+        src = self.load(apiurl, cookies=False,
                         get={"file_id": match.group(1), "auth": "Zf9SnQh9WiReEsb18akjvQGqT0I830e8", "bz": "1",
                              "md5": "1"}, decode=True).strip()
-        if not html and n <= 3:
+        if not src and n <= 3:
             sleep(0.2)
             self.download_api_data(n + 1)
             return
 
-        self.logDebug("APIDATA: " + html)
+        self.logDebug("Netload: APIDATA: " + src)
         self.api_data = {}
-        if html and ";" in html and html not in ("unknown file_data", "unknown_server_data", "No input file specified."):
-            lines = html.split(";")
-            self.api_data['exists'] = True
-            self.api_data['fileid'] = lines[0]
-            self.api_data['filename'] = lines[1]
-            self.api_data['size'] = lines[2]
-            self.api_data['status'] = lines[3]
-            if self.api_data['status'] == "online":
-                self.api_data['checksum'] = lines[4].strip()
+        if src and ";" in src and src not in ("unknown file_data", "unknown_server_data", "No input file specified."):
+            lines = src.split(";")
+            self.api_data["exists"] = True
+            self.api_data["fileid"] = lines[0]
+            self.api_data["filename"] = lines[1]
+            self.api_data["size"] = lines[2]
+            self.api_data["status"] = lines[3]
+            if self.api_data["status"] == "online":
+                self.api_data["checksum"] = lines[4].strip()
             else:
                 self.api_data = False  # check manually since api data is useless sometimes
 
@@ -137,22 +134,36 @@ class NetloadIn(Hoster):
         else:
             self.api_data = False
 
-
     def final_wait(self, page):
         wait_time = self.get_wait_time(page)
         self.setWait(wait_time)
-        self.logDebug("Final wait %d seconds" % wait_time)
+        self.logDebug("Netload: final wait %d seconds" % wait_time)
         self.wait()
         self.url = self.get_file_url(page)
 
+    def check_free_wait(self,page):
+    	#self.logDebug("Page in check_free_wait: " + page)
+	if ">An access request has been made from IP address <" in page:
+	  wait = self.get_wait_time(page)
+	  if wait == 0:
+	    self.logDebug("Netload: Wait was 0 setting 30")
+	    wait = 30
+	  self.logInfo(_("Netload: waiting between downloads %d s." % wait))
+	  self.wantReconnect = True
+	  self.setWait(wait)
+	  self.wait()
+	  return True
+	else:
+	  return False
+
 
     def download_html(self):
-        self.logDebug("Entering download_html")
+        self.logDebug("Netload: Entering download_html (%s)" % self.url)
         page = self.load(self.url, decode=True)
         t = time() + 30
 
         if "/share/templates/download_hddcrash.tpl" in page:
-            self.logError(_("Netload HDD Crash"))
+            self.logError("Netload HDD Crash")
             self.fail(_("File temporarily not available"))
 
         if not self.api_data:
@@ -161,7 +172,7 @@ class NetloadIn(Hoster):
             if "* The file was deleted" in page:
                 self.offline()
 
-            name = re.search(r'class="dl_first_filename">([^<]+)', page, re.M)
+            name = re.search(r'class="dl_first_filename">([^<]+)', page, re.MULTILINE)
             # the found filename is not truncated
             if name:
                 name = name.group(1).strip()
@@ -169,92 +180,84 @@ class NetloadIn(Hoster):
                     self.pyfile.name = name
 
         captchawaited = False
-        for i in xrange(10):
+        for i in range(2):
 
             if not page:
                 page = self.load(self.url)
                 t = time() + 30
 
             if "/share/templates/download_hddcrash.tpl" in page:
-                self.logError(_("Netload HDD Crash"))
+                self.logError("Netload HDD Crash")
                 self.fail(_("File temporarily not available"))
 
-            self.logDebug("Try number %d " % i)
+            self.logDebug("Netload: try number %d " % i)
 
             if ">Your download is being prepared.<" in page:
-                self.logDebug("We will prepare your download")
+                self.logDebug("Netload: We will prepare your download")
                 self.final_wait(page)
                 return True
-            if ">An access request has been made from IP address <" in page:
-                wait = self.get_wait_time(page)
-                if not wait:
-                    self.logDebug("Wait was 0 setting 30")
-                    wait = 30 * 60
-                self.logInfo(_("Waiting between downloads %d seconds") % wait)
-                self.setWait(wait, True)
-                self.wait()
-
-                return self.download_html()
 
-            self.logDebug("Trying to find captcha")
+            self.logDebug("Netload: Trying to find captcha")
 
             try:
-                url_captcha_html = "http://netload.in/" + re.search('(index.php\?id=10&amp;.*&amp;captcha=1)',
-                                                                    page).group(1).replace("amp;", "")
-            except:
+		url_captcha_html = "http://netload.in/" + re.search('(index.php\?id=10&amp;.*&amp;captcha=1)', page).group(1).replace("amp;", "")
+	    except Exception as e:
+	    	self.logDebug("exception during Captcha regex: %s" % e.message)
                 page = None
                 continue
 
-            try:
-                page = self.load(url_captcha_html, cookies=True)
-                captcha_url = "http://netload.in/" + re.search('(share/includes/captcha.php\?t=\d*)', page).group(1)
-            except:
-                self.logDebug("Could not find captcha, try again from beginning")
-                captchawaited = False
-                continue
-
-            file_id = re.search('<input name="file_id" type="hidden" value="(.*)" />', page).group(1)
-            if not captchawaited:
-                wait = self.get_wait_time(page)
-                if i == 0:
-                    self.pyfile.waitUntil = time()  # dont wait contrary to time on website
-                else:
-                    self.pyfile.waitUntil = t
-                self.logInfo(_("Waiting for captcha %d seconds") % (self.pyfile.waitUntil - time()))
-                #self.setWait(wait)
-                self.wait()
-                captchawaited = True
-
-            captcha = self.decryptCaptcha(captcha_url)
-            page = self.load("http://netload.in/index.php?id=10", post={"file_id": file_id, "captcha_check": captcha},
-                             cookies=True)
-
-        return False
-
+	    self.html = self.load(url_captcha_html)
+
+	    recaptcha = ReCaptcha(self)
+	    for _ in xrange(5):
+	    	challenge, response = recaptcha.challenge(self.RECAPTCHA_KEY)
+		#self.logDebug("recaptcha challenge, response, fileID: " + challenge + ", " + response + ", " + self.api_data["fileid"])
+	    	response_page = self.load("http://www.netload.in/index.php?id=10",
+				     post={'captcha_check': '1',
+				     'recaptcha_challenge_field': challenge,
+				     'recaptcha_response_field': response,
+				     'file_id': self.api_data["fileid"],
+				     'Download_Next': ''})
+
+		if "Orange_Link" in response_page:
+		  break
+
+	    if self.check_free_wait(response_page):
+	      self.logDebug("Had to wait for next free slot:")
+	    else:
+	      download_url = self.get_file_url(response_page)
+	      self.logDebug("Download URL after get_file: " + download_url)
+	    if not download_url.startswith("http://"):
+	    	self.error("download url: %s" % download_url)
+	    self.wait()
+
+	    self.url = download_url
+	    #self.logDebug("download_url: " + download_url)
+	    break
+        return True
 
     def get_file_url(self, page):
+    	#self.logDebug("Page: "+ page + "ENDPAGE\n")
         try:
-            file_url_pattern = r'<a class="Orange_Link" href="(http://.+)".?>Or click here'
+            file_url_pattern = r"<a class=\"Orange_Link\" href=\"(http://.+)\".?>Or click here"
             attempt = re.search(file_url_pattern, page)
             if attempt is not None:
                 return attempt.group(1)
             else:
-                self.logDebug("Backup try for final link")
-                file_url_pattern = r'<a href="(.+)" class="Orange_Link">Click here'
+                self.logDebug("Netload: Backup try for final link")
+                file_url_pattern = r"<a href=\"(.+)\" class=\"Orange_Link\">Click here"
                 attempt = re.search(file_url_pattern, page)
                 return "http://netload.in/" + attempt.group(1)
-        except:
-            self.logDebug("Getting final link failed")
+        except Exception as e:
+            self.logDebug("Netload: Getting final link failed; " + e.message)
             return None
 
-
     def get_wait_time(self, page):
         wait_seconds = int(re.search(r"countdown\((.+),'change\(\)'\)", page).group(1)) / 100
-        return wait_seconds
-
+	return wait_seconds
 
     def proceed(self, url):
-        self.logDebug("Downloading..")
+        self.logDebug("Netload: Downloading.." + url)
 
         self.download(url, disposition=True)
 
-- 
cgit v1.2.3


From 17c5924c57a7e3c243b3915094f10dba6c47ff73 Mon Sep 17 00:00:00 2001
From: Christian Plesche <chris-19@users.noreply.github.com>
Date: Sun, 23 Nov 2014 20:53:05 +0100
Subject: Update NetloadIn.py

some code cleanup
---
 module/plugins/hoster/NetloadIn.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'module/plugins')

diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py
index 4efd3839b..a912b438a 100644
--- a/module/plugins/hoster/NetloadIn.py
+++ b/module/plugins/hoster/NetloadIn.py
@@ -158,7 +158,7 @@ class NetloadIn(Hoster):
 
 
     def download_html(self):
-        self.logDebug("Netload: Entering download_html (%s)" % self.url)
+        #self.logDebug("Netload: Entering download_html (%s)" % self.url)
         page = self.load(self.url, decode=True)
         t = time() + 30
 
@@ -180,7 +180,7 @@ class NetloadIn(Hoster):
                     self.pyfile.name = name
 
         captchawaited = False
-        for i in range(2):
+        for i in range(5):
 
             if not page:
                 page = self.load(self.url)
@@ -258,11 +258,9 @@ class NetloadIn(Hoster):
 
     def proceed(self, url):
         self.logDebug("Netload: Downloading.." + url)
-
         self.download(url, disposition=True)
 
         check = self.checkDownload({"empty": re.compile(r"^$"), "offline": re.compile("The file was deleted")})
-
         if check == "empty":
             self.logInfo(_("Downloaded File was empty"))
             self.retry()
-- 
cgit v1.2.3


From be53f2b117c091770ce58f5d2ab479fc1bb57c47 Mon Sep 17 00:00:00 2001
From: Christian Plesche <chris-19@users.noreply.github.com>
Date: Sun, 23 Nov 2014 23:54:55 +0100
Subject: Update NetloadIn.py

restart after waiting period
---
 module/plugins/hoster/NetloadIn.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

(limited to 'module/plugins')

diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py
index a912b438a..204862144 100644
--- a/module/plugins/hoster/NetloadIn.py
+++ b/module/plugins/hoster/NetloadIn.py
@@ -223,17 +223,18 @@ class NetloadIn(Hoster):
 		  break
 
 	    if self.check_free_wait(response_page):
-	      self.logDebug("Had to wait for next free slot:")
+	      self.logDebug("Had to wait for next free slot, trying again")
+	      return self.download_html()
 	    else:
 	      download_url = self.get_file_url(response_page)
 	      self.logDebug("Download URL after get_file: " + download_url)
-	    if not download_url.startswith("http://"):
+	      if not download_url.startswith("http://"):
 	    	self.error("download url: %s" % download_url)
-	    self.wait()
+	      self.wait()
 
-	    self.url = download_url
-	    #self.logDebug("download_url: " + download_url)
-	    break
+	      self.url = download_url
+	      #self.logDebug("download_url: " + download_url)
+	      break
         return True
 
     def get_file_url(self, page):
-- 
cgit v1.2.3