summaryrefslogtreecommitdiffstats
path: root/module/plugins/internal
diff options
context:
space:
mode:
authorGravatar Jochen Oberreiter <joberreiter@users.noreply.github.com> 2015-09-26 18:24:29 +0200
committerGravatar Jochen Oberreiter <joberreiter@users.noreply.github.com> 2015-09-26 18:24:29 +0200
commit93a0c1d930520c055eae766b5dad305111a02c4d (patch)
tree8ac3f9e6dcf682775f5b170f2a9ca40eb7c0f8fc /module/plugins/internal
parentSpare plugin updates (diff)
parentMerge pull request #1850 from chaosblog/patch-2 (diff)
downloadpyload-93a0c1d930520c055eae766b5dad305111a02c4d.tar.xz
Merge pull request #1 from pyload/stable
Merge actual version
Diffstat (limited to 'module/plugins/internal')
-rw-r--r--module/plugins/internal/Account.py12
-rw-r--r--module/plugins/internal/Addon.py23
-rw-r--r--module/plugins/internal/Captcha.py21
-rw-r--r--module/plugins/internal/Container.py20
-rw-r--r--module/plugins/internal/Crypter.py15
-rw-r--r--module/plugins/internal/Extractor.py56
-rw-r--r--module/plugins/internal/Hoster.py310
-rw-r--r--module/plugins/internal/OCR.py6
-rw-r--r--module/plugins/internal/Plugin.py116
-rw-r--r--module/plugins/internal/SevenZip.py32
-rw-r--r--module/plugins/internal/SimpleCrypter.py13
-rw-r--r--module/plugins/internal/SimpleHoster.py77
-rw-r--r--module/plugins/internal/UnRar.py43
-rw-r--r--module/plugins/internal/UnZip.py14
-rw-r--r--module/plugins/internal/XFSAccount.py21
-rw-r--r--module/plugins/internal/XFSCrypter.py4
-rw-r--r--module/plugins/internal/XFSHoster.py14
17 files changed, 415 insertions, 382 deletions
diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py
index 2713e8da4..de338cd33 100644
--- a/module/plugins/internal/Account.py
+++ b/module/plugins/internal/Account.py
@@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size
class Account(Plugin):
__name__ = "Account"
__type__ = "account"
- __version__ = "0.17"
+ __version__ = "0.18"
__status__ = "testing"
__description__ = """Base account plugin"""
@@ -190,7 +190,7 @@ class Account(Plugin):
def get_info(self, user, reload=False):
"""
Retrieve account infos for an user, do **not** overwrite this method!\\
- just use it to retrieve infos in hoster plugins. see `parse_info`
+ just use it to retrieve infos in hoster plugins. see `grab_info`
:param user: username
:param reload: reloads cached account information
@@ -235,7 +235,7 @@ class Account(Plugin):
try:
self.req = self.get_request(user)
- extra_info = self.parse_info(user, info['login']['password'], info, self.req)
+ extra_info = self.grab_info(user, info['login']['password'], info, self.req)
if extra_info and isinstance(extra_info, dict):
info['data'].update(extra_info)
@@ -253,7 +253,7 @@ class Account(Plugin):
return info
- def parse_info(self, user, password, info, req):
+ def grab_info(self, user, password, info, req):
"""
This should be overwritten in account plugin
and retrieving account information for user
@@ -270,8 +270,8 @@ class Account(Plugin):
return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.items()]
- def login_fail(self, reason=_("Login handshake has failed")):
- return self.fail(reason)
+ def fail_login(self, msg=_("Login handshake has failed")):
+ return self.fail(msg)
def get_request(self, user=None):
diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py
index 45ca98eac..5150e88f6 100644
--- a/module/plugins/internal/Addon.py
+++ b/module/plugins/internal/Addon.py
@@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
-import traceback
-
from module.plugins.internal.Plugin import Plugin
@@ -25,7 +23,7 @@ def threaded(fn):
class Addon(Plugin):
__name__ = "Addon"
__type__ = "hook" #@TODO: Change to `addon` in 0.4.10
- __version__ = "0.04"
+ __version__ = "0.06"
__status__ = "testing"
__config__ = [] #: [("name", "type", "desc", "default")]
@@ -57,6 +55,12 @@ class Addon(Plugin):
self.init_events()
+ #@TODO: Remove in 0.4.10
+ def _log(self, level, plugintype, pluginname, messages):
+ plugintype = "addon" if plugintype is "hook" else plugintype
+ return super(Addon, self)._log(level, plugintype, pluginname, messages)
+
+
def init_events(self):
if self.event_map:
for event, funcs in self.event_map.items():
@@ -97,8 +101,6 @@ class Addon(Plugin):
except Exception, e:
self.log_error(_("Error executing periodical task: %s") % e)
- if self.pyload.debug:
- traceback.print_exc()
self.cb = self.pyload.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded)
@@ -107,20 +109,17 @@ class Addon(Plugin):
pass
- def __repr__(self):
- return "<Addon %s>" % self.__name__
-
-
- def is_activated(self):
+ @property
+ def activated(self):
"""
Checks if addon is activated
"""
return self.get_config("activated")
- #: Deprecated method, use `is_activated` instead (Remove in 0.4.10)
+ #: Deprecated method, use `activated` property instead (Remove in 0.4.10)
def isActivated(self, *args, **kwargs):
- return self.is_activated(*args, **kwargs)
+ return self.activated
def deactivate(self):
diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py
index c08050ee8..d2be21a58 100644
--- a/module/plugins/internal/Captcha.py
+++ b/module/plugins/internal/Captcha.py
@@ -4,7 +4,6 @@ from __future__ import with_statement
import os
import time
-import traceback
from module.plugins.internal.Plugin import Plugin
@@ -12,7 +11,7 @@ from module.plugins.internal.Plugin import Plugin
class Captcha(Plugin):
__name__ = "Captcha"
__type__ = "captcha"
- __version__ = "0.42"
+ __version__ = "0.44"
__status__ = "testing"
__description__ = """Base anti-captcha plugin"""
@@ -50,18 +49,18 @@ class Captcha(Plugin):
pass
- def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False,
+ def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False,
input_type='jpg', output_type='textual', ocr=True, timeout=120):
img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode)
return self._decrypt(img, input_type, output_type, ocr, timeout)
#@TODO: Definitely choose a better name for this method!
- def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=False, timeout=120):
+ def _decrypt(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120):
"""
Loads a captcha and decrypts it with ocr, plugin, user input
- :param raw: image raw data
+ :param data: image raw data
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
@@ -77,7 +76,7 @@ class Captcha(Plugin):
time_ref = ("%.2f" % time.time())[-6:].replace(".", "")
with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img:
- tmp_img.write(raw)
+ tmp_img.write(data)
if ocr:
if isinstance(ocr, basestring):
@@ -90,14 +89,13 @@ class Captcha(Plugin):
captchaManager = self.pyload.captchaManager
try:
- self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type)
+ self.task = captchaManager.newTask(data, input_type, tmp_img.name, output_type)
captchaManager.handleCaptcha(self.task)
self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10
while self.task.isWaiting():
- if self.plugin.pyfile.abort:
- self.plugin.abort()
+ self.plugin.check_abort()
time.sleep(1)
finally:
@@ -108,7 +106,7 @@ class Captcha(Plugin):
elif not self.task.result:
self.invalid()
- self.plugin.retry(reason=_("No captcha result obtained in appropiate time"))
+ self.plugin.retry(msg=_("No captcha result obtained in appropiate time"))
result = self.task.result
@@ -118,9 +116,8 @@ class Captcha(Plugin):
except OSError, e:
self.log_warning(_("Error removing: %s") % tmp_img.name, e)
- traceback.print_exc()
- self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here?
+ #self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here?
return result
diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py
index 729592a0d..430590421 100644
--- a/module/plugins/internal/Container.py
+++ b/module/plugins/internal/Container.py
@@ -4,7 +4,6 @@ from __future__ import with_statement
import os
import re
-import traceback
from module.plugins.internal.Crypter import Crypter
from module.plugins.internal.Plugin import exists
@@ -14,7 +13,7 @@ from module.utils import save_join as fs_join
class Container(Crypter):
__name__ = "Container"
__type__ = "container"
- __version__ = "0.06"
+ __version__ = "0.07"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -44,11 +43,6 @@ class Container(Crypter):
self._create_packages()
- #: Deprecated method, use `_load2disk` instead (Remove in 0.4.10)
- def loadToDisk(self, *args, **kwargs):
- return self._load2disk(*args, **kwargs)
-
-
def _load2disk(self):
"""
Loads container to disk if its stored remotely and overwrite url,
@@ -63,20 +57,18 @@ class Container(Crypter):
f.write(content)
except IOError, e:
- self.fail(str(e)) #@TODO: Remove `str` in 0.4.10
+ self.fail(e)
else:
self.pyfile.name = os.path.basename(self.pyfile.url)
+
if not exists(self.pyfile.url):
if exists(fs_join(pypath, self.pyfile.url)):
self.pyfile.url = fs_join(pypath, self.pyfile.url)
else:
self.fail(_("File not exists"))
-
-
- #: Deprecated method, use `delete_tmp` instead (Remove in 0.4.10)
- def deleteTmp(self, *args, **kwargs):
- return self.delete_tmp(*args, **kwargs)
+ else:
+ self.data = self.pyfile.url
def delete_tmp(self):
@@ -87,5 +79,3 @@ class Container(Crypter):
os.remove(self.pyfile.url)
except OSError, e:
self.log_warning(_("Error removing: %s") % self.pyfile.url, e)
- if self.pyload.debug:
- traceback.print_exc()
diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py
index d0e8eb1b4..2033b67df 100644
--- a/module/plugins/internal/Crypter.py
+++ b/module/plugins/internal/Crypter.py
@@ -1,15 +1,13 @@
# -*- coding: utf-8 -*-
-import urlparse
-
-from module.plugins.internal.Hoster import Hoster, _fixurl
+from module.plugins.internal.Hoster import Hoster, parse_name
from module.utils import save_path as safe_filename
class Crypter(Hoster):
__name__ = "Crypter"
__type__ = "crypter"
- __version__ = "0.07"
+ __version__ = "0.08"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -78,13 +76,14 @@ class Crypter(Hoster):
"%d links" % len(links),
"Saved to folder: %s" % folder if folder else "Saved to download folder")
- pid = self.pyload.api.addPackage(name, map(self.fixurl, links), package_queue)
+ links = map(self.fixurl, links)
+ pid = self.pyload.api.addPackage(name, links, package_queue)
if package_password:
self.pyload.api.setPackageData(pid, {'password': package_password})
#: Workaround to do not break API addPackage method
- set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""})
+ set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""})
if use_subfolder:
if not subfolder_per_package:
@@ -93,9 +92,9 @@ class Crypter(Hoster):
elif not folder_per_package or name is not folder:
if not folder:
- folder = urlparse.urlparse(_fixurl(name)).path.split("/")[-1]
+ folder = parse_name(name)
- set_folder(safe_filename(folder))
+ set_folder(folder)
self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder})
elif folder_per_package:
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 7f5212090..f21fe473c 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -22,7 +22,7 @@ class PasswordError(Exception):
class Extractor(Plugin):
__name__ = "Extractor"
__type__ = "extractor"
- __version__ = "0.33"
+ __version__ = "0.34"
__status__ = "testing"
__description__ = """Base extractor plugin"""
@@ -43,15 +43,9 @@ class Extractor(Plugin):
@classmethod
- def is_multipart(cls, filename):
- return False
-
-
- @classmethod
def find(cls):
"""
Check if system statisfy dependencies
- :return: boolean
"""
pass
@@ -72,9 +66,15 @@ class Extractor(Plugin):
if pname not in processed:
processed.append(pname)
targets.append((fname, id, fout))
+
return targets
+ @property
+ def target(self):
+ return fs_encode(self.filename)
+
+
def __init__(self, plugin, filename, out,
fullpath=True,
overwrite=False,
@@ -119,53 +119,29 @@ class Extractor(Plugin):
(self.__name__,) + messages)
- def check(self):
+ def verify(self, password=None):
"""
- Quick Check by listing content of archive.
- Raises error if password is needed, integrity is questionable or else.
-
- :raises PasswordError
- :raises CRCError
- :raises ArchiveError
+ Testing with Extractors built-in method
+ Raise error if password is needed, integrity is questionable or else
"""
- raise NotImplementedError
-
-
- def verify(self):
- """
- Testing with Extractors buildt-in method
- Raises error if password is needed, integrity is questionable or else.
-
- :raises PasswordError
- :raises CRCError
- :raises ArchiveError
- """
- raise NotImplementedError
+ pass
def repair(self):
- return None
+ return False
def extract(self, password=None):
"""
- Extract the archive. Raise specific errors in case of failure.
-
- :param progress: Progress function, call this to update status
- :param password password to use
- :raises PasswordError
- :raises CRCError
- :raises ArchiveError
- :return:
+ Extract the archive
+ Raise specific errors in case of failure
"""
raise NotImplementedError
- def get_delete_files(self):
+ def items(self):
"""
- Return list of files to delete, do *not* delete them here.
-
- :return: List with paths of files to delete
+ Return list of archive parts
"""
return [self.filename]
diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py
index b397a92a6..5d0a64f1a 100644
--- a/module/plugins/internal/Hoster.py
+++ b/module/plugins/internal/Hoster.py
@@ -7,21 +7,21 @@ import mimetypes
import os
import random
import time
-import traceback
import urlparse
from module.plugins.internal.Captcha import Captcha
from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip,
- chunks, encode, exists, fixurl as _fixurl, replace_patterns,
- seconds_to_midnight, set_cookie, set_cookies, parse_html_form,
- parse_html_tag_attr_value, timestamp)
+ chunks, decode, encode, exists, parse_html_form,
+ parse_html_tag_attr_value, parse_name,
+ replace_patterns, seconds_to_midnight,
+ set_cookie, set_cookies, timestamp)
from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename
#@TODO: Remove in 0.4.10
def parse_fileInfo(klass, url="", html=""):
info = klass.get_info(url, html)
- return info['name'], info['size'], info['status'], info['url']
+ return encode(info['name']), info['size'], info['status'], info['url']
#@TODO: Remove in 0.4.10
@@ -44,7 +44,7 @@ def create_getInfo(klass):
class Hoster(Plugin):
__name__ = "Hoster"
__type__ = "hoster"
- __version__ = "0.20"
+ __version__ = "0.28"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -74,7 +74,6 @@ class Hoster(Plugin):
#: Account handler instance, see :py:class:`Account`
self.account = None
- self.user = None
self.req = None #: Browser instance, see `network.Browser`
#: Associated pyfile instance, see `PyFile`
@@ -105,15 +104,21 @@ class Hoster(Plugin):
self.init()
+ def _log(self, level, plugintype, pluginname, messages):
+ log = getattr(self.pyload.log, level)
+ msg = u" | ".join(decode(a).strip() for a in messages if a)
+ log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s"
+ % {'plugintype': plugintype.upper(),
+ 'pluginname': pluginname,
+ 'id' : self.pyfile.id,
+ 'msg' : msg})
+
+
@classmethod
def get_info(cls, url="", html=""):
- url = _fixurl(url)
- url_p = urlparse.urlparse(url)
- return {'name' : (url_p.path.split('/')[-1] or
- url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
- url_p.netloc.split('.', 1)[0]),
+ return {'name' : parse_name(url),
'size' : 0,
- 'status': 3 if url else 8,
+ 'status': 3 if url.strip() else 8,
'url' : url}
@@ -132,34 +137,26 @@ class Hoster(Plugin):
def _setup(self):
+ #@TODO: Remove in 0.4.10
+ self.html = ""
+ self.last_download = ""
+ self.pyfile.error = ""
+
if self.account:
- self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user)
+ self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user)
self.chunk_limit = -1 #: -1 for unlimited
self.resume_download = True
- self.premium = self.account.is_premium(self.user)
+ self.premium = self.account.premium
else:
self.req = self.pyload.requestFactory.getRequest(self.__name__)
self.chunk_limit = 1
self.resume_download = False
self.premium = False
+ return self.setup()
- def load_account(self):
- if self.req:
- self.req.close()
-
- if not self.account:
- self.account = self.pyload.accountManager.getAccountPlugin(self.__name__)
- if self.account:
- if not self.user:
- self.user = self.account.select()[0]
-
- if not self.user or not self.account.is_logged(self.user, True):
- self.account = False
-
-
- def preprocessing(self, thread):
+ def _process(self, thread):
"""
Handles important things to do before starting
"""
@@ -172,19 +169,36 @@ class Hoster(Plugin):
self.retry_free = False
self._setup()
- self.setup()
+ self.pyfile.setStatus("starting")
self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10
- if self.pyfile.abort:
- self.abort()
+ self.check_abort()
- self.pyfile.setStatus("starting")
self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__)
-
return self.process(self.pyfile)
+ #: Deprecated method, use `_process` instead (Remove in 0.4.10)
+ def preprocessing(self, *args, **kwargs):
+ return self._process(*args, **kwargs)
+
+
+ def load_account(self):
+ if self.req:
+ self.req.close()
+
+ if not self.account:
+ self.account = self.pyload.accountManager.getAccountPlugin(self.__name__)
+
+ if self.account:
+ if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10
+ self.account.user = self.account.select()[0]
+
+ if not hasattr(self.account, 'logged'):
+ self.account = False
+
+
def process(self, pyfile):
"""
The 'main' method of every plugin, you **have to** overwrite it
@@ -193,12 +207,13 @@ class Hoster(Plugin):
def set_reconnect(self, reconnect):
- reconnect = bool(reconnect)
+ if reconnect:
+ self.log_info(_("Requesting line reconnection..."))
+ else:
+ self.log_debug("Reconnect: %s" % reconnect)
- self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled"))
self.log_debug("Previous wantReconnect: %s" % self.wantReconnect)
-
- self.wantReconnect = reconnect
+ self.wantReconnect = bool(reconnect)
def set_wait(self, seconds, reconnect=None):
@@ -211,7 +226,7 @@ class Hoster(Plugin):
wait_time = max(int(seconds), 1)
wait_until = time.time() + wait_time + 1
- self.log_info(_("WAIT %d seconds") % wait_time)
+ self.log_info(_("Waiting %d seconds...") % wait_time)
self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil)
self.pyfile.waitUntil = wait_until
@@ -242,15 +257,12 @@ class Hoster(Plugin):
self.log_warning("Ignore reconnection due logged account")
while pyfile.waitUntil > time.time():
- if pyfile.abort:
- self.abort()
-
+ self.check_abort()
time.sleep(2)
else:
while pyfile.waitUntil > time.time():
- if pyfile.abort:
- self.abort()
+ self.check_abort()
if self.thread.m.reconnecting.isSet():
self.waiting = False
@@ -264,91 +276,113 @@ class Hoster(Plugin):
pyfile.status = status #@NOTE: Remove in 0.4.10
- def skip(self, reason=""):
+ def skip(self, msg=""):
"""
- Skip and give reason
+ Skip and give msg
"""
- raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10
+ raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10
- def abort(self, reason=""):
+ #@TODO: Remove in 0.4.10
+ def fail(self, msg):
"""
- Abort and give reason
+ Fail and give msg
"""
- #@TODO: Remove in 0.4.10
- if reason:
- self.pyfile.error = encode(reason)
+ msg = msg.strip()
+
+ if msg:
+ self.pyfile.error = msg
+ else:
+ msg = self.pyfile.error
+
+ raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10
+
+
+ def error(self, msg="", type=_("Parse")):
+ type = _("%s error") % type.strip().capitalize() if type else _("Unknown")
+ msg = _("%(type)s: %(msg)s | Plugin may be out of date"
+ % {'type': type, 'msg': msg or self.pyfile.error})
+
+ self.fail(msg)
+
+
+ def abort(self, msg=""):
+ """
+ Abort and give msg
+ """
+ if msg: #@TODO: Remove in 0.4.10
+ self.pyfile.error = encode(msg)
raise Abort
- def offline(self, reason=""):
+ #@TODO: Recheck in 0.4.10
+ def offline(self, msg=""):
"""
Fail and indicate file is offline
"""
- #@TODO: Remove in 0.4.10
- if reason:
- self.pyfile.error = encode(reason)
-
- raise Fail("offline")
+ self.fail("offline")
- def temp_offline(self, reason=""):
+ #@TODO: Recheck in 0.4.10
+ def temp_offline(self, msg=""):
"""
Fail and indicates file ist temporary offline, the core may take consequences
"""
- #@TODO: Remove in 0.4.10
- if reason:
- self.pyfile.error = encode(reason)
+ self.fail("temp. offline")
- raise Fail("temp. offline")
-
- def retry(self, max_tries=5, wait_time=1, reason=""):
+ def retry(self, attemps=5, delay=1, msg=""):
"""
Retries and begin again from the beginning
- :param max_tries: number of maximum retries
- :param wait_time: time to wait in seconds
- :param reason: reason for retrying, will be passed to fail if max_tries reached
+ :param attemps: number of maximum retries
+ :param delay: time to wait in seconds
+ :param msg: msg for retrying, will be passed to fail if attemps value was reached
"""
id = inspect.currentframe().f_back.f_lineno
if id not in self.retries:
self.retries[id] = 0
- if 0 < max_tries <= self.retries[id]:
- self.fail(reason or _("Max retries reached"))
+ if 0 < attemps <= self.retries[id]:
+ self.fail(msg or _("Max retries reached"))
- self.wait(wait_time, False)
+ self.wait(delay, False)
self.retries[id] += 1
- raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
+ raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10
- def restart(self, reason=None, nopremium=False):
- if not reason:
- reason = _("Fallback to free download") if nopremium else _("Restart")
+ def restart(self, msg=None, nopremium=False):
+ if not msg:
+ msg = _("Fallback to free download") if nopremium else _("Restart")
if nopremium:
if self.premium:
self.retry_free = True
else:
- self.fail("%s | %s" % (reason, _("Download was already free")))
+ self.fail("%s | %s" % (msg, _("Download was already free")))
- raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
+ raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10
- def fixurl(self, url):
- url = _fixurl(url)
+ def fixurl(self, url, baseurl=None):
+ if not baseurl:
+ baseurl = self.pyfile.url
if not urlparse.urlparse(url).scheme:
- url_p = urlparse.urlparse(self.pyfile.url)
+ url_p = urlparse.urlparse(baseurl)
baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
url = urlparse.urljoin(baseurl, url)
return url
+ def load(self, *args, **kwargs):
+ self.check_abort()
+ return super(Hoster, self).load(*args, **kwargs)
+
+
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True):
"""
Downloads the content at url to download folder
@@ -362,20 +396,13 @@ class Hoster(Plugin):
the filename will be changed if needed
:return: The location where the file was saved
"""
- if self.pyfile.abort:
- self.abort()
-
- url = self.fixurl(url)
-
- if not url or not isinstance(url, basestring):
- self.fail(_("No url given"))
+ self.check_abort()
if self.pyload.debug:
self.log_debug("DOWNLOAD URL " + url,
*["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
- name = _fixurl(self.pyfile.name)
- self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name
+ self.pyfile.name = parse_name(self.pyfile.name) #: Safe check
self.captcha.correct()
self.check_for_same_files()
@@ -388,6 +415,7 @@ class Hoster(Plugin):
if not exists(download_location):
try:
os.makedirs(download_location)
+
except Exception, e:
self.fail(e)
@@ -398,8 +426,7 @@ class Hoster(Plugin):
self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename)
- if self.pyfile.abort:
- self.abort()
+ self.check_abort()
try:
newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
@@ -410,9 +437,9 @@ class Hoster(Plugin):
#@TODO: Recheck in 0.4.10
if disposition and newname:
- finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0]
+ finalname = parse_name(newname).split(' filename*=')[0]
- if finalname != newname != self.pyfile.name:
+ if finalname != newname:
try:
os.rename(fs_join(location, newname), fs_join(location, finalname))
@@ -421,8 +448,9 @@ class Hoster(Plugin):
finalname = newname
self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname))
- self.pyfile.name = finalname
- filename = os.path.join(location, finalname)
+
+ self.pyfile.name = finalname
+ filename = os.path.join(location, finalname)
self.set_permissions(fs_encode(filename))
@@ -431,9 +459,55 @@ class Hoster(Plugin):
return self.last_download
- def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576):
+ def check_abort(self):
+ if not self.pyfile.abort:
+ return
+
+ if self.pyfile.hasStatus("failed"):
+ self.fail()
+
+ elif self.pyfile.hasStatus("skipped"):
+ self.skip(self.pyfile.statusname)
+
+ elif self.pyfile.hasStatus("offline"):
+ self.offline()
+
+ elif self.pyfile.hasStatus("temp. offline"):
+ self.temp_offline()
+
+ else:
+ self.abort()
+
+
+ def check_filesize(self, file_size, size_tolerance=1024):
+ """
+ Checks the file size of the last downloaded file
+
+ :param file_size: expected file size
+ :param size_tolerance: size check tolerance
+ """
+ if not self.last_download:
+ return
+
+ download_size = os.stat(fs_encode(self.last_download)).st_size
+
+ if download_size < 1:
+ self.fail(_("Empty file"))
+
+ elif file_size > 0:
+ diff = abs(file_size - download_size)
+
+ if diff > size_tolerance:
+ self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s")
+ % (file_size, download_size))
+
+ elif diff != 0:
+ self.log_warning(_("File size is not equal to expected size"))
+
+
+ def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024):
"""
- Checks the content of the last downloaded file, re match is saved to `lastCheck`
+ Checks the content of the last downloaded file, re match is saved to `last_check`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param delete: delete if matched
@@ -446,26 +520,10 @@ class Hoster(Plugin):
last_download = fs_encode(self.last_download)
if not self.last_download or not exists(last_download):
- self.last_download = ""
self.fail(self.pyfile.error or _("No file downloaded"))
try:
- download_size = os.stat(last_download).st_size
-
- if download_size < 1:
- do_delete = True
- self.fail(_("Empty file"))
-
- elif file_size > 0:
- diff = abs(file_size - download_size)
-
- if diff > size_tolerance:
- do_delete = True
- self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s")
- % (file_size, download_size))
-
- elif diff != 0:
- self.log_warning(_("File size is not equal to expected size"))
+ self.check_filesize(file_size, size_tolerance)
with open(last_download, "rb") as f:
content = f.read(read_size)
@@ -491,12 +549,10 @@ class Hoster(Plugin):
except OSError, e:
self.log_warning(_("Error removing: %s") % last_download, e)
- if self.pyload.debug:
- traceback.print_exc()
else:
+ self.log_info(_("File deleted: ") + self.last_download)
self.last_download = ""
- self.log_info(_("File deleted"))
def direct_link(self, url, follow_location=None):
@@ -519,7 +575,7 @@ class Hoster(Plugin):
except Exception: #: Bad bad bad... rewrite this part in 0.4.10
res = self.load(url,
just_header=True,
- req=self.pyload.requestFactory.getRequest())
+ req=self.pyload.requestFactory.getRequest(self.__name__))
header = {'code': req.code}
for line in res.splitlines():
@@ -543,12 +599,7 @@ class Hoster(Plugin):
link = url
elif 'location' in header and header['location']:
- location = header['location']
-
- if not urlparse.urlparse(location).scheme:
- url_p = urlparse.urlparse(url)
- baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
- location = urlparse.urljoin(baseurl, location)
+ location = self.fixurl(header['location'], url)
if 'code' in header and header['code'] == 302:
link = location
@@ -558,7 +609,7 @@ class Hoster(Plugin):
continue
else:
- extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1]
+ extension = os.path.splitext(parse_name(url))[-1]
if 'content-type' in header and header['content-type']:
mimetype = header['content-type'].split(';')[0].strip()
@@ -579,6 +630,7 @@ class Hoster(Plugin):
else:
try:
self.log_error(_("Too many redirects"))
+
except Exception:
pass
@@ -593,15 +645,17 @@ class Hoster(Plugin):
if not self.account:
return True
- traffic = self.account.get_data(self.user, True)['trafficleft']
+ traffic = self.account.get_data(self.account.user, True)['trafficleft']
if traffic is None:
return False
+
elif traffic == -1:
return True
+
else:
size = self.pyfile.size / 1024
- self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic))
+ self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic))
return size <= traffic
diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py
index b24b3058b..3e5afae69 100644
--- a/module/plugins/internal/OCR.py
+++ b/module/plugins/internal/OCR.py
@@ -12,7 +12,6 @@ import logging
import os
import subprocess
# import tempfile
-import traceback
from module.plugins.internal.Plugin import Plugin
from module.utils import save_join as fs_join
@@ -128,6 +127,7 @@ class OCR(Plugin):
try:
with open(tmpTxt.name, 'r') as f:
self.result_captcha = f.read().replace("\n", "")
+
except Exception:
self.result_captcha = ""
@@ -137,10 +137,9 @@ class OCR(Plugin):
os.remove(tmpTxt.name)
if subset and (digits or lowercase or uppercase):
os.remove(tmpSub.name)
+
except OSError, e:
self.log_warning(e)
- if self.pyload.debug:
- traceback.print_exc()
def recognize(self, name):
@@ -194,6 +193,7 @@ class OCR(Plugin):
count += 1
if pixels[x, y - 1] != 255:
count += 1
+
except Exception:
pass
diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py
index 7b45c40a8..51192d8c9 100644
--- a/module/plugins/internal/Plugin.py
+++ b/module/plugins/internal/Plugin.py
@@ -6,7 +6,10 @@ import datetime
import inspect
import os
import re
+import sys
+import traceback
import urllib
+import urlparse
if os.name != "nt":
import grp
@@ -22,7 +25,7 @@ def decode(string, encoding='utf8'):
if type(string) is str:
return string.decode(encoding, "replace")
else:
- return string
+ return unicode(string)
#@TODO: Move to utils in 0.4.10
@@ -31,7 +34,7 @@ def encode(string, encoding='utf8'):
if type(string) is unicode:
return string.encode(encoding, "replace")
else:
- return string
+ return str(string)
#@TODO: Move to utils in 0.4.10
@@ -47,8 +50,19 @@ def exists(path):
#@TODO: Move to utils in 0.4.10
-def fixurl(url):
- return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/')
+def parse_name(url):
+ url = urllib.unquote(url)
+ url = url.decode('unicode-escape')
+ url = html_unescape(url)
+ url = urllib.quote(url)
+
+ url_p = urlparse.urlparse(url.strip().rstrip('/'))
+
+ name = (url_p.path.split('/')[-1] or
+ url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
+ url_p.netloc.split('.', 1)[0])
+
+ return urllib.unquote(name)
#@TODO: Move to utils in 0.4.10
@@ -56,22 +70,35 @@ def timestamp():
return int(time.time() * 1000)
-def seconds_to_midnight(gmt=0):
- now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt)
-
- if now.hour == 0 and now.minute < 10:
- midnight = now
+#@TODO: Move to utils in 0.4.10
+def which(program):
+ """
+ Works exactly like the unix command which
+ Courtesy of http://stackoverflow.com/a/377028/675646
+ """
+ isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK)
+
+ fpath, fname = os.path.split(program)
+
+ if fpath:
+ if isExe(program):
+ return program
else:
- midnight = now + datetime.timedelta(days=1)
+ for path in os.environ['PATH'].split(os.pathsep):
+ exe_file = os.path.join(path.strip('"'), program)
+ if isExe(exe_file):
+ return exe_file
- td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now
- if hasattr(td, 'total_seconds'):
- res = td.total_seconds()
- else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds
- res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
+def seconds_to_midnight(utc=None):
+ if utc is None:
+ now = datetime.datetime.today()
+ else:
+ now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc)
- return int(res)
+ midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1)
+
+ return (midnight - now).seconds
def replace_patterns(string, ruleslist):
@@ -145,8 +172,8 @@ def chunks(iterable, size):
class Plugin(object):
__name__ = "Plugin"
- __type__ = "hoster"
- __version__ = "0.30"
+ __type__ = "plugin"
+ __version__ = "0.37"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -165,6 +192,11 @@ class Plugin(object):
self.init()
+ def __repr__(self):
+ return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize(),
+ 'name': self.__name__}
+
+
def _init(self, core):
self.pyload = core
self.info = {} #: Provide information in dict here
@@ -180,33 +212,39 @@ class Plugin(object):
def _log(self, level, plugintype, pluginname, messages):
log = getattr(self.pyload.log, level)
- msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a))
- log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s"
+ msg = u" | ".join(decode(a).strip() for a in messages if a)
+ log("%(plugintype)s %(pluginname)s: %(msg)s"
% {'plugintype': plugintype.upper(),
'pluginname': pluginname,
- 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "",
'msg' : msg})
def log_debug(self, *args):
- if self.pyload.debug:
- return self._log("debug", self.__type__, self.__name__, args)
+ if not self.pyload.debug:
+ return
+ self._log("debug", self.__type__, self.__name__, args)
def log_info(self, *args):
- return self._log("info", self.__type__, self.__name__, args)
+ self._log("info", self.__type__, self.__name__, args)
def log_warning(self, *args):
- return self._log("warning", self.__type__, self.__name__, args)
+ self._log("warning", self.__type__, self.__name__, args)
+ if self.pyload.debug:
+ traceback.print_exc()
def log_error(self, *args):
- return self._log("error", self.__type__, self.__name__, args)
+ self._log("error", self.__type__, self.__name__, args)
+ if self.pyload.debug:
+ traceback.print_exc()
def log_critical(self, *args):
return self._log("critical", self.__type__, self.__name__, args)
+ if self.pyload.debug:
+ traceback.print_exc()
def set_permissions(self, path):
@@ -287,21 +325,10 @@ class Plugin(object):
self.pyload.db.delStorage(self.__name__, key)
- def fail(self, reason):
+ def fail(self, msg):
"""
- Fail and give reason
+ Fail and give msg
"""
- raise Fail(encode(reason)) #@TODO: Remove `encode` in 0.4.10
-
-
- def error(self, reason="", type=_("Parse")):
- if not reason:
- type = _("Unknown")
-
- msg = _("%s error") % type.strip().capitalize() if type else _("Error")
- msg += (": %s" % reason.strip()) if reason else ""
- msg += _(" | Plugin may be out of date")
-
raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10
@@ -318,14 +345,6 @@ class Plugin(object):
:param decode: Wether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
- if hasattr(self, 'pyfile') and self.pyfile.abort:
- self.abort()
-
- url = fixurl(url)
-
- if not url or not isinstance(url, basestring):
- self.fail(_("No url given"))
-
if self.pyload.debug:
self.log_debug("LOAD URL " + url,
*["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
@@ -345,7 +364,7 @@ class Plugin(object):
#@TODO: Move to network in 0.4.10
if isinstance(decode, basestring):
- res = decode(res, decode)
+ res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10
if self.pyload.debug:
frame = inspect.currentframe()
@@ -391,6 +410,7 @@ class Plugin(object):
"""
try:
self.req.close()
+
except Exception:
pass
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index 5811c28de..b79256536 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join
class SevenZip(UnRar):
__name__ = "SevenZip"
- __version__ = "0.14"
+ __version__ = "0.15"
__status__ = "testing"
__description__ = """7-Zip extractor plugin"""
@@ -55,42 +55,28 @@ class SevenZip(UnRar):
return True
- def verify(self, password):
+ def verify(self, password=None):
#: 7z can't distinguish crc and pw error in test
- p = self.call_cmd("l", "-slt", fs_encode(self.filename))
+ p = self.call_cmd("l", "-slt", self.target)
out, err = p.communicate()
if self.re_wrongpwd.search(out):
raise PasswordError
- if self.re_wrongpwd.search(err):
+ elif self.re_wrongpwd.search(err):
raise PasswordError
- if self.re_wrongcrc.search(err):
- raise CRCError(err)
-
-
-
- def check(self, password):
- p = self.call_cmd("l", "-slt", fs_encode(self.filename))
- out, err = p.communicate()
-
- #: Check if output or error macthes the 'wrong password'-Regexp
- if self.re_wrongpwd.search(out):
- raise PasswordError
-
- if self.re_wrongcrc.search(out):
+ elif self.re_wrongcrc.search(out):
raise CRCError(_("Header protected"))
-
- def repair(self):
- return False
+ elif self.re_wrongcrc.search(err):
+ raise CRCError(err)
def extract(self, password=None):
command = "x" if self.fullpath else "e"
- p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password)
+ p = self.call_cmd(command, '-o' + self.out, self.target, password=password)
renice(p.pid, self.renice)
@@ -117,7 +103,7 @@ class SevenZip(UnRar):
def list(self, password=None):
command = "l" if self.fullpath else "l"
- p = self.call_cmd(command, fs_encode(self.filename), password=password)
+ p = self.call_cmd(command, self.target, password=password)
out, err = p.communicate()
if "Can not open" in err:
diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py
index 6a3f91a5b..8c5d3599d 100644
--- a/module/plugins/internal/SimpleCrypter.py
+++ b/module/plugins/internal/SimpleCrypter.py
@@ -4,17 +4,16 @@ import re
from module.plugins.internal.Crypter import Crypter
from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies
-from module.utils import fixup, html_unescape
class SimpleCrypter(Crypter, SimpleHoster):
__name__ = "SimpleCrypter"
__type__ = "crypter"
- __version__ = "0.60"
+ __version__ = "0.62"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
- __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package']
+ __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True),
("subfolder_per_pack", "bool", "Create a subfolder for each package", True)]
__description__ = """Simple decrypter plugin"""
@@ -90,6 +89,11 @@ class SimpleCrypter(Crypter, SimpleHoster):
self.log_error(_("Too many redirects"))
+ def prepare(self):
+ self.links = []
+ return super(SimpleCrypter, self).prepare()
+
+
def decrypt(self, pyfile):
self.prepare()
self.check_info() #@TODO: Remove in 0.4.10
@@ -132,7 +136,8 @@ class SimpleCrypter(Crypter, SimpleHoster):
def handle_pages(self, pyfile):
try:
pages = int(re.search(self.PAGES_PATTERN, self.html).group(1))
- except Exception:
+
+ except AttributeError:
pages = 1
for p in xrange(2, pages + 1):
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index 69f88081a..0f030e000 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -5,13 +5,12 @@ from __future__ import with_statement
import os
import re
import time
-import urlparse
from module.PyFile import statusMap as _statusMap
from module.network.HTTPRequest import BadHeader
from module.network.RequestFactory import getURL as get_url
from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo
-from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookie, set_cookies
+from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies
from module.utils import fixup, fs_encode, parseFileSize as parse_size
@@ -22,12 +21,13 @@ statusMap = dict((v, k) for k, v in _statusMap.items())
class SimpleHoster(Hoster):
__name__ = "SimpleHoster"
__type__ = "hoster"
- __version__ = "1.81"
+ __version__ = "1.86"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
- __config__ = [("use_premium", "bool", "Use premium account if available" , True),
- ("fallback" , "bool", "Fallback to free download if premium fails", True)]
+ __config__ = [("use_premium" , "bool", "Use premium account if available" , True),
+ ("premium_fallback", "bool", "Fallback to free download if premium fails", True),
+ ("chk_filesize" , "bool", "Check file size" , True)]
__description__ = """Simple hoster plugin"""
__license__ = "GPLv3"
@@ -90,7 +90,7 @@ class SimpleHoster(Hoster):
LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download
example: LINK_PREMIUM_PATTERN = r'<div class="link"><a href="(.+?)"'
"""
- NAME_REPLACEMENTS = [("&#?\w+;", fixup)]
+ NAME_REPLACEMENTS = []
SIZE_REPLACEMENTS = []
URL_REPLACEMENTS = []
@@ -124,7 +124,7 @@ class SimpleHoster(Hoster):
try:
info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here
- except Exception:
+ except AttributeError:
info['pattern'] = {}
if not html and not online:
@@ -174,8 +174,8 @@ class SimpleHoster(Hoster):
info['status'] = 2
if 'N' in info['pattern']:
- info['name'] = replace_patterns(fixurl(info['pattern']['N']),
- cls.NAME_REPLACEMENTS)
+ name = replace_patterns(info['pattern']['N'], cls.NAME_REPLACEMENTS)
+ info['name'] = parse_name(name)
if 'S' in info['pattern']:
size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'],
@@ -201,19 +201,15 @@ class SimpleHoster(Hoster):
def prepare(self):
- self.pyfile.error = "" #@TODO: Remove in 0.4.10
- self.html = "" #@TODO: Recheck in 0.4.10
- self.link = "" #@TODO: Recheck in 0.4.10
- self.last_download = ""
- self.direct_dl = False
- self.leech_dl = False
-
- if not self.get_config('use_premium', True):
+ self.link = ""
+ self.direct_dl = False
+ self.leech_dl = False
+
+ if not self.get_config('use_premium', True) and self.premium:
self.restart(nopremium=True)
if self.LOGIN_PREMIUM and not self.premium:
self.fail(_("Required premium account not found"))
- self.LOGIN_ACCOUNT = True
if self.LOGIN_ACCOUNT and not self.account:
self.fail(_("Required account not found"))
@@ -294,7 +290,7 @@ class SimpleHoster(Hoster):
self.check_file()
except Fail, e: #@TODO: Move to PluginThread in 0.4.10
- if self.get_config('fallback', True) and self.premium:
+ if self.get_config('premium_fallback', True) and self.premium:
self.log_warning(_("Premium download failed"), e)
self.restart(nopremium=True)
@@ -307,17 +303,18 @@ class SimpleHoster(Hoster):
if self.captcha.task and not self.last_download:
self.captcha.invalid()
- self.retry(10, reason=_("Wrong captcha"))
+ self.retry(10, msg=_("Wrong captcha"))
- # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size
- # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB
elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')},
- file_size=self.info['size'] if 'size' in self.info else 0,
- size_tolerance=10485760,
- delete=False): #@TODO: Make `delete` settable in 0.4.10
+ delete=True):
self.error(_("Empty file"))
else:
+ if self.get_config('chk_filesize', False) and 'size' in self.info:
+ # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size
+ # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB
+ self.check_filesize(self.info['size'], size_tolerance=10485760)
+
self.log_debug("Using default check rules...")
for r, p in self.FILE_ERRORS:
errmsg = self.check_download({r: re.compile(p)})
@@ -326,12 +323,13 @@ class SimpleHoster(Hoster):
try:
errmsg += " | " + self.last_check.group(1).strip()
+
except Exception:
pass
self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry"))
self.wantReconnect = True
- self.retry(wait_time=60, reason=errmsg)
+ self.retry(delay=60, msg=errmsg)
else:
if self.CHECK_FILE:
self.log_debug("Using custom check rules...")
@@ -340,7 +338,7 @@ class SimpleHoster(Hoster):
self.check_errors()
self.log_info(_("No errors found"))
- self.pyfile.error = ""
+ self.pyfile.error = "" #@TODO: Recheck in 0.4.10
def check_errors(self):
@@ -362,14 +360,15 @@ class SimpleHoster(Hoster):
m = re.search(self.DL_LIMIT_PATTERN, self.html)
try:
errmsg = m.group(1).strip()
- except Exception:
+
+ except AttributeError:
errmsg = m.group(0).strip()
self.info['error'] = re.sub(r'<.*?>', " ", errmsg)
self.log_warning(self.info['error'])
if re.search('da(il)?y|today', errmsg, re.I):
- wait_time = seconds_to_midnight(gmt=2)
+ wait_time = seconds_to_midnight()
else:
wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in
re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I))
@@ -385,7 +384,8 @@ class SimpleHoster(Hoster):
if m:
try:
errmsg = m.group(1).strip()
- except Exception:
+
+ except AttributeError:
errmsg = m.group(0).strip()
self.info['error'] = re.sub(r'<.*?>', " ", errmsg)
@@ -393,7 +393,7 @@ class SimpleHoster(Hoster):
if re.search('limit|wait|slot', errmsg, re.I):
if re.search("da(il)?y|today", errmsg):
- wait_time = seconds_to_midnight(gmt=2)
+ wait_time = seconds_to_midnight()
else:
wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in
re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I))
@@ -406,7 +406,7 @@ class SimpleHoster(Hoster):
elif re.search('captcha|code', errmsg, re.I):
self.captcha.invalid()
- self.retry(10, reason=_("Wrong captcha"))
+ self.retry(10, msg=_("Wrong captcha"))
elif re.search('countdown|expired', errmsg, re.I):
self.retry(10, 60, _("Link expired"))
@@ -421,23 +421,22 @@ class SimpleHoster(Hoster):
self.offline()
elif re.search('filename', errmsg, re.I):
- url_p = urlparse.urlparse(self.pyfile.url)
- self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.split('/')[0])
- self.retry(1, reason=_("Wrong url"))
+ self.fail(_("Wrong url"))
elif re.search('premium', errmsg, re.I):
self.fail(_("File can be downloaded by premium users only"))
else:
self.wantReconnect = True
- self.retry(wait_time=60, reason=errmsg)
+ self.retry(delay=60, msg=errmsg)
elif hasattr(self, 'WAIT_PATTERN'):
m = re.search(self.WAIT_PATTERN, self.html)
if m:
try:
waitmsg = m.group(1).strip()
- except Exception:
+
+ except AttributeError:
waitmsg = m.group(0).strip()
wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in
@@ -483,8 +482,8 @@ class SimpleHoster(Hoster):
self.log_debug("Previous file info: %s" % old_info)
try:
- url = self.info['url'].strip()
- name = self.info['name'].strip()
+ url = self.info['url']
+ name = self.info['name']
except KeyError:
pass
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 0386991d9..88c490750 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -22,7 +22,7 @@ def renice(pid, value):
class UnRar(Extractor):
__name__ = "UnRar"
- __version__ = "1.25"
+ __version__ = "1.26"
__status__ = "testing"
__description__ = """Rar extractor plugin"""
@@ -84,20 +84,8 @@ class UnRar(Extractor):
return True if cls.re_multipart.search(filename) else False
- def verify(self, password):
- p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password)
- self._progress(p)
- err = p.stderr.read().strip()
-
- if self.re_wrongpwd.search(err):
- raise PasswordError
-
- if self.re_wrongcrc.search(err):
- raise CRCError(err)
-
-
- def check(self, password):
- p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password)
+ def verify(self, password=None):
+ p = self.call_cmd("l", "-v", self.target, password=password)
out, err = p.communicate()
if self.re_wrongpwd.search(err):
@@ -113,13 +101,28 @@ class UnRar(Extractor):
def repair(self):
- p = self.call_cmd("rc", fs_encode(self.filename))
+ p = self.call_cmd("rc", self.target)
#: Communicate and retrieve stderr
self._progress(p)
err = p.stderr.read().strip()
+
if err or p.returncode:
- return False
+ p = self.call_cmd("r", self.target)
+
+ # communicate and retrieve stderr
+ self._progress(p)
+ err = p.stderr.read().strip()
+
+ if err or p.returncode:
+ return False
+
+ else:
+ dir = os.path.dirname(filename)
+ name = re_filefixed.search(out).group(1)
+
+ self.filename = os.path.join(dir, name)
+
return True
@@ -145,7 +148,7 @@ class UnRar(Extractor):
def extract(self, password=None):
command = "x" if self.fullpath else "e"
- p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password)
+ p = self.call_cmd(command, self.target, self.out, password=password)
renice(p.pid, self.renice)
@@ -169,7 +172,7 @@ class UnRar(Extractor):
self.files = self.list(password)
- def get_delete_files(self):
+ def items(self):
dir, name = os.path.split(self.filename)
#: Actually extracted file
@@ -185,7 +188,7 @@ class UnRar(Extractor):
def list(self, password=None):
command = "vb" if self.fullpath else "lb"
- p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password)
+ p = self.call_cmd(command, "-v", self.target, password=password)
out, err = p.communicate()
if "Cannot open" in err:
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index 9a01611bf..ac197a80d 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -12,7 +12,7 @@ from module.utils import fs_encode
class UnZip(Extractor):
__name__ = "UnZip"
- __version__ = "1.15"
+ __version__ = "1.16"
__status__ = "testing"
__description__ = """Zip extractor plugin"""
@@ -30,17 +30,13 @@ class UnZip(Extractor):
def list(self, password=None):
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+ with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
z.setpassword(password)
return z.namelist()
- def check(self, password):
- pass
-
-
- def verify(self):
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+ def verify(self, password=None):
+ with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
badfile = z.testzip()
if badfile:
@@ -51,7 +47,7 @@ class UnZip(Extractor):
def extract(self, password=None):
try:
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+ with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
z.setpassword(password)
badfile = z.testzip()
diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py
index 5a4cc35fb..bb5cbcf50 100644
--- a/module/plugins/internal/XFSAccount.py
+++ b/module/plugins/internal/XFSAccount.py
@@ -4,6 +4,7 @@ import re
import time
import urlparse
+from module.common.json_layer import json_loads
from module.plugins.internal.Account import Account
from module.plugins.internal.Plugin import parse_html_form, set_cookie
@@ -11,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie
class XFSAccount(Account):
__name__ = "XFSAccount"
__type__ = "account"
- __version__ = "0.43"
+ __version__ = "0.46"
__status__ = "testing"
__description__ = """XFileSharing account plugin"""
@@ -39,7 +40,7 @@ class XFSAccount(Account):
LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<'
- def parse_info(self, user, password, data, req):
+ def grab_info(self, user, password, data, req):
validuntil = None
trafficleft = None
leechtraffic = None
@@ -144,13 +145,13 @@ class XFSAccount(Account):
self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN
if self.COOKIES:
- if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")):
+ if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES:
self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
else:
set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english")
if not self.HOSTER_URL:
- self.login_fail(_("Missing HOSTER_URL"))
+ self.fail_login(_("Missing HOSTER_URL"))
else:
self.HOSTER_URL = self.HOSTER_URL.rstrip('/') + "/"
@@ -174,5 +175,13 @@ class XFSAccount(Account):
html = self.load(url, post=inputs, cookies=self.COOKIES)
- if re.search(self.LOGIN_FAIL_PATTERN, html):
- self.login_fail()
+ try:
+ json = json_loads(html)
+
+ except ValueError:
+ if re.search(self.LOGIN_FAIL_PATTERN, html):
+ self.fail_login()
+
+ else:
+ if not 'success' in json or not json['success']:
+ self.fail_login()
diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py
index 4c059d647..12a48d4a3 100644
--- a/module/plugins/internal/XFSCrypter.py
+++ b/module/plugins/internal/XFSCrypter.py
@@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo
class XFSCrypter(SimpleCrypter):
__name__ = "XFSCrypter"
__type__ = "crypter"
- __version__ = "0.13"
+ __version__ = "0.14"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -42,7 +42,7 @@ class XFSCrypter(SimpleCrypter):
self.fail(_("Missing HOSTER_DOMAIN"))
if self.COOKIES:
- if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")):
+ if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES:
self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
else:
set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english")
diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py
index 5e0830dc6..729c9a0ee 100644
--- a/module/plugins/internal/XFSHoster.py
+++ b/module/plugins/internal/XFSHoster.py
@@ -14,7 +14,7 @@ from module.utils import html_unescape
class XFSHoster(SimpleHoster):
__name__ = "XFSHoster"
__type__ = "hoster"
- __version__ = "0.57"
+ __version__ = "0.60"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -73,7 +73,7 @@ class XFSHoster(SimpleHoster):
self.fail(_("Missing HOSTER_DOMAIN"))
if self.COOKIES:
- if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")):
+ if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES:
self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
else:
set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english")
@@ -150,7 +150,7 @@ class XFSHoster(SimpleHoster):
action, inputs = self.parse_html_form('F1')
if not inputs:
- self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found"))
+ self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found"))
self.log_debug(inputs)
@@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster):
self.retry(20, 3 * 60, _("Can not leech file"))
elif 'today' in stmsg:
- self.retry(wait_time=seconds_to_midnight(gmt=2), reason=_("You've used all Leech traffic today"))
+ self.retry(delay=seconds_to_midnight(), msg=_("You've used all Leech traffic today"))
else:
self.fail(stmsg)
@@ -188,7 +188,7 @@ class XFSHoster(SimpleHoster):
if not inputs:
action, inputs = self.parse_html_form('F1')
if not inputs:
- self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found"))
+ self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found"))
self.log_debug(inputs)
@@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster):
try:
captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1)
- except Exception:
+ except AttributeError:
captcha_key = recaptcha.detect_key()
else:
@@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster):
try:
captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1)
- except Exception:
+ except AttributeError:
captcha_key = solvemedia.detect_key()
else: