summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar Walter Purcaro <vuolter@users.noreply.github.com> 2015-09-29 21:43:22 +0200
committerGravatar Walter Purcaro <vuolter@users.noreply.github.com> 2015-09-29 21:43:22 +0200
commit8af39bfa2c409d4fd07d430485117bf60fdcc935 (patch)
tree9340941b94be82a9aec40544eef7e8f1aeb766e9
parentAccount class completely rewritten + plugins updated (diff)
downloadpyload-8af39bfa2c409d4fd07d430485117bf60fdcc935.tar.xz
Update internal plugins
-rw-r--r--module/plugins/internal/Addon.py1
-rw-r--r--module/plugins/internal/Container.py1
-rw-r--r--module/plugins/internal/Crypter.py6
-rw-r--r--module/plugins/internal/Extractor.py3
-rw-r--r--module/plugins/internal/Hook.py2
-rw-r--r--module/plugins/internal/Hoster.py244
-rw-r--r--module/plugins/internal/MultiAccount.py (renamed from module/plugins/internal/MultiHook.py)59
-rw-r--r--module/plugins/internal/MultiCrypter.py4
-rw-r--r--module/plugins/internal/MultiHoster.py14
-rw-r--r--module/plugins/internal/Plugin.py60
-rw-r--r--module/plugins/internal/SevenZip.py6
-rw-r--r--module/plugins/internal/SimpleCrypter.py6
-rw-r--r--module/plugins/internal/SimpleHoster.py51
-rw-r--r--module/plugins/internal/UnRar.py6
-rw-r--r--module/plugins/internal/UnZip.py1
-rw-r--r--module/plugins/internal/XFSCrypter.py18
-rw-r--r--module/plugins/internal/XFSHoster.py26
17 files changed, 279 insertions, 229 deletions
diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py
index 5150e88f6..3a252fdfb 100644
--- a/module/plugins/internal/Addon.py
+++ b/module/plugins/internal/Addon.py
@@ -26,7 +26,6 @@ class Addon(Plugin):
__version__ = "0.06"
__status__ = "testing"
- __config__ = [] #: [("name", "type", "desc", "default")]
__threaded__ = [] #@TODO: Remove in 0.4.10
__description__ = """Base addon plugin"""
diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py
index 430590421..524b55839 100644
--- a/module/plugins/internal/Container.py
+++ b/module/plugins/internal/Container.py
@@ -17,7 +17,6 @@ class Container(Crypter):
__status__ = "testing"
__pattern__ = r'^unmatchable$'
- __config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base container decrypter plugin"""
__license__ = "GPLv3"
diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py
index 2033b67df..ad5bcc74e 100644
--- a/module/plugins/internal/Crypter.py
+++ b/module/plugins/internal/Crypter.py
@@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename
class Crypter(Hoster):
__name__ = "Crypter"
__type__ = "crypter"
- __version__ = "0.08"
+ __version__ = "0.09"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -83,7 +83,7 @@ class Crypter(Hoster):
self.pyload.api.setPackageData(pid, {'password': package_password})
#: Workaround to do not break API addPackage method
- set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""})
+ set_folder = lambda x="": self.pyload.api.setPackageData(pid, {'folder': safe_filename(x)})
if use_subfolder:
if not subfolder_per_package:
@@ -98,4 +98,4 @@ class Crypter(Hoster):
self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder})
elif folder_per_package:
- set_folder(None)
+ set_folder()
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index f21fe473c..3ab5d6a0d 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -5,6 +5,7 @@ import re
from module.PyFile import PyFile
from module.plugins.internal.Plugin import Plugin
+from module.utils import fs_encode
class ArchiveError(Exception):
@@ -22,7 +23,7 @@ class PasswordError(Exception):
class Extractor(Plugin):
__name__ = "Extractor"
__type__ = "extractor"
- __version__ = "0.34"
+ __version__ = "0.35"
__status__ = "testing"
__description__ = """Base extractor plugin"""
diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py
index 1f566f824..8ae731a7f 100644
--- a/module/plugins/internal/Hook.py
+++ b/module/plugins/internal/Hook.py
@@ -9,8 +9,6 @@ class Hook(Addon):
__version__ = "0.13"
__status__ = "testing"
- __config__ = [] #: [("name", "type", "desc", "default")]
-
__description__ = """Base hook plugin"""
__license__ = "GPLv3"
__authors__ = [("mkaay" , "mkaay@mkaay.de" ),
diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py
index 5d0a64f1a..bc340e78f 100644
--- a/module/plugins/internal/Hoster.py
+++ b/module/plugins/internal/Hoster.py
@@ -11,8 +11,8 @@ import urlparse
from module.plugins.internal.Captcha import Captcha
from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip,
- chunks, decode, encode, exists, parse_html_form,
- parse_html_tag_attr_value, parse_name,
+ chunks, decode, encode, exists, fixurl,
+ parse_html_form, parse_html_tag_attr_value, parse_name,
replace_patterns, seconds_to_midnight,
set_cookie, set_cookies, timestamp)
from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename
@@ -41,21 +41,27 @@ def create_getInfo(klass):
return get_info
+#@NOTE: `check_abort` decorator
+def check_abort(fn):
+
+ def wrapper(self, *args, **kwargs):
+ self.check_abort()
+ return fn(self, *args, **kwargs)
+
+ return wrapper
+
+
class Hoster(Plugin):
__name__ = "Hoster"
__type__ = "hoster"
- __version__ = "0.28"
+ __version__ = "0.31"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
- __config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base hoster plugin"""
__license__ = "GPLv3"
- __authors__ = [("RaNaN" , "RaNaN@pyload.org" ),
- ("spoob" , "spoob@pyload.org" ),
- ("mkaay" , "mkaay@mkaay.de" ),
- ("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
def __init__(self, pyfile):
@@ -74,7 +80,7 @@ class Hoster(Plugin):
#: Account handler instance, see :py:class:`Account`
self.account = None
- self.req = None #: Browser instance, see `network.Browser`
+ self.user = None #@TODO: Remove in 0.4.10
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
@@ -98,7 +104,7 @@ class Hoster(Plugin):
#: Dict of the amount of retries already made
self.retries = {}
- self.retry_free = False #@TODO: Recheck in 0.4.10
+ self.force_free = False #@TODO: Recheck in 0.4.10
self._setup()
self.init()
@@ -116,9 +122,10 @@ class Hoster(Plugin):
@classmethod
def get_info(cls, url="", html=""):
+ url = fixurl(url, unquote=True)
return {'name' : parse_name(url),
'size' : 0,
- 'status': 3 if url.strip() else 8,
+ 'status': 3 if url else 8,
'url' : url}
@@ -142,6 +149,11 @@ class Hoster(Plugin):
self.last_download = ""
self.pyfile.error = ""
+ try:
+ self.req.close()
+ except Exception:
+ pass
+
if self.account:
self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user)
self.chunk_limit = -1 #: -1 for unlimited
@@ -162,16 +174,16 @@ class Hoster(Plugin):
"""
self.thread = thread
- if self.retry_free:
+ if self.force_free:
self.account = False
else:
self.load_account() #@TODO: Move to PluginThread in 0.4.10
- self.retry_free = False
+ self.force_free = False
self._setup()
+ # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10
self.pyfile.setStatus("starting")
- self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10
self.check_abort()
@@ -185,17 +197,17 @@ class Hoster(Plugin):
def load_account(self):
- if self.req:
- self.req.close()
-
if not self.account:
self.account = self.pyload.accountManager.getAccountPlugin(self.__name__)
- if self.account:
- if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10
- self.account.user = self.account.select()[0]
+ if not self.account:
+ self.account = False
+ self.user = None #@TODO: Remove in 0.4.10
- if not hasattr(self.account, 'logged'):
+ else:
+ self.account.choose()
+ self.user = self.account.user #@TODO: Remove in 0.4.10
+ if self.account.user is None:
self.account = False
@@ -207,12 +219,8 @@ class Hoster(Plugin):
def set_reconnect(self, reconnect):
- if reconnect:
- self.log_info(_("Requesting line reconnection..."))
- else:
- self.log_debug("Reconnect: %s" % reconnect)
-
- self.log_debug("Previous wantReconnect: %s" % self.wantReconnect)
+ self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"),
+ "Previous wantReconnect: %s" % self.wantReconnect)
self.wantReconnect = bool(reconnect)
@@ -226,8 +234,8 @@ class Hoster(Plugin):
wait_time = max(int(seconds), 1)
wait_until = time.time() + wait_time + 1
- self.log_info(_("Waiting %d seconds...") % wait_time)
- self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil)
+ self.log_debug("WAIT set to %d seconds" % wait_time,
+ "Previous waitUntil: %f" % self.pyfile.waitUntil)
self.pyfile.waitUntil = wait_until
@@ -249,13 +257,17 @@ class Hoster(Plugin):
self.waiting = True
- status = pyfile.status #@NOTE: Remove in 0.4.10
+ status = pyfile.status #@NOTE: Recheck in 0.4.10
pyfile.setStatus("waiting")
- if not self.wantReconnect or self.account:
+ self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time())
+
+ if self.wantReconnect:
+ self.log_info(_("Requiring reconnection..."))
if self.account:
self.log_warning("Ignore reconnection due logged account")
+ if not self.wantReconnect or self.account:
while pyfile.waitUntil > time.time():
self.check_abort()
time.sleep(2)
@@ -263,24 +275,24 @@ class Hoster(Plugin):
else:
while pyfile.waitUntil > time.time():
self.check_abort()
+ self.thread.m.reconnecting.wait(1)
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
- self.thread.m.reconnecting.wait(2)
time.sleep(2)
self.waiting = False
- pyfile.status = status #@NOTE: Remove in 0.4.10
+ pyfile.status = status #@NOTE: Recheck in 0.4.10
def skip(self, msg=""):
"""
Skip and give msg
"""
- raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10
+ raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10
#@TODO: Remove in 0.4.10
@@ -293,7 +305,7 @@ class Hoster(Plugin):
if msg:
self.pyfile.error = msg
else:
- msg = self.pyfile.error
+ msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName())
raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10
@@ -359,30 +371,33 @@ class Hoster(Plugin):
if nopremium:
if self.premium:
- self.retry_free = True
+ self.force_free = True
else:
self.fail("%s | %s" % (msg, _("Download was already free")))
raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10
- def fixurl(self, url, baseurl=None):
+ def fixurl(self, url, baseurl=None, unquote=None):
+ url = fixurl(url)
+
if not baseurl:
- baseurl = self.pyfile.url
+ baseurl = fixurl(self.pyfile.url)
if not urlparse.urlparse(url).scheme:
url_p = urlparse.urlparse(baseurl)
baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
url = urlparse.urljoin(baseurl, url)
- return url
+ return fixurl(url, unquote)
+ @check_abort
def load(self, *args, **kwargs):
- self.check_abort()
return super(Hoster, self).load(*args, **kwargs)
+ @check_abort
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True):
"""
Downloads the content at url to download folder
@@ -396,16 +411,18 @@ class Hoster(Plugin):
the filename will be changed if needed
:return: The location where the file was saved
"""
- self.check_abort()
-
if self.pyload.debug:
self.log_debug("DOWNLOAD URL " + url,
- *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
+ *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")])
+
+ url = self.fixurl(url, unquote=True)
self.pyfile.name = parse_name(self.pyfile.name) #: Safe check
self.captcha.correct()
- self.check_for_same_files()
+
+ if self.pyload.config.get("download", "skip_existing"):
+ self.check_filedupe()
self.pyfile.setStatus("downloading")
@@ -441,7 +458,9 @@ class Hoster(Plugin):
if finalname != newname:
try:
- os.rename(fs_join(location, newname), fs_join(location, finalname))
+ oldname_enc = fs_join(download_location, newname)
+ newname_enc = fs_join(download_location, finalname)
+ os.rename(oldname_enc, newname_enc)
except OSError, e:
self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e)
@@ -463,16 +482,16 @@ class Hoster(Plugin):
if not self.pyfile.abort:
return
- if self.pyfile.hasStatus("failed"):
+ if self.pyfile.status is 8:
self.fail()
- elif self.pyfile.hasStatus("skipped"):
+ elif self.pyfile.status is 4:
self.skip(self.pyfile.statusname)
- elif self.pyfile.hasStatus("offline"):
+ elif self.pyfile.status is 1:
self.offline()
- elif self.pyfile.hasStatus("temp. offline"):
+ elif self.pyfile.status is 6:
self.temp_offline()
else:
@@ -489,7 +508,8 @@ class Hoster(Plugin):
if not self.last_download:
return
- download_size = os.stat(fs_encode(self.last_download)).st_size
+ download_location = fs_encode(self.last_download)
+ download_size = os.stat(download_location).st_size
if download_size < 1:
self.fail(_("Empty file"))
@@ -505,7 +525,7 @@ class Hoster(Plugin):
self.log_warning(_("File size is not equal to expected size"))
- def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024):
+ def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024):
"""
Checks the content of the last downloaded file, re match is saved to `last_check`
@@ -552,7 +572,66 @@ class Hoster(Plugin):
else:
self.log_info(_("File deleted: ") + self.last_download)
- self.last_download = ""
+ self.last_download = "" #: Recheck in 0.4.10
+
+
+ def check_traffic(self):
+ if not self.account:
+ return True
+
+ traffic = self.account.get_data(refresh=True)['trafficleft']
+
+ if traffic is None:
+ return False
+
+ elif traffic is -1:
+ return True
+
+ else:
+ size = self.pyfile.size / 1024 #@TODO: Remove in 0.4.10
+ self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) #@TODO: Rewrite in 0.4.10
+ return size <= traffic
+
+
+ def check_filedupe(self):
+ """
+ Checks if same file was/is downloaded within same package
+
+ :param starting: indicates that the current download is going to start
+ :raises Skip:
+ """
+ pack = self.pyfile.package()
+
+ for pyfile in self.pyload.files.cache.values():
+ if pyfile is self.pyfile:
+ continue
+
+ if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder:
+ continue
+
+ if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading)
+ self.skip(pyfile.pluginname)
+
+ download_folder = self.pyload.config.get("general", "download_folder")
+ package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else ""
+ download_location = fs_join(download_folder, package_folder, self.pyfile.name)
+
+ if not exists(download_location):
+ return
+
+ pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name)
+ if pyfile:
+ self.skip(pyfile[0])
+
+ size = os.stat(download_location).st_size
+ if size >= self.pyfile.size:
+ self.skip(_("File exists"))
+
+
+ #: Deprecated method, use `check_filedupe` instead (Remove in 0.4.10)
+ def checkForSameFiles(self, *args, **kwargs):
+ if self.pyload.config.get("download", "skip_existing"):
+ return self.check_filedupe()
def direct_link(self, url, follow_location=None):
@@ -598,10 +677,10 @@ class Hoster(Plugin):
if 'content-disposition' in header:
link = url
- elif 'location' in header and header['location']:
+ elif header.get('location'):
location = self.fixurl(header['location'], url)
- if 'code' in header and header['code'] == 302:
+ if header.get('code') == 302:
link = location
if follow_location:
@@ -611,7 +690,7 @@ class Hoster(Plugin):
else:
extension = os.path.splitext(parse_name(url))[-1]
- if 'content-type' in header and header['content-type']:
+ if header.get('content-type'):
mimetype = header['content-type'].split(';')[0].strip()
elif extension:
@@ -641,63 +720,8 @@ class Hoster(Plugin):
return parse_html_form(attr_str, self.html, input_names)
- def check_traffic_left(self):
- if not self.account:
- return True
-
- traffic = self.account.get_data(self.account.user, True)['trafficleft']
-
- if traffic is None:
- return False
-
- elif traffic == -1:
- return True
-
- else:
- size = self.pyfile.size / 1024
- self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic))
- return size <= traffic
-
-
def get_password(self):
"""
Get the password the user provided in the package
"""
return self.pyfile.package().password or ""
-
-
- #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10)
- def checkForSameFiles(self, *args, **kwargs):
- return self.check_for_same_files(*args, **kwargs)
-
-
- def check_for_same_files(self, starting=False):
- """
- Checks if same file was/is downloaded within same package
-
- :param starting: indicates that the current download is going to start
- :raises Skip:
- """
- pack = self.pyfile.package()
-
- for pyfile in self.pyload.files.cache.values():
- if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder:
- if pyfile.status in (0, 12): #: Finished or downloading
- self.skip(pyfile.pluginname)
- elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before
- self.skip(pyfile.pluginname)
-
- download_folder = self.pyload.config.get("general", "download_folder")
- location = fs_join(download_folder, pack.folder, self.pyfile.name)
-
- if starting and self.pyload.config.get("download", "skip_existing") and exists(location):
- size = os.stat(location).st_size
- if size >= self.pyfile.size:
- self.skip("File exists")
-
- pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
- if pyfile:
- if exists(location):
- self.skip(pyfile[0])
-
- self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name)
diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiAccount.py
index 42a1985b5..b38670ce7 100644
--- a/module/plugins/internal/MultiHook.py
+++ b/module/plugins/internal/MultiAccount.py
@@ -2,30 +2,29 @@
import re
import time
-import traceback
-from module.plugins.internal.Hook import Hook
+from module.plugins.internal.Account import Account
from module.utils import decode, remove_chars
-class MultiHook(Hook):
- __name__ = "MultiHook"
- __type__ = "hook"
- __version__ = "0.54"
+class MultiAccount(Account):
+ __name__ = "MultiAccount"
+ __type__ = "account"
+ __version__ = "0.02"
__status__ = "testing"
- __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
- ("pluginlist" , "str" , "Plugin list (comma separated)", "" ),
- ("reload" , "bool" , "Reload plugin list" , True ),
- ("reloadinterval", "int" , "Reload interval in hours" , 12 )]
+ __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
+ ("pluginlist" , "str" , "Plugin list (comma separated)", "" ),
+ ("reload" , "bool" , "Reload plugin list" , True ),
+ ("reloadinterval", "int" , "Reload interval in hours" , 12 )]
- __description__ = """Hook plugin for multi hoster/crypter"""
+ __description__ = """Multi hoster account plugin"""
__license__ = "GPLv3"
__authors__ = [("pyLoad Team" , "admin@pyload.org" ),
("Walter Purcaro", "vuolter@gmail.com")]
- MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour
+ REFRESH_INTERVAL = 1 * 60 * 60 #: 1 hour
DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"),
(r'bayfiles\.net' , "bayfiles.com" ),
@@ -55,6 +54,34 @@ class MultiHook(Hook):
(r'^0' , "zero" )]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
def init(self):
self.plugins = []
self.supported = []
@@ -102,7 +129,7 @@ class MultiHook(Hook):
for _i in xrange(5):
try:
- pluginset = self._plugin_set(self.get_hosters())
+ pluginset = self._plugin_set(self.grab_hosters())
break
except Exception, e:
@@ -110,7 +137,7 @@ class MultiHook(Hook):
time.sleep(60)
else:
self.log_error(_("No hoster list retrieved"))
- self.interval = self.MIN_RELOAD_INTERVAL
+ self.interval = self.REFRESH_INTERVAL
return list()
try:
@@ -144,7 +171,7 @@ class MultiHook(Hook):
return set(plugins)
- def get_hosters(self):
+ def grab_hosters(self, user, password, data):
"""
Load list of supported hoster
@@ -160,7 +187,7 @@ class MultiHook(Hook):
self.load_account()
if self.get_config('reload', True):
- self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL)
+ self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.REFRESH_INTERVAL)
else:
self.pyload.scheduler.removeJob(self.cb)
self.cb = None
diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py
index ca7b03941..ae8785116 100644
--- a/module/plugins/internal/MultiCrypter.py
+++ b/module/plugins/internal/MultiCrypter.py
@@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter
class MultiCrypter(SimpleCrypter):
__name__ = "MultiCrypter"
__type__ = "hoster"
- __version__ = "0.02"
+ __version__ = "0.03"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -19,7 +19,7 @@ class MultiCrypter(SimpleCrypter):
def init(self):
- self.CRYPTER_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name']
+ self.PLUGIN_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name']
def _log(self, level, plugintype, pluginname, messages):
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py
index c0c928a45..fbfab1ade 100644
--- a/module/plugins/internal/MultiHoster.py
+++ b/module/plugins/internal/MultiHoster.py
@@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r
class MultiHoster(SimpleHoster):
__name__ = "MultiHoster"
__type__ = "hoster"
- __version__ = "0.50"
+ __version__ = "0.51"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -21,21 +21,21 @@ class MultiHoster(SimpleHoster):
__authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
- HOSTER_NAME = None
+ PLUGIN_NAME = None
LEECH_HOSTER = False
LOGIN_ACCOUNT = True
def init(self):
- self.HOSTER_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name']
+ self.PLUGIN_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name']
def _log(self, level, plugintype, pluginname, messages):
return super(MultiHoster, self)._log(level,
plugintype,
pluginname,
- (self.HOSTER_NAME,) + messages)
+ (self.PLUGIN_NAME,) + messages)
def setup(self):
@@ -83,11 +83,11 @@ class MultiHoster(SimpleHoster):
self.check_errors()
self.check_status(getinfo=False)
- if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()):
+ if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()):
self.log_info(_("Processing as premium download..."))
self.handle_premium(pyfile)
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()):
+ elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()):
self.log_info(_("Processing as free download..."))
self.handle_free(pyfile)
@@ -95,7 +95,7 @@ class MultiHoster(SimpleHoster):
self.log_info(_("Downloading file..."))
self.download(self.link, disposition=self.DISPOSITION)
- self.check_file()
+ self.check_download()
except Fail, e: #@TODO: Move to PluginThread in 0.4.10
if self.premium:
diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py
index 51192d8c9..f1cd74fcb 100644
--- a/module/plugins/internal/Plugin.py
+++ b/module/plugins/internal/Plugin.py
@@ -6,7 +6,6 @@ import datetime
import inspect
import os
import re
-import sys
import traceback
import urllib
import urlparse
@@ -28,6 +27,11 @@ def decode(string, encoding='utf8'):
return unicode(string)
+#@TODO: Remove in 0.4.10
+def _decode(*args, **kwargs):
+ return decode(*args, **kwargs)
+
+
#@TODO: Move to utils in 0.4.10
def encode(string, encoding='utf8'):
""" Decode string to utf8 """
@@ -49,18 +53,25 @@ def exists(path):
return False
-#@TODO: Move to utils in 0.4.10
-def parse_name(url):
- url = urllib.unquote(url)
- url = url.decode('unicode-escape')
- url = html_unescape(url)
- url = urllib.quote(url)
+#@TODO: Recheck in 0.4.10
+def fixurl(url, unquote=None):
+ newurl = urllib.unquote(url)
+
+ if unquote is None:
+ unquote = newurl == url
+
+ newurl = html_unescape(newurl.decode('unicode-escape')).strip()
- url_p = urlparse.urlparse(url.strip().rstrip('/'))
+ return newurl if unquote else urllib.quote(newurl)
- name = (url_p.path.split('/')[-1] or
- url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
- url_p.netloc.split('.', 1)[0])
+
+#@TODO: Recheck in 0.4.10
+def parse_name(string):
+ path = fixurl(decode(string), unquote=False)
+ url_p = urlparse.urlparse(path.rstrip('/'))
+ name = (url_p.path.split('/')[-1] or
+ url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
+ url_p.netloc.split('.', 1)[0])
return urllib.unquote(name)
@@ -96,7 +107,7 @@ def seconds_to_midnight(utc=None):
else:
now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc)
- midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1)
+ midnight = now.replace(hour=0, minute=1, second=0, microsecond=0) + datetime.timedelta(days=1)
return (midnight - now).seconds
@@ -173,7 +184,7 @@ def chunks(iterable, size):
class Plugin(object):
__name__ = "Plugin"
__type__ = "plugin"
- __version__ = "0.37"
+ __version__ = "0.38"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -199,8 +210,8 @@ class Plugin(object):
def _init(self, core):
self.pyload = core
- self.info = {} #: Provide information in dict here
- self.req = None
+ self.info = {} #: Provide information in dict here
+ self.req = None #: Browser instance, see `network.Browser`
def init(self):
@@ -231,8 +242,6 @@ class Plugin(object):
def log_warning(self, *args):
self._log("warning", self.__type__, self.__name__, args)
- if self.pyload.debug:
- traceback.print_exc()
def log_error(self, *args):
@@ -278,7 +287,7 @@ class Plugin(object):
return min(self.pyload.config.get("download", "chunks"), self.chunk_limit)
- def set_config(self, option, value):
+ def set_config(self, option, value, plugin=None):
"""
Set config value for current plugin
@@ -286,7 +295,7 @@ class Plugin(object):
:param value:
:return:
"""
- self.pyload.config.setPlugin(self.__name__, option, value)
+ self.pyload.config.setPlugin(plugin or self.__name__, option, value)
def get_config(self, option, default="", plugin=None):
@@ -347,7 +356,9 @@ class Plugin(object):
"""
if self.pyload.debug:
self.log_debug("LOAD URL " + url,
- *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
+ *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")])
+
+ url = fixurl(url, unquote=True) #: Recheck in 0.4.10
if req is None:
req = self.req or self.pyload.requestFactory.getRequest(self.__name__)
@@ -364,7 +375,7 @@ class Plugin(object):
#@TODO: Move to network in 0.4.10
if isinstance(decode, basestring):
- res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10
+ res = _decode(res, decode) #@NOTE: Use `utils.decode()` in 0.4.10
if self.pyload.debug:
frame = inspect.currentframe()
@@ -409,11 +420,12 @@ class Plugin(object):
Clean everything and remove references
"""
try:
+ self.req.clearCookies()
self.req.close()
except Exception:
pass
- for a in ("pyfile", "thread", "html", "req"):
- if hasattr(self, a):
- setattr(self, a, None)
+ for attr in ("account", "html", "pyfile", "req", "thread"):
+ if hasattr(self, attr):
+ setattr(self, attr, None)
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index b79256536..9d3a71a21 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -5,12 +5,12 @@ import re
import subprocess
from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice
-from module.utils import fs_encode, save_join as fs_join
+from module.utils import save_join as fs_join
class SevenZip(UnRar):
__name__ = "SevenZip"
- __version__ = "0.15"
+ __version__ = "0.16"
__status__ = "testing"
__description__ = """7-Zip extractor plugin"""
@@ -128,7 +128,7 @@ class SevenZip(UnRar):
args.append("-y")
#: Set a password
- if "password" in kwargs and kwargs['password']:
+ if kwargs.get('password'):
args.append("-p%s" % kwargs['password'])
else:
args.append("-p-")
diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py
index 8c5d3599d..53d69b2a5 100644
--- a/module/plugins/internal/SimpleCrypter.py
+++ b/module/plugins/internal/SimpleCrypter.py
@@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r
class SimpleCrypter(Crypter, SimpleHoster):
__name__ = "SimpleCrypter"
__type__ = "crypter"
- __version__ = "0.62"
+ __version__ = "0.63"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -81,7 +81,7 @@ class SimpleCrypter(Crypter, SimpleHoster):
self.log_debug("Redirect #%d to: %s" % (i, redirect))
header = self.load(redirect, just_header=True)
- if 'location' in header and header['location']:
+ if header.get('location'):
self.link = header['location']
else:
break
@@ -137,7 +137,7 @@ class SimpleCrypter(Crypter, SimpleHoster):
try:
pages = int(re.search(self.PAGES_PATTERN, self.html).group(1))
- except AttributeError:
+ except (AttributeError, IndexError, ValueError):
pages = 1
for p in xrange(2, pages + 1):
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index 0f030e000..00b63ac12 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -6,7 +6,6 @@ import os
import re
import time
-from module.PyFile import statusMap as _statusMap
from module.network.HTTPRequest import BadHeader
from module.network.RequestFactory import getURL as get_url
from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo
@@ -14,14 +13,10 @@ from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_pat
from module.utils import fixup, fs_encode, parseFileSize as parse_size
-#@TODO: Adapt and move to PyFile in 0.4.10
-statusMap = dict((v, k) for k, v in _statusMap.items())
-
-
class SimpleHoster(Hoster):
__name__ = "SimpleHoster"
__type__ = "hoster"
- __version__ = "1.86"
+ __version__ = "1.87"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -124,7 +119,7 @@ class SimpleHoster(Hoster):
try:
info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here
- except AttributeError:
+ except (AttributeError, IndexError):
info['pattern'] = {}
if not html and not online:
@@ -164,7 +159,7 @@ class SimpleHoster(Hoster):
if all(True for k in pdict if k not in info['pattern']):
info['pattern'].update(pdict)
- except AttributeError:
+ except (AttributeError, IndexError):
continue
else:
@@ -275,11 +270,11 @@ class SimpleHoster(Hoster):
if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10
self.check_info()
- if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()):
+ if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()):
self.log_info(_("Processing as premium download..."))
self.handle_premium(pyfile)
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()):
+ elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()):
self.log_info(_("Processing as free download..."))
self.handle_free(pyfile)
@@ -287,7 +282,7 @@ class SimpleHoster(Hoster):
self.log_info(_("Downloading file..."))
self.download(self.link, disposition=self.DISPOSITION)
- self.check_file()
+ self.check_download()
except Fail, e: #@TODO: Move to PluginThread in 0.4.10
if self.get_config('premium_fallback', True) and self.premium:
@@ -298,26 +293,26 @@ class SimpleHoster(Hoster):
raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10
- def check_file(self):
- self.log_info(_("Checking file..."))
+ def check_download(self):
+ self.log_info(_("Checking downloaded file..."))
if self.captcha.task and not self.last_download:
self.captcha.invalid()
self.retry(10, msg=_("Wrong captcha"))
- elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')},
+ elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')},
delete=True):
self.error(_("Empty file"))
else:
- if self.get_config('chk_filesize', False) and 'size' in self.info:
+ if self.get_config('chk_filesize', False) and self.info.get('size'):
# 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size
# For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB
self.check_filesize(self.info['size'], size_tolerance=10485760)
self.log_debug("Using default check rules...")
for r, p in self.FILE_ERRORS:
- errmsg = self.check_download({r: re.compile(p)})
+ errmsg = self.check_file({r: re.compile(p)})
if errmsg is not None:
errmsg = errmsg.strip().capitalize()
@@ -338,7 +333,6 @@ class SimpleHoster(Hoster):
self.check_errors()
self.log_info(_("No errors found"))
- self.pyfile.error = "" #@TODO: Recheck in 0.4.10
def check_errors(self):
@@ -361,7 +355,7 @@ class SimpleHoster(Hoster):
try:
errmsg = m.group(1).strip()
- except AttributeError:
+ except (AttributeError, IndexError):
errmsg = m.group(0).strip()
self.info['error'] = re.sub(r'<.*?>', " ", errmsg)
@@ -385,7 +379,7 @@ class SimpleHoster(Hoster):
try:
errmsg = m.group(1).strip()
- except AttributeError:
+ except (AttributeError, IndexError):
errmsg = m.group(0).strip()
self.info['error'] = re.sub(r'<.*?>', " ", errmsg)
@@ -436,7 +430,7 @@ class SimpleHoster(Hoster):
try:
waitmsg = m.group(1).strip()
- except AttributeError:
+ except (AttributeError, IndexError):
waitmsg = m.group(0).strip()
wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in
@@ -455,22 +449,19 @@ class SimpleHoster(Hoster):
self.log_debug("Previous file info: %s" % old_info)
try:
- status = self.info['status'] or None
+ status = self.info['status'] or 14
- if status == 1:
+ if status is 1:
self.offline()
- elif status == 6:
+ elif status is 6:
self.temp_offline()
- elif status == 8:
- if 'error' in self.info:
- self.fail(self.info['error'])
- else:
- self.fail(_("File status: " + statusMap[status]))
+ elif status is 8:
+ self.fail()
finally:
- self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown")))
+ self.log_info(_("File status: ") + self.pyfile.getStatusName(status))
def check_name_size(self, getinfo=True):
@@ -492,7 +483,7 @@ class SimpleHoster(Hoster):
if name and name is not url:
self.pyfile.name = name
- if 'size' in self.info and self.info['size'] > 0:
+ if self.info.get('size') > 0:
self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10
# self.pyfile.sync()
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 88c490750..7efecc5ef 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -8,7 +8,7 @@ from glob import glob
from string import digits
from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
-from module.utils import fs_decode, fs_encode, save_join as fs_join
+from module.utils import fs_decode, save_join as fs_join
def renice(pid, value):
@@ -22,7 +22,7 @@ def renice(pid, value):
class UnRar(Extractor):
__name__ = "UnRar"
- __version__ = "1.26"
+ __version__ = "1.27"
__status__ = "testing"
__description__ = """Rar extractor plugin"""
@@ -229,7 +229,7 @@ class UnRar(Extractor):
args.append("-y")
#: Set a password
- if "password" in kwargs and kwargs['password']:
+ if kwargs.get('password'):
args.append("-p%s" % kwargs['password'])
else:
args.append("-p-")
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index ac197a80d..87cbd568a 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -7,7 +7,6 @@ import sys
import zipfile
from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
-from module.utils import fs_encode
class UnZip(Extractor):
diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py
index 12a48d4a3..8047fd03e 100644
--- a/module/plugins/internal/XFSCrypter.py
+++ b/module/plugins/internal/XFSCrypter.py
@@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo
class XFSCrypter(SimpleCrypter):
__name__ = "XFSCrypter"
__type__ = "crypter"
- __version__ = "0.14"
+ __version__ = "0.15"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -17,7 +17,7 @@ class XFSCrypter(SimpleCrypter):
__authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
- HOSTER_DOMAIN = None
+ PLUGIN_DOMAIN = None
URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')]
@@ -29,22 +29,22 @@ class XFSCrypter(SimpleCrypter):
def prepare(self):
- if not self.HOSTER_DOMAIN:
+ if not self.PLUGIN_DOMAIN:
if self.account:
account = self.account
else:
account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "")
account = self.pyload.accountManager.getAccountPlugin(account_name)
- if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
- self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
+ if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN:
+ self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN
else:
- self.fail(_("Missing HOSTER_DOMAIN"))
+ self.fail(_("Missing PLUGIN_DOMAIN"))
if self.COOKIES:
- if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES:
- self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
+ if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES:
+ self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english"))
else:
- set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english")
+ set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english")
return super(XFSCrypter, self).prepare()
diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py
index 729c9a0ee..8af6b83ed 100644
--- a/module/plugins/internal/XFSHoster.py
+++ b/module/plugins/internal/XFSHoster.py
@@ -14,7 +14,7 @@ from module.utils import html_unescape
class XFSHoster(SimpleHoster):
__name__ = "XFSHoster"
__type__ = "hoster"
- __version__ = "0.60"
+ __version__ = "0.61"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
@@ -26,7 +26,7 @@ class XFSHoster(SimpleHoster):
("Walter Purcaro", "vuolter@gmail.com" )]
- HOSTER_DOMAIN = None
+ PLUGIN_DOMAIN = None
LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy...
@@ -61,26 +61,26 @@ class XFSHoster(SimpleHoster):
"""
Initialize important variables
"""
- if not self.HOSTER_DOMAIN:
+ if not self.PLUGIN_DOMAIN:
if self.account:
account = self.account
else:
account = self.pyload.accountManager.getAccountPlugin(self.__name__)
- if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
- self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
+ if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN:
+ self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN
else:
- self.fail(_("Missing HOSTER_DOMAIN"))
+ self.fail(_("Missing PLUGIN_DOMAIN"))
if self.COOKIES:
- if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES:
- self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
+ if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES:
+ self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english"))
else:
- set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english")
+ set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english")
if not self.LINK_PATTERN:
pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])'
- self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.')
+ self.LINK_PATTERN = pattern % self.PLUGIN_DOMAIN.replace('.', '\.')
super(XFSHoster, self).prepare()
@@ -129,7 +129,7 @@ class XFSHoster(SimpleHoster):
self.fail(_("Only registered or premium users can use url leech feature"))
#: Only tested with easybytez.com
- self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN)
+ self.html = self.load("http://www.%s/" % self.PLUGIN_DOMAIN)
action, inputs = self.parse_html_form()
@@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster):
try:
captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1)
- except AttributeError:
+ except (AttributeError, IndexError):
captcha_key = recaptcha.detect_key()
else:
@@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster):
try:
captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1)
- except AttributeError:
+ except (AttributeError, IndexError):
captcha_key = solvemedia.detect_key()
else: