Merge branch 'stable' into 0.4.10

Conflicts:
	module/plugins/OCR.py
	module/plugins/accounts/BayfilesCom.py
	module/plugins/accounts/CatShareNet.py
	module/plugins/accounts/DebridItaliaCom.py
	module/plugins/accounts/FourSharedCom.py
	module/plugins/accounts/FshareVn.py
	module/plugins/accounts/Keep2ShareCc.py
	module/plugins/accounts/LomafileCom.py
	module/plugins/accounts/RapiduNet.py
	module/plugins/accounts/RyushareCom.py
	module/plugins/accounts/ShareonlineBiz.py
	module/plugins/addon/Checksum.py
	module/plugins/addon/ExternalScripts.py
	module/plugins/addon/ExtractArchive.py
	module/plugins/addon/MergeFiles.py
	module/plugins/addon/RestartSlow.py
	module/plugins/addon/SkipRev.py
	module/plugins/addon/UnSkipOnFail.py
	module/plugins/addon/UpdateManager.py
	module/plugins/addon/WindowsPhoneToastNotify.py
	module/plugins/container/CCF.py
	module/plugins/container/LinkList.py
	module/plugins/container/RSDF.py
	module/plugins/crypter/BitshareCom.py
	module/plugins/crypter/CrockoCom.py
	module/plugins/crypter/DataHu.py
	module/plugins/crypter/DepositfilesCom.py
	module/plugins/crypter/Dereferer.py
	module/plugins/crypter/DevhostStFolder.py
	module/plugins/crypter/DlProtectCom.py
	module/plugins/crypter/DontKnowMe.py
	module/plugins/crypter/EasybytezCom.py
	module/plugins/crypter/FilecloudIo.py
	module/plugins/crypter/FilecryptCc.py
	module/plugins/crypter/FilefactoryCom.py
	module/plugins/crypter/FilerNet.py
	module/plugins/crypter/FilestubeCom.py
	module/plugins/crypter/FiletramCom.py
	module/plugins/crypter/FreakhareCom.py
	module/plugins/crypter/FreetexthostCom.py
	module/plugins/crypter/FshareVn.py
	module/plugins/crypter/ImgurComAlbum.py
	module/plugins/crypter/JunocloudMe.py
	module/plugins/crypter/LinkSaveIn.py
	module/plugins/crypter/MegaRapidCz.py
	module/plugins/crypter/MultiUpOrg.py
	module/plugins/crypter/NetfolderIn.py
	module/plugins/crypter/NosvideoCom.py
	module/plugins/crypter/OneKhDe.py
	module/plugins/crypter/PastebinCom.py
	module/plugins/crypter/RapidfileshareNet.py
	module/plugins/crypter/RelinkUs.py
	module/plugins/crypter/TnyCz.py
	module/plugins/crypter/TurbobitNet.py
	module/plugins/crypter/TusfilesNet.py
	module/plugins/crypter/UploadableCh.py
	module/plugins/crypter/UploadedTo.py
	module/plugins/crypter/XFileSharingPro.py
	module/plugins/hooks/AlldebridCom.py
	module/plugins/hooks/BypassCaptcha.py
	module/plugins/hooks/Captcha9Kw.py
	module/plugins/hooks/CaptchaBrotherhood.py
	module/plugins/hooks/ClickAndLoad.py
	module/plugins/hooks/DeathByCaptcha.py
	module/plugins/hooks/DebridItaliaCom.py
	module/plugins/hooks/EasybytezCom.py
	module/plugins/hooks/ExpertDecoders.py
	module/plugins/hooks/FastixRu.py
	module/plugins/hooks/FreeWayMe.py
	module/plugins/hooks/ImageTyperz.py
	module/plugins/hooks/LinkdecrypterCom.py
	module/plugins/hooks/LinksnappyCom.py
	module/plugins/hooks/MegaDebridEu.py
	module/plugins/hooks/MultishareCz.py
	module/plugins/hooks/MyfastfileCom.py
	module/plugins/hooks/OverLoadMe.py
	module/plugins/hooks/PremiumTo.py
	module/plugins/hooks/PremiumizeMe.py
	module/plugins/hooks/RPNetBiz.py
	module/plugins/hooks/RealdebridCom.py
	module/plugins/hooks/RehostTo.py
	module/plugins/hooks/SimplyPremiumCom.py
	module/plugins/hooks/SimplydebridCom.py
	module/plugins/hooks/UnrestrictLi.py
	module/plugins/hooks/XFileSharingPro.py
	module/plugins/hooks/ZeveraCom.py
	module/plugins/hoster/AlldebridCom.py
	module/plugins/hoster/BayfilesCom.py
	module/plugins/hoster/DebridItaliaCom.py
	module/plugins/hoster/DepositfilesCom.py
	module/plugins/hoster/DodanePl.py
	module/plugins/hoster/ExtabitCom.py
	module/plugins/hoster/FastixRu.py
	module/plugins/hoster/FastshareCz.py
	module/plugins/hoster/FileParadoxIn.py
	module/plugins/hoster/FileSharkPl.py
	module/plugins/hoster/FilerNet.py
	module/plugins/hoster/FileserveCom.py
	module/plugins/hoster/FreakshareCom.py
	module/plugins/hoster/FreeWayMe.py
	module/plugins/hoster/GigapetaCom.py
	module/plugins/hoster/IfileIt.py
	module/plugins/hoster/Keep2ShareCc.py
	module/plugins/hoster/LetitbitNet.py
	module/plugins/hoster/LinksnappyCom.py
	module/plugins/hoster/LomafileCom.py
	module/plugins/hoster/MegaCoNz.py
	module/plugins/hoster/MegaDebridEu.py
	module/plugins/hoster/MegaRapidCz.py
	module/plugins/hoster/MyfastfileCom.py
	module/plugins/hoster/OverLoadMe.py
	module/plugins/hoster/PremiumTo.py
	module/plugins/hoster/PremiumizeMe.py
	module/plugins/hoster/RPNetBiz.py
	module/plugins/hoster/RapidgatorNet.py
	module/plugins/hoster/RealdebridCom.py
	module/plugins/hoster/RehostTo.py
	module/plugins/hoster/RyushareCom.py
	module/plugins/hoster/SendmywayCom.py
	module/plugins/hoster/ShareonlineBiz.py
	module/plugins/hoster/SimplyPremiumCom.py
	module/plugins/hoster/SimplydebridCom.py
	module/plugins/hoster/TusfilesNet.py
	module/plugins/hoster/UnibytesCom.py
	module/plugins/hoster/UnrestrictLi.py
	module/plugins/hoster/UploadedTo.py
	module/plugins/hoster/WebshareCz.py
	module/plugins/hoster/XFileSharingPro.py
	module/plugins/hoster/YoutubeCom.py
	module/plugins/hoster/ZeveraCom.py
	module/plugins/hoster/ZippyshareCom.py
	module/plugins/internal/AbstractExtractor.py
	module/plugins/internal/BasePlugin.py
	module/plugins/internal/CaptchaService.py
	module/plugins/internal/DeadCrypter.py
	module/plugins/internal/DeadHoster.py
	module/plugins/internal/MultiHoster.py
	module/plugins/internal/SimpleCrypter.py
	module/plugins/internal/SimpleHoster.py
	module/plugins/internal/UnRar.py
	module/plugins/internal/UnZip.py
	module/plugins/internal/XFSCrypter.py
	module/plugins/internal/XFSHoster.py

1 files changed, 308 insertions, 0 deletions

diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py
new file mode 100644
index 000000000..652443098
--- /dev/null
+++ b/module/plugins/internal/MultiHook.py
@@ -0,0 +1,308 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from time import sleep
+
+from module.plugins.Hook import Hook
+from module.utils import decode, remove_chars
+
+
+class MultiHook(Hook):
+    __name__    = "MultiHook"
+    __type__    = "hook"
+    __version__ = "0.37"
+
+    __config__ = [("pluginmode"    , "all;listed;unlisted", "Use for plugins"                     , "all"),
+                  ("pluginlist"    , "str"                , "Plugin list (comma separated)"       , ""   ),
+                  ("revertfailed"  , "bool"               , "Revert to standard download if fails", True ),
+                  ("retry"         , "int"                , "Number of retries before revert"     , 10   ),
+                  ("retryinterval" , "int"                , "Retry interval in minutes"           , 1    ),
+                  ("reload"        , "bool"               , "Reload plugin list"                  , True ),
+                  ("reloadinterval", "int"                , "Reload interval in hours"            , 12   )]
+
+    __description__ = """Hook plugin for multi hoster/crypter"""
+    __license__     = "GPLv3"
+    __authors__     = [("pyLoad Team", "admin@pyload.org"),
+                       ("Walter Purcaro", "vuolter@gmail.com")]
+
+
+    MIN_INTERVAL = 1 * 60 * 60
+
+    DOMAIN_REPLACEMENTS = [(r'180upload\.com'  , "hundredeightyupload.com"),
+                           (r'1fichier\.com'   , "onefichier.com"         ),
+                           (r'2shared\.com'    , "twoshared.com"          ),
+                           (r'4shared\.com'    , "fourshared.com"         ),
+                           (r'bayfiles\.net'   , "bayfiles.com"           ),
+                           (r'cloudnator\.com' , "shragle.com"            ),
+                           (r'dfiles\.eu'      , "depositfiles.com"       ),
+                           (r'easy-share\.com' , "crocko.com"             ),
+                           (r'freakshare\.net' , "freakshare.com"         ),
+                           (r'hellshare\.com'  , "hellshare.cz"           ),
+                           (r'ifile\.it'       , "filecloud.io"           ),
+                           (r'nowdownload\.\w+', "nowdownload.sx"         ),
+                           (r'nowvideo\.\w+'   , "nowvideo.sx"            ),
+                           (r'putlocker\.com'  , "firedrive.com"          ),
+                           (r'share-?rapid\.cz', "multishare.cz"          ),
+                           (r'ul\.to'          , "uploaded.to"            ),
+                           (r'uploaded\.net'   , "uploaded.to"            ),
+                           (r'uploadhero\.co'  , "uploadhero.com"         ),
+                           (r'zshares\.net'    , "zshare.net"             ),
+                           (r'(\d+.+)'         , "X\1"                    )]
+
+
+    def setup(self):
+        self.plugins       = []
+        self.supported     = []
+        self.new_supported = []
+
+        self.account      = None
+        self.pluginclass  = None
+        self.pluginmodule = None
+        self.pluginname   = None
+        self.plugintype   = None
+
+        self._initPlugin()
+
+
+    def _initPlugin(self):
+        plugin, type = self.core.pluginManager.findPlugin(self.__name__)
+
+        if not plugin:
+            self.logWarning("Hook plugin will be deactivated due missing plugin reference")
+            self.setConfig('activated', False)
+        else:
+            self.pluginname   = self.__name__
+            self.plugintype   = type
+            self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__)
+            self.pluginclass  = getattr(self.pluginmodule, self.__name__)
+
+
+    def _loadAccount(self):
+        self.account = self.core.accountManager.getAccountPlugin(self.pluginname)
+
+        if self.account and not self.account.canUse():
+            self.account = None
+
+        if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT:
+            self.logWarning("Hook plugin will be deactivated due missing account reference")
+            self.setConfig('activated', False)
+
+
+    def coreReady(self):
+        self._loadAccount()
+
+
+    def getURL(self, *args, **kwargs):  #@TODO: Remove in 0.4.10
+        """ see HTTPRequest for argument list """
+        h = pyreq.getHTTPRequest(timeout=120)
+        try:
+            if not 'decode' in kwargs:
+                kwargs['decode'] = True
+            rep = h.load(*args, **kwargs)
+        finally:
+            h.close()
+
+        return rep
+
+
+    def getConfig(self, option, default=''):
+        """getConfig with default value - sublass may not implements all config options"""
+        try:
+            return self.getConf(option)
+
+        except KeyError:
+            return default
+
+
+    def pluginsCached(self):
+        if self.plugins:
+            return self.plugins
+
+        for _i in xrange(3):
+            try:
+                pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters())
+
+            except Exception, e:
+                self.logError(e, "Waiting 1 minute and retry")
+                sleep(60)
+
+            else:
+                break
+        else:
+            return list()
+
+        try:
+            configmode = self.getConfig("pluginmode", 'all')
+            if configmode in ("listed", "unlisted"):
+                pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',')
+                configset  = self._pluginSet(pluginlist)
+
+                if configmode == "listed":
+                    pluginset &= configset
+                else:
+                    pluginset -= configset
+
+        except Exception, e:
+            self.logError(e)
+
+        self.plugins = list(pluginset)
+
+        return self.plugins
+
+
+    def _pluginSet(self, plugins):
+        plugins = set((decode(x).strip().lower() for x in plugins if '.' in x))
+
+        for rf, rt in self.DOMAIN_REPLACEMENTS:
+            regex = re.compile(rf)
+            for p in filter(lambda x: regex.match(x), plugins):
+                plugins.remove(p)
+                plugins.add(re.sub(rf, rt, p))
+
+        plugins.discard('')
+
+        return plugins
+
+
+    def getHosters(self):
+        """Load list of supported hoster
+
+        :return: List of domain names
+        """
+        raise NotImplementedError
+
+
+    def getCrypters(self):
+        """Load list of supported crypters
+
+        :return: List of domain names
+        """
+        raise NotImplementedError
+
+
+    def periodical(self):
+        """reload plugin list periodically"""
+        self.logInfo(_("Reloading supported %s list") % self.plugintype)
+
+        old_supported = self.supported
+
+        self.supported     = []
+        self.new_supported = []
+        self.plugins       = []
+
+        self.overridePlugins()
+
+        old_supported = [plugin for plugin in old_supported if plugin not in self.supported]
+
+        if old_supported:
+            self.logDebug("Unload: %s" % ", ".join(old_supported))
+            for plugin in old_supported:
+                self.unloadPlugin(plugin)
+
+        if self.getConfig("reload", True):
+            self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL)
+        else:
+            self.core.scheduler.removeJob(self.cb)
+            self.cb = None
+
+
+    def overridePlugins(self):
+        excludedList = []
+
+        if self.plugintype == "hoster":
+            pluginMap    = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys())
+            accountList  = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium]
+        else:
+            pluginMap    = {}
+            accountList  = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()]
+
+        for plugin in self.pluginsCached():
+            name = remove_chars(plugin, "-.")
+
+            if name in accountList:
+                excludedList.append(plugin)
+            else:
+                if name in pluginMap:
+                    self.supported.append(pluginMap[name])
+                else:
+                    self.new_supported.append(plugin)
+
+        if not self.supported and not self.new_supported:
+            self.logError(_("No %s loaded") % self.plugintype)
+            return
+
+        # inject plugin plugin
+        self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported))))
+
+        for plugin in self.supported:
+            hdict = self.core.pluginManager.plugins[self.plugintype][plugin]
+            hdict['new_module'] = self.pluginmodule
+            hdict['new_name']   = self.pluginname
+
+        if excludedList:
+            self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList))))
+
+        if self.new_supported:
+            plugins = sorted(self.new_supported)
+
+            self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins)))
+
+            # create new regexp
+            regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins])
+            if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__:
+                regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp)
+
+            self.logDebug("Regexp: %s" % regexp)
+
+            hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname]
+            hdict['pattern'] = regexp
+            hdict['re']      = re.compile(regexp)
+
+
+    def unloadPlugin(self, plugin):
+        hdict = self.core.pluginManager.plugins[self.plugintype][plugin]
+        if "module" in hdict:
+            del hdict['module']
+
+        if "new_module" in hdict:
+            del hdict['new_module']
+            del hdict['new_name']
+
+
+    def unload(self):
+        """Remove override for all plugins. Scheduler job is removed by hookmanager"""
+        for plugin in self.supported:
+            self.unloadPlugin(plugin)
+
+        # reset pattern
+        hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname]
+
+        hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$')
+        hdict['re']      = re.compile(hdict['pattern'])
+
+
+    def downloadFailed(self, pyfile):
+        """remove plugin override if download fails but not if file is offline/temp.offline"""
+        if pyfile.status != 8 or not self.getConfig("revertfailed", True):
+            return
+
+        hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname]
+        if "new_name" in hdict and hdict['new_name'] == self.pluginname:
+            if pyfile.error == "MultiHook":
+                self.logDebug("Unload MultiHook", pyfile.pluginname, hdict)
+                self.unloadPlugin(pyfile.pluginname)
+                pyfile.setStatus("queued")
+                pyfile.sync()
+            else:
+                retries   = max(self.getConfig("retry", 10), 0)
+                wait_time = max(self.getConfig("retryinterval", 1), 0)
+
+                if 0 < retries > pyfile.plugin.retries:
+                    self.logInfo(_("Retrying: %s") % pyfile.name)
+                    pyfile.setCustomStatus("MultiHook", "queued")
+                    pyfile.sync()
+
+                    pyfile.plugin.retries += 1
+                    pyfile.plugin.setWait(wait_time)
+                    pyfile.plugin.wait()