From 8e7d14bae4d3c836f029a1235eb227380acc3f75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 16 Feb 2015 21:59:10 +0100 Subject: Fix plugins to work on 0.4.10 --- pyload/Core.py | 632 +++ pyload/__init__.py | 102 + pyload/api/__init__.py | 973 ++++ pyload/api/types.py | 381 ++ pyload/cli/AddPackage.py | 50 + pyload/cli/Cli.py | 570 ++ pyload/cli/Handler.py | 32 + pyload/cli/ManageFiles.py | 178 + pyload/cli/__init__.py | 4 + pyload/config/Parser.py | 353 ++ pyload/config/Setup.py | 553 ++ pyload/config/__init__.py | 1 + pyload/config/default.conf | 65 + pyload/database/DatabaseBackend.py | 290 + pyload/database/File.py | 875 +++ pyload/database/Storage.py | 34 + pyload/database/User.py | 93 + pyload/database/__init__.py | 7 + pyload/datatype/File.py | 270 + pyload/datatype/Package.py | 64 + pyload/datatype/__init__.py | 1 + pyload/manager/Account.py | 191 + pyload/manager/Addon.py | 304 + pyload/manager/Captcha.py | 138 + pyload/manager/Event.py | 104 + pyload/manager/Plugin.py | 404 ++ pyload/manager/Remote.py | 76 + pyload/manager/Thread.py | 302 + pyload/manager/__init__.py | 1 + pyload/manager/event/Scheduler.py | 126 + pyload/manager/event/__init__.py | 1 + pyload/manager/thread/Addon.py | 69 + pyload/manager/thread/Decrypter.py | 101 + pyload/manager/thread/Download.py | 213 + pyload/manager/thread/Info.py | 225 + pyload/manager/thread/Plugin.py | 130 + pyload/manager/thread/Server.py | 111 + pyload/manager/thread/__init__.py | 1 + pyload/network/Browser.py | 132 + pyload/network/Bucket.py | 52 + pyload/network/CookieJar.py | 42 + pyload/network/HTTPChunk.py | 294 + pyload/network/HTTPDownload.py | 312 + pyload/network/HTTPRequest.py | 302 + pyload/network/JsEngine.py | 256 + pyload/network/RequestFactory.py | 121 + pyload/network/XDCCRequest.py | 144 + pyload/network/__init__.py | 1 + pyload/plugin/Account.py | 307 + pyload/plugin/Addon.py | 182 + pyload/plugin/Captcha.py | 34 + pyload/plugin/Container.py | 66 + pyload/plugin/Crypter.py | 107 + pyload/plugin/Extractor.py | 140 + pyload/plugin/Hook.py | 15 + pyload/plugin/Hoster.py | 21 + pyload/plugin/OCR.py | 319 ++ pyload/plugin/Plugin.py | 753 +++ pyload/plugin/__init__.py | 1 + pyload/plugin/account/AlldebridCom.py | 64 + pyload/plugin/account/BackinNet.py | 16 + pyload/plugin/account/BillionuploadsCom.py | 16 + pyload/plugin/account/BitshareCom.py | 35 + pyload/plugin/account/CatShareNet.py | 62 + pyload/plugin/account/CloudzillaTo.py | 37 + pyload/plugin/account/CramitIn.py | 16 + pyload/plugin/account/CzshareCom.py | 54 + pyload/plugin/account/DebridItaliaCom.py | 45 + pyload/plugin/account/DepositfilesCom.py | 37 + pyload/plugin/account/DropboxCom.py | 35 + pyload/plugin/account/EasybytezCom.py | 19 + pyload/plugin/account/EuroshareEu.py | 41 + pyload/plugin/account/ExashareCom.py | 16 + pyload/plugin/account/FastixRu.py | 41 + pyload/plugin/account/FastshareCz.py | 53 + pyload/plugin/account/File4SafeCom.py | 18 + pyload/plugin/account/FileParadoxIn.py | 16 + pyload/plugin/account/FilecloudIo.py | 59 + pyload/plugin/account/FilefactoryCom.py | 49 + pyload/plugin/account/FilejungleCom.py | 50 + pyload/plugin/account/FileomCom.py | 16 + pyload/plugin/account/FilerNet.py | 59 + pyload/plugin/account/FilerioCom.py | 16 + pyload/plugin/account/FilesMailRu.py | 32 + pyload/plugin/account/FileserveCom.py | 44 + pyload/plugin/account/FourSharedCom.py | 35 + pyload/plugin/account/FreakshareCom.py | 53 + pyload/plugin/account/FreeWayMe.py | 52 + pyload/plugin/account/FshareVn.py | 63 + pyload/plugin/account/Ftp.py | 17 + pyload/plugin/account/HellshareCz.py | 79 + pyload/plugin/account/Http.py | 17 + pyload/plugin/account/HugefilesNet.py | 16 + pyload/plugin/account/HundredEightyUploadCom.py | 16 + pyload/plugin/account/JunkyvideoCom.py | 16 + pyload/plugin/account/JunocloudMe.py | 16 + pyload/plugin/account/Keep2ShareCc.py | 69 + pyload/plugin/account/LetitbitNet.py | 34 + pyload/plugin/account/LinestorageCom.py | 17 + pyload/plugin/account/LinksnappyCom.py | 57 + pyload/plugin/account/MegaDebridEu.py | 39 + pyload/plugin/account/MegaRapidCz.py | 60 + pyload/plugin/account/MegasharesCom.py | 48 + pyload/plugin/account/MovReelCom.py | 19 + pyload/plugin/account/MultihostersCom.py | 16 + pyload/plugin/account/MultishareCz.py | 44 + pyload/plugin/account/MyfastfileCom.py | 37 + pyload/plugin/account/NetloadIn.py | 44 + pyload/plugin/account/NoPremiumPl.py | 82 + pyload/plugin/account/NosuploadCom.py | 16 + pyload/plugin/account/NovafileCom.py | 16 + pyload/plugin/account/NowVideoSx.py | 57 + pyload/plugin/account/OboomCom.py | 65 + pyload/plugin/account/OneFichierCom.py | 60 + pyload/plugin/account/OverLoadMe.py | 43 + pyload/plugin/account/PremiumTo.py | 38 + pyload/plugin/account/PremiumizeMe.py | 49 + pyload/plugin/account/PutdriveCom.py | 16 + pyload/plugin/account/QuickshareCz.py | 43 + pyload/plugin/account/RPNetBiz.py | 51 + pyload/plugin/account/RapideoPl.py | 81 + pyload/plugin/account/RapidfileshareNet.py | 18 + pyload/plugin/account/RapidgatorNet.py | 72 + pyload/plugin/account/RapiduNet.py | 66 + pyload/plugin/account/RarefileNet.py | 16 + pyload/plugin/account/RealdebridCom.py | 40 + pyload/plugin/account/RehostTo.py | 54 + pyload/plugin/account/RyushareCom.py | 16 + pyload/plugin/account/SafesharingEu.py | 16 + pyload/plugin/account/SecureUploadEu.py | 16 + pyload/plugin/account/SendmywayCom.py | 16 + pyload/plugin/account/ShareonlineBiz.py | 65 + pyload/plugin/account/SimplyPremiumCom.py | 48 + pyload/plugin/account/SimplydebridCom.py | 35 + pyload/plugin/account/SmoozedCom.py | 62 + pyload/plugin/account/StahnuTo.py | 35 + pyload/plugin/account/StreamcloudEu.py | 16 + pyload/plugin/account/TurbobitNet.py | 43 + pyload/plugin/account/TusfilesNet.py | 23 + pyload/plugin/account/UlozTo.py | 50 + pyload/plugin/account/UnrestrictLi.py | 44 + pyload/plugin/account/UploadableCh.py | 34 + pyload/plugin/account/UploadcCom.py | 16 + pyload/plugin/account/UploadedTo.py | 71 + pyload/plugin/account/UploadheroCom.py | 42 + pyload/plugin/account/UploadingCom.py | 63 + pyload/plugin/account/UptoboxCom.py | 18 + pyload/plugin/account/VidPlayNet.py | 16 + pyload/plugin/account/WebshareCz.py | 68 + pyload/plugin/account/XFileSharingPro.py | 34 + pyload/plugin/account/YibaishiwuCom.py | 40 + pyload/plugin/account/ZeveraCom.py | 78 + pyload/plugin/account/__init__.py | 1 + pyload/plugin/addon/AndroidPhoneNotify.py | 75 + pyload/plugin/addon/Checksum.py | 195 + pyload/plugin/addon/ClickAndLoad.py | 119 + pyload/plugin/addon/DeleteFinished.py | 79 + pyload/plugin/addon/DownloadScheduler.py | 77 + pyload/plugin/addon/ExternalScripts.py | 150 + pyload/plugin/addon/ExtractArchive.py | 504 ++ pyload/plugin/addon/HotFolder.py | 71 + pyload/plugin/addon/IRCInterface.py | 431 ++ pyload/plugin/addon/JustPremium.py | 46 + pyload/plugin/addon/MergeFiles.py | 85 + pyload/plugin/addon/MultiHome.py | 81 + pyload/plugin/addon/RestartFailed.py | 45 + pyload/plugin/addon/RestartSlow.py | 57 + pyload/plugin/addon/SkipRev.py | 93 + pyload/plugin/addon/UnSkipOnFail.py | 90 + pyload/plugin/addon/UpdateManager.py | 306 + pyload/plugin/addon/WindowsPhoneNotify.py | 91 + pyload/plugin/addon/XMPPInterface.py | 252 + pyload/plugin/addon/__init__.py | 1 + pyload/plugin/captcha/AdYouLike.py | 108 + pyload/plugin/captcha/AdsCaptcha.py | 79 + pyload/plugin/captcha/ReCaptcha.py | 208 + pyload/plugin/captcha/SolveMedia.py | 113 + pyload/plugin/captcha/__init__.py | 1 + pyload/plugin/container/CCF.py | 49 + pyload/plugin/container/DLC.py | 72 + pyload/plugin/container/RSDF.py | 52 + pyload/plugin/container/TXT.py | 69 + pyload/plugin/container/__init__.py | 1 + pyload/plugin/crypter/BitshareCom.py | 21 + pyload/plugin/crypter/C1NeonCom.py | 16 + pyload/plugin/crypter/ChipDe.py | 29 + pyload/plugin/crypter/CloudzillaTo.py | 36 + pyload/plugin/crypter/CrockoCom.py | 20 + pyload/plugin/crypter/CryptItCom.py | 16 + pyload/plugin/crypter/CzshareCom.py | 32 + pyload/plugin/crypter/DDLMusicOrg.py | 51 + pyload/plugin/crypter/DailymotionBatch.py | 106 + pyload/plugin/crypter/DataHu.py | 40 + pyload/plugin/crypter/DdlstorageCom.py | 17 + pyload/plugin/crypter/DepositfilesCom.py | 20 + pyload/plugin/crypter/Dereferer.py | 17 + pyload/plugin/crypter/DevhostSt.py | 54 + pyload/plugin/crypter/DlProtectCom.py | 65 + pyload/plugin/crypter/DontKnowMe.py | 17 + pyload/plugin/crypter/DuckCryptInfo.py | 59 + pyload/plugin/crypter/DuploadOrg.py | 16 + pyload/plugin/crypter/EasybytezCom.py | 20 + pyload/plugin/crypter/EmbeduploadCom.py | 60 + pyload/plugin/crypter/FilebeerInfo.py | 16 + pyload/plugin/crypter/FilecloudIo.py | 21 + pyload/plugin/crypter/FilecryptCc.py | 182 + pyload/plugin/crypter/FilefactoryCom.py | 28 + pyload/plugin/crypter/FilerNet.py | 24 + pyload/plugin/crypter/FileserveCom.py | 38 + pyload/plugin/crypter/FilesonicCom.py | 15 + pyload/plugin/crypter/FilestubeCom.py | 21 + pyload/plugin/crypter/FiletramCom.py | 22 + pyload/plugin/crypter/FiredriveCom.py | 16 + pyload/plugin/crypter/FourChanOrg.py | 27 + pyload/plugin/crypter/FreakhareCom.py | 38 + pyload/plugin/crypter/FreetexthostCom.py | 27 + pyload/plugin/crypter/FshareVn.py | 20 + pyload/plugin/crypter/Go4UpCom.py | 46 + pyload/plugin/crypter/GooGl.py | 32 + pyload/plugin/crypter/HoerbuchIn.py | 62 + pyload/plugin/crypter/HotfileCom.py | 16 + pyload/plugin/crypter/ILoadTo.py | 16 + pyload/plugin/crypter/ImgurComAlbum.py | 27 + pyload/plugin/crypter/LetitbitNet.py | 33 + pyload/plugin/crypter/LinkCryptWs.py | 322 ++ pyload/plugin/crypter/LinkSaveIn.py | 22 + pyload/plugin/crypter/LinkdecrypterCom.py | 68 + pyload/plugin/crypter/LixIn.py | 62 + pyload/plugin/crypter/LofCc.py | 16 + pyload/plugin/crypter/MBLinkInfo.py | 17 + pyload/plugin/crypter/MediafireCom.py | 58 + pyload/plugin/crypter/MegaCoNz.py | 32 + pyload/plugin/crypter/MegaRapidCz.py | 20 + pyload/plugin/crypter/MegauploadCom.py | 15 + pyload/plugin/crypter/Movie2KTo.py | 16 + pyload/plugin/crypter/MultiUpOrg.py | 38 + pyload/plugin/crypter/MultiloadCz.py | 42 + pyload/plugin/crypter/MultiuploadCom.py | 15 + pyload/plugin/crypter/NCryptIn.py | 310 + pyload/plugin/crypter/NetfolderIn.py | 70 + pyload/plugin/crypter/NosvideoCom.py | 21 + pyload/plugin/crypter/OneKhDe.py | 41 + pyload/plugin/crypter/OronCom.py | 16 + pyload/plugin/crypter/PastebinCom.py | 21 + pyload/plugin/crypter/QuickshareCz.py | 31 + pyload/plugin/crypter/RSLayerCom.py | 16 + pyload/plugin/crypter/RelinkUs.py | 293 + pyload/plugin/crypter/SafelinkingNet.py | 81 + pyload/plugin/crypter/SecuredIn.py | 16 + pyload/plugin/crypter/SexuriaCom.py | 94 + pyload/plugin/crypter/ShareLinksBiz.py | 279 + pyload/plugin/crypter/SharingmatrixCom.py | 15 + pyload/plugin/crypter/SpeedLoadOrg.py | 16 + pyload/plugin/crypter/StealthTo.py | 16 + pyload/plugin/crypter/TnyCz.py | 27 + pyload/plugin/crypter/TrailerzoneInfo.py | 16 + pyload/plugin/crypter/TurbobitNet.py | 44 + pyload/plugin/crypter/TusfilesNet.py | 43 + pyload/plugin/crypter/UlozTo.py | 46 + pyload/plugin/crypter/UploadableCh.py | 24 + pyload/plugin/crypter/UploadedTo.py | 34 + pyload/plugin/crypter/WiiReloadedOrg.py | 16 + pyload/plugin/crypter/WuploadCom.py | 15 + pyload/plugin/crypter/XFileSharingPro.py | 52 + pyload/plugin/crypter/XupPl.py | 25 + pyload/plugin/crypter/YoutubeBatch.py | 148 + pyload/plugin/crypter/__init__.py | 1 + pyload/plugin/extractor/SevenZip.py | 155 + pyload/plugin/extractor/UnRar.py | 248 + pyload/plugin/extractor/UnZip.py | 68 + pyload/plugin/extractor/__init__.py | 1 + pyload/plugin/hook/AlldebridCom.py | 29 + pyload/plugin/hook/BypassCaptcha.py | 135 + pyload/plugin/hook/Captcha9Kw.py | 251 + pyload/plugin/hook/CaptchaBrotherhood.py | 167 + pyload/plugin/hook/DeathByCaptcha.py | 214 + pyload/plugin/hook/DebridItaliaCom.py | 28 + pyload/plugin/hook/EasybytezCom.py | 32 + pyload/plugin/hook/ExpertDecoders.py | 96 + pyload/plugin/hook/FastixRu.py | 31 + pyload/plugin/hook/FreeWayMe.py | 27 + pyload/plugin/hook/ImageTyperz.py | 153 + pyload/plugin/hook/LinkdecrypterCom.py | 25 + pyload/plugin/hook/LinksnappyCom.py | 29 + pyload/plugin/hook/MegaDebridEu.py | 35 + pyload/plugin/hook/MultihostersCom.py | 18 + pyload/plugin/hook/MultishareCz.py | 31 + pyload/plugin/hook/MyfastfileCom.py | 30 + pyload/plugin/hook/NoPremiumPl.py | 31 + pyload/plugin/hook/OverLoadMe.py | 31 + pyload/plugin/hook/PremiumTo.py | 29 + pyload/plugin/hook/PremiumizeMe.py | 40 + pyload/plugin/hook/PutdriveCom.py | 18 + pyload/plugin/hook/RPNetBiz.py | 38 + pyload/plugin/hook/RapideoPl.py | 31 + pyload/plugin/hook/RealdebridCom.py | 29 + pyload/plugin/hook/RehostTo.py | 29 + pyload/plugin/hook/SimplyPremiumCom.py | 31 + pyload/plugin/hook/SimplydebridCom.py | 26 + pyload/plugin/hook/SmoozedCom.py | 26 + pyload/plugin/hook/UnrestrictLi.py | 30 + pyload/plugin/hook/XFileSharingPro.py | 110 + pyload/plugin/hook/ZeveraCom.py | 27 + pyload/plugin/hook/__init__.py | 1 + pyload/plugin/hoster/AlldebridCom.py | 75 + pyload/plugin/hoster/AndroidfilehostCom.py | 60 + pyload/plugin/hoster/BasketbuildCom.py | 58 + pyload/plugin/hoster/BayfilesCom.py | 15 + pyload/plugin/hoster/BezvadataCz.py | 91 + pyload/plugin/hoster/BillionuploadsCom.py | 19 + pyload/plugin/hoster/BitshareCom.py | 155 + pyload/plugin/hoster/BoltsharingCom.py | 15 + pyload/plugin/hoster/CatShareNet.py | 59 + pyload/plugin/hoster/CloudzerNet.py | 17 + pyload/plugin/hoster/CloudzillaTo.py | 58 + pyload/plugin/hoster/CramitIn.py | 20 + pyload/plugin/hoster/CrockoCom.py | 63 + pyload/plugin/hoster/CyberlockerCh.py | 15 + pyload/plugin/hoster/CzshareCom.py | 158 + pyload/plugin/hoster/DailymotionCom.py | 125 + pyload/plugin/hoster/DataHu.py | 31 + pyload/plugin/hoster/DataportCz.py | 54 + pyload/plugin/hoster/DateiTo.py | 79 + pyload/plugin/hoster/DdlstorageCom.py | 16 + pyload/plugin/hoster/DebridItaliaCom.py | 40 + pyload/plugin/hoster/DepositfilesCom.py | 112 + pyload/plugin/hoster/DevhostSt.py | 33 + pyload/plugin/hoster/DlFreeFr.py | 137 + pyload/plugin/hoster/DodanePl.py | 15 + pyload/plugin/hoster/DuploadOrg.py | 15 + pyload/plugin/hoster/EasybytezCom.py | 21 + pyload/plugin/hoster/EdiskCz.py | 53 + pyload/plugin/hoster/EgoFilesCom.py | 15 + pyload/plugin/hoster/EnteruploadCom.py | 15 + pyload/plugin/hoster/EpicShareNet.py | 15 + pyload/plugin/hoster/EuroshareEu.py | 64 + pyload/plugin/hoster/ExashareCom.py | 35 + pyload/plugin/hoster/ExtabitCom.py | 75 + pyload/plugin/hoster/FastixRu.py | 63 + pyload/plugin/hoster/FastshareCz.py | 76 + pyload/plugin/hoster/FileApeCom.py | 15 + pyload/plugin/hoster/FileSharkPl.py | 130 + pyload/plugin/hoster/FileStoreTo.py | 34 + pyload/plugin/hoster/FilebeerInfo.py | 15 + pyload/plugin/hoster/FilecloudIo.py | 123 + pyload/plugin/hoster/FilefactoryCom.py | 85 + pyload/plugin/hoster/FilejungleCom.py | 29 + pyload/plugin/hoster/FileomCom.py | 30 + pyload/plugin/hoster/FilepostCom.py | 130 + pyload/plugin/hoster/FilepupNet.py | 44 + pyload/plugin/hoster/FilerNet.py | 71 + pyload/plugin/hoster/FilerioCom.py | 20 + pyload/plugin/hoster/FilesMailRu.py | 105 + pyload/plugin/hoster/FileserveCom.py | 216 + pyload/plugin/hoster/FileshareInUa.py | 15 + pyload/plugin/hoster/FilesonicCom.py | 16 + pyload/plugin/hoster/FilezyNet.py | 15 + pyload/plugin/hoster/FiredriveCom.py | 15 + pyload/plugin/hoster/FlyFilesNet.py | 45 + pyload/plugin/hoster/FourSharedCom.py | 59 + pyload/plugin/hoster/FreakshareCom.py | 180 + pyload/plugin/hoster/FreeWayMe.py | 50 + pyload/plugin/hoster/FreevideoCz.py | 15 + pyload/plugin/hoster/FshareVn.py | 110 + pyload/plugin/hoster/Ftp.py | 78 + pyload/plugin/hoster/GamefrontCom.py | 90 + pyload/plugin/hoster/GigapetaCom.py | 61 + pyload/plugin/hoster/GooIm.py | 34 + pyload/plugin/hoster/GoogledriveCom.py | 62 + pyload/plugin/hoster/HellshareCz.py | 32 + pyload/plugin/hoster/HellspyCz.py | 15 + pyload/plugin/hoster/HotfileCom.py | 18 + pyload/plugin/hoster/HugefilesNet.py | 22 + pyload/plugin/hoster/HundredEightyUploadCom.py | 15 + pyload/plugin/hoster/IFileWs.py | 15 + pyload/plugin/hoster/IcyFilesCom.py | 15 + pyload/plugin/hoster/IfileIt.py | 15 + pyload/plugin/hoster/IfolderRu.py | 73 + pyload/plugin/hoster/JumbofilesCom.py | 33 + pyload/plugin/hoster/JunocloudMe.py | 21 + pyload/plugin/hoster/Keep2ShareCc.py | 109 + pyload/plugin/hoster/KickloadCom.py | 15 + pyload/plugin/hoster/KingfilesNet.py | 75 + pyload/plugin/hoster/LemUploadsCom.py | 15 + pyload/plugin/hoster/LetitbitNet.py | 155 + pyload/plugin/hoster/LinksnappyCom.py | 55 + pyload/plugin/hoster/LoadTo.py | 65 + pyload/plugin/hoster/LomafileCom.py | 16 + pyload/plugin/hoster/LuckyShareNet.py | 71 + pyload/plugin/hoster/MediafireCom.py | 128 + pyload/plugin/hoster/MegaCoNz.py | 217 + pyload/plugin/hoster/MegaDebridEu.py | 86 + pyload/plugin/hoster/MegaFilesSe.py | 15 + pyload/plugin/hoster/MegaRapidCz.py | 65 + pyload/plugin/hoster/MegacrypterCom.py | 57 + pyload/plugin/hoster/MegareleaseOrg.py | 16 + pyload/plugin/hoster/MegasharesCom.py | 110 + pyload/plugin/hoster/MegauploadCom.py | 15 + pyload/plugin/hoster/MegavideoCom.py | 16 + pyload/plugin/hoster/MovReelCom.py | 18 + pyload/plugin/hoster/MultihostersCom.py | 15 + pyload/plugin/hoster/MultishareCz.py | 51 + pyload/plugin/hoster/MyfastfileCom.py | 36 + pyload/plugin/hoster/MystoreTo.py | 42 + pyload/plugin/hoster/MyvideoDe.py | 49 + pyload/plugin/hoster/NahrajCz.py | 15 + pyload/plugin/hoster/NarodRu.py | 63 + pyload/plugin/hoster/NetloadIn.py | 298 + pyload/plugin/hoster/NitroflareCom.py | 107 + pyload/plugin/hoster/NoPremiumPl.py | 103 + pyload/plugin/hoster/NosuploadCom.py | 39 + pyload/plugin/hoster/NovafileCom.py | 26 + pyload/plugin/hoster/NowDownloadSx.py | 61 + pyload/plugin/hoster/NowVideoSx.py | 41 + pyload/plugin/hoster/OboomCom.py | 145 + pyload/plugin/hoster/OneFichierCom.py | 58 + pyload/plugin/hoster/OronCom.py | 16 + pyload/plugin/hoster/OverLoadMe.py | 73 + pyload/plugin/hoster/PandaplaNet.py | 15 + pyload/plugin/hoster/PornhostCom.py | 79 + pyload/plugin/hoster/PornhubCom.py | 89 + pyload/plugin/hoster/PotloadCom.py | 15 + pyload/plugin/hoster/PremiumTo.py | 52 + pyload/plugin/hoster/PremiumizeMe.py | 57 + pyload/plugin/hoster/PromptfileCom.py | 42 + pyload/plugin/hoster/PrzeklejPl.py | 15 + pyload/plugin/hoster/PutdriveCom.py | 15 + pyload/plugin/hoster/QuickshareCz.py | 86 + pyload/plugin/hoster/RPNetBiz.py | 77 + pyload/plugin/hoster/RapideoPl.py | 103 + pyload/plugin/hoster/RapidfileshareNet.py | 22 + pyload/plugin/hoster/RapidgatorNet.py | 169 + pyload/plugin/hoster/RapiduNet.py | 83 + pyload/plugin/hoster/RarefileNet.py | 20 + pyload/plugin/hoster/RealdebridCom.py | 76 + pyload/plugin/hoster/RedtubeCom.py | 62 + pyload/plugin/hoster/RehostTo.py | 25 + pyload/plugin/hoster/RemixshareCom.py | 57 + pyload/plugin/hoster/RgHostNet.py | 23 + pyload/plugin/hoster/SafesharingEu.py | 18 + pyload/plugin/hoster/SecureUploadEu.py | 18 + pyload/plugin/hoster/SendspaceCom.py | 58 + pyload/plugin/hoster/Share4WebCom.py | 18 + pyload/plugin/hoster/Share76Com.py | 15 + pyload/plugin/hoster/ShareFilesCo.py | 15 + pyload/plugin/hoster/SharebeesCom.py | 15 + pyload/plugin/hoster/ShareonlineBiz.py | 184 + pyload/plugin/hoster/ShareplaceCom.py | 89 + pyload/plugin/hoster/SharingmatrixCom.py | 16 + pyload/plugin/hoster/ShragleCom.py | 16 + pyload/plugin/hoster/SimplyPremiumCom.py | 78 + pyload/plugin/hoster/SimplydebridCom.py | 45 + pyload/plugin/hoster/SmoozedCom.py | 63 + pyload/plugin/hoster/SockshareCom.py | 17 + pyload/plugin/hoster/SoundcloudCom.py | 57 + pyload/plugin/hoster/SpeedLoadOrg.py | 15 + pyload/plugin/hoster/SpeedfileCz.py | 15 + pyload/plugin/hoster/SpeedyshareCom.py | 41 + pyload/plugin/hoster/StorageTo.py | 15 + pyload/plugin/hoster/StreamCz.py | 71 + pyload/plugin/hoster/StreamcloudEu.py | 28 + pyload/plugin/hoster/TurbobitNet.py | 164 + pyload/plugin/hoster/TurbouploadCom.py | 15 + pyload/plugin/hoster/TusfilesNet.py | 37 + pyload/plugin/hoster/TwoSharedCom.py | 29 + pyload/plugin/hoster/UlozTo.py | 155 + pyload/plugin/hoster/UloziskoSk.py | 68 + pyload/plugin/hoster/UnibytesCom.py | 70 + pyload/plugin/hoster/UnrestrictLi.py | 82 + pyload/plugin/hoster/UpleaCom.py | 56 + pyload/plugin/hoster/UploadStationCom.py | 16 + pyload/plugin/hoster/UploadableCh.py | 76 + pyload/plugin/hoster/UploadboxCom.py | 15 + pyload/plugin/hoster/UploadedTo.py | 117 + pyload/plugin/hoster/UploadhereCom.py | 15 + pyload/plugin/hoster/UploadheroCom.py | 73 + pyload/plugin/hoster/UploadingCom.py | 96 + pyload/plugin/hoster/UploadkingCom.py | 15 + pyload/plugin/hoster/UpstoreNet.py | 69 + pyload/plugin/hoster/UptoboxCom.py | 29 + pyload/plugin/hoster/VeehdCom.py | 81 + pyload/plugin/hoster/VeohCom.py | 51 + pyload/plugin/hoster/VidPlayNet.py | 21 + pyload/plugin/hoster/VimeoCom.py | 71 + pyload/plugin/hoster/Vipleech4UCom.py | 15 + pyload/plugin/hoster/WarserverCz.py | 15 + pyload/plugin/hoster/WebshareCz.py | 60 + pyload/plugin/hoster/WrzucTo.py | 49 + pyload/plugin/hoster/WuploadCom.py | 16 + pyload/plugin/hoster/X7To.py | 15 + pyload/plugin/hoster/XFileSharingPro.py | 59 + pyload/plugin/hoster/XHamsterCom.py | 129 + pyload/plugin/hoster/XVideosCom.py | 28 + pyload/plugin/hoster/XdadevelopersCom.py | 35 + pyload/plugin/hoster/Xdcc.py | 210 + pyload/plugin/hoster/YibaishiwuCom.py | 56 + pyload/plugin/hoster/YoupornCom.py | 60 + pyload/plugin/hoster/YourfilesTo.py | 87 + pyload/plugin/hoster/YoutubeCom.py | 193 + pyload/plugin/hoster/ZDF.py | 59 + pyload/plugin/hoster/ZShareNet.py | 16 + pyload/plugin/hoster/ZeveraCom.py | 31 + pyload/plugin/hoster/ZippyshareCom.py | 62 + pyload/plugin/hoster/__init__.py | 1 + pyload/plugin/internal/BasePlugin.py | 92 + pyload/plugin/internal/DeadCrypter.py | 27 + pyload/plugin/internal/DeadHoster.py | 27 + pyload/plugin/internal/MultiHook.py | 308 + pyload/plugin/internal/MultiHoster.py | 85 + pyload/plugin/internal/SimpleCrypter.py | 157 + pyload/plugin/internal/SimpleDereferer.py | 98 + pyload/plugin/internal/SimpleHoster.py | 701 +++ pyload/plugin/internal/UpdateManager.py | 306 + pyload/plugin/internal/XFSAccount.py | 174 + pyload/plugin/internal/XFSCrypter.py | 45 + pyload/plugin/internal/XFSHoster.py | 326 ++ pyload/plugin/internal/__init__.py | 1 + pyload/plugin/ocr/GigasizeCom.py | 24 + pyload/plugin/ocr/LinksaveIn.py | 158 + pyload/plugin/ocr/NetloadIn.py | 29 + pyload/plugin/ocr/ShareonlineBiz.py | 39 + pyload/plugin/ocr/__init__.py | 1 + pyload/remote/ClickAndLoadBackend.py | 156 + pyload/remote/SocketBackend.py | 25 + pyload/remote/ThriftBackend.py | 42 + pyload/remote/__init__.py | 3 + pyload/remote/socketbackend/__init__.py | 1 + pyload/remote/socketbackend/create_ttypes.py | 88 + pyload/remote/thriftbackend/Processor.py | 77 + pyload/remote/thriftbackend/Protocol.py | 30 + pyload/remote/thriftbackend/Socket.py | 129 + pyload/remote/thriftbackend/ThriftClient.py | 91 + pyload/remote/thriftbackend/ThriftTest.py | 94 + pyload/remote/thriftbackend/Transport.py | 37 + pyload/remote/thriftbackend/__init__.py | 1 + pyload/remote/thriftbackend/pyload.thrift | 337 ++ pyload/remote/thriftbackend/thriftgen/__init__.py | 1 + .../thriftbackend/thriftgen/pyload/Pyload-remote | 570 ++ .../thriftbackend/thriftgen/pyload/Pyload.py | 5533 ++++++++++++++++++ .../thriftbackend/thriftgen/pyload/__init__.py | 3 + .../thriftbackend/thriftgen/pyload/constants.py | 10 + .../thriftbackend/thriftgen/pyload/ttypes.py | 834 +++ pyload/utils/__init__.py | 263 + pyload/utils/packagetools.py | 155 + pyload/utils/printer.py | 16 + pyload/utils/pylgettext.py | 60 + pyload/webui/__init__.py | 130 + pyload/webui/app/__init__.py | 3 + pyload/webui/app/api.py | 101 + pyload/webui/app/cnl.py | 167 + pyload/webui/app/json.py | 311 + pyload/webui/app/pyload.py | 530 ++ pyload/webui/app/utils.py | 123 + pyload/webui/filters.py | 61 + pyload/webui/middlewares.py | 132 + pyload/webui/servers/lighttpd_default.conf | 154 + pyload/webui/servers/nginx_default.conf | 87 + pyload/webui/themes/Dark/css/MooDialog.css | 94 + pyload/webui/themes/Dark/css/base.css | 962 ++++ pyload/webui/themes/Dark/css/log.css | 75 + pyload/webui/themes/Dark/css/pathchooser.css | 68 + pyload/webui/themes/Dark/css/window.css | 92 + .../themes/Dark/img/MooDialog/dialog-close.png | Bin 0 -> 689 bytes .../themes/Dark/img/MooDialog/dialog-error.png | Bin 0 -> 1472 bytes .../themes/Dark/img/MooDialog/dialog-question.png | Bin 0 -> 2073 bytes .../themes/Dark/img/MooDialog/dialog-warning.png | Bin 0 -> 1651 bytes pyload/webui/themes/Dark/img/button.png | Bin 0 -> 569 bytes pyload/webui/themes/Dark/img/dark-bg.jpg | Bin 0 -> 40930 bytes .../webui/themes/Dark/img/default/add_folder.png | Bin 0 -> 571 bytes .../webui/themes/Dark/img/default/ajax-loader.gif | Bin 0 -> 404 bytes .../themes/Dark/img/default/arrow_refresh.png | Bin 0 -> 685 bytes .../webui/themes/Dark/img/default/arrow_right.png | Bin 0 -> 349 bytes .../webui/themes/Dark/img/default/big_button.gif | Bin 0 -> 1905 bytes .../themes/Dark/img/default/big_button_over.gif | Bin 0 -> 728 bytes pyload/webui/themes/Dark/img/default/body.png | Bin 0 -> 402 bytes pyload/webui/themes/Dark/img/default/closebtn.gif | Bin 0 -> 254 bytes pyload/webui/themes/Dark/img/default/cog.png | Bin 0 -> 512 bytes .../webui/themes/Dark/img/default/control_add.png | Bin 0 -> 446 bytes .../themes/Dark/img/default/control_add_blue.png | Bin 0 -> 845 bytes .../themes/Dark/img/default/control_cancel.png | Bin 0 -> 3349 bytes .../Dark/img/default/control_cancel_blue.png | Bin 0 -> 787 bytes .../themes/Dark/img/default/control_pause.png | Bin 0 -> 598 bytes .../themes/Dark/img/default/control_pause_blue.png | Bin 0 -> 721 bytes .../webui/themes/Dark/img/default/control_play.png | Bin 0 -> 592 bytes .../themes/Dark/img/default/control_play_blue.png | Bin 0 -> 717 bytes .../webui/themes/Dark/img/default/control_stop.png | Bin 0 -> 403 bytes .../themes/Dark/img/default/control_stop_blue.png | Bin 0 -> 695 bytes pyload/webui/themes/Dark/img/default/delete.png | Bin 0 -> 715 bytes .../webui/themes/Dark/img/default/drag_corner.gif | Bin 0 -> 76 bytes pyload/webui/themes/Dark/img/default/error.png | Bin 0 -> 701 bytes pyload/webui/themes/Dark/img/default/folder.png | Bin 0 -> 537 bytes pyload/webui/themes/Dark/img/default/full.png | Bin 0 -> 3543 bytes .../webui/themes/Dark/img/default/head-login.png | Bin 0 -> 1288 bytes .../Dark/img/default/head-menu-collector.png | Bin 0 -> 1953 bytes .../themes/Dark/img/default/head-menu-config.png | Bin 0 -> 1802 bytes .../Dark/img/default/head-menu-development.png | Bin 0 -> 876 bytes .../themes/Dark/img/default/head-menu-download.png | Bin 0 -> 721 bytes .../themes/Dark/img/default/head-menu-home.png | Bin 0 -> 920 bytes .../themes/Dark/img/default/head-menu-index.png | Bin 0 -> 482 bytes .../themes/Dark/img/default/head-menu-news.png | Bin 0 -> 628 bytes .../themes/Dark/img/default/head-menu-queue.png | Bin 0 -> 2629 bytes .../themes/Dark/img/default/head-menu-recent.png | Bin 0 -> 932 bytes .../themes/Dark/img/default/head-menu-wiki.png | Bin 0 -> 1204 bytes .../Dark/img/default/head-search-noshadow.png | Bin 0 -> 1187 bytes pyload/webui/themes/Dark/img/default/head_bg1.png | Bin 0 -> 125 bytes pyload/webui/themes/Dark/img/default/images.png | Bin 0 -> 661 bytes pyload/webui/themes/Dark/img/default/notice.png | Bin 0 -> 778 bytes .../webui/themes/Dark/img/default/package_go.png | Bin 0 -> 898 bytes .../Dark/img/default/page-tools-backlinks.png | Bin 0 -> 540 bytes .../themes/Dark/img/default/page-tools-edit.png | Bin 0 -> 574 bytes .../Dark/img/default/page-tools-revisions.png | Bin 0 -> 603 bytes pyload/webui/themes/Dark/img/default/parseUri.png | Bin 0 -> 666 bytes pyload/webui/themes/Dark/img/default/pencil.png | Bin 0 -> 450 bytes pyload/webui/themes/Dark/img/default/reconnect.png | Bin 0 -> 755 bytes .../webui/themes/Dark/img/default/status_None.png | Bin 0 -> 7613 bytes .../themes/Dark/img/default/status_downloading.png | Bin 0 -> 943 bytes .../themes/Dark/img/default/status_failed.png | Bin 0 -> 701 bytes .../themes/Dark/img/default/status_finished.png | Bin 0 -> 781 bytes .../themes/Dark/img/default/status_offline.png | Bin 0 -> 700 bytes .../webui/themes/Dark/img/default/status_proc.png | Bin 0 -> 512 bytes .../webui/themes/Dark/img/default/status_queue.png | Bin 0 -> 7613 bytes .../themes/Dark/img/default/status_waiting.png | Bin 0 -> 889 bytes pyload/webui/themes/Dark/img/default/success.png | Bin 0 -> 781 bytes .../themes/Dark/img/default/tabs-border-bottom.png | Bin 0 -> 163 bytes .../Dark/img/default/user-actions-logout.png | Bin 0 -> 799 bytes .../Dark/img/default/user-actions-profile.png | Bin 0 -> 628 bytes pyload/webui/themes/Dark/img/default/user-info.png | Bin 0 -> 3963 bytes pyload/webui/themes/Dark/img/pyload-logo.png | Bin 0 -> 6947 bytes pyload/webui/themes/Dark/img/tab-background.png | Bin 0 -> 3044 bytes pyload/webui/themes/Dark/js/render/admin.coffee | 58 + pyload/webui/themes/Dark/js/render/admin.min.js | 3 + pyload/webui/themes/Dark/js/render/base.coffee | 177 + pyload/webui/themes/Dark/js/render/base.min.js | 3 + pyload/webui/themes/Dark/js/render/package.js | 376 ++ pyload/webui/themes/Dark/js/render/settings.coffee | 107 + pyload/webui/themes/Dark/js/render/settings.min.js | 3 + pyload/webui/themes/Dark/js/static/MooDialog.js | 140 + .../webui/themes/Dark/js/static/MooDialog.min.js | 1 + pyload/webui/themes/Dark/js/static/MooDropMenu.js | 86 + .../webui/themes/Dark/js/static/MooDropMenu.min.js | 1 + .../webui/themes/Dark/js/static/mootools-core.js | 5977 ++++++++++++++++++++ .../themes/Dark/js/static/mootools-core.min.js | 491 ++ .../webui/themes/Dark/js/static/mootools-more.js | 2856 ++++++++++ .../themes/Dark/js/static/mootools-more.min.js | 226 + pyload/webui/themes/Dark/js/static/purr.js | 309 + pyload/webui/themes/Dark/js/static/purr.min.js | 1 + pyload/webui/themes/Dark/js/static/tinytab.js | 43 + pyload/webui/themes/Dark/js/static/tinytab.min.js | 1 + pyload/webui/themes/Dark/tml/admin.html | 98 + pyload/webui/themes/Dark/tml/base.html | 177 + pyload/webui/themes/Dark/tml/captcha.html | 42 + pyload/webui/themes/Dark/tml/downloads.html | 29 + pyload/webui/themes/Dark/tml/folder.html | 15 + pyload/webui/themes/Dark/tml/home.html | 263 + pyload/webui/themes/Dark/tml/info.html | 76 + pyload/webui/themes/Dark/tml/login.html | 37 + pyload/webui/themes/Dark/tml/logout.html | 9 + pyload/webui/themes/Dark/tml/logs.html | 41 + pyload/webui/themes/Dark/tml/pathchooser.html | 76 + pyload/webui/themes/Dark/tml/queue.html | 104 + pyload/webui/themes/Dark/tml/settings.html | 204 + pyload/webui/themes/Dark/tml/settings_item.html | 48 + pyload/webui/themes/Dark/tml/window.html | 52 + pyload/webui/themes/Default/css/MooDialog.css | 91 + pyload/webui/themes/Default/css/base.css | 902 +++ pyload/webui/themes/Default/css/log.css | 71 + pyload/webui/themes/Default/css/pathchooser.css | 68 + pyload/webui/themes/Default/css/window.css | 73 + .../themes/Default/img/MooDialog/dialog-close.png | Bin 0 -> 689 bytes .../themes/Default/img/MooDialog/dialog-error.png | Bin 0 -> 1472 bytes .../Default/img/MooDialog/dialog-question.png | Bin 0 -> 2073 bytes .../Default/img/MooDialog/dialog-warning.png | Bin 0 -> 1651 bytes pyload/webui/themes/Default/img/add_folder.png | Bin 0 -> 571 bytes pyload/webui/themes/Default/img/ajax-loader.gif | Bin 0 -> 404 bytes pyload/webui/themes/Default/img/arrow_refresh.png | Bin 0 -> 685 bytes pyload/webui/themes/Default/img/arrow_right.png | Bin 0 -> 349 bytes pyload/webui/themes/Default/img/big_button.gif | Bin 0 -> 1905 bytes .../webui/themes/Default/img/big_button_over.gif | Bin 0 -> 728 bytes pyload/webui/themes/Default/img/body.png | Bin 0 -> 402 bytes pyload/webui/themes/Default/img/button.png | Bin 0 -> 452 bytes pyload/webui/themes/Default/img/closebtn.gif | Bin 0 -> 254 bytes pyload/webui/themes/Default/img/cog.png | Bin 0 -> 512 bytes pyload/webui/themes/Default/img/control_add.png | Bin 0 -> 446 bytes .../webui/themes/Default/img/control_add_blue.png | Bin 0 -> 845 bytes pyload/webui/themes/Default/img/control_cancel.png | Bin 0 -> 3349 bytes .../themes/Default/img/control_cancel_blue.png | Bin 0 -> 787 bytes pyload/webui/themes/Default/img/control_pause.png | Bin 0 -> 598 bytes .../themes/Default/img/control_pause_blue.png | Bin 0 -> 721 bytes pyload/webui/themes/Default/img/control_play.png | Bin 0 -> 592 bytes .../webui/themes/Default/img/control_play_blue.png | Bin 0 -> 717 bytes pyload/webui/themes/Default/img/control_stop.png | Bin 0 -> 403 bytes .../webui/themes/Default/img/control_stop_blue.png | Bin 0 -> 695 bytes pyload/webui/themes/Default/img/delete.png | Bin 0 -> 715 bytes pyload/webui/themes/Default/img/drag_corner.gif | Bin 0 -> 76 bytes pyload/webui/themes/Default/img/error.png | Bin 0 -> 701 bytes pyload/webui/themes/Default/img/folder.png | Bin 0 -> 537 bytes pyload/webui/themes/Default/img/full.png | Bin 0 -> 3543 bytes pyload/webui/themes/Default/img/head-login.png | Bin 0 -> 1288 bytes .../themes/Default/img/head-menu-collector.png | Bin 0 -> 1953 bytes .../webui/themes/Default/img/head-menu-config.png | Bin 0 -> 1802 bytes .../themes/Default/img/head-menu-development.png | Bin 0 -> 876 bytes .../themes/Default/img/head-menu-download.png | Bin 0 -> 721 bytes pyload/webui/themes/Default/img/head-menu-home.png | Bin 0 -> 920 bytes .../webui/themes/Default/img/head-menu-index.png | Bin 0 -> 482 bytes pyload/webui/themes/Default/img/head-menu-news.png | Bin 0 -> 628 bytes .../webui/themes/Default/img/head-menu-queue.png | Bin 0 -> 2629 bytes .../webui/themes/Default/img/head-menu-recent.png | Bin 0 -> 932 bytes pyload/webui/themes/Default/img/head-menu-wiki.png | Bin 0 -> 1204 bytes .../themes/Default/img/head-search-noshadow.png | Bin 0 -> 1187 bytes pyload/webui/themes/Default/img/head_bg1.png | Bin 0 -> 125 bytes pyload/webui/themes/Default/img/images.png | Bin 0 -> 661 bytes pyload/webui/themes/Default/img/notice.png | Bin 0 -> 778 bytes pyload/webui/themes/Default/img/package_go.png | Bin 0 -> 898 bytes .../themes/Default/img/page-tools-backlinks.png | Bin 0 -> 540 bytes .../webui/themes/Default/img/page-tools-edit.png | Bin 0 -> 574 bytes .../themes/Default/img/page-tools-revisions.png | Bin 0 -> 603 bytes pyload/webui/themes/Default/img/parseUri.png | Bin 0 -> 666 bytes pyload/webui/themes/Default/img/pencil.png | Bin 0 -> 450 bytes pyload/webui/themes/Default/img/pyload-logo.png | Bin 0 -> 8457 bytes pyload/webui/themes/Default/img/reconnect.png | Bin 0 -> 755 bytes pyload/webui/themes/Default/img/status_None.png | Bin 0 -> 7613 bytes .../themes/Default/img/status_downloading.png | Bin 0 -> 943 bytes pyload/webui/themes/Default/img/status_failed.png | Bin 0 -> 701 bytes .../webui/themes/Default/img/status_finished.png | Bin 0 -> 781 bytes pyload/webui/themes/Default/img/status_offline.png | Bin 0 -> 700 bytes pyload/webui/themes/Default/img/status_proc.png | Bin 0 -> 512 bytes pyload/webui/themes/Default/img/status_queue.png | Bin 0 -> 7613 bytes pyload/webui/themes/Default/img/status_waiting.png | Bin 0 -> 889 bytes pyload/webui/themes/Default/img/success.png | Bin 0 -> 781 bytes pyload/webui/themes/Default/img/tab-background.png | Bin 0 -> 179 bytes .../themes/Default/img/tabs-border-bottom.png | Bin 0 -> 163 bytes .../themes/Default/img/user-actions-logout.png | Bin 0 -> 799 bytes .../themes/Default/img/user-actions-profile.png | Bin 0 -> 628 bytes pyload/webui/themes/Default/img/user-info.png | Bin 0 -> 3963 bytes pyload/webui/themes/Default/js/render/admin.coffee | 58 + pyload/webui/themes/Default/js/render/admin.min.js | 3 + pyload/webui/themes/Default/js/render/base.coffee | 177 + pyload/webui/themes/Default/js/render/base.min.js | 3 + pyload/webui/themes/Default/js/render/package.js | 376 ++ .../webui/themes/Default/js/render/settings.coffee | 107 + .../webui/themes/Default/js/render/settings.min.js | 3 + pyload/webui/themes/Default/js/static/MooDialog.js | 140 + .../themes/Default/js/static/MooDialog.min.js | 1 + .../webui/themes/Default/js/static/MooDropMenu.js | 86 + .../themes/Default/js/static/MooDropMenu.min.js | 1 + .../themes/Default/js/static/mootools-core.js | 5977 ++++++++++++++++++++ .../themes/Default/js/static/mootools-core.min.js | 491 ++ .../themes/Default/js/static/mootools-more.js | 2856 ++++++++++ .../themes/Default/js/static/mootools-more.min.js | 226 + pyload/webui/themes/Default/js/static/purr.js | 309 + pyload/webui/themes/Default/js/static/purr.min.js | 1 + pyload/webui/themes/Default/js/static/tinytab.js | 43 + .../webui/themes/Default/js/static/tinytab.min.js | 1 + pyload/webui/themes/Default/tml/admin.html | 98 + pyload/webui/themes/Default/tml/base.html | 177 + pyload/webui/themes/Default/tml/captcha.html | 42 + pyload/webui/themes/Default/tml/downloads.html | 29 + pyload/webui/themes/Default/tml/filemanager.html | 76 + pyload/webui/themes/Default/tml/folder.html | 15 + pyload/webui/themes/Default/tml/home.html | 263 + pyload/webui/themes/Default/tml/info.html | 81 + pyload/webui/themes/Default/tml/login.html | 36 + pyload/webui/themes/Default/tml/logout.html | 9 + pyload/webui/themes/Default/tml/logs.html | 41 + pyload/webui/themes/Default/tml/pathchooser.html | 76 + pyload/webui/themes/Default/tml/queue.html | 104 + pyload/webui/themes/Default/tml/settings.html | 204 + pyload/webui/themes/Default/tml/settings_item.html | 48 + pyload/webui/themes/Default/tml/window.html | 46 + pyload/webui/themes/Flat/css/MooDialog.css | 85 + pyload/webui/themes/Flat/css/base.css | 866 +++ pyload/webui/themes/Flat/css/log.css | 72 + pyload/webui/themes/Flat/css/pathchooser.css | 68 + pyload/webui/themes/Flat/css/window.css | 70 + .../themes/Flat/img/MooDialog/dialog-close.png | Bin 0 -> 689 bytes .../themes/Flat/img/MooDialog/dialog-error.png | Bin 0 -> 1472 bytes .../themes/Flat/img/MooDialog/dialog-question.png | Bin 0 -> 2073 bytes .../themes/Flat/img/MooDialog/dialog-warning.png | Bin 0 -> 1651 bytes pyload/webui/themes/Flat/img/arrow_refresh.png | Bin 0 -> 119032 bytes pyload/webui/themes/Flat/img/arrow_right.png | Bin 0 -> 136967 bytes pyload/webui/themes/Flat/img/button.png | Bin 0 -> 569 bytes pyload/webui/themes/Flat/img/cog.png | Bin 0 -> 137406 bytes pyload/webui/themes/Flat/img/control_add.png | Bin 0 -> 116941 bytes pyload/webui/themes/Flat/img/control_add_blue.png | Bin 0 -> 116941 bytes pyload/webui/themes/Flat/img/control_cancel.png | Bin 0 -> 116939 bytes .../webui/themes/Flat/img/control_cancel_blue.png | Bin 0 -> 116939 bytes pyload/webui/themes/Flat/img/control_pause.png | Bin 0 -> 134855 bytes .../webui/themes/Flat/img/control_pause_blue.png | Bin 0 -> 134855 bytes pyload/webui/themes/Flat/img/control_play.png | Bin 0 -> 134904 bytes pyload/webui/themes/Flat/img/control_play_blue.png | Bin 0 -> 134904 bytes pyload/webui/themes/Flat/img/control_stop.png | Bin 0 -> 134835 bytes pyload/webui/themes/Flat/img/control_stop_blue.png | Bin 0 -> 134835 bytes .../webui/themes/Flat/img/default/add_folder.png | Bin 0 -> 571 bytes .../webui/themes/Flat/img/default/ajax-loader.gif | Bin 0 -> 404 bytes .../webui/themes/Flat/img/default/big_button.gif | Bin 0 -> 1905 bytes .../themes/Flat/img/default/big_button_over.gif | Bin 0 -> 728 bytes pyload/webui/themes/Flat/img/default/body.png | Bin 0 -> 402 bytes pyload/webui/themes/Flat/img/default/closebtn.gif | Bin 0 -> 254 bytes .../webui/themes/Flat/img/default/drag_corner.gif | Bin 0 -> 76 bytes pyload/webui/themes/Flat/img/default/full.png | Bin 0 -> 3543 bytes .../themes/Flat/img/default/head-menu-recent.png | Bin 0 -> 932 bytes pyload/webui/themes/Flat/img/default/head_bg1.png | Bin 0 -> 125 bytes pyload/webui/themes/Flat/img/default/images.png | Bin 0 -> 661 bytes pyload/webui/themes/Flat/img/default/parseUri.png | Bin 0 -> 666 bytes .../webui/themes/Flat/img/default/pyload-logo.png | Bin 0 -> 8457 bytes .../themes/Flat/img/default/tab-background.png | Bin 0 -> 179 bytes .../themes/Flat/img/default/tabs-border-bottom.png | Bin 0 -> 163 bytes pyload/webui/themes/Flat/img/delete.png | Bin 0 -> 117658 bytes pyload/webui/themes/Flat/img/error.png | Bin 0 -> 137673 bytes pyload/webui/themes/Flat/img/folder.png | Bin 0 -> 134669 bytes pyload/webui/themes/Flat/img/head-login.png | Bin 0 -> 137406 bytes .../webui/themes/Flat/img/head-menu-collector.png | Bin 0 -> 134985 bytes pyload/webui/themes/Flat/img/head-menu-config.png | Bin 0 -> 137664 bytes .../themes/Flat/img/head-menu-development.png | Bin 0 -> 135818 bytes .../webui/themes/Flat/img/head-menu-download.png | Bin 0 -> 137664 bytes pyload/webui/themes/Flat/img/head-menu-home.png | Bin 0 -> 139387 bytes pyload/webui/themes/Flat/img/head-menu-index.png | Bin 0 -> 136511 bytes pyload/webui/themes/Flat/img/head-menu-news.png | Bin 0 -> 136511 bytes pyload/webui/themes/Flat/img/head-menu-queue.png | Bin 0 -> 136269 bytes pyload/webui/themes/Flat/img/head-menu-wiki.png | Bin 0 -> 137217 bytes .../webui/themes/Flat/img/head-search-noshadow.png | Bin 0 -> 137217 bytes pyload/webui/themes/Flat/img/notice.png | Bin 0 -> 3061 bytes pyload/webui/themes/Flat/img/package_go.png | Bin 0 -> 136299 bytes .../webui/themes/Flat/img/page-tools-backlinks.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/page-tools-edit.png | Bin 0 -> 138112 bytes .../webui/themes/Flat/img/page-tools-revisions.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/pencil.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/reconnect.png | Bin 0 -> 3063 bytes pyload/webui/themes/Flat/img/status_None.png | Bin 0 -> 138112 bytes .../webui/themes/Flat/img/status_downloading.png | Bin 0 -> 3061 bytes pyload/webui/themes/Flat/img/status_failed.png | Bin 0 -> 137673 bytes pyload/webui/themes/Flat/img/status_finished.png | Bin 0 -> 117658 bytes pyload/webui/themes/Flat/img/status_offline.png | Bin 0 -> 137673 bytes pyload/webui/themes/Flat/img/status_proc.png | Bin 0 -> 137406 bytes pyload/webui/themes/Flat/img/status_queue.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/status_waiting.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/success.png | Bin 0 -> 117658 bytes .../webui/themes/Flat/img/user-actions-logout.png | Bin 0 -> 138112 bytes .../webui/themes/Flat/img/user-actions-profile.png | Bin 0 -> 138112 bytes pyload/webui/themes/Flat/img/user-info.png | Bin 0 -> 3080 bytes pyload/webui/themes/Flat/js/render/admin.coffee | 58 + pyload/webui/themes/Flat/js/render/admin.min.js | 3 + pyload/webui/themes/Flat/js/render/base.coffee | 177 + pyload/webui/themes/Flat/js/render/base.min.js | 3 + pyload/webui/themes/Flat/js/render/filemanager.js | 291 + pyload/webui/themes/Flat/js/render/package.js | 376 ++ pyload/webui/themes/Flat/js/render/settings.coffee | 107 + pyload/webui/themes/Flat/js/render/settings.min.js | 3 + pyload/webui/themes/Flat/js/static/MooDialog.js | 140 + .../webui/themes/Flat/js/static/MooDialog.min.js | 1 + pyload/webui/themes/Flat/js/static/MooDropMenu.js | 86 + .../webui/themes/Flat/js/static/MooDropMenu.min.js | 1 + .../webui/themes/Flat/js/static/mootools-core.js | 5977 ++++++++++++++++++++ .../themes/Flat/js/static/mootools-core.min.js | 491 ++ .../webui/themes/Flat/js/static/mootools-more.js | 2856 ++++++++++ .../themes/Flat/js/static/mootools-more.min.js | 226 + pyload/webui/themes/Flat/js/static/purr.js | 309 + pyload/webui/themes/Flat/js/static/purr.min.js | 1 + pyload/webui/themes/Flat/js/static/tinytab.js | 43 + pyload/webui/themes/Flat/js/static/tinytab.min.js | 1 + pyload/webui/themes/Flat/tml/admin.html | 98 + pyload/webui/themes/Flat/tml/base.html | 180 + pyload/webui/themes/Flat/tml/captcha.html | 42 + pyload/webui/themes/Flat/tml/downloads.html | 29 + pyload/webui/themes/Flat/tml/filemanager.html | 78 + pyload/webui/themes/Flat/tml/folder.html | 15 + pyload/webui/themes/Flat/tml/home.html | 266 + pyload/webui/themes/Flat/tml/info.html | 81 + pyload/webui/themes/Flat/tml/login.html | 36 + pyload/webui/themes/Flat/tml/logout.html | 9 + pyload/webui/themes/Flat/tml/logs.html | 41 + pyload/webui/themes/Flat/tml/pathchooser.html | 76 + pyload/webui/themes/Flat/tml/queue.html | 104 + pyload/webui/themes/Flat/tml/settings.html | 204 + pyload/webui/themes/Flat/tml/settings_item.html | 48 + pyload/webui/themes/Flat/tml/window.html | 46 + 876 files changed, 90067 insertions(+) create mode 100644 pyload/Core.py create mode 100644 pyload/__init__.py create mode 100644 pyload/api/__init__.py create mode 100644 pyload/api/types.py create mode 100644 pyload/cli/AddPackage.py create mode 100644 pyload/cli/Cli.py create mode 100644 pyload/cli/Handler.py create mode 100644 pyload/cli/ManageFiles.py create mode 100644 pyload/cli/__init__.py create mode 100644 pyload/config/Parser.py create mode 100644 pyload/config/Setup.py create mode 100644 pyload/config/__init__.py create mode 100644 pyload/config/default.conf create mode 100644 pyload/database/DatabaseBackend.py create mode 100644 pyload/database/File.py create mode 100644 pyload/database/Storage.py create mode 100644 pyload/database/User.py create mode 100644 pyload/database/__init__.py create mode 100644 pyload/datatype/File.py create mode 100644 pyload/datatype/Package.py create mode 100644 pyload/datatype/__init__.py create mode 100644 pyload/manager/Account.py create mode 100644 pyload/manager/Addon.py create mode 100644 pyload/manager/Captcha.py create mode 100644 pyload/manager/Event.py create mode 100644 pyload/manager/Plugin.py create mode 100644 pyload/manager/Remote.py create mode 100644 pyload/manager/Thread.py create mode 100644 pyload/manager/__init__.py create mode 100644 pyload/manager/event/Scheduler.py create mode 100644 pyload/manager/event/__init__.py create mode 100644 pyload/manager/thread/Addon.py create mode 100644 pyload/manager/thread/Decrypter.py create mode 100644 pyload/manager/thread/Download.py create mode 100644 pyload/manager/thread/Info.py create mode 100644 pyload/manager/thread/Plugin.py create mode 100644 pyload/manager/thread/Server.py create mode 100644 pyload/manager/thread/__init__.py create mode 100644 pyload/network/Browser.py create mode 100644 pyload/network/Bucket.py create mode 100644 pyload/network/CookieJar.py create mode 100644 pyload/network/HTTPChunk.py create mode 100644 pyload/network/HTTPDownload.py create mode 100644 pyload/network/HTTPRequest.py create mode 100644 pyload/network/JsEngine.py create mode 100644 pyload/network/RequestFactory.py create mode 100644 pyload/network/XDCCRequest.py create mode 100644 pyload/network/__init__.py create mode 100644 pyload/plugin/Account.py create mode 100644 pyload/plugin/Addon.py create mode 100644 pyload/plugin/Captcha.py create mode 100644 pyload/plugin/Container.py create mode 100644 pyload/plugin/Crypter.py create mode 100644 pyload/plugin/Extractor.py create mode 100644 pyload/plugin/Hook.py create mode 100644 pyload/plugin/Hoster.py create mode 100644 pyload/plugin/OCR.py create mode 100644 pyload/plugin/Plugin.py create mode 100644 pyload/plugin/__init__.py create mode 100644 pyload/plugin/account/AlldebridCom.py create mode 100644 pyload/plugin/account/BackinNet.py create mode 100644 pyload/plugin/account/BillionuploadsCom.py create mode 100644 pyload/plugin/account/BitshareCom.py create mode 100644 pyload/plugin/account/CatShareNet.py create mode 100644 pyload/plugin/account/CloudzillaTo.py create mode 100644 pyload/plugin/account/CramitIn.py create mode 100644 pyload/plugin/account/CzshareCom.py create mode 100644 pyload/plugin/account/DebridItaliaCom.py create mode 100644 pyload/plugin/account/DepositfilesCom.py create mode 100644 pyload/plugin/account/DropboxCom.py create mode 100644 pyload/plugin/account/EasybytezCom.py create mode 100644 pyload/plugin/account/EuroshareEu.py create mode 100644 pyload/plugin/account/ExashareCom.py create mode 100644 pyload/plugin/account/FastixRu.py create mode 100644 pyload/plugin/account/FastshareCz.py create mode 100644 pyload/plugin/account/File4SafeCom.py create mode 100644 pyload/plugin/account/FileParadoxIn.py create mode 100644 pyload/plugin/account/FilecloudIo.py create mode 100644 pyload/plugin/account/FilefactoryCom.py create mode 100644 pyload/plugin/account/FilejungleCom.py create mode 100644 pyload/plugin/account/FileomCom.py create mode 100644 pyload/plugin/account/FilerNet.py create mode 100644 pyload/plugin/account/FilerioCom.py create mode 100644 pyload/plugin/account/FilesMailRu.py create mode 100644 pyload/plugin/account/FileserveCom.py create mode 100644 pyload/plugin/account/FourSharedCom.py create mode 100644 pyload/plugin/account/FreakshareCom.py create mode 100644 pyload/plugin/account/FreeWayMe.py create mode 100644 pyload/plugin/account/FshareVn.py create mode 100644 pyload/plugin/account/Ftp.py create mode 100644 pyload/plugin/account/HellshareCz.py create mode 100644 pyload/plugin/account/Http.py create mode 100644 pyload/plugin/account/HugefilesNet.py create mode 100644 pyload/plugin/account/HundredEightyUploadCom.py create mode 100644 pyload/plugin/account/JunkyvideoCom.py create mode 100644 pyload/plugin/account/JunocloudMe.py create mode 100644 pyload/plugin/account/Keep2ShareCc.py create mode 100644 pyload/plugin/account/LetitbitNet.py create mode 100644 pyload/plugin/account/LinestorageCom.py create mode 100644 pyload/plugin/account/LinksnappyCom.py create mode 100644 pyload/plugin/account/MegaDebridEu.py create mode 100644 pyload/plugin/account/MegaRapidCz.py create mode 100644 pyload/plugin/account/MegasharesCom.py create mode 100644 pyload/plugin/account/MovReelCom.py create mode 100644 pyload/plugin/account/MultihostersCom.py create mode 100644 pyload/plugin/account/MultishareCz.py create mode 100644 pyload/plugin/account/MyfastfileCom.py create mode 100644 pyload/plugin/account/NetloadIn.py create mode 100644 pyload/plugin/account/NoPremiumPl.py create mode 100644 pyload/plugin/account/NosuploadCom.py create mode 100644 pyload/plugin/account/NovafileCom.py create mode 100644 pyload/plugin/account/NowVideoSx.py create mode 100644 pyload/plugin/account/OboomCom.py create mode 100644 pyload/plugin/account/OneFichierCom.py create mode 100644 pyload/plugin/account/OverLoadMe.py create mode 100644 pyload/plugin/account/PremiumTo.py create mode 100644 pyload/plugin/account/PremiumizeMe.py create mode 100644 pyload/plugin/account/PutdriveCom.py create mode 100644 pyload/plugin/account/QuickshareCz.py create mode 100644 pyload/plugin/account/RPNetBiz.py create mode 100644 pyload/plugin/account/RapideoPl.py create mode 100644 pyload/plugin/account/RapidfileshareNet.py create mode 100644 pyload/plugin/account/RapidgatorNet.py create mode 100644 pyload/plugin/account/RapiduNet.py create mode 100644 pyload/plugin/account/RarefileNet.py create mode 100644 pyload/plugin/account/RealdebridCom.py create mode 100644 pyload/plugin/account/RehostTo.py create mode 100644 pyload/plugin/account/RyushareCom.py create mode 100644 pyload/plugin/account/SafesharingEu.py create mode 100644 pyload/plugin/account/SecureUploadEu.py create mode 100644 pyload/plugin/account/SendmywayCom.py create mode 100644 pyload/plugin/account/ShareonlineBiz.py create mode 100644 pyload/plugin/account/SimplyPremiumCom.py create mode 100644 pyload/plugin/account/SimplydebridCom.py create mode 100644 pyload/plugin/account/SmoozedCom.py create mode 100644 pyload/plugin/account/StahnuTo.py create mode 100644 pyload/plugin/account/StreamcloudEu.py create mode 100644 pyload/plugin/account/TurbobitNet.py create mode 100644 pyload/plugin/account/TusfilesNet.py create mode 100644 pyload/plugin/account/UlozTo.py create mode 100644 pyload/plugin/account/UnrestrictLi.py create mode 100644 pyload/plugin/account/UploadableCh.py create mode 100644 pyload/plugin/account/UploadcCom.py create mode 100644 pyload/plugin/account/UploadedTo.py create mode 100644 pyload/plugin/account/UploadheroCom.py create mode 100644 pyload/plugin/account/UploadingCom.py create mode 100644 pyload/plugin/account/UptoboxCom.py create mode 100644 pyload/plugin/account/VidPlayNet.py create mode 100644 pyload/plugin/account/WebshareCz.py create mode 100644 pyload/plugin/account/XFileSharingPro.py create mode 100644 pyload/plugin/account/YibaishiwuCom.py create mode 100644 pyload/plugin/account/ZeveraCom.py create mode 100644 pyload/plugin/account/__init__.py create mode 100644 pyload/plugin/addon/AndroidPhoneNotify.py create mode 100644 pyload/plugin/addon/Checksum.py create mode 100644 pyload/plugin/addon/ClickAndLoad.py create mode 100644 pyload/plugin/addon/DeleteFinished.py create mode 100644 pyload/plugin/addon/DownloadScheduler.py create mode 100644 pyload/plugin/addon/ExternalScripts.py create mode 100644 pyload/plugin/addon/ExtractArchive.py create mode 100644 pyload/plugin/addon/HotFolder.py create mode 100644 pyload/plugin/addon/IRCInterface.py create mode 100644 pyload/plugin/addon/JustPremium.py create mode 100644 pyload/plugin/addon/MergeFiles.py create mode 100644 pyload/plugin/addon/MultiHome.py create mode 100644 pyload/plugin/addon/RestartFailed.py create mode 100644 pyload/plugin/addon/RestartSlow.py create mode 100644 pyload/plugin/addon/SkipRev.py create mode 100644 pyload/plugin/addon/UnSkipOnFail.py create mode 100644 pyload/plugin/addon/UpdateManager.py create mode 100644 pyload/plugin/addon/WindowsPhoneNotify.py create mode 100644 pyload/plugin/addon/XMPPInterface.py create mode 100644 pyload/plugin/addon/__init__.py create mode 100644 pyload/plugin/captcha/AdYouLike.py create mode 100644 pyload/plugin/captcha/AdsCaptcha.py create mode 100644 pyload/plugin/captcha/ReCaptcha.py create mode 100644 pyload/plugin/captcha/SolveMedia.py create mode 100644 pyload/plugin/captcha/__init__.py create mode 100644 pyload/plugin/container/CCF.py create mode 100644 pyload/plugin/container/DLC.py create mode 100644 pyload/plugin/container/RSDF.py create mode 100644 pyload/plugin/container/TXT.py create mode 100644 pyload/plugin/container/__init__.py create mode 100644 pyload/plugin/crypter/BitshareCom.py create mode 100644 pyload/plugin/crypter/C1NeonCom.py create mode 100644 pyload/plugin/crypter/ChipDe.py create mode 100644 pyload/plugin/crypter/CloudzillaTo.py create mode 100644 pyload/plugin/crypter/CrockoCom.py create mode 100644 pyload/plugin/crypter/CryptItCom.py create mode 100644 pyload/plugin/crypter/CzshareCom.py create mode 100644 pyload/plugin/crypter/DDLMusicOrg.py create mode 100644 pyload/plugin/crypter/DailymotionBatch.py create mode 100644 pyload/plugin/crypter/DataHu.py create mode 100644 pyload/plugin/crypter/DdlstorageCom.py create mode 100644 pyload/plugin/crypter/DepositfilesCom.py create mode 100644 pyload/plugin/crypter/Dereferer.py create mode 100644 pyload/plugin/crypter/DevhostSt.py create mode 100644 pyload/plugin/crypter/DlProtectCom.py create mode 100644 pyload/plugin/crypter/DontKnowMe.py create mode 100644 pyload/plugin/crypter/DuckCryptInfo.py create mode 100644 pyload/plugin/crypter/DuploadOrg.py create mode 100644 pyload/plugin/crypter/EasybytezCom.py create mode 100644 pyload/plugin/crypter/EmbeduploadCom.py create mode 100644 pyload/plugin/crypter/FilebeerInfo.py create mode 100644 pyload/plugin/crypter/FilecloudIo.py create mode 100644 pyload/plugin/crypter/FilecryptCc.py create mode 100644 pyload/plugin/crypter/FilefactoryCom.py create mode 100644 pyload/plugin/crypter/FilerNet.py create mode 100644 pyload/plugin/crypter/FileserveCom.py create mode 100644 pyload/plugin/crypter/FilesonicCom.py create mode 100644 pyload/plugin/crypter/FilestubeCom.py create mode 100644 pyload/plugin/crypter/FiletramCom.py create mode 100644 pyload/plugin/crypter/FiredriveCom.py create mode 100644 pyload/plugin/crypter/FourChanOrg.py create mode 100644 pyload/plugin/crypter/FreakhareCom.py create mode 100644 pyload/plugin/crypter/FreetexthostCom.py create mode 100644 pyload/plugin/crypter/FshareVn.py create mode 100644 pyload/plugin/crypter/Go4UpCom.py create mode 100644 pyload/plugin/crypter/GooGl.py create mode 100644 pyload/plugin/crypter/HoerbuchIn.py create mode 100644 pyload/plugin/crypter/HotfileCom.py create mode 100644 pyload/plugin/crypter/ILoadTo.py create mode 100644 pyload/plugin/crypter/ImgurComAlbum.py create mode 100644 pyload/plugin/crypter/LetitbitNet.py create mode 100644 pyload/plugin/crypter/LinkCryptWs.py create mode 100644 pyload/plugin/crypter/LinkSaveIn.py create mode 100644 pyload/plugin/crypter/LinkdecrypterCom.py create mode 100644 pyload/plugin/crypter/LixIn.py create mode 100644 pyload/plugin/crypter/LofCc.py create mode 100644 pyload/plugin/crypter/MBLinkInfo.py create mode 100644 pyload/plugin/crypter/MediafireCom.py create mode 100644 pyload/plugin/crypter/MegaCoNz.py create mode 100644 pyload/plugin/crypter/MegaRapidCz.py create mode 100644 pyload/plugin/crypter/MegauploadCom.py create mode 100644 pyload/plugin/crypter/Movie2KTo.py create mode 100644 pyload/plugin/crypter/MultiUpOrg.py create mode 100644 pyload/plugin/crypter/MultiloadCz.py create mode 100644 pyload/plugin/crypter/MultiuploadCom.py create mode 100644 pyload/plugin/crypter/NCryptIn.py create mode 100644 pyload/plugin/crypter/NetfolderIn.py create mode 100644 pyload/plugin/crypter/NosvideoCom.py create mode 100644 pyload/plugin/crypter/OneKhDe.py create mode 100644 pyload/plugin/crypter/OronCom.py create mode 100644 pyload/plugin/crypter/PastebinCom.py create mode 100644 pyload/plugin/crypter/QuickshareCz.py create mode 100644 pyload/plugin/crypter/RSLayerCom.py create mode 100644 pyload/plugin/crypter/RelinkUs.py create mode 100644 pyload/plugin/crypter/SafelinkingNet.py create mode 100644 pyload/plugin/crypter/SecuredIn.py create mode 100644 pyload/plugin/crypter/SexuriaCom.py create mode 100644 pyload/plugin/crypter/ShareLinksBiz.py create mode 100644 pyload/plugin/crypter/SharingmatrixCom.py create mode 100644 pyload/plugin/crypter/SpeedLoadOrg.py create mode 100644 pyload/plugin/crypter/StealthTo.py create mode 100644 pyload/plugin/crypter/TnyCz.py create mode 100644 pyload/plugin/crypter/TrailerzoneInfo.py create mode 100644 pyload/plugin/crypter/TurbobitNet.py create mode 100644 pyload/plugin/crypter/TusfilesNet.py create mode 100644 pyload/plugin/crypter/UlozTo.py create mode 100644 pyload/plugin/crypter/UploadableCh.py create mode 100644 pyload/plugin/crypter/UploadedTo.py create mode 100644 pyload/plugin/crypter/WiiReloadedOrg.py create mode 100644 pyload/plugin/crypter/WuploadCom.py create mode 100644 pyload/plugin/crypter/XFileSharingPro.py create mode 100644 pyload/plugin/crypter/XupPl.py create mode 100644 pyload/plugin/crypter/YoutubeBatch.py create mode 100644 pyload/plugin/crypter/__init__.py create mode 100644 pyload/plugin/extractor/SevenZip.py create mode 100644 pyload/plugin/extractor/UnRar.py create mode 100644 pyload/plugin/extractor/UnZip.py create mode 100644 pyload/plugin/extractor/__init__.py create mode 100644 pyload/plugin/hook/AlldebridCom.py create mode 100644 pyload/plugin/hook/BypassCaptcha.py create mode 100644 pyload/plugin/hook/Captcha9Kw.py create mode 100644 pyload/plugin/hook/CaptchaBrotherhood.py create mode 100644 pyload/plugin/hook/DeathByCaptcha.py create mode 100644 pyload/plugin/hook/DebridItaliaCom.py create mode 100644 pyload/plugin/hook/EasybytezCom.py create mode 100644 pyload/plugin/hook/ExpertDecoders.py create mode 100644 pyload/plugin/hook/FastixRu.py create mode 100644 pyload/plugin/hook/FreeWayMe.py create mode 100644 pyload/plugin/hook/ImageTyperz.py create mode 100644 pyload/plugin/hook/LinkdecrypterCom.py create mode 100644 pyload/plugin/hook/LinksnappyCom.py create mode 100644 pyload/plugin/hook/MegaDebridEu.py create mode 100644 pyload/plugin/hook/MultihostersCom.py create mode 100644 pyload/plugin/hook/MultishareCz.py create mode 100644 pyload/plugin/hook/MyfastfileCom.py create mode 100644 pyload/plugin/hook/NoPremiumPl.py create mode 100644 pyload/plugin/hook/OverLoadMe.py create mode 100644 pyload/plugin/hook/PremiumTo.py create mode 100644 pyload/plugin/hook/PremiumizeMe.py create mode 100644 pyload/plugin/hook/PutdriveCom.py create mode 100644 pyload/plugin/hook/RPNetBiz.py create mode 100644 pyload/plugin/hook/RapideoPl.py create mode 100644 pyload/plugin/hook/RealdebridCom.py create mode 100644 pyload/plugin/hook/RehostTo.py create mode 100644 pyload/plugin/hook/SimplyPremiumCom.py create mode 100644 pyload/plugin/hook/SimplydebridCom.py create mode 100644 pyload/plugin/hook/SmoozedCom.py create mode 100644 pyload/plugin/hook/UnrestrictLi.py create mode 100644 pyload/plugin/hook/XFileSharingPro.py create mode 100644 pyload/plugin/hook/ZeveraCom.py create mode 100644 pyload/plugin/hook/__init__.py create mode 100644 pyload/plugin/hoster/AlldebridCom.py create mode 100644 pyload/plugin/hoster/AndroidfilehostCom.py create mode 100644 pyload/plugin/hoster/BasketbuildCom.py create mode 100644 pyload/plugin/hoster/BayfilesCom.py create mode 100644 pyload/plugin/hoster/BezvadataCz.py create mode 100644 pyload/plugin/hoster/BillionuploadsCom.py create mode 100644 pyload/plugin/hoster/BitshareCom.py create mode 100644 pyload/plugin/hoster/BoltsharingCom.py create mode 100644 pyload/plugin/hoster/CatShareNet.py create mode 100644 pyload/plugin/hoster/CloudzerNet.py create mode 100644 pyload/plugin/hoster/CloudzillaTo.py create mode 100644 pyload/plugin/hoster/CramitIn.py create mode 100644 pyload/plugin/hoster/CrockoCom.py create mode 100644 pyload/plugin/hoster/CyberlockerCh.py create mode 100644 pyload/plugin/hoster/CzshareCom.py create mode 100644 pyload/plugin/hoster/DailymotionCom.py create mode 100644 pyload/plugin/hoster/DataHu.py create mode 100644 pyload/plugin/hoster/DataportCz.py create mode 100644 pyload/plugin/hoster/DateiTo.py create mode 100644 pyload/plugin/hoster/DdlstorageCom.py create mode 100644 pyload/plugin/hoster/DebridItaliaCom.py create mode 100644 pyload/plugin/hoster/DepositfilesCom.py create mode 100644 pyload/plugin/hoster/DevhostSt.py create mode 100644 pyload/plugin/hoster/DlFreeFr.py create mode 100644 pyload/plugin/hoster/DodanePl.py create mode 100644 pyload/plugin/hoster/DuploadOrg.py create mode 100644 pyload/plugin/hoster/EasybytezCom.py create mode 100644 pyload/plugin/hoster/EdiskCz.py create mode 100644 pyload/plugin/hoster/EgoFilesCom.py create mode 100644 pyload/plugin/hoster/EnteruploadCom.py create mode 100644 pyload/plugin/hoster/EpicShareNet.py create mode 100644 pyload/plugin/hoster/EuroshareEu.py create mode 100644 pyload/plugin/hoster/ExashareCom.py create mode 100644 pyload/plugin/hoster/ExtabitCom.py create mode 100644 pyload/plugin/hoster/FastixRu.py create mode 100644 pyload/plugin/hoster/FastshareCz.py create mode 100644 pyload/plugin/hoster/FileApeCom.py create mode 100644 pyload/plugin/hoster/FileSharkPl.py create mode 100644 pyload/plugin/hoster/FileStoreTo.py create mode 100644 pyload/plugin/hoster/FilebeerInfo.py create mode 100644 pyload/plugin/hoster/FilecloudIo.py create mode 100644 pyload/plugin/hoster/FilefactoryCom.py create mode 100644 pyload/plugin/hoster/FilejungleCom.py create mode 100644 pyload/plugin/hoster/FileomCom.py create mode 100644 pyload/plugin/hoster/FilepostCom.py create mode 100644 pyload/plugin/hoster/FilepupNet.py create mode 100644 pyload/plugin/hoster/FilerNet.py create mode 100644 pyload/plugin/hoster/FilerioCom.py create mode 100644 pyload/plugin/hoster/FilesMailRu.py create mode 100644 pyload/plugin/hoster/FileserveCom.py create mode 100644 pyload/plugin/hoster/FileshareInUa.py create mode 100644 pyload/plugin/hoster/FilesonicCom.py create mode 100644 pyload/plugin/hoster/FilezyNet.py create mode 100644 pyload/plugin/hoster/FiredriveCom.py create mode 100644 pyload/plugin/hoster/FlyFilesNet.py create mode 100644 pyload/plugin/hoster/FourSharedCom.py create mode 100644 pyload/plugin/hoster/FreakshareCom.py create mode 100644 pyload/plugin/hoster/FreeWayMe.py create mode 100644 pyload/plugin/hoster/FreevideoCz.py create mode 100644 pyload/plugin/hoster/FshareVn.py create mode 100644 pyload/plugin/hoster/Ftp.py create mode 100644 pyload/plugin/hoster/GamefrontCom.py create mode 100644 pyload/plugin/hoster/GigapetaCom.py create mode 100644 pyload/plugin/hoster/GooIm.py create mode 100644 pyload/plugin/hoster/GoogledriveCom.py create mode 100644 pyload/plugin/hoster/HellshareCz.py create mode 100644 pyload/plugin/hoster/HellspyCz.py create mode 100644 pyload/plugin/hoster/HotfileCom.py create mode 100644 pyload/plugin/hoster/HugefilesNet.py create mode 100644 pyload/plugin/hoster/HundredEightyUploadCom.py create mode 100644 pyload/plugin/hoster/IFileWs.py create mode 100644 pyload/plugin/hoster/IcyFilesCom.py create mode 100644 pyload/plugin/hoster/IfileIt.py create mode 100644 pyload/plugin/hoster/IfolderRu.py create mode 100644 pyload/plugin/hoster/JumbofilesCom.py create mode 100644 pyload/plugin/hoster/JunocloudMe.py create mode 100644 pyload/plugin/hoster/Keep2ShareCc.py create mode 100644 pyload/plugin/hoster/KickloadCom.py create mode 100644 pyload/plugin/hoster/KingfilesNet.py create mode 100644 pyload/plugin/hoster/LemUploadsCom.py create mode 100644 pyload/plugin/hoster/LetitbitNet.py create mode 100644 pyload/plugin/hoster/LinksnappyCom.py create mode 100644 pyload/plugin/hoster/LoadTo.py create mode 100644 pyload/plugin/hoster/LomafileCom.py create mode 100644 pyload/plugin/hoster/LuckyShareNet.py create mode 100644 pyload/plugin/hoster/MediafireCom.py create mode 100644 pyload/plugin/hoster/MegaCoNz.py create mode 100644 pyload/plugin/hoster/MegaDebridEu.py create mode 100644 pyload/plugin/hoster/MegaFilesSe.py create mode 100644 pyload/plugin/hoster/MegaRapidCz.py create mode 100644 pyload/plugin/hoster/MegacrypterCom.py create mode 100644 pyload/plugin/hoster/MegareleaseOrg.py create mode 100644 pyload/plugin/hoster/MegasharesCom.py create mode 100644 pyload/plugin/hoster/MegauploadCom.py create mode 100644 pyload/plugin/hoster/MegavideoCom.py create mode 100644 pyload/plugin/hoster/MovReelCom.py create mode 100644 pyload/plugin/hoster/MultihostersCom.py create mode 100644 pyload/plugin/hoster/MultishareCz.py create mode 100644 pyload/plugin/hoster/MyfastfileCom.py create mode 100644 pyload/plugin/hoster/MystoreTo.py create mode 100644 pyload/plugin/hoster/MyvideoDe.py create mode 100644 pyload/plugin/hoster/NahrajCz.py create mode 100644 pyload/plugin/hoster/NarodRu.py create mode 100644 pyload/plugin/hoster/NetloadIn.py create mode 100644 pyload/plugin/hoster/NitroflareCom.py create mode 100644 pyload/plugin/hoster/NoPremiumPl.py create mode 100644 pyload/plugin/hoster/NosuploadCom.py create mode 100644 pyload/plugin/hoster/NovafileCom.py create mode 100644 pyload/plugin/hoster/NowDownloadSx.py create mode 100644 pyload/plugin/hoster/NowVideoSx.py create mode 100644 pyload/plugin/hoster/OboomCom.py create mode 100644 pyload/plugin/hoster/OneFichierCom.py create mode 100644 pyload/plugin/hoster/OronCom.py create mode 100644 pyload/plugin/hoster/OverLoadMe.py create mode 100644 pyload/plugin/hoster/PandaplaNet.py create mode 100644 pyload/plugin/hoster/PornhostCom.py create mode 100644 pyload/plugin/hoster/PornhubCom.py create mode 100644 pyload/plugin/hoster/PotloadCom.py create mode 100644 pyload/plugin/hoster/PremiumTo.py create mode 100644 pyload/plugin/hoster/PremiumizeMe.py create mode 100644 pyload/plugin/hoster/PromptfileCom.py create mode 100644 pyload/plugin/hoster/PrzeklejPl.py create mode 100644 pyload/plugin/hoster/PutdriveCom.py create mode 100644 pyload/plugin/hoster/QuickshareCz.py create mode 100644 pyload/plugin/hoster/RPNetBiz.py create mode 100644 pyload/plugin/hoster/RapideoPl.py create mode 100644 pyload/plugin/hoster/RapidfileshareNet.py create mode 100644 pyload/plugin/hoster/RapidgatorNet.py create mode 100644 pyload/plugin/hoster/RapiduNet.py create mode 100644 pyload/plugin/hoster/RarefileNet.py create mode 100644 pyload/plugin/hoster/RealdebridCom.py create mode 100644 pyload/plugin/hoster/RedtubeCom.py create mode 100644 pyload/plugin/hoster/RehostTo.py create mode 100644 pyload/plugin/hoster/RemixshareCom.py create mode 100644 pyload/plugin/hoster/RgHostNet.py create mode 100644 pyload/plugin/hoster/SafesharingEu.py create mode 100644 pyload/plugin/hoster/SecureUploadEu.py create mode 100644 pyload/plugin/hoster/SendspaceCom.py create mode 100644 pyload/plugin/hoster/Share4WebCom.py create mode 100644 pyload/plugin/hoster/Share76Com.py create mode 100644 pyload/plugin/hoster/ShareFilesCo.py create mode 100644 pyload/plugin/hoster/SharebeesCom.py create mode 100644 pyload/plugin/hoster/ShareonlineBiz.py create mode 100644 pyload/plugin/hoster/ShareplaceCom.py create mode 100644 pyload/plugin/hoster/SharingmatrixCom.py create mode 100644 pyload/plugin/hoster/ShragleCom.py create mode 100644 pyload/plugin/hoster/SimplyPremiumCom.py create mode 100644 pyload/plugin/hoster/SimplydebridCom.py create mode 100644 pyload/plugin/hoster/SmoozedCom.py create mode 100644 pyload/plugin/hoster/SockshareCom.py create mode 100644 pyload/plugin/hoster/SoundcloudCom.py create mode 100644 pyload/plugin/hoster/SpeedLoadOrg.py create mode 100644 pyload/plugin/hoster/SpeedfileCz.py create mode 100644 pyload/plugin/hoster/SpeedyshareCom.py create mode 100644 pyload/plugin/hoster/StorageTo.py create mode 100644 pyload/plugin/hoster/StreamCz.py create mode 100644 pyload/plugin/hoster/StreamcloudEu.py create mode 100644 pyload/plugin/hoster/TurbobitNet.py create mode 100644 pyload/plugin/hoster/TurbouploadCom.py create mode 100644 pyload/plugin/hoster/TusfilesNet.py create mode 100644 pyload/plugin/hoster/TwoSharedCom.py create mode 100644 pyload/plugin/hoster/UlozTo.py create mode 100644 pyload/plugin/hoster/UloziskoSk.py create mode 100644 pyload/plugin/hoster/UnibytesCom.py create mode 100644 pyload/plugin/hoster/UnrestrictLi.py create mode 100644 pyload/plugin/hoster/UpleaCom.py create mode 100644 pyload/plugin/hoster/UploadStationCom.py create mode 100644 pyload/plugin/hoster/UploadableCh.py create mode 100644 pyload/plugin/hoster/UploadboxCom.py create mode 100644 pyload/plugin/hoster/UploadedTo.py create mode 100644 pyload/plugin/hoster/UploadhereCom.py create mode 100644 pyload/plugin/hoster/UploadheroCom.py create mode 100644 pyload/plugin/hoster/UploadingCom.py create mode 100644 pyload/plugin/hoster/UploadkingCom.py create mode 100644 pyload/plugin/hoster/UpstoreNet.py create mode 100644 pyload/plugin/hoster/UptoboxCom.py create mode 100644 pyload/plugin/hoster/VeehdCom.py create mode 100644 pyload/plugin/hoster/VeohCom.py create mode 100644 pyload/plugin/hoster/VidPlayNet.py create mode 100644 pyload/plugin/hoster/VimeoCom.py create mode 100644 pyload/plugin/hoster/Vipleech4UCom.py create mode 100644 pyload/plugin/hoster/WarserverCz.py create mode 100644 pyload/plugin/hoster/WebshareCz.py create mode 100644 pyload/plugin/hoster/WrzucTo.py create mode 100644 pyload/plugin/hoster/WuploadCom.py create mode 100644 pyload/plugin/hoster/X7To.py create mode 100644 pyload/plugin/hoster/XFileSharingPro.py create mode 100644 pyload/plugin/hoster/XHamsterCom.py create mode 100644 pyload/plugin/hoster/XVideosCom.py create mode 100644 pyload/plugin/hoster/XdadevelopersCom.py create mode 100644 pyload/plugin/hoster/Xdcc.py create mode 100644 pyload/plugin/hoster/YibaishiwuCom.py create mode 100644 pyload/plugin/hoster/YoupornCom.py create mode 100644 pyload/plugin/hoster/YourfilesTo.py create mode 100644 pyload/plugin/hoster/YoutubeCom.py create mode 100644 pyload/plugin/hoster/ZDF.py create mode 100644 pyload/plugin/hoster/ZShareNet.py create mode 100644 pyload/plugin/hoster/ZeveraCom.py create mode 100644 pyload/plugin/hoster/ZippyshareCom.py create mode 100644 pyload/plugin/hoster/__init__.py create mode 100644 pyload/plugin/internal/BasePlugin.py create mode 100644 pyload/plugin/internal/DeadCrypter.py create mode 100644 pyload/plugin/internal/DeadHoster.py create mode 100644 pyload/plugin/internal/MultiHook.py create mode 100644 pyload/plugin/internal/MultiHoster.py create mode 100644 pyload/plugin/internal/SimpleCrypter.py create mode 100644 pyload/plugin/internal/SimpleDereferer.py create mode 100644 pyload/plugin/internal/SimpleHoster.py create mode 100644 pyload/plugin/internal/UpdateManager.py create mode 100644 pyload/plugin/internal/XFSAccount.py create mode 100644 pyload/plugin/internal/XFSCrypter.py create mode 100644 pyload/plugin/internal/XFSHoster.py create mode 100644 pyload/plugin/internal/__init__.py create mode 100644 pyload/plugin/ocr/GigasizeCom.py create mode 100644 pyload/plugin/ocr/LinksaveIn.py create mode 100644 pyload/plugin/ocr/NetloadIn.py create mode 100644 pyload/plugin/ocr/ShareonlineBiz.py create mode 100644 pyload/plugin/ocr/__init__.py create mode 100644 pyload/remote/ClickAndLoadBackend.py create mode 100644 pyload/remote/SocketBackend.py create mode 100644 pyload/remote/ThriftBackend.py create mode 100644 pyload/remote/__init__.py create mode 100644 pyload/remote/socketbackend/__init__.py create mode 100644 pyload/remote/socketbackend/create_ttypes.py create mode 100644 pyload/remote/thriftbackend/Processor.py create mode 100644 pyload/remote/thriftbackend/Protocol.py create mode 100644 pyload/remote/thriftbackend/Socket.py create mode 100644 pyload/remote/thriftbackend/ThriftClient.py create mode 100644 pyload/remote/thriftbackend/ThriftTest.py create mode 100644 pyload/remote/thriftbackend/Transport.py create mode 100644 pyload/remote/thriftbackend/__init__.py create mode 100644 pyload/remote/thriftbackend/pyload.thrift create mode 100644 pyload/remote/thriftbackend/thriftgen/__init__.py create mode 100644 pyload/remote/thriftbackend/thriftgen/pyload/Pyload-remote create mode 100644 pyload/remote/thriftbackend/thriftgen/pyload/Pyload.py create mode 100644 pyload/remote/thriftbackend/thriftgen/pyload/__init__.py create mode 100644 pyload/remote/thriftbackend/thriftgen/pyload/constants.py create mode 100644 pyload/remote/thriftbackend/thriftgen/pyload/ttypes.py create mode 100644 pyload/utils/__init__.py create mode 100644 pyload/utils/packagetools.py create mode 100644 pyload/utils/printer.py create mode 100644 pyload/utils/pylgettext.py create mode 100644 pyload/webui/__init__.py create mode 100644 pyload/webui/app/__init__.py create mode 100644 pyload/webui/app/api.py create mode 100644 pyload/webui/app/cnl.py create mode 100644 pyload/webui/app/json.py create mode 100644 pyload/webui/app/pyload.py create mode 100644 pyload/webui/app/utils.py create mode 100644 pyload/webui/filters.py create mode 100644 pyload/webui/middlewares.py create mode 100644 pyload/webui/servers/lighttpd_default.conf create mode 100644 pyload/webui/servers/nginx_default.conf create mode 100644 pyload/webui/themes/Dark/css/MooDialog.css create mode 100644 pyload/webui/themes/Dark/css/base.css create mode 100644 pyload/webui/themes/Dark/css/log.css create mode 100644 pyload/webui/themes/Dark/css/pathchooser.css create mode 100644 pyload/webui/themes/Dark/css/window.css create mode 100644 pyload/webui/themes/Dark/img/MooDialog/dialog-close.png create mode 100644 pyload/webui/themes/Dark/img/MooDialog/dialog-error.png create mode 100644 pyload/webui/themes/Dark/img/MooDialog/dialog-question.png create mode 100644 pyload/webui/themes/Dark/img/MooDialog/dialog-warning.png create mode 100644 pyload/webui/themes/Dark/img/button.png create mode 100644 pyload/webui/themes/Dark/img/dark-bg.jpg create mode 100644 pyload/webui/themes/Dark/img/default/add_folder.png create mode 100644 pyload/webui/themes/Dark/img/default/ajax-loader.gif create mode 100644 pyload/webui/themes/Dark/img/default/arrow_refresh.png create mode 100644 pyload/webui/themes/Dark/img/default/arrow_right.png create mode 100644 pyload/webui/themes/Dark/img/default/big_button.gif create mode 100644 pyload/webui/themes/Dark/img/default/big_button_over.gif create mode 100644 pyload/webui/themes/Dark/img/default/body.png create mode 100644 pyload/webui/themes/Dark/img/default/closebtn.gif create mode 100644 pyload/webui/themes/Dark/img/default/cog.png create mode 100644 pyload/webui/themes/Dark/img/default/control_add.png create mode 100644 pyload/webui/themes/Dark/img/default/control_add_blue.png create mode 100644 pyload/webui/themes/Dark/img/default/control_cancel.png create mode 100644 pyload/webui/themes/Dark/img/default/control_cancel_blue.png create mode 100644 pyload/webui/themes/Dark/img/default/control_pause.png create mode 100644 pyload/webui/themes/Dark/img/default/control_pause_blue.png create mode 100644 pyload/webui/themes/Dark/img/default/control_play.png create mode 100644 pyload/webui/themes/Dark/img/default/control_play_blue.png create mode 100644 pyload/webui/themes/Dark/img/default/control_stop.png create mode 100644 pyload/webui/themes/Dark/img/default/control_stop_blue.png create mode 100644 pyload/webui/themes/Dark/img/default/delete.png create mode 100644 pyload/webui/themes/Dark/img/default/drag_corner.gif create mode 100644 pyload/webui/themes/Dark/img/default/error.png create mode 100644 pyload/webui/themes/Dark/img/default/folder.png create mode 100644 pyload/webui/themes/Dark/img/default/full.png create mode 100644 pyload/webui/themes/Dark/img/default/head-login.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-collector.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-config.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-development.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-download.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-home.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-index.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-news.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-queue.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-recent.png create mode 100644 pyload/webui/themes/Dark/img/default/head-menu-wiki.png create mode 100644 pyload/webui/themes/Dark/img/default/head-search-noshadow.png create mode 100644 pyload/webui/themes/Dark/img/default/head_bg1.png create mode 100644 pyload/webui/themes/Dark/img/default/images.png create mode 100644 pyload/webui/themes/Dark/img/default/notice.png create mode 100644 pyload/webui/themes/Dark/img/default/package_go.png create mode 100644 pyload/webui/themes/Dark/img/default/page-tools-backlinks.png create mode 100644 pyload/webui/themes/Dark/img/default/page-tools-edit.png create mode 100644 pyload/webui/themes/Dark/img/default/page-tools-revisions.png create mode 100644 pyload/webui/themes/Dark/img/default/parseUri.png create mode 100644 pyload/webui/themes/Dark/img/default/pencil.png create mode 100644 pyload/webui/themes/Dark/img/default/reconnect.png create mode 100644 pyload/webui/themes/Dark/img/default/status_None.png create mode 100644 pyload/webui/themes/Dark/img/default/status_downloading.png create mode 100644 pyload/webui/themes/Dark/img/default/status_failed.png create mode 100644 pyload/webui/themes/Dark/img/default/status_finished.png create mode 100644 pyload/webui/themes/Dark/img/default/status_offline.png create mode 100644 pyload/webui/themes/Dark/img/default/status_proc.png create mode 100644 pyload/webui/themes/Dark/img/default/status_queue.png create mode 100644 pyload/webui/themes/Dark/img/default/status_waiting.png create mode 100644 pyload/webui/themes/Dark/img/default/success.png create mode 100644 pyload/webui/themes/Dark/img/default/tabs-border-bottom.png create mode 100644 pyload/webui/themes/Dark/img/default/user-actions-logout.png create mode 100644 pyload/webui/themes/Dark/img/default/user-actions-profile.png create mode 100644 pyload/webui/themes/Dark/img/default/user-info.png create mode 100644 pyload/webui/themes/Dark/img/pyload-logo.png create mode 100644 pyload/webui/themes/Dark/img/tab-background.png create mode 100644 pyload/webui/themes/Dark/js/render/admin.coffee create mode 100644 pyload/webui/themes/Dark/js/render/admin.min.js create mode 100644 pyload/webui/themes/Dark/js/render/base.coffee create mode 100644 pyload/webui/themes/Dark/js/render/base.min.js create mode 100644 pyload/webui/themes/Dark/js/render/package.js create mode 100644 pyload/webui/themes/Dark/js/render/settings.coffee create mode 100644 pyload/webui/themes/Dark/js/render/settings.min.js create mode 100644 pyload/webui/themes/Dark/js/static/MooDialog.js create mode 100644 pyload/webui/themes/Dark/js/static/MooDialog.min.js create mode 100644 pyload/webui/themes/Dark/js/static/MooDropMenu.js create mode 100644 pyload/webui/themes/Dark/js/static/MooDropMenu.min.js create mode 100644 pyload/webui/themes/Dark/js/static/mootools-core.js create mode 100644 pyload/webui/themes/Dark/js/static/mootools-core.min.js create mode 100644 pyload/webui/themes/Dark/js/static/mootools-more.js create mode 100644 pyload/webui/themes/Dark/js/static/mootools-more.min.js create mode 100644 pyload/webui/themes/Dark/js/static/purr.js create mode 100644 pyload/webui/themes/Dark/js/static/purr.min.js create mode 100644 pyload/webui/themes/Dark/js/static/tinytab.js create mode 100644 pyload/webui/themes/Dark/js/static/tinytab.min.js create mode 100644 pyload/webui/themes/Dark/tml/admin.html create mode 100644 pyload/webui/themes/Dark/tml/base.html create mode 100644 pyload/webui/themes/Dark/tml/captcha.html create mode 100644 pyload/webui/themes/Dark/tml/downloads.html create mode 100644 pyload/webui/themes/Dark/tml/folder.html create mode 100644 pyload/webui/themes/Dark/tml/home.html create mode 100644 pyload/webui/themes/Dark/tml/info.html create mode 100644 pyload/webui/themes/Dark/tml/login.html create mode 100644 pyload/webui/themes/Dark/tml/logout.html create mode 100644 pyload/webui/themes/Dark/tml/logs.html create mode 100644 pyload/webui/themes/Dark/tml/pathchooser.html create mode 100644 pyload/webui/themes/Dark/tml/queue.html create mode 100644 pyload/webui/themes/Dark/tml/settings.html create mode 100644 pyload/webui/themes/Dark/tml/settings_item.html create mode 100644 pyload/webui/themes/Dark/tml/window.html create mode 100644 pyload/webui/themes/Default/css/MooDialog.css create mode 100644 pyload/webui/themes/Default/css/base.css create mode 100644 pyload/webui/themes/Default/css/log.css create mode 100644 pyload/webui/themes/Default/css/pathchooser.css create mode 100644 pyload/webui/themes/Default/css/window.css create mode 100644 pyload/webui/themes/Default/img/MooDialog/dialog-close.png create mode 100644 pyload/webui/themes/Default/img/MooDialog/dialog-error.png create mode 100644 pyload/webui/themes/Default/img/MooDialog/dialog-question.png create mode 100644 pyload/webui/themes/Default/img/MooDialog/dialog-warning.png create mode 100644 pyload/webui/themes/Default/img/add_folder.png create mode 100644 pyload/webui/themes/Default/img/ajax-loader.gif create mode 100644 pyload/webui/themes/Default/img/arrow_refresh.png create mode 100644 pyload/webui/themes/Default/img/arrow_right.png create mode 100644 pyload/webui/themes/Default/img/big_button.gif create mode 100644 pyload/webui/themes/Default/img/big_button_over.gif create mode 100644 pyload/webui/themes/Default/img/body.png create mode 100644 pyload/webui/themes/Default/img/button.png create mode 100644 pyload/webui/themes/Default/img/closebtn.gif create mode 100644 pyload/webui/themes/Default/img/cog.png create mode 100644 pyload/webui/themes/Default/img/control_add.png create mode 100644 pyload/webui/themes/Default/img/control_add_blue.png create mode 100644 pyload/webui/themes/Default/img/control_cancel.png create mode 100644 pyload/webui/themes/Default/img/control_cancel_blue.png create mode 100644 pyload/webui/themes/Default/img/control_pause.png create mode 100644 pyload/webui/themes/Default/img/control_pause_blue.png create mode 100644 pyload/webui/themes/Default/img/control_play.png create mode 100644 pyload/webui/themes/Default/img/control_play_blue.png create mode 100644 pyload/webui/themes/Default/img/control_stop.png create mode 100644 pyload/webui/themes/Default/img/control_stop_blue.png create mode 100644 pyload/webui/themes/Default/img/delete.png create mode 100644 pyload/webui/themes/Default/img/drag_corner.gif create mode 100644 pyload/webui/themes/Default/img/error.png create mode 100644 pyload/webui/themes/Default/img/folder.png create mode 100644 pyload/webui/themes/Default/img/full.png create mode 100644 pyload/webui/themes/Default/img/head-login.png create mode 100644 pyload/webui/themes/Default/img/head-menu-collector.png create mode 100644 pyload/webui/themes/Default/img/head-menu-config.png create mode 100644 pyload/webui/themes/Default/img/head-menu-development.png create mode 100644 pyload/webui/themes/Default/img/head-menu-download.png create mode 100644 pyload/webui/themes/Default/img/head-menu-home.png create mode 100644 pyload/webui/themes/Default/img/head-menu-index.png create mode 100644 pyload/webui/themes/Default/img/head-menu-news.png create mode 100644 pyload/webui/themes/Default/img/head-menu-queue.png create mode 100644 pyload/webui/themes/Default/img/head-menu-recent.png create mode 100644 pyload/webui/themes/Default/img/head-menu-wiki.png create mode 100644 pyload/webui/themes/Default/img/head-search-noshadow.png create mode 100644 pyload/webui/themes/Default/img/head_bg1.png create mode 100644 pyload/webui/themes/Default/img/images.png create mode 100644 pyload/webui/themes/Default/img/notice.png create mode 100644 pyload/webui/themes/Default/img/package_go.png create mode 100644 pyload/webui/themes/Default/img/page-tools-backlinks.png create mode 100644 pyload/webui/themes/Default/img/page-tools-edit.png create mode 100644 pyload/webui/themes/Default/img/page-tools-revisions.png create mode 100644 pyload/webui/themes/Default/img/parseUri.png create mode 100644 pyload/webui/themes/Default/img/pencil.png create mode 100644 pyload/webui/themes/Default/img/pyload-logo.png create mode 100644 pyload/webui/themes/Default/img/reconnect.png create mode 100644 pyload/webui/themes/Default/img/status_None.png create mode 100644 pyload/webui/themes/Default/img/status_downloading.png create mode 100644 pyload/webui/themes/Default/img/status_failed.png create mode 100644 pyload/webui/themes/Default/img/status_finished.png create mode 100644 pyload/webui/themes/Default/img/status_offline.png create mode 100644 pyload/webui/themes/Default/img/status_proc.png create mode 100644 pyload/webui/themes/Default/img/status_queue.png create mode 100644 pyload/webui/themes/Default/img/status_waiting.png create mode 100644 pyload/webui/themes/Default/img/success.png create mode 100644 pyload/webui/themes/Default/img/tab-background.png create mode 100644 pyload/webui/themes/Default/img/tabs-border-bottom.png create mode 100644 pyload/webui/themes/Default/img/user-actions-logout.png create mode 100644 pyload/webui/themes/Default/img/user-actions-profile.png create mode 100644 pyload/webui/themes/Default/img/user-info.png create mode 100644 pyload/webui/themes/Default/js/render/admin.coffee create mode 100644 pyload/webui/themes/Default/js/render/admin.min.js create mode 100644 pyload/webui/themes/Default/js/render/base.coffee create mode 100644 pyload/webui/themes/Default/js/render/base.min.js create mode 100644 pyload/webui/themes/Default/js/render/package.js create mode 100644 pyload/webui/themes/Default/js/render/settings.coffee create mode 100644 pyload/webui/themes/Default/js/render/settings.min.js create mode 100644 pyload/webui/themes/Default/js/static/MooDialog.js create mode 100644 pyload/webui/themes/Default/js/static/MooDialog.min.js create mode 100644 pyload/webui/themes/Default/js/static/MooDropMenu.js create mode 100644 pyload/webui/themes/Default/js/static/MooDropMenu.min.js create mode 100644 pyload/webui/themes/Default/js/static/mootools-core.js create mode 100644 pyload/webui/themes/Default/js/static/mootools-core.min.js create mode 100644 pyload/webui/themes/Default/js/static/mootools-more.js create mode 100644 pyload/webui/themes/Default/js/static/mootools-more.min.js create mode 100644 pyload/webui/themes/Default/js/static/purr.js create mode 100644 pyload/webui/themes/Default/js/static/purr.min.js create mode 100644 pyload/webui/themes/Default/js/static/tinytab.js create mode 100644 pyload/webui/themes/Default/js/static/tinytab.min.js create mode 100644 pyload/webui/themes/Default/tml/admin.html create mode 100644 pyload/webui/themes/Default/tml/base.html create mode 100644 pyload/webui/themes/Default/tml/captcha.html create mode 100644 pyload/webui/themes/Default/tml/downloads.html create mode 100644 pyload/webui/themes/Default/tml/filemanager.html create mode 100644 pyload/webui/themes/Default/tml/folder.html create mode 100644 pyload/webui/themes/Default/tml/home.html create mode 100644 pyload/webui/themes/Default/tml/info.html create mode 100644 pyload/webui/themes/Default/tml/login.html create mode 100644 pyload/webui/themes/Default/tml/logout.html create mode 100644 pyload/webui/themes/Default/tml/logs.html create mode 100644 pyload/webui/themes/Default/tml/pathchooser.html create mode 100644 pyload/webui/themes/Default/tml/queue.html create mode 100644 pyload/webui/themes/Default/tml/settings.html create mode 100644 pyload/webui/themes/Default/tml/settings_item.html create mode 100644 pyload/webui/themes/Default/tml/window.html create mode 100644 pyload/webui/themes/Flat/css/MooDialog.css create mode 100644 pyload/webui/themes/Flat/css/base.css create mode 100644 pyload/webui/themes/Flat/css/log.css create mode 100644 pyload/webui/themes/Flat/css/pathchooser.css create mode 100644 pyload/webui/themes/Flat/css/window.css create mode 100644 pyload/webui/themes/Flat/img/MooDialog/dialog-close.png create mode 100644 pyload/webui/themes/Flat/img/MooDialog/dialog-error.png create mode 100644 pyload/webui/themes/Flat/img/MooDialog/dialog-question.png create mode 100644 pyload/webui/themes/Flat/img/MooDialog/dialog-warning.png create mode 100644 pyload/webui/themes/Flat/img/arrow_refresh.png create mode 100644 pyload/webui/themes/Flat/img/arrow_right.png create mode 100644 pyload/webui/themes/Flat/img/button.png create mode 100644 pyload/webui/themes/Flat/img/cog.png create mode 100644 pyload/webui/themes/Flat/img/control_add.png create mode 100644 pyload/webui/themes/Flat/img/control_add_blue.png create mode 100644 pyload/webui/themes/Flat/img/control_cancel.png create mode 100644 pyload/webui/themes/Flat/img/control_cancel_blue.png create mode 100644 pyload/webui/themes/Flat/img/control_pause.png create mode 100644 pyload/webui/themes/Flat/img/control_pause_blue.png create mode 100644 pyload/webui/themes/Flat/img/control_play.png create mode 100644 pyload/webui/themes/Flat/img/control_play_blue.png create mode 100644 pyload/webui/themes/Flat/img/control_stop.png create mode 100644 pyload/webui/themes/Flat/img/control_stop_blue.png create mode 100644 pyload/webui/themes/Flat/img/default/add_folder.png create mode 100644 pyload/webui/themes/Flat/img/default/ajax-loader.gif create mode 100644 pyload/webui/themes/Flat/img/default/big_button.gif create mode 100644 pyload/webui/themes/Flat/img/default/big_button_over.gif create mode 100644 pyload/webui/themes/Flat/img/default/body.png create mode 100644 pyload/webui/themes/Flat/img/default/closebtn.gif create mode 100644 pyload/webui/themes/Flat/img/default/drag_corner.gif create mode 100644 pyload/webui/themes/Flat/img/default/full.png create mode 100644 pyload/webui/themes/Flat/img/default/head-menu-recent.png create mode 100644 pyload/webui/themes/Flat/img/default/head_bg1.png create mode 100644 pyload/webui/themes/Flat/img/default/images.png create mode 100644 pyload/webui/themes/Flat/img/default/parseUri.png create mode 100644 pyload/webui/themes/Flat/img/default/pyload-logo.png create mode 100644 pyload/webui/themes/Flat/img/default/tab-background.png create mode 100644 pyload/webui/themes/Flat/img/default/tabs-border-bottom.png create mode 100644 pyload/webui/themes/Flat/img/delete.png create mode 100644 pyload/webui/themes/Flat/img/error.png create mode 100644 pyload/webui/themes/Flat/img/folder.png create mode 100644 pyload/webui/themes/Flat/img/head-login.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-collector.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-config.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-development.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-download.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-home.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-index.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-news.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-queue.png create mode 100644 pyload/webui/themes/Flat/img/head-menu-wiki.png create mode 100644 pyload/webui/themes/Flat/img/head-search-noshadow.png create mode 100644 pyload/webui/themes/Flat/img/notice.png create mode 100644 pyload/webui/themes/Flat/img/package_go.png create mode 100644 pyload/webui/themes/Flat/img/page-tools-backlinks.png create mode 100644 pyload/webui/themes/Flat/img/page-tools-edit.png create mode 100644 pyload/webui/themes/Flat/img/page-tools-revisions.png create mode 100644 pyload/webui/themes/Flat/img/pencil.png create mode 100644 pyload/webui/themes/Flat/img/reconnect.png create mode 100644 pyload/webui/themes/Flat/img/status_None.png create mode 100644 pyload/webui/themes/Flat/img/status_downloading.png create mode 100644 pyload/webui/themes/Flat/img/status_failed.png create mode 100644 pyload/webui/themes/Flat/img/status_finished.png create mode 100644 pyload/webui/themes/Flat/img/status_offline.png create mode 100644 pyload/webui/themes/Flat/img/status_proc.png create mode 100644 pyload/webui/themes/Flat/img/status_queue.png create mode 100644 pyload/webui/themes/Flat/img/status_waiting.png create mode 100644 pyload/webui/themes/Flat/img/success.png create mode 100644 pyload/webui/themes/Flat/img/user-actions-logout.png create mode 100644 pyload/webui/themes/Flat/img/user-actions-profile.png create mode 100644 pyload/webui/themes/Flat/img/user-info.png create mode 100644 pyload/webui/themes/Flat/js/render/admin.coffee create mode 100644 pyload/webui/themes/Flat/js/render/admin.min.js create mode 100644 pyload/webui/themes/Flat/js/render/base.coffee create mode 100644 pyload/webui/themes/Flat/js/render/base.min.js create mode 100644 pyload/webui/themes/Flat/js/render/filemanager.js create mode 100644 pyload/webui/themes/Flat/js/render/package.js create mode 100644 pyload/webui/themes/Flat/js/render/settings.coffee create mode 100644 pyload/webui/themes/Flat/js/render/settings.min.js create mode 100644 pyload/webui/themes/Flat/js/static/MooDialog.js create mode 100644 pyload/webui/themes/Flat/js/static/MooDialog.min.js create mode 100644 pyload/webui/themes/Flat/js/static/MooDropMenu.js create mode 100644 pyload/webui/themes/Flat/js/static/MooDropMenu.min.js create mode 100644 pyload/webui/themes/Flat/js/static/mootools-core.js create mode 100644 pyload/webui/themes/Flat/js/static/mootools-core.min.js create mode 100644 pyload/webui/themes/Flat/js/static/mootools-more.js create mode 100644 pyload/webui/themes/Flat/js/static/mootools-more.min.js create mode 100644 pyload/webui/themes/Flat/js/static/purr.js create mode 100644 pyload/webui/themes/Flat/js/static/purr.min.js create mode 100644 pyload/webui/themes/Flat/js/static/tinytab.js create mode 100644 pyload/webui/themes/Flat/js/static/tinytab.min.js create mode 100644 pyload/webui/themes/Flat/tml/admin.html create mode 100644 pyload/webui/themes/Flat/tml/base.html create mode 100644 pyload/webui/themes/Flat/tml/captcha.html create mode 100644 pyload/webui/themes/Flat/tml/downloads.html create mode 100644 pyload/webui/themes/Flat/tml/filemanager.html create mode 100644 pyload/webui/themes/Flat/tml/folder.html create mode 100644 pyload/webui/themes/Flat/tml/home.html create mode 100644 pyload/webui/themes/Flat/tml/info.html create mode 100644 pyload/webui/themes/Flat/tml/login.html create mode 100644 pyload/webui/themes/Flat/tml/logout.html create mode 100644 pyload/webui/themes/Flat/tml/logs.html create mode 100644 pyload/webui/themes/Flat/tml/pathchooser.html create mode 100644 pyload/webui/themes/Flat/tml/queue.html create mode 100644 pyload/webui/themes/Flat/tml/settings.html create mode 100644 pyload/webui/themes/Flat/tml/settings_item.html create mode 100644 pyload/webui/themes/Flat/tml/window.html (limited to 'pyload') diff --git a/pyload/Core.py b/pyload/Core.py new file mode 100644 index 000000000..fe4ae566e --- /dev/null +++ b/pyload/Core.py @@ -0,0 +1,632 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay, sebnapi, spoob +# @version: v0.4.10 + +CURRENT_VERSION = '0.4.10' + +import __builtin__ + +from getopt import getopt, GetoptError +import pyload.utils.pylgettext as gettext +from imp import find_module +import logging +import logging.handlers +import os +from os import _exit, execl, getcwd, makedirs, remove, sep, walk, chdir, close +from os.path import exists, join +import signal +import subprocess +import sys +from sys import argv, executable, exit +from time import time, sleep +from traceback import print_exc + +from pyload.manager.Account import AccountManager +from pyload.manager.Captcha import CaptchaManager +from pyload.config.Parser import ConfigParser +from pyload.manager.Plugin import PluginManager +from pyload.manager.Event import PullManager +from pyload.network.RequestFactory import RequestFactory +from pyload.manager.thread.Server import WebServer +from pyload.manager.event.Scheduler import Scheduler +from pyload.network.JsEngine import JsEngine +from pyload import remote +from pyload.manager.Remote import RemoteManager +from pyload.database import DatabaseBackend, FileHandler + +from pyload.utils import freeSpace, formatSize, get_console_encoding + +from codecs import getwriter + +enc = get_console_encoding(sys.stdout.encoding) +sys.stdout = getwriter(enc)(sys.stdout, errors="replace") + +# TODO List +# - configurable auth system ldap/mysql +# - cron job like sheduler + +class Core(object): + """pyLoad Core, one tool to rule them all... (the filehosters) :D""" + + def __init__(self): + self.doDebug = False + self.running = False + self.daemon = False + self.remote = True + self.arg_links = [] + self.pidfile = "pyload.pid" + self.deleteLinks = False # will delete links on startup + + if len(argv) > 1: + try: + options, args = getopt(argv[1:], 'vchdusqp:', + ["version", "clear", "clean", "help", "debug", "user", + "setup", "configdir=", "changedir", "daemon", + "quit", "status", "no-remote","pidfile="]) + + for option, argument in options: + if option in ("-v", "--version"): + print "pyLoad", CURRENT_VERSION + exit() + elif option in ("-p", "--pidfile"): + self.pidfile = argument + elif option == "--daemon": + self.daemon = True + elif option in ("-c", "--clear"): + self.deleteLinks = True + elif option in ("-h", "--help"): + self.print_help() + exit() + elif option in ("-d", "--debug"): + self.doDebug = True + elif option in ("-u", "--user"): + from pyload.config.Setup import SetupAssistant as Setup + + self.config = ConfigParser() + s = Setup(self.config) + s.set_user() + exit() + elif option in ("-s", "--setup"): + from pyload.config.Setup import SetupAssistant as Setup + + self.config = ConfigParser() + s = Setup(self.config) + s.start() + exit() + elif option == "--changedir": + from pyload.config.Setup import SetupAssistant as Setup + + self.config = ConfigParser() + s = Setup(self.config) + s.conf_path(True) + exit() + elif option in ("-q", "--quit"): + self.quitInstance() + exit() + elif option == "--status": + pid = self.isAlreadyRunning() + if self.isAlreadyRunning(): + print pid + exit(0) + else: + print "false" + exit(1) + elif option == "--clean": + self.cleanTree() + exit() + elif option == "--no-remote": + self.remote = False + + except GetoptError: + print 'Unknown Argument(s) "%s"' % " ".join(argv[1:]) + self.print_help() + exit() + + def print_help(self): + print + print "pyLoad v%s 2008-2015 the pyLoad Team" % CURRENT_VERSION + print + if sys.argv[0].endswith(".py"): + print "Usage: python pyload.py [options]" + else: + print "Usage: pyload [options]" + print + print "" + print " -v, --version", " " * 10, "Print version to terminal" + print " -c, --clear", " " * 12, "Delete all saved packages/links" + #print " -a, --add=", " " * 2, "Add the specified links" + print " -u, --user", " " * 13, "Manages users" + print " -d, --debug", " " * 12, "Enable debug mode" + print " -s, --setup", " " * 12, "Run Setup Assistant" + print " --configdir=", " " * 6, "Run with as config directory" + print " -p, --pidfile=", " " * 3, "Set pidfile to " + print " --changedir", " " * 12, "Change config dir permanently" + print " --daemon", " " * 15, "Daemonmize after start" + print " --no-remote", " " * 12, "Disable remote access (saves RAM)" + print " --status", " " * 15, "Display pid if running or False" + print " --clean", " " * 16, "Remove .pyc/.pyo files" + print " -q, --quit", " " * 13, "Quit running pyLoad instance" + print " -h, --help", " " * 13, "Display this help screen" + print + + def toggle_pause(self): + if self.threadManager.pause: + self.threadManager.pause = False + return False + elif not self.threadManager.pause: + self.threadManager.pause = True + return True + + def quit(self, a, b): + self.shutdown() + self.log.info(_("Received Quit signal")) + _exit(1) + + def writePidFile(self): + self.deletePidFile() + pid = os.getpid() + f = open(self.pidfile, "wb") + f.write(str(pid)) + f.close() + + def deletePidFile(self): + if self.checkPidFile(): + self.log.debug("Deleting old pidfile %s" % self.pidfile) + os.remove(self.pidfile) + + def checkPidFile(self): + """ return pid as int or 0""" + if os.path.isfile(self.pidfile): + f = open(self.pidfile, "rb") + pid = f.read().strip() + f.close() + if pid: + pid = int(pid) + return pid + + return 0 + + def isAlreadyRunning(self): + pid = self.checkPidFile() + if not pid or os.name == "nt": return False + try: + os.kill(pid, 0) # 0 - default signal (does nothing) + except Exception: + return 0 + + return pid + + def quitInstance(self): + if os.name == "nt": + print "Not supported on windows." + return + + pid = self.isAlreadyRunning() + if not pid: + print "No pyLoad running." + return + + try: + os.kill(pid, 3) #SIGUIT + + t = time() + print "waiting for pyLoad to quit" + + while exists(self.pidfile) and t + 10 > time(): + sleep(0.25) + + if not exists(self.pidfile): + print "pyLoad successfully stopped" + else: + os.kill(pid, 9) #SIGKILL + print "pyLoad did not respond" + print "Kill signal was send to process with id %s" % pid + + except Exception: + print "Error quitting pyLoad" + + + def cleanTree(self): + for path, dirs, files in walk(self.path("")): + for f in files: + if not f.endswith(".pyo") and not f.endswith(".pyc"): + continue + + if "_25" in f or "_26" in f or "_27" in f: + continue + + print join(path, f) + remove(join(path, f)) + + def start(self, rpc=True, web=True): + """ starts the fun :D """ + + self.version = CURRENT_VERSION + + if not exists("pyload.conf"): + from pyload.config.Setup import SetupAssistant as Setup + + print "This is your first start, running configuration assistent now." + self.config = ConfigParser() + s = Setup(self.config) + res = False + try: + res = s.start() + except SystemExit: + pass + except KeyboardInterrupt: + print "\nSetup interrupted" + except Exception: + res = False + print_exc() + print "Setup failed" + if not res: + remove("pyload.conf") + + exit() + + try: signal.signal(signal.SIGQUIT, self.quit) + except Exception: pass + + self.config = ConfigParser() + + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("pyLoad", self.path("locale"), + languages=[self.config['general']['language'], "en"], fallback=True) + translation.install(True) + + self.debug = self.doDebug or self.config['general']['debug_mode'] + self.remote &= self.config['remote']['activated'] + + pid = self.isAlreadyRunning() + if pid: + print _("pyLoad already running with pid %s") % pid + exit() + + if os.name != "nt" and self.config["general"]["renice"]: + os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid())) + + if self.config["permission"]["change_group"]: + if os.name != "nt": + try: + from grp import getgrnam + + group = getgrnam(self.config["permission"]["group"]) + os.setgid(group[2]) + except Exception, e: + print _("Failed changing group: %s") % e + + if self.config["permission"]["change_user"]: + if os.name != "nt": + try: + from pwd import getpwnam + + user = getpwnam(self.config["permission"]["user"]) + os.setuid(user[2]) + except Exception, e: + print _("Failed changing user: %s") % e + + self.check_file(self.config['log']['log_folder'], _("folder for logs"), True) + + if self.debug: + self.init_logger(logging.DEBUG) # logging level + else: + self.init_logger(logging.INFO) # logging level + + self.do_kill = False + self.do_restart = False + self.shuttedDown = False + + self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION) + self.log.info(_("Using home directory: %s") % getcwd()) + + self.writePidFile() + + #@TODO refractor + + remote.activated = self.remote + self.log.debug("Remote activated: %s" % self.remote) + + self.check_install("Crypto", _("pycrypto to decode container files")) + #img = self.check_install("Image", _("Python Image Library (PIL) for captcha reading")) + #self.check_install("pycurl", _("pycurl to download any files"), True, True) + self.check_file("tmp", _("folder for temporary files"), True) + #tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if os.name != "nt" else True + + self.captcha = True # checks seems to fail, although tesseract is available + + self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True) + + if self.config['ssl']['activated']: + self.check_install("OpenSSL", _("OpenSSL for secure connection")) + + self.setupDB() + if self.config.oldRemoteData: + self.log.info(_("Moving old user config to DB")) + self.db.addUser(self.config.oldRemoteData["username"], self.config.oldRemoteData["password"]) + + self.log.info(_("Please check your logindata with ./pyload.py -u")) + + if self.deleteLinks: + self.log.info(_("All links removed")) + self.db.purgeLinks() + + self.requestFactory = RequestFactory(self) + __builtin__.pyreq = self.requestFactory + + self.lastClientConnected = 0 + + # later imported because they would trigger api import, and remote value not set correctly + from pyload import api + from pyload.manager.Addon import AddonManager + from pyload.manager.Thread import ThreadManager + + if api.activated != self.remote: + self.log.warning("Import error: API remote status not correct.") + + self.api = api.Api(self) + + self.scheduler = Scheduler(self) + + #hell yeah, so many important managers :D + self.pluginManager = PluginManager(self) + self.pullManager = PullManager(self) + self.accountManager = AccountManager(self) + self.threadManager = ThreadManager(self) + self.captchaManager = CaptchaManager(self) + self.addonManager = AddonManager(self) + self.remoteManager = RemoteManager(self) + + self.js = JsEngine(self) + + self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload()) + + if rpc: + self.remoteManager.startBackends() + + if web: + self.init_webserver() + + spaceLeft = freeSpace(self.config["general"]["download_folder"]) + + self.log.info(_("Free space: %s") % formatSize(spaceLeft)) + + self.config.save() #save so config files gets filled + + link_file = join(pypath, "links.txt") + + if exists(link_file): + f = open(link_file, "rb") + if f.read().strip(): + self.api.addPackage("links.txt", [link_file], 1) + f.close() + + link_file = "links.txt" + if exists(link_file): + f = open(link_file, "rb") + if f.read().strip(): + self.api.addPackage("links.txt", [link_file], 1) + f.close() + + #self.scheduler.addJob(0, self.accountManager.getAccountInfos) + self.log.info(_("Activating Accounts...")) + self.accountManager.getAccountInfos() + + self.threadManager.pause = False + self.running = True + + self.log.info(_("Activating Plugins...")) + self.addonManager.coreReady() + + self.log.info(_("pyLoad is up and running")) + + locals().clear() + + while True: + sleep(2) + if self.do_restart: + self.log.info(_("restarting pyLoad")) + self.restart() + if self.do_kill: + self.shutdown() + self.log.info(_("pyLoad quits")) + self.removeLogger() + _exit(0) #@TODO thrift blocks shutdown + + self.threadManager.work() + self.scheduler.work() + + def setupDB(self): + self.db = DatabaseBackend(self) # the backend + self.db.setup() + + self.files = FileHandler(self) + self.db.manager = self.files #ugly? + + def init_webserver(self): + if self.config['webinterface']['activated']: + self.webserver = WebServer(self) + self.webserver.start() + + def init_logger(self, level): + console = logging.StreamHandler(sys.stdout) + frm = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%d.%m.%Y %H:%M:%S") + console.setFormatter(frm) + self.log = logging.getLogger("log") # settable in config + + if self.config['log']['file_log']: + if self.config['log']['log_rotate']: + file_handler = logging.handlers.RotatingFileHandler(join(self.config['log']['log_folder'], 'log.txt'), + maxBytes=self.config['log']['log_size'] * 1024, + backupCount=int(self.config['log']['log_count']), + encoding="utf8") + else: + file_handler = logging.FileHandler(join(self.config['log']['log_folder'], 'log.txt'), encoding="utf8") + + file_handler.setFormatter(frm) + self.log.addHandler(file_handler) + + self.log.addHandler(console) #if console logging + self.log.setLevel(level) + + def removeLogger(self): + for h in list(self.log.handlers): + self.log.removeHandler(h) + h.close() + + def check_install(self, check_name, legend, python=True, essential=False): + """check wether needed tools are installed""" + try: + if python: + find_module(check_name) + else: + pipe = subprocess.PIPE + subprocess.Popen(check_name, stdout=pipe, stderr=pipe) + + return True + except Exception: + if essential: + self.log.info(_("Install %s") % legend) + exit() + + return False + + def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False): + """check wether needed files exists""" + tmp_names = [] + if not type(check_names) == list: + tmp_names.append(check_names) + else: + tmp_names.extend(check_names) + file_created = True + file_exists = True + for tmp_name in tmp_names: + if not exists(tmp_name): + file_exists = False + if empty: + try: + if folder: + tmp_name = tmp_name.replace("/", sep) + makedirs(tmp_name) + else: + open(tmp_name, "w") + except Exception: + file_created = False + else: + file_created = False + + if not file_exists and not quiet: + if file_created: + #self.log.info( _("%s created") % description ) + pass + else: + if not empty: + self.log.warning( + _("could not find %(desc)s: %(name)s") % {"desc": description, "name": tmp_name}) + else: + print _("could not create %(desc)s: %(name)s") % {"desc": description, "name": tmp_name} + if essential: + exit() + + def isClientConnected(self): + return (self.lastClientConnected + 30) > time() + + def restart(self): + self.shutdown() + chdir(owd) + # close some open fds + for i in range(3, 50): + try: + close(i) + except : + pass + + execl(executable, executable, *sys.argv) + _exit(0) + + def shutdown(self): + self.log.info(_("shutting down...")) + try: + if self.config['webinterface']['activated'] and hasattr(self, "webserver"): + self.webserver.quit() + + for thread in self.threadManager.threads: + thread.put("quit") + pyfiles = self.files.cache.values() + + for pyfile in pyfiles: + pyfile.abortDownload() + + self.addonManager.coreExiting() + + except Exception: + if self.debug: + print_exc() + self.log.info(_("error while shutting down")) + + finally: + self.files.syncSave() + self.shuttedDown = True + + self.deletePidFile() + + + def path(self, *args): + return join(pypath, *args) + + +def deamon(): + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError, e: + print >> sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror) + sys.exit(1) + + # decouple from parent environment + os.setsid() + os.umask(0) + + # do second fork + try: + pid = os.fork() + if pid > 0: + # exit from second parent, print eventual PID before + print "Daemon PID %d" % pid + sys.exit(0) + except OSError, e: + print >> sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror) + sys.exit(1) + + # Iterate through and close some file descriptors. + for fd in range(0, 3): + try: + os.close(fd) + except OSError: # ERROR, fd wasn't open to begin with (ignored) + pass + + os.open(os.devnull, os.O_RDWR) # standard input (0) + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) + + pyload_core = Core() + pyload_core.start() + + +def main(): + if "--daemon" in sys.argv: + deamon() + else: + pyload_core = Core() + try: + pyload_core.start() + except KeyboardInterrupt: + pyload_core.shutdown() + pyload_core.log.info(_("killed pyLoad from Terminal")) + pyload_core.removeLogger() + _exit(1) + +# And so it begins... +if __name__ == "__main__": + main() diff --git a/pyload/__init__.py b/pyload/__init__.py new file mode 100644 index 000000000..e29c81ad7 --- /dev/null +++ b/pyload/__init__.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import __builtin__ + +import os +import platform +import sys + +from codecs import getwriter + +from pyload.utils import get_console_encoding + + +__all__ = ["__status_code__", "__status__", "__version_info__", "__version__", "__author_name__", "__author_mail__", "__license__"] + +__version_info__ = (0, 4, 10) +__version__ = '.'.join(map(str, __version_info__)) + +__status_code__ = 4 +__status__ = {1: "Planning", + 2: "Pre-Alpha", + 3: "Alpha", + 4: "Beta", + 5: "Production/Stable", + 6: "Mature", + 7: "Inactive"}[__status_code__] #: PyPI Development Status Classifiers + +__description__ = "Fast, lightweight and full featured download manager" + +__license__ = "GNU General Public License v3" + +__website__ = "http://pyload.org" + +__authors__ = [("Marius" , "mkaay@mkaay.de" ), + ("RaNaN" , "Mast3rRaNaN@hotmail.de"), + ("Stefano" , "l.stickell@yahoo.it" ), + ("Walter Purcaro", "vuolter@gmail.com" ), + ("himbrr" , "himbrr@himbrr.ws" ), + ("sebnapi" , "" ), + ("spoob" , "spoob@gmx.de" ), + ("zoidberg10" , "zoidberg@mujmail.cz" )] + + +################################# InitHomeDir ################################# + +__builtin__.owd = os.path.abspath("") #: original working directory +__builtin__.homedir = os.path.expanduser("~") +__builtin__.rootdir = os.path.abspath(os.path.join(__file__, "..")) +__builtin__.configdir = "" +__builtin__.pypath = os.path.abspath(os.path.join(rootdir, "..")) + + +if "64" in platform.machine(): + sys.path.append(os.path.join(pypath, "lib64")) +sys.path.append(os.path.join(pypath, "lib")) + +sys.stdout = getwriter(get_console_encoding(sys.stdout.encoding))(sys.stdout, errors="replace") + +if homedir == "~" and os.name == "nt": + import ctypes + + CSIDL_APPDATA = 26 + + _SHGetFolderPath = ctypes.windll.shell32.SHGetFolderPathW + _SHGetFolderPath.argtypes = [ctypes.wintypes.HWND, + ctypes.c_int, + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.LPCWSTR] + + path_buf = ctypes.wintypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + + _SHGetFolderPath(0, CSIDL_APPDATA, 0, 0, path_buf) + + __builtin__.homedir = path_buf.value + +try: + p = os.path.join(rootdir, "config", "configdir") + + with open(p, "rb") as f: + configdir = f.read().strip() + +except IOError: + if os.name == "posix": + configdir = os.path.join(homedir, ".pyload") + else: + configdir = os.path.join(homedir, "pyload") + +try: + if not os.path.exists(configdir): + os.makedirs(configdir, 0700) + + os.chdir(configdir) + +except IOError, e: + print >> sys.stderr, "configdir init failed: %d (%s)" % (e.errno, e.strerror) + sys.exit(1) + +else: + __builtin__.configdir = configdir diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py new file mode 100644 index 000000000..387481da2 --- /dev/null +++ b/pyload/api/__init__.py @@ -0,0 +1,973 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from base64 import standard_b64encode +from os.path import join +from time import time +import re + +from urlparse import urlparse + +from pyload.datatype.File import PyFile +from pyload.utils.packagetools import parseNames +from pyload.network.RequestFactory import getURL +from pyload.remote import activated +from pyload.utils import compare_time, freeSpace, safe_filename + +if activated: + try: + from thrift.protocol import TBase + from pyload.remote.thriftbackend.thriftgen.pyload.ttypes import * + from pyload.remote.thriftbackend.thriftgen.pyload.Pyload import Iface + + BaseObject = TBase + + except ImportError: + from pyload.api.types import * + + print "Thrift not imported" + +else: + from pyload.api.types import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +permMap = {} + + +# decorator only called on init, never initialized, so has no effect on runtime +def permission(bits): + class _Dec(object): + def __new__(cls, func, *args, **kwargs): + permMap[func.__name__] = bits + return func + return _Dec + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&\[\]\|]*)", re.IGNORECASE) + + +class PERMS(object): + ALL = 0 # requires no permission, but login + ADD = 1 # can add packages + DELETE = 2 # can delete packages + STATUS = 4 # see and change server status + LIST = 16 # see queue and collector + MODIFY = 32 # moddify some attribute of downloads + DOWNLOAD = 64 # can download from webinterface + SETTINGS = 128 # can access settings + ACCOUNTS = 256 # can access accounts + LOGS = 512 # can see server logs + + +class ROLE(object): + ADMIN = 0 # admin has all permissions implicit + USER = 1 + + +def has_permission(userperms, perms): + # bytewise or perms before if needed + return perms == (userperms & perms) + + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api or via thrift backend. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usuable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via webinterface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + EXTERNAL = Iface # let the json api know which methods are external + + def __init__(self, core): + self.core = core + + def _convertPyFile(self, p): + fdata = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], + p["format_size"], p["status"], p["statusmsg"], + p["package"], p["error"], p["order"]) + return fdata + + def _convertConfigFormat(self, c): + sections = {} + for sectionName, sub in c.iteritems(): + section = ConfigSection(sectionName, sub["desc"]) + items = [] + for key, data in sub.iteritems(): + if key in ("desc", "outline"): + continue + item = ConfigItem() + item.name = key + item.description = data["desc"] + item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] + item.type = data["type"] + items.append(item) + section.items = items + sections[sectionName] = section + if "outline" in sub: + section.outline = sub["outline"] + return sections + + @permission(PERMS.SETTINGS) + def getConfigValue(self, category, option, section="core"): + """Retrieve config value. + + :param category: name of category, or plugin + :param option: config option + :param section: 'plugin' or 'core' + :return: config value as string + """ + if section == "core": + value = self.core.config[category][option] + else: + value = self.core.config.getPlugin(category, option) + return str(value) + + @permission(PERMS.SETTINGS) + def setConfigValue(self, category, option, value, section="core"): + """Set new config value. + + :param category: + :param option: + :param value: new config value + :param section: 'plugin' or 'core + """ + self.core.addonManager.dispatchEvent("config-changed", category, option, value, section) + if section == "core": + self.core.config[category][option] = value + if option in ("limit_speed", "max_speed"): # not so nice to update the limit + self.core.requestFactory.updateBucket() + elif section == "plugin": + self.core.config.setPlugin(category, option, value) + + @permission(PERMS.SETTINGS) + def getConfig(self): + """Retrieves complete config of core. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.config) + + def getConfigDict(self): + """Retrieves complete config in dict format, not for RPC. + + :return: dict + """ + return self.core.config.config + + @permission(PERMS.SETTINGS) + def getPluginConfig(self): + """Retrieves complete config for all plugins. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.plugin) + + def getPluginConfigDict(self): + """Plugin config as dict, not for RPC. + + :return: dict + """ + return self.core.config.plugin + + @permission(PERMS.STATUS) + def pauseServer(self): + """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + @permission(PERMS.STATUS) + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + @permission(PERMS.STATUS) + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + @permission(PERMS.STATUS) + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config["reconnect"]["activated"] ^= True + return self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusServer(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: + serverStatus.speed += pyfile.getSpeed() # bytes/s + return serverStatus + + @permission(PERMS.STATUS) + def freeSpace(self): + """Available free space at download directory in bytes""" + return freeSpace(self.core.config["general"]["download_folder"]) + + @permission(PERMS.ALL) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + def kill(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + @permission(PERMS.LOGS) + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = join(self.core.config['log']['log_folder'], 'log.txt') + try: + fh = open(filename, "r") + lines = fh.readlines() + fh.close() + if offset >= len(lines): + return [] + return lines[offset:] + except Exception: + return ['No log available'] + + @permission(PERMS.STATUS) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return compare_time(start, end) + + @permission(PERMS.STATUS) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config['reconnect']['startTime'].split(":") + end = self.core.config['reconnect']['endTime'].split(":") + return compare_time(start, end) and self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusDownloads(self): + """ Status off all currently running downloads. + + :return: list of `DownloadStatus` + """ + data = [] + for pyfile in self.core.threadManager.getActiveFiles(): + if not isinstance(pyfile, PyFile): + continue + data.append(DownloadInfo( + pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), + pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), + pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), + pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) + return data + + @permission(PERMS.ADD) + def addPackage(self, name, links, dest=Destination.Queue): + """Adds a package, with links to desired destination. + + :param name: name of the new package + :param links: list of urls + :param dest: `Destination` + :return: package id of the new package + """ + if self.core.config['general']['folder_per_package']: + folder = urlparse(name).path.split("/")[-1] + else: + folder = "" + + folder = safe_filename(folder) + + pid = self.core.files.addPackage(name, folder, dest) + + self.core.files.addLinks(links, pid) + + self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) + + self.core.files.save() + + return pid + + @permission(PERMS.ADD) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + # remove duplicates + return self.checkURLs(set(urls)) + + @permission(PERMS.ADD) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + return plugins + + @permission(PERMS.ADD) + def checkOnlineStatus(self, urls): + """ initiates online status check + + :param urls: + :return: initial set of data as `OnlineCheck` instance containing the result id + """ + data = self.core.pluginManager.parseUrls(urls) + + rid = self.core.threadManager.createResultThread(data, False) + + tmp = [(url, (url, OnlineStatus(url, (plugintype, pluginname), "unknown", 3, 0))) for url, plugintype, pluginname in data] + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + return OnlineCheck(rid, result) + + @permission(PERMS.ADD) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submited container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") + th.write(str(data)) + th.close() + return self.checkOnlineStatus(urls + [th.name]) + + @permission(PERMS.ADD) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + if "ALL_INFO_FETCHED" in result: + del result["ALL_INFO_FETCHED"] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + @permission(PERMS.ADD) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + return parseNames((x, x) for x in links) + + @permission(PERMS.ADD) + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + + @permission(PERMS.ADD) + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(links) + self.core.threadManager.createResultThread(data, True) + + @permission(PERMS.LIST) + def getPackageData(self, pid): + """Returns complete information about package, and included files. + + :param pid: package id + :return: `PackageData` with .links attribute + """ + data = self.core.files.getPackageData(int(pid)) + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + links=[self._convertPyFile(x) for x in data["links"].itervalues()]) + + @permission(PERMS.LIST) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :return: `PackageData` with .fid attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + fids=[int(x) for x in data["links"]]) + + @permission(PERMS.LIST) + def getFileData(self, fid): + """Get complete information about a specific file. + + :param fid: file id + :return: `FileData` + """ + info = self.core.files.getFileData(int(fid)) + if not info: + raise FileDoesNotExists(fid) + return self._convertPyFile(info.values()[0]) + + @permission(PERMS.DELETE) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for fid in fids: + self.core.files.deleteLink(int(fid)) + self.core.files.save() + + @permission(PERMS.DELETE) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for pid in pids: + self.core.files.deletePackage(int(pid)) + self.core.files.save() + + @permission(PERMS.LIST) + def getQueue(self): + """Returns info about queue and packages, **not** about files, see `getQueueData` \ + or `getPackageData` instead. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getQueueData(self): + """Return complete data about everything in queue, this is very expensive use it sparely.\ + See `getQueue` for alternative. + + :return: list of `PackageData` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getCollector(self): + """same as `getQueue` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] + + @permission(PERMS.LIST) + def getCollectorData(self): + """same as `getQueueData` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] + + @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + self.core.files.addLinks(links, int(pid)) + self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) + self.core.files.save() + + @permission(PERMS.MODIFY) + def pushToQueue(self, pid): + """Moves package from Collector to Queue. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Queue) + + @permission(PERMS.MODIFY) + def pullFromQueue(self, pid): + """Moves package from Queue to Collector. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Collector) + + @permission(PERMS.MODIFY) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(int(pid)) + + @permission(PERMS.MODIFY) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(int(fid)) + + @permission(PERMS.MODIFY) + def recheckPackage(self, pid): + """Proofes online status of all files in a package, also a default action when package is added. + + :param pid: + :return: + """ + self.core.files.reCheckPackage(int(pid)) + + @permission(PERMS.MODIFY) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def setPackageName(self, pid, name): + """Renames a package. + + :param pid: package id + :param name: new package name + """ + pack = self.core.files.getPackage(pid) + pack.name = name + pack.sync() + + @permission(PERMS.MODIFY) + def movePackage(self, destination, pid): + """Set a new package location. + + :param destination: `Destination` + :param pid: package id + """ + if destination in (0, 1): + self.core.files.setPackageLocation(pid, destination) + + @permission(PERMS.MODIFY) + def moveFiles(self, fids, pid): + """Move multiple files to another package + + :param fids: list of file ids + :param pid: destination package + :return: + """ + # TODO: implement + pass + + + @permission(PERMS.ADD) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") + th.write(str(data)) + th.close() + self.addPackage(th.name, [th.name], Destination.Queue) + + @permission(PERMS.MODIFY) + def orderPackage(self, pid, position): + """Gives a package a new position. + + :param pid: package id + :param position: + """ + self.core.files.reorderPackage(pid, position) + + @permission(PERMS.MODIFY) + def orderFile(self, fid, position): + """Gives a new position to a file within its package. + + :param fid: file id + :param position: + """ + self.core.files.reorderFile(fid, position) + + @permission(PERMS.MODIFY) + def setPackageData(self, pid, data): + """Allows to modify several package attributes. + + :param pid: package id + :param data: dict that maps attribute to desired value + """ + package = self.core.files.getPackage(pid) + if not package: + raise PackageDoesNotExists(pid) + for key, value in data.iteritems(): + if key == "id": + continue + setattr(package, key, value) + package.sync() + self.core.files.save() + + @permission(PERMS.DELETE) + def deleteFinished(self): + """Deletes all finished files and completly finished packages. + + :return: list of deleted package ids + """ + return self.core.files.deleteFinishedLinks() + + @permission(PERMS.MODIFY) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + @permission(PERMS.LIST) + def getPackageOrder(self, destination): + """Returns information about package order. + + :param destination: `Destination` + :return: dict mapping order to package id + """ + packs = self.core.files.getInfoData(destination) + order = {} + for pid in packs: + pack = self.core.files.getPackageData(int(pid)) + while pack["order"] in order.keys(): # just in case + pack["order"] += 1 + order[pack["order"]] = pack["id"] + return order + + @permission(PERMS.LIST) + def getFileOrder(self, pid): + """Information about file order within package. + + :param pid: + :return: dict mapping order to file id + """ + rawdata = self.core.files.getPackageData(int(pid)) + order = {} + for id, pyfile in rawdata["links"].iteritems(): + while pyfile["order"] in order.keys(): # just in case + pyfile["order"] += 1 + order[pyfile["order"]] = pyfile["id"] + return order + + + @permission(PERMS.STATUS) + def isCaptchaWaiting(self): + """Indicates wether a captcha task is available + + :return: bool + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + return not task is None + + @permission(PERMS.STATUS) + def getCaptchaTask(self, exclusive=False): + """Returns a captcha task + + :param exclusive: unused + :return: `CaptchaTask` + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser(exclusive=exclusive) + data, type, result = task.getCaptcha() + ctask = CaptchaTask(int(task.id), standard_b64encode(data), type, result) + return ctask + return CaptchaTask(-1) + + @permission(PERMS.STATUS) + def getCaptchaTaskStatus(self, tid): + """Get information about captcha task + + :param tid: task id + :return: string + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTaskByID(tid) + return task.getStatus() if task else "" + + @permission(PERMS.STATUS) + def setCaptchaResult(self, tid, result): + """Set result for a captcha task + + :param tid: task id + :param result: captcha result + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTaskByID(tid) + if task: + task.setResult(result) + self.core.captchaManager.removeTask(task) + + @permission(PERMS.STATUS) + def getEvents(self, uuid): + """Lists occured events, may be affected to changes in future. + + :param uuid: + :return: list of `Events` + """ + events = self.core.pullManager.getEvents(uuid) + new_events = [] + + def convDest(d): + return Destination.Queue if d == "queue" else Destination.Collector + + for e in events: + event = EventInfo() + event.eventname = e[0] + if e[0] in ("update", "remove", "insert"): + event.id = e[3] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[1]) + elif e[0] == "order": + if e[1]: + event.id = e[1] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[3]) + elif e[0] == "reload": + event.destination = convDest(e[1]) + new_events.append(event) + return new_events + + @permission(PERMS.ACCOUNTS) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAccountInfos(False, refresh) + for group in accs.values(): + accounts = [AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], + acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) + for acc in group] + return accounts or [] + + @permission(PERMS.ALL) + def getAccountTypes(self): + """All available account types. + + :return: list + """ + return self.core.accountManager.accounts.keys() + + @permission(PERMS.ACCOUNTS) + def updateAccount(self, plugin, account, password=None, options=None): + """Changes pw/options for specific account.""" + self.core.accountManager.updateAccount(plugin, account, password, options or {}) + + @permission(PERMS.ACCOUNTS) + def removeAccount(self, plugin, account): + """Remove account from pyload. + + :param plugin: pluginname + :param account: accountname + """ + self.core.accountManager.removeAccount(plugin, account) + + @permission(PERMS.ALL) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return bool(self.checkAuth(username, password, remoteip)) + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": + return "local" + else: + return self.core.db.checkAuth(username, password) + + def isAuthorized(self, func, userdata): + """checks if the user is authorized for specific method + + :param func: function name + :param userdata: dictionary of user data + :return: boolean + """ + if userdata == "local" or userdata["role"] == ROLE.ADMIN: + return True + elif func in permMap and has_permission(userdata["permission"], permMap[func]): + return True + else: + return False + + @permission(PERMS.ALL) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if user: + return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) + else: + return UserData() + + def getAllUserData(self): + """returns all known user and info""" + return dict((user, UserData(user, data["email"], data["role"], data["permission"], data["template"])) for user, data + in self.core.db.getAllUserData().iteritems()) + + @permission(PERMS.STATUS) + def getServices(self): + """ A dict of available services, these can be defined by addon plugins. + + :return: dict with this style: {"plugin": {"method": "description"}} + """ + return dict((plugin, funcs) for plugin, funcs in self.core.addonManager.methods.iteritems()) + + @permission(PERMS.STATUS) + def hasService(self, plugin, func): + """Checks wether a service is available. + + :param plugin: + :param func: + :return: bool + """ + cont = self.core.addonManager.methods + return plugin in cont and func in cont[plugin] + + @permission(PERMS.STATUS) + def call(self, info): + """Calls a service (a method in addon plugin). + + :param info: `ServiceCall` + :return: result + :raises: ServiceDoesNotExists, when its not available + :raises: ServiceException, when a exception was raised + """ + plugin = info.plugin + func = info.func + args = info.arguments + parse = info.parseArguments + if not self.hasService(plugin, func): + raise ServiceDoesNotExists(plugin, func) + try: + ret = self.core.addonManager.callRPC(plugin, func, args, parse) + except Exception, e: + raise ServiceException(e.message) + + @permission(PERMS.STATUS) + def getAllInfo(self): + """Returns all information stored by addon plugins. Values are always strings + + :return: {"plugin": {"name": value}} + """ + return self.core.addonManager.getAllInfo() + + @permission(PERMS.STATUS) + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.addonManager.getInfo(plugin) + + def changePassword(self, user, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(user, oldpw, newpw) + + def setUserPermission(self, user, perm, role): + self.core.db.setPermission(user, perm) + self.core.db.setRole(user, role) diff --git a/pyload/api/types.py b/pyload/api/types.py new file mode 100644 index 000000000..81385bf9f --- /dev/null +++ b/pyload/api/types.py @@ -0,0 +1,381 @@ +# -*- coding: utf-8 -*- +# Autogenerated by pyload +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + +class BaseObject(object): + __slots__ = [] + +class Destination(object): + Collector = 0 + Queue = 1 + +class DownloadStatus(object): + Aborted = 9 + Custom = 11 + Decrypting = 10 + Downloading = 12 + Failed = 8 + Finished = 0 + Offline = 1 + Online = 2 + Processing = 13 + Queued = 3 + Skipped = 4 + Starting = 7 + TempOffline = 6 + Unknown = 14 + Waiting = 5 + +class ElementType(object): + File = 1 + Package = 0 + +class Input(object): + BOOL = 4 + CHOICE = 6 + CLICK = 5 + LIST = 8 + MULTIPLE = 7 + NONE = 0 + PASSWORD = 3 + TABLE = 9 + TEXT = 1 + TEXTBOX = 2 + +class Output(object): + CAPTCHA = 1 + NOTIFICATION = 4 + QUESTION = 2 + +class AccountInfo(BaseObject): + __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + + def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): + self.validuntil = validuntil + self.login = login + self.options = options + self.valid = valid + self.trafficleft = trafficleft + self.maxtraffic = maxtraffic + self.premium = premium + self.type = type + +class CaptchaTask(BaseObject): + __slots__ = ['tid', 'data', 'type', 'resultType'] + + def __init__(self, tid=None, data=None, type=None, resultType=None): + self.tid = tid + self.data = data + self.type = type + self.resultType = resultType + +class ConfigItem(BaseObject): + __slots__ = ['name', 'description', 'value', 'type'] + + def __init__(self, name=None, description=None, value=None, type=None): + self.name = name + self.description = description + self.value = value + self.type = type + +class ConfigSection(BaseObject): + __slots__ = ['name', 'description', 'items', 'outline'] + + def __init__(self, name=None, description=None, items=None, outline=None): + self.name = name + self.description = description + self.items = items + self.outline = outline + +class DownloadInfo(BaseObject): + __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] + + def __init__(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): + self.fid = fid + self.name = name + self.speed = speed + self.eta = eta + self.format_eta = format_eta + self.bleft = bleft + self.size = size + self.format_size = format_size + self.percent = percent + self.status = status + self.statusmsg = statusmsg + self.format_wait = format_wait + self.wait_until = wait_until + self.packageID = packageID + self.packageName = packageName + self.plugin = plugin + +class EventInfo(BaseObject): + __slots__ = ['eventname', 'id', 'type', 'destination'] + + def __init__(self, eventname=None, id=None, type=None, destination=None): + self.eventname = eventname + self.id = id + self.type = type + self.destination = destination + +class FileData(BaseObject): + __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] + + def __init__(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): + self.fid = fid + self.url = url + self.name = name + self.plugin = plugin + self.size = size + self.format_size = format_size + self.status = status + self.statusmsg = statusmsg + self.packageID = packageID + self.error = error + self.order = order + +class FileDoesNotExists(Exception): + __slots__ = ['fid'] + + def __init__(self, fid=None): + self.fid = fid + +class InteractionTask(BaseObject): + __slots__ = ['iid', 'input', 'structure', 'preset', 'output', 'data', 'title', 'description', 'plugin'] + + def __init__(self, iid=None, input=None, structure=None, preset=None, output=None, data=None, title=None, description=None, plugin=None): + self.iid = iid + self.input = input + self.structure = structure + self.preset = preset + self.output = output + self.data = data + self.title = title + self.description = description + self.plugin = plugin + +class OnlineCheck(BaseObject): + __slots__ = ['rid', 'data'] + + def __init__(self, rid=None, data=None): + self.rid = rid + self.data = data + +class OnlineStatus(BaseObject): + __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] + + def __init__(self, name=None, plugin=(None, None), packagename=None, status=None, size=None): + self.name = name + self.plugin = plugin + self.packagename = packagename + self.status = status + self.size = size + +class PackageData(BaseObject): + __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] + + def __init__(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): + self.pid = pid + self.name = name + self.folder = folder + self.site = site + self.password = password + self.dest = dest + self.order = order + self.linksdone = linksdone + self.sizedone = sizedone + self.sizetotal = sizetotal + self.linkstotal = linkstotal + self.links = links + self.fids = fids + +class PackageDoesNotExists(Exception): + __slots__ = ['pid'] + + def __init__(self, pid=None): + self.pid = pid + +class ServerStatus(BaseObject): + __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] + + def __init__(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): + self.pause = pause + self.active = active + self.queue = queue + self.total = total + self.speed = speed + self.download = download + self.reconnect = reconnect + +class ServiceCall(BaseObject): + __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] + + def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): + self.plugin = plugin + self.func = func + self.arguments = arguments + self.parseArguments = parseArguments + +class ServiceDoesNotExists(Exception): + __slots__ = ['plugin', 'func'] + + def __init__(self, plugin=None, func=None): + self.plugin = plugin + self.func = func + +class ServiceException(Exception): + __slots__ = ['msg'] + + def __init__(self, msg=None): + self.msg = msg + +class UserData(BaseObject): + __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] + + def __init__(self, name=None, email=None, role=None, permission=None, templateName=None): + self.name = name + self.email = email + self.role = role + self.permission = permission + self.templateName = templateName + +class Iface(object): + def addFiles(self, pid, links): + pass + def addPackage(self, name, links, dest): + pass + def call(self, info): + pass + def checkOnlineStatus(self, urls): + pass + def checkOnlineStatusContainer(self, urls, filename, data): + pass + def checkURLs(self, urls): + pass + def deleteFiles(self, fids): + pass + def deleteFinished(self): + pass + def deletePackages(self, pids): + pass + def freeSpace(self): + pass + def generateAndAddPackages(self, links, dest): + pass + def generatePackages(self, links): + pass + def getAccountTypes(self): + pass + def getAccounts(self, refresh): + pass + def getAllInfo(self): + pass + def getAllUserData(self): + pass + def getCaptchaTask(self, exclusive): + pass + def getCaptchaTaskStatus(self, tid): + pass + def getCollector(self): + pass + def getCollectorData(self): + pass + def getConfig(self): + pass + def getConfigValue(self, category, option, section): + pass + def getEvents(self, uuid): + pass + def getFileData(self, fid): + pass + def getFileOrder(self, pid): + pass + def getInfoByPlugin(self, plugin): + pass + def getLog(self, offset): + pass + def getPackageData(self, pid): + pass + def getPackageInfo(self, pid): + pass + def getPackageOrder(self, destination): + pass + def getPluginConfig(self): + pass + def getQueue(self): + pass + def getQueueData(self): + pass + def getServerVersion(self): + pass + def getServices(self): + pass + def getUserData(self, username, password): + pass + def hasService(self, plugin, func): + pass + def isCaptchaWaiting(self): + pass + def isTimeDownload(self): + pass + def isTimeReconnect(self): + pass + def kill(self): + pass + def login(self, username, password): + pass + def moveFiles(self, fids, pid): + pass + def movePackage(self, destination, pid): + pass + def orderFile(self, fid, position): + pass + def orderPackage(self, pid, position): + pass + def parseURLs(self, html, url): + pass + def pauseServer(self): + pass + def pollResults(self, rid): + pass + def pullFromQueue(self, pid): + pass + def pushToQueue(self, pid): + pass + def recheckPackage(self, pid): + pass + def removeAccount(self, plugin, account): + pass + def restart(self): + pass + def restartFailed(self): + pass + def restartFile(self, fid): + pass + def restartPackage(self, pid): + pass + def setCaptchaResult(self, tid, result): + pass + def setConfigValue(self, category, option, value, section): + pass + def setPackageData(self, pid, data): + pass + def setPackageName(self, pid, name): + pass + def statusDownloads(self): + pass + def statusServer(self): + pass + def stopAllDownloads(self): + pass + def stopDownloads(self, fids): + pass + def togglePause(self): + pass + def toggleReconnect(self): + pass + def unpauseServer(self): + pass + def updateAccount(self, plugin, account, password, options): + pass + def uploadContainer(self, filename, data): + pass diff --git a/pyload/cli/AddPackage.py b/pyload/cli/AddPackage.py new file mode 100644 index 000000000..f7e30b64e --- /dev/null +++ b/pyload/cli/AddPackage.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from pyload.cli.Handler import Handler +from pyload.utils.printer import * + + +class AddPackage(Handler): + """ let the user add packages """ + + def init(self): + self.name = "" + self.urls = [] + + def onEnter(self, inp): + if inp == "0": + self.cli.reset() + + if not self.name: + self.name = inp + self.setInput() + elif inp == "END": + #add package + self.client.addPackage(self.name, self.urls, 1) + self.cli.reset() + else: + if inp.strip(): + self.urls.append(inp) + self.setInput() + + def renderBody(self, line): + println(line, white(_("Add Package:"))) + println(line + 1, "") + line += 2 + + if not self.name: + println(line, _("Enter a name for the new package")) + println(line + 1, "") + line += 2 + else: + println(line, _("Package: %s") % self.name) + println(line + 1, _("Parse the links you want to add.")) + println(line + 2, _("Type %s when done.") % mag("END")) + println(line + 3, _("Links added: ") + mag(len(self.urls))) + line += 4 + + println(line, "") + println(line + 1, mag("0.") + _(" back to main menu")) + + return line + 2 diff --git a/pyload/cli/Cli.py b/pyload/cli/Cli.py new file mode 100644 index 000000000..d8c8602fc --- /dev/null +++ b/pyload/cli/Cli.py @@ -0,0 +1,570 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from __future__ import with_statement + +from getopt import GetoptError, getopt + +import pyload.utils.pylgettext as gettext +import os +from os import _exit +from os.path import join, exists, abspath, basename +import sys +from sys import exit +from threading import Thread, Lock +from time import sleep +from traceback import print_exc + +from pyload.config.Parser import ConfigParser + +from codecs import getwriter + +if os.name == "nt": + enc = "cp850" +else: + enc = "utf8" + +sys.stdout = getwriter(enc)(sys.stdout, errors="replace") + +from pyload.cli.printer import * +from pyload.cli import AddPackage, ManageFiles + +from pyload.api import Destination +from pyload.utils import formatSize, decode +from pyload.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed +from Getch import Getch +from rename_process import renameProcess + +class Cli(object): + def __init__(self, client, command): + self.client = client + self.command = command + + if not self.command: + renameProcess('pyload-cli') + self.getch = Getch() + self.input = "" + self.inputline = 0 + self.lastLowestLine = 0 + self.menuline = 0 + + self.lock = Lock() + + #processor funcions, these will be changed dynamically depending on control flow + self.headerHandler = self #the download status + self.bodyHandler = self #the menu section + self.inputHandler = self + + os.system("clear") + println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + println(2, "") + + self.thread = RefreshThread(self) + self.thread.start() + + self.start() + else: + self.processCommand() + + def reset(self): + """ reset to initial main menu """ + self.input = "" + self.headerHandler = self.bodyHandler = self.inputHandler = self + + def start(self): + """ main loop. handle input """ + while True: + #inp = raw_input() + inp = self.getch.impl() + if ord(inp) == 3: + os.system("clear") + sys.exit() # ctrl + c + elif ord(inp) == 13: #enter + try: + self.lock.acquire() + self.inputHandler.onEnter(self.input) + + except Exception, e: + println(2, red(e)) + finally: + self.lock.release() + + elif ord(inp) == 127: + self.input = self.input[:-1] #backspace + try: + self.lock.acquire() + self.inputHandler.onBackSpace() + finally: + self.lock.release() + + elif ord(inp) == 27: #ugly symbol + pass + else: + self.input += inp + try: + self.lock.acquire() + self.inputHandler.onChar(inp) + finally: + self.lock.release() + + self.inputline = self.bodyHandler.renderBody(self.menuline) + self.renderFooter(self.inputline) + + + def refresh(self): + """refresh screen""" + + println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + println(2, "") + + self.lock.acquire() + + self.menuline = self.headerHandler.renderHeader(3) + 1 + println(self.menuline - 1, "") + self.inputline = self.bodyHandler.renderBody(self.menuline) + self.renderFooter(self.inputline) + + self.lock.release() + + + def setInput(self, string=""): + self.input = string + + def setHandler(self, klass): + #create new handler with reference to cli + self.bodyHandler = self.inputHandler = klass(self) + self.input = "" + + def renderHeader(self, line): + """ prints download status """ + #print updated information + # print "\033[J" #clear screen + # self.println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + # self.println(2, "") + # self.println(3, white(_("%s Downloads:") % (len(data)))) + + data = self.client.statusDownloads() + speed = 0 + + println(line, white(_("%s Downloads:") % (len(data)))) + line += 1 + + for download in data: + if download.status == 12: # downloading + percent = download.percent + z = percent / 4 + speed += download.speed + println(line, cyan(download.name)) + line += 1 + println(line, + blue("[") + yellow(z * "#" + (25 - z) * " ") + blue("] ") + green(str(percent) + "%") + _( + " Speed: ") + green(formatSize(download.speed) + "/s") + _(" Size: ") + green( + download.format_size) + _(" Finished in: ") + green(download.format_eta) + _( + " ID: ") + green(download.fid)) + line += 1 + if download.status == 5: + println(line, cyan(download.name)) + line += 1 + println(line, _("waiting: ") + green(download.format_wait)) + line += 1 + + println(line, "") + line += 1 + status = self.client.statusServer() + if status.pause: + paused = _("Status:") + " " + red(_("paused")) + else: + paused = _("Status:") + " " + red(_("running")) + + println(line,"%s %s: %s %s: %s %s: %s" % ( + paused, _("total Speed"), red(formatSize(speed) + "/s"), _("Files in queue"), red( + status.queue), _("Total"), red(status.total))) + + return line + 1 + + def renderBody(self, line): + """ prints initial menu """ + println(line, white(_("Menu:"))) + println(line + 1, "") + println(line + 2, mag("1.") + _(" Add Links")) + println(line + 3, mag("2.") + _(" Manage Queue")) + println(line + 4, mag("3.") + _(" Manage Collector")) + println(line + 5, mag("4.") + _(" (Un)Pause Server")) + println(line + 6, mag("5.") + _(" Kill Server")) + println(line + 7, mag("6.") + _(" Quit")) + + return line + 8 + + def renderFooter(self, line): + """ prints out the input line with input """ + println(line, "") + line += 1 + + println(line, white(" Input: ") + decode(self.input)) + + #clear old output + if line < self.lastLowestLine: + for i in range(line + 1, self.lastLowestLine + 1): + println(i, "") + + self.lastLowestLine = line + + #set cursor to position + print "\033[" + str(self.inputline) + ";0H" + + def onChar(self, char): + """ default no special handling for single chars """ + if char == "1": + self.setHandler(AddPackage) + elif char == "2": + self.setHandler(ManageFiles) + elif char == "3": + self.setHandler(ManageFiles) + self.bodyHandler.target = Destination.Collector + elif char == "4": + self.client.togglePause() + self.setInput() + elif char == "5": + self.client.kill() + self.client.close() + sys.exit() + elif char == "6": + os.system('clear') + sys.exit() + + def onEnter(self, inp): + pass + + def onBackSpace(self): + pass + + def processCommand(self): + command = self.command[0] + args = [] + if len(self.command) > 1: + args = self.command[1:] + + if command == "status": + files = self.client.statusDownloads() + + if not files: + print "No downloads running." + + for download in files: + if download.status == 12: # downloading + print print_status(download) + print "\tDownloading: %s @ %s/s\t %s (%s%%)" % ( + download.format_eta, formatSize(download.speed), formatSize(download.size - download.bleft), + download.percent) + elif download.status == 5: + print print_status(download) + print "\tWaiting: %s" % download.format_wait + else: + print print_status(download) + + elif command == "queue": + print_packages(self.client.getQueueData()) + + elif command == "collector": + print_packages(self.client.getCollectorData()) + + elif command == "add": + if len(args) < 2: + print _("Please use this syntax: add ...") + return + + self.client.addPackage(args[0], args[1:], Destination.Queue) + + elif command == "add_coll": + if len(args) < 2: + print _("Please use this syntax: add ...") + return + + self.client.addPackage(args[0], args[1:], Destination.Collector) + + elif command == "del_file": + self.client.deleteFiles([int(x) for x in args]) + print "Files deleted." + + elif command == "del_package": + self.client.deletePackages([int(x) for x in args]) + print "Packages deleted." + + elif command == "move": + for pid in args: + pack = self.client.getPackageInfo(int(pid)) + self.client.movePackage((pack.dest + 1) % 2, pack.pid) + + elif command == "check": + print _("Checking %d links:") % len(args) + print + rid = self.client.checkOnlineStatus(args).rid + self.printOnlineCheck(self.client, rid) + + + elif command == "check_container": + path = args[0] + if not exists(join(owd, path)): + print _("File does not exists.") + return + + f = open(join(owd, path), "rb") + content = f.read() + f.close() + + rid = self.client.checkOnlineStatusContainer([], basename(f.name), content).rid + self.printOnlineCheck(self.client, rid) + + + elif command == "pause": + self.client.pause() + + elif command == "unpause": + self.client.unpause() + + elif command == "toggle": + self.client.togglePause() + + elif command == "kill": + self.client.kill() + elif command == "restart_file": + for x in args: + self.client.restartFile(int(x)) + print "Files restarted." + elif command == "restart_package": + for pid in args: + self.client.restartPackage(int(pid)) + print "Packages restarted." + + else: + print_commands() + + def printOnlineCheck(self, client, rid): + while True: + sleep(1) + result = client.pollResults(rid) + for url, status in result.data.iteritems(): + if status.status == 2: check = "Online" + elif status.status == 1: check = "Offline" + else: check = "Unknown" + + print "%-45s %-12s\t %-15s\t %s" % (status.name, formatSize(status.size), status.plugin, check) + + if result.rid == -1: break + + +class RefreshThread(Thread): + def __init__(self, cli): + Thread.__init__(self) + self.setDaemon(True) + self.cli = cli + + def run(self): + while True: + sleep(1) + try: + self.cli.refresh() + except ConnectionClosed: + os.system("clear") + print _("pyLoad was terminated") + _exit(0) + except Exception, e: + println(2, red(str(e))) + self.cli.reset() + print_exc() + + +def print_help(config): + print + print "pyLoad CLI Copyright (c) 2008-2015 the pyLoad Team" + print + print "Usage: [python] pyload-cli.py [options] [command]" + print + print "" + print "See pyload-cli.py -c for a complete listing." + print + print "" + print " -i, --interactive", " Start in interactive mode" + print + print " -u, --username=", " " * 2, "Specify Username" + print " --pw=", " " * 2, "Password" + print " -a, --address=", " " * 3, "Specify address (current=%s)" % config["addr"] + print " -p, --port", " " * 7, "Specify port (current=%s)" % config["port"] + print + print " -l, --language", " " * 3, "Set user interface language (current=%s)" % config["language"] + print " -h, --help", " " * 7, "Display this help screen" + print " -c, --commands", " " * 3, "List all available commands" + print + + +def print_packages(data): + for pack in data: + print "Package %s (#%s):" % (pack.name, pack.pid) + for download in pack.links: + print "\t" + print_file(download) + print + + +def print_file(download): + return "#%(id)-6d %(name)-30s %(statusmsg)-10s %(plugin)-8s" % { + "id": download.fid, + "name": download.name, + "statusmsg": download.statusmsg, + "plugin": download.plugin + } + + +def print_status(download): + return "#%(id)-6s %(name)-40s Status: %(statusmsg)-10s Size: %(size)s" % { + "id": download.fid, + "name": download.name, + "statusmsg": download.statusmsg, + "size": download.format_size + } + + +def print_commands(): + commands = [("status", _("Prints server status")), + ("queue", _("Prints downloads in queue")), + ("collector", _("Prints downloads in collector")), + ("add ...", _("Adds package to queue")), + ("add_coll ...", _("Adds package to collector")), + ("del_file ...", _("Delete Files from Queue/Collector")), + ("del_package ...", _("Delete Packages from Queue/Collector")), + ("move ...", _("Move Packages from Queue to Collector or vice versa")), + ("restart_file ...", _("Restart files")), + ("restart_package ...", _("Restart packages")), + ("check ...", _("Check online status, works with local container")), + ("check_container path", _("Checks online status of a container file")), + ("pause", _("Pause the server")), + ("unpause", _("continue downloads")), + ("toggle", _("Toggle pause/unpause")), + ("kill", _("kill server")), ] + + print _("List of commands:") + print + for c in commands: + print "%-35s %s" % c + + +def writeConfig(opts): + try: + with open(join(homedir, ".pyload-cli"), "w") as cfgfile: + cfgfile.write("[cli]") + for opt in opts: + cfgfile.write("%s=%s\n" % (opt, opts[opt])) + except Exception: + print _("Couldn't write user config file") + + +def main(): + config = {"addr": "127.0.0.1", "port": "7227", "language": "en"} + try: + config["language"] = os.environ["LANG"][0:2] + except Exception: + pass + + if (not exists(join(pypath, "locale", config["language"]))) or config["language"] == "": + config["language"] = "en" + + configFile = ConfigParser.ConfigParser() + configFile.read(join(homedir, ".pyload-cli")) + + if configFile.has_section("cli"): + for opt in configFile.items("cli"): + config[opt[0]] = opt[1] + + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("Cli", join(pypath, "locale"), + languages=[config["language"], "en"], fallback=True) + translation.install(unicode=True) + + interactive = False + command = None + username = "" + password = "" + + shortOptions = 'iu:p:a:hcl:' + longOptions = ['interactive', "username=", "pw=", "address=", "port=", "help", "commands", "language="] + + try: + opts, extraparams = getopt(sys.argv[1:], shortOptions, longOptions) + for option, params in opts: + if option in ("-i", "--interactive"): + interactive = True + elif option in ("-u", "--username"): + username = params + elif option in ("-a", "--address"): + config["addr"] = params + elif option in ("-p", "--port"): + config["port"] = params + elif option in ("-l", "--language"): + config["language"] = params + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("Cli", join(pypath, "locale"), + languages=[config["language"], "en"], fallback=True) + translation.install(unicode=True) + elif option in ("-h", "--help"): + print_help(config) + exit() + elif option in ("--pw"): + password = params + elif option in ("-c", "--comands"): + print_commands() + exit() + + except GetoptError: + print 'Unknown Argument(s) "%s"' % " ".join(sys.argv[1:]) + print_help(config) + exit() + + if len(extraparams) >= 1: + command = extraparams + + client = False + + if interactive: + try: + client = ThriftClient(config["addr"], int(config["port"]), username, password) + except WrongLogin: + pass + except NoSSL: + print _("You need py-openssl to connect to this pyLoad Core.") + exit() + except NoConnection: + config["addr"] = False + config["port"] = False + + if not client: + if not config["addr"]: config["addr"] = raw_input(_("Address: ")) + if not config["port"]: config["port"] = raw_input(_("Port: ")) + if not username: username = raw_input(_("Username: ")) + if not password: + from getpass import getpass + + password = getpass(_("Password: ")) + + try: + client = ThriftClient(config["addr"], int(config["port"]), username, password) + except WrongLogin: + print _("Login data is wrong.") + except NoConnection: + print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config["addr"], + "port": config["port"]}) + + else: + try: + client = ThriftClient(config["addr"], int(config["port"]), username, password) + except WrongLogin: + print _("Login data is wrong.") + except NoConnection: + print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config["addr"], + "port": config["port"]}) + except NoSSL: + print _("You need py-openssl to connect to this pyLoad core.") + + if interactive and command: print _("Interactive mode ignored since you passed some commands.") + + if client: + writeConfig(config) + cli = Cli(client, command) diff --git a/pyload/cli/Handler.py b/pyload/cli/Handler.py new file mode 100644 index 000000000..4aab3f0e2 --- /dev/null +++ b/pyload/cli/Handler.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +class Handler(object): + def __init__(self, cli): + self.cli = cli + self.init() + + client = property(lambda self: self.cli.client) + input = property(lambda self: self.cli.input) + + def init(self): + pass + + def onChar(self, char): + pass + + def onBackSpace(self): + pass + + def onEnter(self, inp): + pass + + def setInput(self, inp=""): + self.cli.setInput(inp) + + def backspace(self): + self.cli.setInput(self.input[:-1]) + + def renderBody(self, line): + """ gets the line where to render output and should return the line number below its content """ + return line + 1 diff --git a/pyload/cli/ManageFiles.py b/pyload/cli/ManageFiles.py new file mode 100644 index 000000000..ca1070113 --- /dev/null +++ b/pyload/cli/ManageFiles.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from itertools import islice +from time import time + +from pyload.cli.Handler import Handler +from pyload.utils.printer import * + +from pyload.api import Destination, PackageData + + +class ManageFiles(Handler): + """ possibility to manage queue/collector """ + + def init(self): + self.target = Destination.Queue + self.pos = 0 #position in queue + self.package = -1 #choosen package + self.mode = "" # move/delete/restart + self.cache = None + self.links = None + self.time = 0 + + def onChar(self, char): + if char in ("m", "d", "r"): + self.mode = char + self.setInput() + elif char == "p": + self.pos = max(0, self.pos - 5) + self.backspace() + elif char == "n": + self.pos += 5 + self.backspace() + + def onBackSpace(self): + if not self.input and self.mode: + self.mode = "" + if not self.input and self.package > -1: + self.package = -1 + + def onEnter(self, input): + if input == "0": + self.cli.reset() + elif self.package < 0 and self.mode: + #mode select + packs = self.parseInput(input) + if self.mode == "m": + [self.client.movePackage((self.target + 1) % 2, x) for x in packs] + elif self.mode == "d": + self.client.deletePackages(packs) + elif self.mode == "r": + [self.client.restartPackage(x) for x in packs] + + elif self.mode: + #edit links + links = self.parseInput(input, False) + + if self.mode == "d": + self.client.deleteFiles(links) + elif self.mode == "r": + map(self.client.restartFile, links) + + else: + #look into package + try: + self.package = int(input) + except Exception: + pass + + self.cache = None + self.links = None + self.pos = 0 + self.mode = "" + self.setInput() + + def renderBody(self, line): + if self.package < 0: + println(line, white(_("Manage Packages:"))) + else: + println(line, white((_("Manage Links:")))) + line += 1 + + if self.mode: + if self.mode == "m": + println(line, _("What do you want to move?")) + elif self.mode == "d": + println(line, _("What do you want to delete?")) + elif self.mode == "r": + println(line, _("What do you want to restart?")) + + println(line + 1, "Enter single number, comma seperated numbers or ranges. eg. 1, 2, 3 or 1-3.") + line += 2 + else: + println(line, _("Choose what yout want to do or enter package number.")) + println(line + 1, ("%s - %%s, %s - %%s, %s - %%s" % (mag("d"), mag("m"), mag("r"))) % ( + _("delete"), _("move"), _("restart"))) + line += 2 + + if self.package < 0: + #print package info + pack = self.getPackages() + i = 0 + for value in islice(pack, self.pos, self.pos + 5): + try: + println(line, mag(str(value.pid)) + ": " + value.name) + line += 1 + i += 1 + except Exception, e: + pass + for x in range(5 - i): + println(line, "") + line += 1 + else: + #print links info + pack = self.getLinks() + i = 0 + for value in islice(pack.links, self.pos, self.pos + 5): + try: + println(line, mag(value.fid) + ": %s | %s | %s" % ( + value.name, value.statusmsg, value.plugin)) + line += 1 + i += 1 + except Exception, e: + pass + for x in range(5 - i): + println(line, "") + line += 1 + + println(line, mag("p") + _(" - previous") + " | " + mag("n") + _(" - next")) + println(line + 1, mag("0.") + _(" back to main menu")) + return line + 2 + + def getPackages(self): + if self.cache and self.time + 2 < time(): + return self.cache + + if self.target == Destination.Queue: + data = self.client.getQueue() + else: + data = self.client.getCollector() + + self.cache = data + self.time = time() + + return data + + def getLinks(self): + if self.links and self.time + 1 < time(): + return self.links + try: + data = self.client.getPackageData(self.package) + except Exception: + data = PackageData(links=[]) + self.links = data + self.time = time() + return data + + def parseInput(self, inp, package=True): + inp = inp.strip() + if "-" in inp: + l, n, h = inp.partition("-") + l = int(l) + h = int(h) + r = range(l, h + 1) + + ret = [] + if package: + for p in self.cache: + if p.pid in r: + ret.append(p.pid) + else: + for l in self.links.links: + if l.lid in r: + ret.append(l.lid) + return ret + else: + return [int(x) for x in inp.split(",")] diff --git a/pyload/cli/__init__.py b/pyload/cli/__init__.py new file mode 100644 index 000000000..a64fc0c0c --- /dev/null +++ b/pyload/cli/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from pyload.cli.AddPackage import AddPackage +from pyload.cli.ManageFiles import ManageFiles diff --git a/pyload/config/Parser.py b/pyload/config/Parser.py new file mode 100644 index 000000000..e21eaba9f --- /dev/null +++ b/pyload/config/Parser.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from time import sleep +from os.path import exists, join +from shutil import copy + +from traceback import print_exc +from pyload.utils import chmod, encode, decode + + +CONF_VERSION = 1 + + +class ConfigParser(object): + """ + holds and manage the configuration + + current dict layout: + + { + + section: { + option: { + value: + type: + desc: + } + desc: + + } + """ + + def __init__(self): + """Constructor""" + self.config = {} #: the config values + self.plugin = {} #: the config for plugins + self.oldRemoteData = {} + + self.pluginCB = None #: callback when plugin config value is changed + + self.checkVersion() + + self.readConfig() + + + def checkVersion(self, n=0): + """determines if config need to be copied""" + try: + if not exists("pyload.conf"): + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + + if not exists("plugin.conf"): + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + + f = open("pyload.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + print "Old version of config was replaced" + + f = open("plugin.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + print "Old version of plugin-config replaced" + except Exception: + if n >= 3: + raise + sleep(0.3) + self.checkVersion(n + 1) + + + def readConfig(self): + """reads the config file""" + self.config = self.parseConfig(join(pypath, "pyload", "config", "default.conf")) + self.plugin = self.parseConfig("plugin.conf") + + try: + homeconf = self.parseConfig("pyload.conf") + if "username" in homeconf["remote"]: + if "password" in homeconf["remote"]: + self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], + "password": homeconf["remote"]["username"]["value"]} + del homeconf["remote"]["password"] + del homeconf["remote"]["username"] + self.updateValues(homeconf, self.config) + except Exception: + print "Config Warning" + print_exc() + + + def parseConfig(self, config): + """parses a given configfile""" + + f = open(config) + + config = f.read() + + config = config.splitlines()[1:] + + conf = {} + + section, option, value, typ, desc = "", "", "", "", "" + + listmode = False + + for line in config: + comment = line.rfind("#") + if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): + line = line.rpartition("#") #: removes comments + if line[1]: + line = line[0] + else: + line = line[2] + + line = line.strip() + + try: + if line == "": + continue + elif line.endswith(":"): + section, none, desc = line[:-1].partition('-') + section = section.strip() + desc = desc.replace('"', "").strip() + conf[section] = {"desc": desc} + else: + if listmode: + if line.endswith("]"): + listmode = False + line = line.replace("]", "") + + value += [self.cast(typ, x.strip()) for x in line.split(",") if x] + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + + else: + content, none, value = line.partition("=") + + content, none, desc = content.partition(":") + + desc = desc.replace('"', "").strip() + + typ, none, option = content.strip().rpartition(" ") + + value = value.strip() + + if value.startswith("["): + if value.endswith("]"): + listmode = False + value = value[:-1] + else: + listmode = True + + value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] + else: + value = self.cast(typ, value) + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + except Exception, e: + print "Config Warning" + print_exc() + + f.close() + return conf + + + def updateValues(self, config, dest): + """sets the config values from a parsed config file to values in destination""" + + for section in config.iterkeys(): + if section in dest: + for option in config[section].iterkeys(): + if option in ("desc", "outline"): + continue + + if option in dest[section]: + dest[section][option]["value"] = config[section][option]["value"] + + # else: + # dest[section][option] = config[section][option] + + + # else: + # dest[section] = config[section] + + + def saveConfig(self, config, filename): + """saves config to filename""" + with open(filename, "wb") as f: + chmod(filename, 0600) + f.write("version: %i \n" % CONF_VERSION) + for section in config.iterkeys(): + f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) + + for option, data in config[section].iteritems(): + if option in ("desc", "outline"): continue + + if isinstance(data["value"], list): + value = "[ \n" + for x in data["value"]: + value += "\t\t" + str(x) + ",\n" + value += "\t\t]\n" + else: + if isinstance(data["value"], basestring): + value = data["value"] + "\n" + else: + value = str(data["value"]) + "\n" + try: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) + except UnicodeEncodeError: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], encode(value))) + + + def cast(self, typ, value): + """cast value to given format""" + if not isinstance(value, basestring): + return value + + elif typ == "int": + return int(value) + elif typ == "bool": + return value.lower() in ("1", "true", "on", "an", "yes") + elif typ == "time": + if not value: + value = "0:00" + if not ":" in value: + value += ":00" + return value + elif typ in ("str", "file", "folder"): + return encode(value) + else: + return value + + + def save(self): + """saves the configs to disk""" + self.saveConfig(self.config, "pyload.conf") + self.saveConfig(self.plugin, "plugin.conf") + + + def __getitem__(self, section): + """provides dictonary like access: c['section']['option']""" + return Section(self, section) + + + def get(self, section, option): + """get value""" + value = self.config[section][option]["value"] + return decode(value) + + + def set(self, section, option, value): + """set value""" + + value = self.cast(self.config[section][option]["type"], value) + + self.config[section][option]["value"] = value + self.save() + + + def getPlugin(self, plugin, option): + """gets a value for a plugin""" + value = self.plugin[plugin][option]["value"] + return encode(value) + + + def setPlugin(self, plugin, option, value): + """sets a value for a plugin""" + + value = self.cast(self.plugin[plugin][option]["type"], value) + + if self.pluginCB: self.pluginCB(plugin, option, value) + + self.plugin[plugin][option]["value"] = value + self.save() + + + def getMetaData(self, section, option): + """ get all config data for an option """ + return self.config[section][option] + + + def addPluginConfig(self, name, config, outline=""): + """adds config options with tuples (name, type, desc, default)""" + if name not in self.plugin: + conf = {"desc": name, + "outline": outline} + self.plugin[name] = conf + else: + conf = self.plugin[name] + conf["outline"] = outline + + for item in config: + if item[0] in conf: + conf[item[0]]["type"] = item[1] + conf[item[0]]["desc"] = item[2] + else: + conf[item[0]] = { + "desc": item[2], + "type": item[1], + "value": self.cast(item[1], item[3]) + } + + values = [x[0] for x in config] + ["desc", "outline"] + # delete old values + for item in conf.keys(): + if item not in values: + del conf[item] + + + def deleteConfig(self, name): + """Removes a plugin config""" + if name in self.plugin: + del self.plugin[name] + + +class Section(object): + """provides dictionary like access for configparser""" + + def __init__(self, parser, section): + """Constructor""" + self.parser = parser + self.section = section + + + def __getitem__(self, item): + """getitem""" + return self.parser.get(self.section, item) + + + def __setitem__(self, item, value): + """setitem""" + self.parser.set(self.section, item, value) diff --git a/pyload/config/Setup.py b/pyload/config/Setup.py new file mode 100644 index 000000000..8b88daa48 --- /dev/null +++ b/pyload/config/Setup.py @@ -0,0 +1,553 @@ +# -*- coding: utf-8 -*- +# @author: vuolter + +from __future__ import with_statement + +import __builtin__ + +import os +import sys + +from getpass import getpass +from os import chdir, makedirs, path +from subprocess import PIPE, call + +from pyload.network.JsEngine import JsEngine +from pyload.utils import get_console_encoding, load_translation, safe_join, versiontuple + + +class SetupAssistant(object): + """ pyLoads initial setup configuration assistant """ + + def __init__(self, config): + self.config = config + self.lang = "en" + self.stdin_encoding = get_console_encoding(sys.stdin.encoding) + + + def start(self): + print + langs = sorted(self.config.getMetaData("general", "language")['type'].split(";")) + self.lang = self.ask(u"Choose setup language", "en", langs) + + load_translation("setup", self.lang) + + #Input shorthand for yes + self.yes = _("y") + #Input shorthand for no + self.no = _("n") + + # print + # print _("Would you like to configure pyLoad via Webinterface?") + # print _("You need a Browser and a connection to this PC for it.") + # viaweb = self.ask(_("Start initial webinterface for configuration?"), "y", bool=True) + # ... + + print + print + print _("## Welcome to the pyLoad Configuration Assistant ##") + print + print _("It will check your system and make a basic setup in order to run pyLoad.") + print + print _("The value in brackets [] always is the default value,") + print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") + print _( + "Don't forget: You can always rerun this assistant with --setup or -s parameter, when you start pyload.py .") + print _("If you have any problems with this assistant hit STRG-C,") + print _("to abort and don't let him start with pyload.py automatically anymore.") + print + print + raw_input(_("When you are ready for system check, hit enter.")) + print + print + + basic, ssl, captcha, web, js = self.system_check() + print + print + + if not basic: + print _("You need pycurl, sqlite and python 2.5, 2.6 or 2.7 to run pyLoad.") + print _("Please correct this and re-run pyLoad.") + print + print _("Setup will now close.") + print + print + raw_input(_("Press Enter to exit.")) + return False + + raw_input(_("System check finished, hit enter to see your status report.")) + print + print + print _("## Status ##") + print + + avail = [] + if self.check_module("Crypto"): + avail.append(_("- container decrypting")) + if ssl: + avail.append(_("- ssl connection")) + if captcha: + avail.append(_("- automatic captcha decryption")) + if web: + avail.append(_("- webinterface")) + if js: + avail.append(_("- extended Click'N'Load")) + + if avail: + print _("AVAILABLE FEATURES:") + for feature in avail: + print feature + print + + if len(avail) < 5: + print _("MISSING FEATURES:") + + if not self.check_module("Crypto"): + print _("- no py-crypto available") + print _("You need this if you want to decrypt container files.") + print + + if not ssl: + print _("- no SSL available") + print _("This is needed if you want to establish a secure connection to core or webinterface.") + print _("If you only want to access locally to pyLoad ssl is not usefull.") + print + + if not captcha: + print _("- no Captcha Recognition available") + print _("Only needed for some hosters and as freeuser.") + print + + if not js: + print _("- no JavaScript engine found") + print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") + print + + print + print _("You can abort the setup now and fix some dependicies if you want.") + else: + print _("NO MISSING FEATURES!") + + print + print + con = self.ask(_("Continue with setup?"), self.yes, bool=True) + + if not con: + return False + + print + print + print _("CURRENT CONFIG PATH: %s") % configdir + print + print _("NOTE: If you use pyLoad on a server or the home partition lives on an iternal flash it may be a good idea to change it.") + confpath = self.ask(_("Do you want to change the config path?"), self.no, bool=True) + if confpath: + print + self.conf_path() + print + + print + print _("Do you want to configure login data and basic settings?") + print _("This is recommend for first run.") + con = self.ask(_("Make basic setup?"), self.yes, bool=True) + + if con: + print + print + self.conf_basic() + + if ssl: + print + print _("Do you want to configure ssl?") + ssl = self.ask(_("Configure ssl?"), self.no, bool=True) + if ssl: + print + print + self.conf_ssl() + + if web: + print + print _("Do you want to configure webinterface?") + web = self.ask(_("Configure webinterface?"), self.yes, bool=True) + if web: + print + print + self.conf_web() + + print + print + print _("Setup finished successfully!") + print + print + raw_input(_("Hit enter to exit and restart pyLoad.")) + return True + + + def system_check(self): + """ make a systemcheck and return the results """ + import platform + + print _("## System Information ##") + print + print _("Platform: ") + platform.platform(aliased=True) + print _("OS: ") + platform.system() or "Unknown" + print _("Python: ") + sys.version.replace("\n", "") + print + print + + print _("## System Check ##") + print + + if (2, 5) > sys.version_info > (2, 7): + python = False + else: + python = True + + self.print_dep("python", python, false="NOT OK") + + curl = self.check_module("pycurl") + self.print_dep("pycurl", curl) + + sqlite = self.check_module("sqlite3") + self.print_dep("sqlite3", sqlite) + + basic = python and curl and sqlite + + print + + crypto = self.check_module("Crypto") + self.print_dep("pycrypto", crypto) + + ssl = self.check_module("OpenSSL") + self.print_dep("py-OpenSSL", ssl) + + print + + pil = self.check_module("Image") + self.print_dep("py-imaging", pil) + + if os.name == "nt": + tesser = self.check_prog([path.join(pypath, "tesseract", "tesseract.exe"), "-v"]) + else: + tesser = self.check_prog(["tesseract", "-v"]) + + self.print_dep("tesseract", tesser) + + captcha = pil and tesser + + print + + try: + import jinja2 + + v = jinja2.__version__ + if v and versiontuple(v) < (2, 5, 0): + jinja = False + else: + jinja = True + except Exception: + jinja = False + jinja_error = "MISSING" + else: + jinja_error = "NOT OK" + + self.print_dep("jinja2", jinja, false=jinja_error) + + beaker = self.check_module("beaker") + self.print_dep("beaker", beaker) + + bjoern = self.check_module("bjoern") + self.print_dep("bjoern", bjoern) + + web = sqlite and beaker + + js = True if JsEngine.find() else False + self.print_dep(_("JS engine"), js) + + if not python: + print + print + if sys.version_info > (2, 7): + print _("WARNING: Your python version is too NEW!") + print _("Please use Python version 2.6/2.7 .") + else: + print _("WARNING: Your python version is too OLD!") + print _("Please use at least Python version 2.5 .") + + if not jinja and jinja_error == "NOT OK": + print + print + print _("WARNING: Your installed jinja2 version %s is too OLD!") % jinja2.__version__ + print _("You can safely continue but if the webinterface is not working,") + print _("please upgrade or uninstall it, because pyLoad self-includes jinja2 libary.") + + return basic, ssl, captcha, web, js + + + def conf_basic(self): + print _("## Basic Setup ##") + + print + print _("The following logindata is valid for CLI and webinterface.") + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + print _("NOTE: Consider a password of 10 or more symbols if you expect to access to your local network from outside (ex. internet).") + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + db.shutdown() + + print + print _("External clients (GUI, CLI or other) need remote access to work over the network.") + print _("However, if you only want to use the webinterface you may disable it to save ram.") + self.config.set("remote", "activated", self.ask(_("Enable remote access"), self.no, bool=True)) + + print + langs = sorted(self.config.getMetaData("general", "language")['type'].split(";")) + self.config.set("general", "language", self.ask(_("Choose system language"), self.lang, langs)) + + print + self.config.set("general", "download_folder", self.ask(_("Download folder"), "Downloads")) + print + self.config.set("download", "max_downloads", self.ask(_("Max parallel downloads"), "3")) + print + reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) + self.config.set("reconnect", "activated", reconnect) + if reconnect: + self.config.set("reconnect", "method", self.ask(_("Reconnect script location"), "./reconnect.sh")) + + + def conf_web(self): + print _("## Webinterface Setup ##") + + print + print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") + self.config.set("webui", "host", self.ask(_("Address"), "0.0.0.0")) + self.config.set("webui", "port", self.ask(_("Port"), "8000")) + print + print _("pyLoad offers several server backends, now following a short explanation.") + print "- auto:", _("Automatically choose the best webserver for your platform.") + print "- builtin:", _("First choice if you plan to use pyLoad just for you.") + print "- threaded:", _("Support SSL connection and can serve simultaneously more client flawlessly.") + print "- fastcgi:", _( + "Can be used by apache, lighttpd, etc.; needs to be properly configured before.") + if os.name != "nt": + print "- lightweight:", _("Very fast alternative to builtin; requires libev and bjoern packages.") + + print + print _("NOTE: In some rare cases the builtin server not works correctly, so if you have troubles with the web interface") + print _("run this setup assistant again and change the builtin server to the threaded.") + + if os.name == "nt": + servers = ["auto", "builtin", "threaded", "fastcgi"] + else: + servers = ["auto", "builtin", "threaded", "fastcgi", "lightweight"] + + self.config.set("webui", "server", self.ask(_("Choose webserver"), "auto", servers)) + + + def conf_ssl(self): + print _("## SSL Setup ##") + print + print _("Execute these commands from pyLoad config folder to make ssl certificates:") + print + print "openssl genrsa -out ssl.key 1024" + print "openssl req -new -key ssl.key -out ssl.csr" + print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " + print + print _("If you're done and everything went fine, you can activate ssl now.") + + ssl = self.ask(_("Activate SSL?"), self.yes, bool=True) + self.config.set("remote", "ssl", ssl) + self.config.set("webui", "ssl", ssl) + + + def set_user(self): + load_translation("setup", self.config.get("general", "language")) + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + + noaction = True + try: + while True: + print _("Select action") + print _("1 - Create/Edit user") + print _("2 - List users") + print _("3 - Remove user") + print _("4 - Quit") + action = raw_input("[1]/2/3/4: ") + if not action in ("1", "2", "3", "4"): + continue + elif action == "1": + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + noaction = False + elif action == "2": + print + print _("Users") + print "-----" + users = db.listUsers() + noaction = False + for user in users: + print user + print "-----" + print + elif action == "3": + print + username = self.ask(_("Username"), "") + if username: + db.removeUser(username) + noaction = False + elif action == "4": + break + finally: + if not noaction: + db.shutdown() + + + def set_configdir(self, configdir, persistent=False): + dirname = path.abspath(configdir) + try: + if not path.exists(dirname): + makedirs(dirname, 0700) + + chdir(dirname) + + if persistent: + c = path.join(rootdir, "config", "configdir") + if not path.exists(c): + makedirs(c, 0700) + + with open(c, "wb") as f: + f.write(dirname) + + except IOError: + return False + + else: + __builtin__.configdir = dirname + return dirname #: return always abspath + + + def conf_path(self): + print _("Setting new config path.") + print _("NOTE: Current configuration will not be transfered!") + + while True: + confdir = self.ask(_("CONFIG PATH"), configdir) + confpath = self.set_configdir(confdir) + print + if not confpath: + print _("Failed to change the current config path!") + print + else: + print _("pyLoad config path successfully changed.") + break + + + def print_dep(self, name, value, false="MISSING", true="OK"): + """ Print Status of dependency """ + if value and isinstance(value, basestring): + msg = "%(dep)-12s %(bool)s (%(info)s)" + else: + msg = "%(dep)-12s %(bool)s" + + print msg % {'dep': name + ':', + 'bool': _(true if value else false).upper(), + 'info': ", ".join(value)} + + + def check_module(self, module): + try: + __import__(module) + return True + except Exception: + return False + + + def check_prog(self, command): + pipe = PIPE + try: + call(command, stdout=pipe, stderr=pipe) + return True + except Exception: + return False + + + def ask(self, qst, default, answers=[], bool=False, password=False): + """ produce one line to asking for input """ + if answers: + info = "(" + + for i, answer in enumerate(answers): + info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) + + info += ")" + elif bool: + if default == self.yes: + info = "([%s]/%s)" % (self.yes, self.no) + else: + info = "(%s/[%s])" % (self.yes, self.no) + else: + info = "[%s]" % default + + if password: + p1 = True + p2 = False + pwlen = 8 + while p1 != p2: + sys.stdout.write(_("Password: ")) + p1 = getpass("") + + if len(p1) < pwlen: + print _("Password too short! Use at least %s symbols." % pwlen) + continue + elif not p1.isalnum(): + print _("Password must be alphanumeric.") + continue + + sys.stdout.write(_("Password (again): ")) + p2 = getpass("") + + if p1 == p2: + return p1 + else: + print _("Passwords did not match.") + + while True: + try: + input = raw_input(qst + " %s: " % info) + except KeyboardInterrupt: + print "\nSetup interrupted" + sys.exit() + + input = input.decode(self.stdin_encoding) + + if input.strip() == "": + input = default + + if bool: + # yes, true, t are inputs for booleans with value true + if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: + return True + # no, false, f are inputs for booleans with value false + elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: + return False + else: + print _("Invalid Input") + print + continue + + if not answers or input in answers: + return input + else: + print _("Invalid Input") diff --git a/pyload/config/__init__.py b/pyload/config/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/config/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/config/default.conf b/pyload/config/default.conf new file mode 100644 index 000000000..6e2fb7749 --- /dev/null +++ b/pyload/config/default.conf @@ -0,0 +1,65 @@ +version: 1 + +remote - "Remote": + int port : "Port" = 7227 + ip listenaddr : "Address" = 0.0.0.0 + bool nolocalauth : "No authentication on local connections" = True + bool activated : "Activated" = True +ssl - "SSL": + bool activated : "Activated"= False + file cert : "SSL Certificate" = ssl.crt + file key : "SSL Key" = ssl.key +webinterface - "Web UI": + bool activated : "Activated" = True + builtin;threaded;fastcgi;lightweight server : "Server" = builtin + bool https : "Use HTTPS" = False + ip host : "IP" = 0.0.0.0 + int port : "Port" = 8001 + Default;Dark;Flat theme : "Theme" = Default + str prefix: "Path Prefix" = +log - "Log": + bool file_log : "File Log" = True + folder log_folder : "Folder" = Logs + int log_count : "Count" = 5 + int log_size : "Size in kb" = 100 + bool log_rotate : "Log Rotate" = True +general - "General": + en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR language : "Language" = en + folder download_folder : "Download Folder" = Downloads + bool debug_mode : "Debug Mode" = False + int min_free_space : "Min Free Space (MB)" = 200 + bool folder_per_package : "Create folder for each package" = True + int renice : "CPU Priority" = 0 + auto;common;pyv8;node;rhino;jsc jsengine : "JS Engine" = auto +download - "Download": + int chunks : "Max connections for one download" = 3 + int max_downloads : "Max Parallel Downloads" = 3 + int max_speed : "Max Download Speed in kb/s" = -1 + bool limit_speed : "Limit Download Speed" = False + str interface : "Download interface to bind (ip or Name)" = None + bool ipv6: "Allow IPv6" = False + bool skip_existing : "Skip already existing files" = False +permission - "Permissions": + bool change_user : "Change user of running process" = False + str user : "Username" = user + str folder : "Folder Permission mode" = 0755 + bool change_file : "Change file mode of downloads" = False + str file : "Filemode for Downloads" = 0644 + bool change_group : "Change group of running process" = False + str group : "Groupname" = users + bool change_dl : "Change Group and User of Downloads" = False +reconnect - "Reconnect": + bool activated : "Use Reconnect" = False + str method : "Method" = None + time startTime : "Start" = 0:00 + time endTime : "End" = 0:00 +downloadTime - "Download Time": + time start : "Start" = 0:00 + time end : "End" = 0:00 +proxy - "Proxy": + str address : "Address" = "localhost" + int port : "Port" = 7070 + http;socks4;socks5 type : "Protocol" = http + str username : "Username" = None + password password : "Password" = None + bool proxy : "Use Proxy" = False diff --git a/pyload/database/DatabaseBackend.py b/pyload/database/DatabaseBackend.py new file mode 100644 index 000000000..4b63dd284 --- /dev/null +++ b/pyload/database/DatabaseBackend.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from threading import Event, Thread +from os import remove +from os.path import exists +from shutil import move + +from Queue import Queue +from traceback import print_exc + +from pyload.utils import chmod + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + +DB_VERSION = 4 + +class style(object): + db = None + + @classmethod + def setDB(cls, db): + cls.db = db + + @classmethod + def inner(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return f(cls.db, *args, **kwargs) + return x + + @classmethod + def queue(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.queue(f, *args, **kwargs) + return x + + @classmethod + def async(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.async(f, *args, **kwargs) + return x + +class DatabaseJob(object): + def __init__(self, f, *args, **kwargs): + self.done = Event() + + self.f = f + self.args = args + self.kwargs = kwargs + + self.result = None + self.exception = False + +# import inspect +# self.frame = inspect.currentframe() + + def __repr__(self): + from os.path import basename + frame = self.frame.f_back + output = "" + for i in range(5): + output += "\t%s:%s, %s\n" % (basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) + frame = frame.f_back + del frame + del self.frame + + return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + + def processJob(self): + try: + self.result = self.f(*self.args, **self.kwargs) + except Exception, e: + print_exc() + try: + print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e + except Exception: + pass + + self.exception = e + finally: + self.done.set() + + def wait(self): + self.done.wait() + +class DatabaseBackend(Thread): + subs = [] + def __init__(self, core): + Thread.__init__(self) + self.setDaemon(True) + self.core = core + + self.jobs = Queue() + + self.setuplock = Event() + + style.setDB(self) + + def setup(self): + self.start() + self.setuplock.wait() + + def run(self): + """main loop, which executes commands""" + convert = self._checkVersion() #returns None or current version + + self.conn = sqlite3.connect("files.db") + chmod("files.db", 0600) + + self.c = self.conn.cursor() #compatibility + + if convert is not None: + self._convertDB(convert) + + self._createTables() + self._migrateUser() + + self.conn.commit() + + self.setuplock.set() + + while True: + j = self.jobs.get() + if j == "quit": + self.c.close() + self.conn.close() + break + j.processJob() + + @style.queue + def shutdown(self): + self.conn.commit() + self.jobs.put("quit") + + def _checkVersion(self): + """ check db version and delete it if needed""" + if not exists("files.version"): + f = open("files.version", "wb") + f.write(str(DB_VERSION)) + f.close() + return + + f = open("files.version", "rb") + v = int(f.read().strip()) + f.close() + if v < DB_VERSION: + if v < 2: + try: + self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + except Exception: + print "Filedatabase was deleted due to incompatible version." + remove("files.version") + move("files.db", "files.backup.db") + f = open("files.version", "wb") + f.write(str(DB_VERSION)) + f.close() + return v + + def _convertDB(self, v): + try: + getattr(self, "_convertV%i" % v)() + except Exception: + try: + self.core.log.error(_("Filedatabase could NOT be converted.")) + except Exception: + print "Filedatabase could NOT be converted." + + #convert scripts start----------------------------------------------------- + + def _convertV2(self): + self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + try: + self.manager.core.log.info(_("Database was converted from v2 to v3.")) + except Exception: + print "Database was converted from v2 to v3." + self._convertV3() + + def _convertV3(self): + self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + try: + self.manager.core.log.info(_("Database was converted from v3 to v4.")) + except Exception: + print "Database was converted from v3 to v4." + + #convert scripts end------------------------------------------------------- + + def _createTables(self): + """create tables for database""" + + self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') + self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + + self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ + SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ + FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ + (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ + FROM packages p JOIN links l ON p.id = l.package AND l.status in (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ + GROUP BY p.id') + + #try to lower ids + self.c.execute('SELECT max(id) FROM LINKS') + fid = self.c.fetchone()[0] + if fid: + fid = int(fid) + else: + fid = 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) + + + self.c.execute('SELECT max(id) FROM packages') + pid = self.c.fetchone()[0] + if pid: + pid = int(pid) + else: + pid = 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) + + self.c.execute('VACUUM') + + + def _migrateUser(self): + if exists("pyload.db"): + try: + self.core.log.info(_("Converting old Django DB")) + except Exception: + print "Converting old Django DB" + conn = sqlite3.connect('pyload.db') + c = conn.cursor() + c.execute("SELECT username, password, email from auth_user WHERE is_superuser") + users = [] + for r in c: + pw = r[1].split("$") + users.append((r[0], pw[1] + pw[2], r[2])) + c.close() + conn.close() + + self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) + move("pyload.db", "pyload.old.db") + + def createCursor(self): + return self.conn.cursor() + + @style.async + def commit(self): + self.conn.commit() + + @style.queue + def syncSave(self): + self.conn.commit() + + @style.async + def rollback(self): + self.conn.rollback() + + def async(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + + def queue(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + job.wait() + return job.result + + @classmethod + def registerSub(cls, klass): + cls.subs.append(klass) + + @classmethod + def unregisterSub(cls, klass): + cls.subs.remove(klass) + + def __getattr__(self, attr): + for sub in DatabaseBackend.subs: + if hasattr(sub, attr): + return getattr(sub, attr) diff --git a/pyload/database/File.py b/pyload/database/File.py new file mode 100644 index 000000000..0b22805a7 --- /dev/null +++ b/pyload/database/File.py @@ -0,0 +1,875 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from threading import RLock +from time import time + +from pyload.utils import formatSize, lock +from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.datatype.Package import PyPackage +from pyload.datatype.File import PyFile +from pyload.database import style, DatabaseBackend + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + + +class FileHandler(object): + """Handles all request made to obtain information, + modify status or other request for links or packages""" + + def __init__(self, core): + """Constructor""" + self.core = core + + # translations + self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] + + self.cache = {} #holds instances for files + self.packageCache = {} # same for packages + #@TODO: purge the cache + + self.jobCache = {} + + self.lock = RLock() #@TODO should be a Lock w/o R + #self.lock._Verbose__verbose = True + + self.filecount = -1 # if an invalid value is set get current value from db + self.queuecount = -1 #number of package to be loaded + self.unchanged = False #determines if any changes was made since last call + + self.db = self.core.db + + def change(func): + def new(*args): + args[0].unchanged = False + args[0].filecount = -1 + args[0].queuecount = -1 + args[0].jobCache = {} + return func(*args) + return new + + #-------------------------------------------------------------------------- + def save(self): + """saves all data to backend""" + self.db.commit() + + #-------------------------------------------------------------------------- + def syncSave(self): + """saves all data to backend and waits until all data are written""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + pyfile.sync() + + pypacks = self.packageCache.values() + for pypack in pypacks: + pypack.sync() + + self.db.syncSave() + + @lock + def getCompleteData(self, queue=1): + """gets a complete data representation""" + + data = self.db.getAllLinks(queue) + packs = self.db.getAllPackages(queue) + + data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) + + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: continue + packs[x.id].update(x.toDict()[x.id]) + + for key, value in data.iteritems(): + if value["package"] in packs: + packs[value["package"]]["links"][key] = value + + return packs + + @lock + def getInfoData(self, queue=1): + """gets a data representation without links""" + + packs = self.db.getAllPackages(queue) + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: continue + packs[x.id].update(x.toDict()[x.id]) + + return packs + + @lock + @change + def addLinks(self, urls, package): + """adds links""" + + self.core.addonManager.dispatchEvent("links-added", urls, package) + + data = self.core.pluginManager.parseUrls(urls) + + self.db.addLinks(data, package) + self.core.threadManager.createInfoThread(data, package) + + #@TODO change from reloadAll event to package update event + self.core.pullManager.addEvent(ReloadAllEvent("collector")) + + #-------------------------------------------------------------------------- + @lock + @change + def addPackage(self, name, folder, queue=0): + """adds a package, default to link collector""" + lastID = self.db.addPackage(name, folder, queue) + p = self.db.getPackage(lastID) + e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") + self.core.pullManager.addEvent(e) + return lastID + + #-------------------------------------------------------------------------- + @lock + @change + def deletePackage(self, id): + """delete package and all contained links""" + + p = self.getPackage(id) + if not p: + if id in self.packageCache: del self.packageCache[id] + return + + oldorder = p.order + queue = p.queue + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + + pyfiles = self.cache.values() + + for pyfile in pyfiles: + if pyfile.packageid == id: + pyfile.abortDownload() + pyfile.release() + + self.db.deletePackage(p) + self.core.pullManager.addEvent(e) + self.core.addonManager.dispatchEvent("package-deleted", id) + + if id in self.packageCache: + del self.packageCache[id] + + packs = self.packageCache.values() + for pack in packs: + if pack.queue == queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + #-------------------------------------------------------------------------- + @lock + @change + def deleteLink(self, id): + """deletes links""" + + f = self.getFile(id) + if not f: + return None + + pid = f.packageid + e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") + + oldorder = f.order + + if id in self.core.threadManager.processingIds(): + self.cache[id].abortDownload() + + if id in self.cache: + del self.cache[id] + + self.db.deleteLink(f) + + self.core.pullManager.addEvent(e) + + p = self.getPackage(pid) + if not len(p.getChildren()): + p.delete() + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == pid and pyfile.order > oldorder: + pyfile.order -= 1 + pyfile.notifyChange() + + #-------------------------------------------------------------------------- + def releaseLink(self, id): + """removes pyfile from cache""" + if id in self.cache: + del self.cache[id] + + #-------------------------------------------------------------------------- + def releasePackage(self, id): + """removes package from cache""" + if id in self.packageCache: + del self.packageCache[id] + + #-------------------------------------------------------------------------- + def updateLink(self, pyfile): + """updates link""" + self.db.updateLink(pyfile) + + e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") + self.core.pullManager.addEvent(e) + + #-------------------------------------------------------------------------- + def updatePackage(self, pypack): + """updates a package""" + self.db.updatePackage(pypack) + + e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") + self.core.pullManager.addEvent(e) + + #-------------------------------------------------------------------------- + def getPackage(self, id): + """return package instance""" + + if id in self.packageCache: + return self.packageCache[id] + else: + return self.db.getPackage(id) + + #-------------------------------------------------------------------------- + def getPackageData(self, id): + """returns dict with package information""" + pack = self.getPackage(id) + + if not pack: + return None + + pack = pack.toDict()[id] + + data = self.db.getPackageData(id) + + tmplist = [] + + cache = self.cache.values() + for x in cache: + if int(x.toDbDict()[x.id]["package"]) == int(id): + tmplist.append((x.id, x.toDbDict()[x.id])) + data.update(tmplist) + + pack["links"] = data + + return pack + + #-------------------------------------------------------------------------- + def getFileData(self, id): + """returns dict with file information""" + if id in self.cache: + return self.cache[id].toDbDict() + + return self.db.getLinkData(id) + + #-------------------------------------------------------------------------- + def getFile(self, id): + """returns pyfile instance""" + if id in self.cache: + return self.cache[id] + else: + return self.db.getFile(id) + + #-------------------------------------------------------------------------- + @lock + def getJob(self, occ): + """get suitable job""" + + #@TODO clean mess + #@TODO improve selection of valid jobs + + if occ in self.jobCache: + if self.jobCache[occ]: + id = self.jobCache[occ].pop() + if id == "empty": + pyfile = None + self.jobCache[occ].append("empty") + else: + pyfile = self.getFile(id) + else: + jobs = self.db.getJob(occ) + jobs.reverse() + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + self.jobCache[occ].extend(jobs) + pyfile = self.getFile(self.jobCache[occ].pop()) + + else: + self.jobCache = {} #better not caching to much + jobs = self.db.getJob(occ) + jobs.reverse() + self.jobCache[occ] = jobs + + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + pyfile = self.getFile(self.jobCache[occ].pop()) + + #@TODO: maybe the new job has to be approved... + + + #pyfile = self.getFile(self.jobCache[occ].pop()) + return pyfile + + @lock + def getDecryptJob(self): + """return job for decrypting""" + if "decrypt" in self.jobCache: + return None + + plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = str(tuple(plugins)) + + jobs = self.db.getPluginJob(plugins) + if jobs: + return self.getFile(jobs[0]) + else: + self.jobCache["decrypt"] = "empty" + return None + + def getFileCount(self): + """returns number of files""" + + if self.filecount == -1: + self.filecount = self.db.filecount(1) + + return self.filecount + + def getQueueCount(self, force=False): + """number of files that have to be processed""" + if self.queuecount == -1 or force: + self.queuecount = self.db.queuecount(1) + + return self.queuecount + + def checkAllLinksFinished(self): + """checks if all files are finished and dispatch event""" + + if not self.getQueueCount(True): + self.core.addonManager.dispatchEvent("all_downloads-finished") + self.core.log.debug("All downloads finished") + return True + + return False + + def checkAllLinksProcessed(self, fid): + """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" + + # reset count so statistic will update (this is called when dl was processed) + self.resetCount() + + if not self.db.processcount(1, fid): + self.core.addonManager.dispatchEvent("all_downloads-processed") + self.core.log.debug("All downloads processed") + return True + + return False + + def resetCount(self): + self.queuecount = -1 + + @lock + @change + def restartPackage(self, id): + """restart package""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == id: + self.restartFile(pyfile.id) + + self.db.restartPackage(id) + + if id in self.packageCache: + self.packageCache[id].setFinished = False + + e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def restartFile(self, id): + """ restart file""" + if id in self.cache: + self.cache[id].status = 3 + self.cache[id].name = self.cache[id].url + self.cache[id].error = "" + self.cache[id].abortDownload() + + + self.db.restartFile(id) + + e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def setPackageLocation(self, id, queue): + """push package to queue""" + + p = self.db.getPackage(id) + oldorder = p.order + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.clearPackageOrder(p) + + p = self.db.getPackage(id) + + p.queue = queue + self.db.updatePackage(p) + + self.db.reorderPackage(p, -1, True) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + self.db.commit() + self.releasePackage(id) + p = self.getPackage(id) + + e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def reorderPackage(self, id, position): + p = self.getPackage(id) + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + self.db.reorderPackage(p, position) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != p.queue or pack.order < 0 or pack == p: continue + if p.order > position: + if pack.order >= position and pack.order < p.order: + pack.order += 1 + pack.notifyChange() + elif p.order < position: + if pack.order <= position and pack.order > p.order: + pack.order -= 1 + pack.notifyChange() + + p.order = position + self.db.commit() + + e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def reorderFile(self, id, position): + f = self.getFileData(id) + f = f[id] + + e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.reorderLink(f, position) + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid != f["package"] or pyfile.order < 0: continue + if f["order"] > position: + if pyfile.order >= position and pyfile.order < f["order"]: + pyfile.order += 1 + pyfile.notifyChange() + elif f["order"] < position: + if pyfile.order <= position and pyfile.order > f["order"]: + pyfile.order -= 1 + pyfile.notifyChange() + + if id in self.cache: + self.cache[id].order = position + + self.db.commit() + + e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") + self.core.pullManager.addEvent(e) + + @change + def updateFileInfo(self, data, pid): + """ updates file info (name, size, status, url)""" + ids = self.db.updateLinkInfo(data) + e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") + self.core.pullManager.addEvent(e) + + def checkPackageFinished(self, pyfile): + """ checks if package is finished and calls AddonManager """ + + ids = self.db.getUnfinished(pyfile.packageid) + if not ids or (pyfile.id in ids and len(ids) == 1): + if not pyfile.package().setFinished: + self.core.log.info(_("Package finished: %s") % pyfile.package().name) + self.core.addonManager.packageFinished(pyfile.package()) + pyfile.package().setFinished = True + + + def reCheckPackage(self, pid): + """ recheck links in package """ + data = self.db.getPackageData(pid) + + urls = [] + + for pyfile in data.itervalues(): + if pyfile["status"] not in (0, 12, 13): + urls.append((pyfile["url"], pyfile["plugin"])) + + self.core.threadManager.createInfoThread(urls, pid) + + @lock + @change + def deleteFinishedLinks(self): + """ deletes finished links and packages, return deleted packages """ + + old_packs = self.getInfoData(0) + old_packs.update(self.getInfoData(1)) + + self.db.deleteFinished() + + new_packs = self.db.getAllPackages(0) + new_packs.update(self.db.getAllPackages(1)) + #get new packages only from db + + deleted = [] + for id in old_packs.iterkeys(): + if id not in new_packs: + deleted.append(id) + self.deletePackage(int(id)) + + return deleted + + @lock + @change + def restartFailed(self): + """ restart all failed links """ + self.db.restartFailed() + +class FileMethods(object): + @style.queue + def filecount(self, queue): + """returns number of files in queue""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue,)) + return self.c.fetchone()[0] + + @style.queue + def queuecount(self, queue): + """ number of files in queue not finished yet""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", (queue,)) + return self.c.fetchone()[0] + + @style.queue + def processcount(self, queue, fid): + """ number of files which have to be proccessed """ + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", (queue, str(fid))) + return self.c.fetchone()[0] + + @style.inner + def _nextPackageOrder(self, queue=0): + self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + @style.inner + def _nextFileOrder(self, package): + self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + @style.queue + def addLink(self, url, name, plugin, package): + order = self._nextFileOrder(package) + self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, (plugintype, pluginname), package, order)) + return self.c.lastrowid + + @style.queue + def addLinks(self, links, package): + """ links is a list of tupels (url, plugin)""" + order = self._nextFileOrder(package) + orders = [order + x for x in range(len(links))] + links = [(x[0], x[0], (x[1], x[2]), package, o) for x, o in zip(links, orders)] + self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + + @style.queue + def addPackage(self, name, folder, queue): + order = self._nextPackageOrder(queue) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + return self.c.lastrowid + + @style.queue + def deletePackage(self, p): + + self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) + self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) + + @style.queue + def deleteLink(self, f): + + self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) + + + @style.queue + def getAllLinks(self, q): + """return information about all links in queue q + + q0 queue + q1 collector + + format: + + { + id: {'name': name, ... 'package': id }, ... + } + + """ + self.c.execute('SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + @style.queue + def getAllPackages(self, q): + """return information about packages in queue q + (only useful in get all data) + + q0 queue + q1 collector + + format: + + { + id: {'name': name ... 'links': {}}, ... + } + """ + self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ + FROM packages p JOIN pstats s ON p.id = s.id \ + WHERE p.queue=? ORDER BY p.packageorder', str(q)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'name': r[1], + 'folder': r[2], + 'site': r[3], + 'password': r[4], + 'queue': r[5], + 'order': r[6], + 'sizetotal': int(r[7]), + 'sizedone': r[8] if r[8] else 0, #these can be None + 'linksdone': r[9] if r[9] else 0, + 'linkstotal': r[10], + 'links': {} + } + + return data + + @style.queue + def getLinkData(self, id): + """get link information as dict""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', (str(id),)) + data = {} + r = self.c.fetchone() + if not r: + return None + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + @style.queue + def getPackageData(self, id): + """get data about links for a package""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.async + def updateLink(self, f): + self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) + + @style.queue + def updatePackage(self, p): + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) + + @style.queue + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status, url) """ + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) + ids = [] + self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) + for r in self.c: + ids.append(int(r[0])) + return ids + + @style.queue + def reorderPackage(self, p, position, noMove=False): + if position == -1: + position = self._nextPackageOrder(p.queue) + if not noMove: + if p.order > position: + self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) + elif p.order < position: + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) + + @style.queue + def reorderLink(self, f, position): + """ reorder link with f as dict for pyfile """ + if f["order"] > position: + self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', (position, f["order"], f["package"])) + elif f["order"] < position: + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"])) + + self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) + + @style.queue + def clearPackageOrder(self, p): + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) + + @style.async + def restartFile(self, id): + self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) + + @style.async + def restartPackage(self, id): + self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) + + @style.queue + def getPackage(self, id): + """return package instance from id""" + self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: return None + return PyPackage(self.manager, id, * r) + + #-------------------------------------------------------------------------- + @style.queue + def getFile(self, id): + """return link instance from id""" + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: return None + return PyFile(self.manager, id, * r) + + + @style.queue + def getJob(self, occ): + """return pyfile ids, which are suitable for download and dont use a occupied plugin""" + + #@TODO improve this hardcoded method + pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #plugins which are processed in collector + + cmd = "(" + for i, item in enumerate(occ): + if i: cmd += ", " + cmd += "'%s'" % item + + cmd += ")" + + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) + + self.c.execute(cmd) # very bad! + + return [x[0] for x in self.c] + + @style.queue + def getPluginJob(self, plugins): + """returns pyfile ids with suited plugins""" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + + self.c.execute(cmd) # very bad! + + return [x[0] for x in self.c] + + @style.queue + def getUnfinished(self, pid): + """return list of max length 3 ids with pyfiles in package not finished or processed""" + + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) + return [r[0] for r in self.c] + + @style.queue + def deleteFinished(self): + self.c.execute("DELETE FROM links WHERE status IN (0, 4)") + self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + + @style.queue + def restartFailed(self): + self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") + + @style.queue + def findDuplicates(self, id, folder, filename): + """ checks if filename exists with different id and same package """ + self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) + return self.c.fetchone() + + @style.queue + def purgeLinks(self): + self.c.execute("DELETE FROM links;") + self.c.execute("DELETE FROM packages;") + +DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py new file mode 100644 index 000000000..75e166d39 --- /dev/null +++ b/pyload/database/Storage.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from pyload.database import style +from pyload.database import DatabaseBackend + +class StorageMethods(object): + @style.queue + def setStorage(db, identifier, key, value): + db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + if db.c.fetchone() is not None: + db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + else: + db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + + @style.queue + def getStorage(db, identifier, key=None): + if key is not None: + db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + row = db.c.fetchone() + if row is not None: + return row[0] + else: + db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) + d = {} + for row in db.c: + d[row[0]] = row[1] + return d + + @style.queue + def delStorage(db, identifier, key): + db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + +DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/database/User.py b/pyload/database/User.py new file mode 100644 index 000000000..67cb62ab9 --- /dev/null +++ b/pyload/database/User.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from hashlib import sha1 +import random + +from pyload.database.DatabaseBackend import DatabaseBackend +from pyload.database.DatabaseBackend import style + +class UserMethods(object): + @style.queue + def checkAuth(db, user, password): + c = db.c + c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) + r = c.fetchone() + if not r: + return {} + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return {"id": r[0], "name": r[1], "role": r[3], + "permission": r[4], "template": r[5], "email": r[6]} + else: + return {} + + @style.queue + def addUser(db, user, password): + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + h = sha1(salt + password) + password = salt + h.hexdigest() + + c = db.c + c.execute('SELECT name FROM users WHERE name=?', (user,)) + if c.fetchone() is not None: + c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + else: + c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + + + @style.queue + def changePassword(db, user, oldpw, newpw): + db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) + r = db.c.fetchone() + if not r: + return False + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + oldpw) + if h.hexdigest() == pw: + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + h = sha1(salt + newpw) + password = salt + h.hexdigest() + + db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + return True + + return False + + + @style.async + def setPermission(db, user, perms): + db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + + @style.async + def setRole(db, user, role): + db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + + + @style.queue + def listUsers(db): + db.c.execute('SELECT name FROM users') + users = [] + for row in db.c: + users.append(row[0]) + return users + + @style.queue + def getAllUserData(db): + db.c.execute("SELECT name, permission, role, template, email FROM users") + user = {} + for r in db.c: + user[r[0]] = {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} + + return user + + @style.queue + def removeUser(db, user): + db.c.execute('DELETE FROM users WHERE name=?', (user,)) + +DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py new file mode 100644 index 000000000..64f049be1 --- /dev/null +++ b/pyload/database/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +from pyload.database.DatabaseBackend import DatabaseBackend, style + +from pyload.database.File import FileHandler +from pyload.database.User import UserMethods +from pyload.database.Storage import StorageMethods diff --git a/pyload/datatype/File.py b/pyload/datatype/File.py new file mode 100644 index 000000000..1df0a8590 --- /dev/null +++ b/pyload/datatype/File.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from pyload.manager.Event import UpdateEvent +from pyload.utils import formatSize, lock + +from time import sleep, time + +from threading import RLock + +statusMap = { + "finished": 0, + "offline": 1, + "online": 2, + "queued": 3, + "skipped": 4, + "waiting": 5, + "temp. offline": 6, + "starting": 7, + "failed": 8, + "aborted": 9, + "decrypting": 10, + "custom": 11, + "downloading": 12, + "processing": 13, + "unknown": 14, +} + + +def setSize(self, value): + self._size = int(value) + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "plugin", + "packageid", "error", "order", "lock", "plugin", "waitUntil", + "active", "abort", "statusname", "reconnected", "progress", + "maxprogress", "pluginmodule", "pluginclass") + + def __init__(self, manager, id, url, name, size, status, error, plugin, package, order): + self.m = manager + + self.id = int(id) + self.url = url + self.name = name + self.size = size + self.status = status + self.plugin = self.plugintype, self.pluginname = plugin + self.packageid = package #should not be used, use package() instead + self.error = error + self.order = order + # database information ends here + + self.lock = RLock() + + self.plugin = None + #self.download = None + + self.waitUntil = 0 # time() + time to wait + + # status attributes + self.active = False #obsolete? + self.abort = False + self.reconnected = False + + self.statusname = None + + self.progress = 0 + self.maxprogress = 100 + + self.m.cache[int(id)] = self + + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + def __repr__(self): + return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginmodule = self.m.core.pluginManager.getPlugin(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.plugintype, self.pluginname)) + self.plugin = self.pluginclass(self) + + @lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute + + :return: + """ + return hasattr(self, "plugin") and self.plugin + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + def setStatus(self, status): + self.status = statusMap[status] + self.sync() #@TODO needed aslong no better job approving exists + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + def hasStatus(self, status): + return statusMap[status] == self.status + + def sync(self): + """sync PyFile instance with database""" + self.m.updateLink(self) + + @lock + def release(self): + """sync and remove from cache""" + # file has valid package + if self.packageid > 0: + self.sync() + + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseLink(self.id) + + def delete(self): + """delete pyfile from database""" + self.m.deleteLink(self.id) + + def toDict(self): + """return dict with all information for interface""" + return self.toDbDict() + + def toDbDict(self): + """return data as dict for databse + + format: + + { + id: {'url': url, 'name': name ... } + } + + """ + return { + self.id: { + 'id': self.id, + 'url': self.url, + 'name': self.name, + 'plugin': self.pluginname, + 'size': self.getSize(), + 'format_size': self.formatSize(), + 'status': self.status, + 'statusmsg': self.getStatusName(), + 'package': self.packageid, + 'error': self.error, + 'order': self.order + } + } + + def abortDownload(self): + """abort pyfile if possible""" + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + sleep(0.1) + + self.abort = False + if self.hasPlugin() and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + def formatWait(self): + """ formats and return wait time in humanreadable format """ + seconds = self.waitUntil - time() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def formatSize(self): + """ formats size to readable format """ + return formatSize(self.getSize()) + + def formatETA(self): + """ formats eta to readable format """ + seconds = self.getETA() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except Exception: + return 0 + + def getETA(self): + """ gets established time of arrival""" + try: + return self.getBytesLeft() / self.getSpeed() + except Exception: + return 0 + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.getSize() - self.plugin.req.arrived + except Exception: + return 0 + + def getPercent(self): + """ get % of download """ + if self.status == 12: + try: + return self.plugin.req.percent + except Exception: + return 0 + else: + return self.progress + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except Exception: + return self.size + + def notifyChange(self): + e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") + self.m.core.pullManager.addEvent(e) + + def setProgress(self, value): + if not value == self.progress: + self.progress = value + self.notifyChange() diff --git a/pyload/datatype/Package.py b/pyload/datatype/Package.py new file mode 100644 index 000000000..bf3edffea --- /dev/null +++ b/pyload/datatype/Package.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from pyload.manager.Event import UpdateEvent +from pyload.utils import safe_filename + +class PyPackage(object): + """ + Represents a package object at runtime + """ + def __init__(self, manager, id, name, folder, site, password, queue, order): + self.m = manager + self.m.packageCache[int(id)] = self + + self.id = int(id) + self.name = name + self._folder = folder + self.site = site + self.password = password + self.queue = queue + self.order = order + self.setFinished = False + + @property + def folder(self): + return safe_filename(self._folder) + + def toDict(self): + """ Returns a dictionary representation of the data. + + :return: dict: {id: { attr: value }} + """ + return { + self.id: { + 'id': self.id, + 'name': self.name, + 'folder': self.folder, + 'site': self.site, + 'password': self.password, + 'queue': self.queue, + 'order': self.order, + 'links': {} + } + } + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.id)["links"] + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + def delete(self): + self.m.deletePackage(self.id) + + def notifyChange(self): + e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") + self.m.core.pullManager.addEvent(e) diff --git a/pyload/datatype/__init__.py b/pyload/datatype/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/datatype/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/manager/Account.py b/pyload/manager/Account.py new file mode 100644 index 000000000..3acf70311 --- /dev/null +++ b/pyload/manager/Account.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from os.path import exists +from shutil import copy + +from threading import Lock + +from pyload.manager.Event import AccountUpdateEvent +from pyload.utils import chmod, lock + +ACC_VERSION = 1 + + +class AccountManager(object): + """manages all accounts""" + + #---------------------------------------------------------------------- + def __init__(self, core): + """Constructor""" + + self.core = core + self.lock = Lock() + + self.initPlugins() + self.saveAccounts() # save to add categories to conf + + + def initPlugins(self): + self.accounts = {} # key = ( plugin ) + self.plugins = {} + + self.initAccountPlugins() + self.loadAccounts() + + + def getAccountPlugin(self, plugin): + """get account instance for plugin or None if anonymous""" + try: + if plugin in self.accounts: + if plugin not in self.plugins: + klass = self.core.pluginManager.loadClass("accounts", plugin) + if klass: + self.plugins[plugin] = klass(self, self.accounts[plugin]) + else: #@NOTE: The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash + raise + + return self.plugins[plugin] + else: + raise + except Exception: + return None + + + def getAccountPlugins(self): + """ get all account instances""" + + plugins = [] + for plugin in self.accounts.keys(): + plugins.append(self.getAccountPlugin(plugin)) + + return plugins + + + #---------------------------------------------------------------------- + def loadAccounts(self): + """loads all accounts available""" + + try: + with open("accounts.conf", "a+") as f: + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f.seek(0) + f.write("version: " + str(ACC_VERSION)) + + self.core.log.warning(_("Account settings deleted, due to new config format")) + return + + except IOError, e: + self.core.log.error(str(e)) + return + + plugin = "" + name = "" + + for line in content[1:]: + line = line.strip() + + if not line: continue + if line.startswith("#"): continue + if line.startswith("version"): continue + + if line.endswith(":") and line.count(":") == 1: + plugin = line[:-1] + self.accounts[plugin] = {} + + elif line.startswith("@"): + try: + option = line[1:].split() + self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + except Exception: + pass + + elif ":" in line: + name, sep, pw = line.partition(":") + self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} + + + #---------------------------------------------------------------------- + def saveAccounts(self): + """save all account information""" + + try: + with open("accounts.conf", "wb") as f: + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin + ":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data['password']) ) + if data['options']: + for option, values in data['options'].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + chmod(f.name, 0600) + + except Exception, e: + self.core.log.error(str(e)) + + + #---------------------------------------------------------------------- + def initAccountPlugins(self): + """init names""" + for name in self.core.pluginManager.getAccountPlugins(): + self.accounts[name] = {} + + + @lock + def updateAccount(self, plugin , user, password=None, options={}): + """add or update account""" + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + updated = p.updateAccounts(user, password, options) + #since accounts is a ref in plugin self.accounts doesnt need to be updated here + + self.saveAccounts() + if updated: p.scheduleRefresh(user, force=False) + + + @lock + def removeAccount(self, plugin, user): + """remove account""" + + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + p.removeAccount(user) + + self.saveAccounts() + + + @lock + def getAccountInfos(self, force=True, refresh=False): + data = {} + + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + force = False + + for p in self.accounts.keys(): + if self.accounts[p]: + p = self.getAccountPlugin(p) + if p: + data[p.__name__] = p.getAllAccounts(force) + else: #@NOTE: When an account has been skipped, p is None + data[p] = [] + else: + data[p] = [] + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) + return data + + + def sendChange(self): + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) diff --git a/pyload/manager/Addon.py b/pyload/manager/Addon.py new file mode 100644 index 000000000..d293109c7 --- /dev/null +++ b/pyload/manager/Addon.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay +# @interface-version: 0.1 + +import __builtin__ + +import traceback +from threading import RLock, Thread + +from types import MethodType + +from pyload.manager.thread.Addon import AddonThread +from pyload.manager.Plugin import literal_eval +from pyload.utils import lock + + +class AddonManager(object): + """Manages addons, delegates and handles Events. + + Every plugin can define events, \ + but some very usefull events are called by the Core. + Contrary to overwriting addon methods you can use event listener, + which provides additional entry point in the control flow. + Only do very short tasks or use threads. + + **Known Events:** + Most addon methods exists as events. These are the additional known events. + + ======================= ============== ================================== + Name Arguments Description + ======================= ============== ================================== + download-preparing fid A download was just queued and will be prepared now. + download-start fid A plugin will immediately starts the download afterwards. + links-added links, pid Someone just added links, you are able to modify the links. + all_downloads-processed Every link was handled, pyload would idle afterwards. + all_downloads-finished Every download in queue is finished. + config-changed The config was changed via the api. + pluginConfigChanged The plugin config changed, due to api or internal process. + ======================= ============== ================================== + + | Notes: + | all_downloads-processed is *always* called before all_downloads-finished. + | config-changed is *always* called before pluginConfigChanged. + + + """ + + def __init__(self, core): + self.core = core + + __builtin__.addonManager = self #: needed to let addons register themself + + self.plugins = [] + self.pluginMap = {} + self.methods = {} #: dict of names and list of methods usable by rpc + + self.events = {} #: contains events + + # registering callback for config event + self.core.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) #@TODO: Rename event pluginConfigChanged + + self.addEvent("pluginConfigChanged", self.manageAddon) + + self.lock = RLock() + self.createIndex() + + + def try_catch(func): + + def new(*args): + try: + return func(*args) + except Exception, e: + args[0].log.error(_("Error executing addon: %s") % e) + if args[0].core.debug: + traceback.print_exc() + + return new + + + def addRPC(self, plugin, func, doc): + plugin = plugin.rpartition(".")[2] + doc = doc.strip() if doc else "" + + if plugin in self.methods: + self.methods[plugin][func] = doc + else: + self.methods[plugin] = {func: doc} + + + def callRPC(self, plugin, func, args, parse): + if not args: + args = tuple() + if parse: + args = tuple([literal_eval(x) for x in args]) + plugin = self.pluginMap[plugin] + f = getattr(plugin, func) + return f(*args) + + + def createIndex(self): + plugins = [] + active = [] + deactive = [] + + for pluginname in self.core.pluginManager.addonPlugins: + try: + # hookClass = getattr(plugin, plugin.__name__) + if self.core.config.getPlugin(pluginname, "activated"): + pluginClass = self.core.pluginManager.loadClass("addon", pluginname) + if not pluginClass: + continue + + plugin = pluginClass(self.core, self) + plugins.append(plugin) + self.pluginMap[pluginClass.__name__] = plugin + if plugin.isActivated(): + active.append(pluginClass.__name__) + else: + deactive.append(pluginname) + + except Exception: + self.core.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) + if self.core.debug: + traceback.print_exc() + + self.core.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) + self.core.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) + + self.plugins = plugins + + + def manageAddon(self, plugin, name, value): + if name == "activated" and value: + self.activateAddon(plugin) + + elif name == "activated" and not value: + self.deactivateAddon(plugin) + + + def activateAddon(self, pluginname): + # check if already loaded + for inst in self.plugins: + if inst.__name__ == pluginname: + return + + pluginClass = self.core.pluginManager.loadClass("addon", pluginname) + + if not pluginClass: + return + + self.core.log.debug("Activate addon: %s" % pluginname) + + addon = pluginClass(self.core, self) + self.plugins.append(addon) + self.pluginMap[pluginClass.__name__] = addon + + addon.activate() + + + def deactivateAddon(self, pluginname): + for plugin in self.plugins: + if plugin.__name__ == pluginname: + addon = plugin + break + else: + return + + self.core.log.debug("Deactivate addon: %s" % pluginname) + + addon.deactivate() + + #remove periodic call + self.core.log.debug("Removed callback: %s" % self.core.scheduler.removeJob(addon.cb)) + + self.plugins.remove(addon) + del self.pluginMap[addon.__name__] + + + @try_catch + def coreReady(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.activate() + + self.dispatchEvent("addon-start") + + + @try_catch + def coreExiting(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.exit() + + self.dispatchEvent("addon-exit") + + + @lock + def downloadPreparing(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadPreparing(pyfile) + + self.dispatchEvent("download-preparing", pyfile) + + + @lock + def downloadFinished(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFinished(pyfile) + + self.dispatchEvent("download-finished", pyfile) + + + @lock + @try_catch + def downloadFailed(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFailed(pyfile) + + self.dispatchEvent("download-failed", pyfile) + + + @lock + def packageFinished(self, package): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.packageFinished(package) + + self.dispatchEvent("package-finished", package) + + + @lock + def beforeReconnecting(self, ip): + for plugin in self.plugins: + plugin.beforeReconnecting(ip) + + self.dispatchEvent("beforeReconnecting", ip) + + + @lock + def afterReconnecting(self, ip): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.afterReconnecting(ip) + + self.dispatchEvent("afterReconnecting", ip) + + + def startThread(self, function, *args, **kwargs): + return AddonThread(self.core.threadManager, function, args, kwargs) + + + def activePlugins(self): + """ returns all active plugins """ + return [x for x in self.plugins if x.isActivated()] + + + def getAllInfo(self): + """returns info stored by addon plugins""" + info = {} + for name, plugin in self.pluginMap.iteritems(): + if plugin.info: + # copy and convert so str + info[name] = dict( + [(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + return info + + + def getInfo(self, plugin): + info = {} + if plugin in self.pluginMap and self.pluginMap[plugin].info: + info = dict((x, str(y) if not isinstance(y, basestring) else y) + for x, y in self.pluginMap[plugin].info.iteritems()) + return info + + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + self.events[event].append(func) + else: + self.events[event] = [func] + + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.core.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + traceback.print_exc() diff --git a/pyload/manager/Captcha.py b/pyload/manager/Captcha.py new file mode 100644 index 000000000..e54eacf30 --- /dev/null +++ b/pyload/manager/Captcha.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from time import time +from traceback import print_exc +from threading import Lock + +from pyload.utils import encode + + +class CaptchaManager(object): + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] # task store, for outgoing tasks only + self.ids = 0 # only for internal purpose + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + def removeTask(self, task): + self.lock.acquire() + if task in self.tasks: + self.tasks.remove(task) + self.lock.release() + + def getTask(self): + self.lock.acquire() + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + self.lock.release() + return task + self.lock.release() + return None + + def getTaskByID(self, tid): + self.lock.acquire() + for task in self.tasks: + if task.id == str(tid): # task ids are strings + self.lock.release() + return task + self.lock.release() + return None + + def handleCaptcha(self, task, timeout=50): + cli = self.core.isClientConnected() + + if cli: #: client connected -> should solve the captcha + task.setWaiting(timeout) #wait 50 sec for response + + for plugin in self.core.addonManager.activePlugins(): + try: + plugin.captchaTask(task) + except Exception: + if self.core.debug: + print_exc() + + if task.handler or cli: #: the captcha was handled + self.tasks.append(task) + return True + task.error = _("No Client connected for captcha decrypting") + return False + + +class CaptchaTask(object): + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #: the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None # error message + self.status = "init" + self.data = {} # handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except Exception: + self.result = None + + def getResult(self): + return encode(self.result) + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or self.timedOut(): + return False + else: + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + for x in self.handler: + x.captchaInvalid(self) + + def correct(self): + for x in self.handler: + x.captchaCorrect(self) + + def __str__(self): + return "" % self.id diff --git a/pyload/manager/Event.py b/pyload/manager/Event.py new file mode 100644 index 000000000..20897290e --- /dev/null +++ b/pyload/manager/Event.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from time import time +from pyload.utils import uniqify + +class PullManager(object): + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + +class Client(object): + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event) + +class UpdateEvent(object): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["update", self.destination, self.type, self.id] + +class RemoveEvent(object): + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["remove", self.destination, self.type, self.id] + +class InsertEvent(object): + def __init__(self, itype, iid, after, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.after = after + self.destination = destination + + def toList(self): + return ["insert", self.destination, self.type, self.id, self.after] + +class ReloadAllEvent(object): + def __init__(self, destination): + assert destination == "queue" or destination == "collector" + self.destination = destination + + def toList(self): + return ["reload", self.destination] + +class AccountUpdateEvent(object): + def toList(self): + return ["account"] + +class ConfigUpdateEvent(object): + def toList(self): + return ["config"] diff --git a/pyload/manager/Plugin.py b/pyload/manager/Plugin.py new file mode 100644 index 000000000..879430be2 --- /dev/null +++ b/pyload/manager/Plugin.py @@ -0,0 +1,404 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import sys + +from itertools import chain +from os import listdir, makedirs +from os.path import isdir, isfile, join, exists, abspath +from sys import version_info +from traceback import print_exc +from urllib import unquote + +from SafeEval import const_eval as literal_eval + + +class PluginManager(object): + ROOT = "pyload.plugin." + USERROOT = "userplugins." + TYPES = ["account", "addon", "container", "crypter", "hook", "hoster", "internal", "ocr"] + + PATTERN = re.compile(r'__pattern__\s*=\s*u?r("|\')([^"\']+)') + VERSION = re.compile(r'__version__\s*=\s*("|\')([\d.]+)') + CONFIG = re.compile(r'__config__\s*=\s*\[([^\]]+)', re.M) + DESC = re.compile(r'__description__\s*=\s*("|"""|\')([^"\']+)') + + + def __init__(self, core): + self.core = core + + self.plugins = {} + self.createIndex() + + #register for import addon + sys.meta_path.append(self) + + + def loadTypes(self): + rootdir = join(pypath, "pyload", "plugins") + userdir = "userplugins" + + types = set().union(*[[d for d in listdir(p) if isdir(join(p, d))] + for p in (rootdir, userdir) if exists(p)]) + + if not types: + self.log.critical(_("No plugins found!")) + + self.TYPES = list(set(self.TYPES) | types) + + + def createIndex(self): + """create information for all plugins available""" + + sys.path.append(abspath("")) + + self.loadTypes() + + for type in self.TYPES: + self.plugins[type] = self.parse(type) + setattr(self, "%sPlugins" % type, self.plugins[type]) + + self.plugins['addon'] = self.addonPlugins.update(self.hookPlugins) + + self.core.log.debug("Created index of plugins") + + + def parse(self, folder, rootplugins={}): + """ + returns dict with information + home contains parsed plugins from pyload. + """ + + plugins = {} + + if rootplugins: + try: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + + for ifile in (join("userplugins", "__init__.py"), + join(pfolder, "__init__.py")): + if not exists(ifile): + f = open(ifile, "wb") + f.close() + + except IOError, e: + self.core.log.critical(str(e)) + return rootplugins + + else: + pfolder = join(pypath, "pyload", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + + try: + with open(join(pfolder, f)) as data: + content = data.read() + + except IOError, e: + self.core.log.error(str(e)) + continue + + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10 + continue + + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10 + continue + + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10 + continue + + name = f[:-3] + if name[-1] == ".": + name = name[:-4] + + version = self.VERSION.findall(content) + if version: + version = float(version[0][1]) + else: + version = 0 + + if rootplugins and name in rootplugins: + if rootplugins[name]['version'] >= version: + continue + + plugins[name] = {} + plugins[name]['version'] = version + + module = f.replace(".pyc", "").replace(".py", "") + + # the plugin is loaded from user directory + plugins[name]['user'] = True if rootplugins else False + plugins[name]['name'] = module + + pattern = self.PATTERN.findall(content) + + if pattern: + pattern = pattern[0][1] + + try: + regexp = re.compile(pattern) + except Exception: + self.core.log.error(_("%s has a invalid pattern") % name) + pattern = r'^unmatchable$' + regexp = re.compile(pattern) + + plugins[name]['pattern'] = pattern + plugins[name]['re'] = regexp + + # internals have no config + if folder == "internal": + self.core.config.deleteConfig(name) + continue + + config = self.CONFIG.findall(content) + if config: + try: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder not in ("account", "internal") and not [True for item in config if item[0] == "activated"]: + config.insert(0, ["activated", "bool", "Activated", False if folder in ("addon", "hook") else True]) + + self.core.config.addPluginConfig(name, config, desc) + except Exception: + self.core.log.error("Invalid config in %s: %s" % (name, config)) + + elif folder in ("addon", "hook"): #force config creation + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.core.config.addPluginConfig(name, config, desc) + except Exception: + self.core.log.error("Invalid config in %s: %s" % (name, config)) + + if not rootplugins and plugins: #: Double check + plugins.update(self.parse(folder, plugins)) + + return plugins + + + def parseUrls(self, urls): + """parse plugins for given list of urls""" + + last = None + res = [] #: tupels of (url, plugintype, pluginname) + + for url in urls: + if type(url) not in (str, unicode, buffer): + continue + + url = unquote(url) + + if last and last[2]['re'].match(url): + res.append((url, last[0], last[1])) + continue + + for type in self.TYPES: + for name, plugin in self.plugins[type]: + + m = None + try: + if 'pattern' in plugin: + m = plugin['re'].match(url) + + except KeyError: + self.core.log.error(_("Plugin [%(type)s] %(name)s skipped due broken pattern") + % {'name': name, 'type': type}) + + if m: + res.append((url, type, name)) + last = (type, name, plugin) + break + else: + res.append((url, "internal", "BasePlugin")) + + return res + + + def findPlugin(self, type, name): + if type not in self.plugins: + return None + + elif name not in self.plugins[type]: + self.core.log.warning(_("Plugin [%(type)s] %(name)s not found | Using plugin: [internal] BasePlugin") + % {'name': name, 'type': type}) + return self.internalPlugins["BasePlugin"] + + else: + return self.plugins[type][name] + + + def getPlugin(self, type, name, original=False): + """return plugin module from hoster|decrypter|container""" + plugin = self.findPlugin(type, name) + + if plugin is None: + return {} + + if "new_module" in plugin and not original: + return plugin['new_module'] + else: + return self.loadModule(type, name) + + + def getPluginName(self, type, name): + """ used to obtain new name if other plugin was injected""" + plugin = self.findPlugin(type, name) + + if plugin is None: + return "" + + if "new_name" in plugin: + return plugin['new_name'] + + return name + + + def loadModule(self, type, name): + """ Returns loaded module for plugin + + :param type: plugin type, subfolder of pyload.plugins + :param name: + """ + plugins = self.plugins[type] + + if name in plugins: + if "module" in plugins[name]: + return plugins[name]['module'] + + try: + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), + plugins[name]['name']) + + except Exception, e: + self.core.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s") + % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)}) + if self.core.debug: + print_exc() + + else: + plugins[name]['module'] = module #: cache import, maybe unneeded + + self.core.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)") + % {'name': name, 'type': type, 'version': plugins[name]['version']}) + return module + + + def loadClass(self, type, name): + """Returns the class of a plugin with the same name""" + module = self.loadModule(type, name) + if module: + return getattr(module, name) + else: + return None + + + def getAccountPlugins(self): + """return list of account plugin names""" + return self.accountPlugins.keys() + + + def find_module(self, fullname, path=None): + #redirecting imports if necesarry + if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins + if fullname.startswith(self.USERROOT): user = 1 + else: user = 0 #used as bool and int + + split = fullname.split(".") + if len(split) != 4 - user: return + type, name = split[2 - user:4 - user] + + if type in self.plugins and name in self.plugins[type]: + #userplugin is a newer version + if not user and self.plugins[type][name]['user']: + return self + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name]['user']: + return self + + + def load_module(self, name, replace=True): + if name not in sys.modules: #could be already in modules + if replace: + if self.ROOT in name: + newname = name.replace(self.ROOT, self.USERROOT) + else: + newname = name.replace(self.USERROOT, self.ROOT) + else: + newname = name + + base, plugin = newname.rsplit(".", 1) + + self.core.log.debug("Redirected import %s -> %s" % (name, newname)) + + module = __import__(newname, globals(), locals(), [plugin]) + #inject under new an old name + sys.modules[name] = module + sys.modules[newname] = module + + return sys.modules[name] + + + def reloadPlugins(self, type_plugins): + """ reload and reindex plugins """ + if not type_plugins: + return None + + self.core.log.debug("Request reload of plugins: %s" % type_plugins) + + reloaded = [] + + as_dict = {} + for t,n in type_plugins: + if t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + for type in as_dict.iterkeys(): + if type in ("addon", "internal"): #: do not reload them because would cause to much side effects + self.core.log.debug("Skipping reload for plugin: [%(type)s] %(name)s" % {'name': plugin, 'type': type}) + continue + + for plugin in as_dict[type]: + if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: + self.core.log.debug(_("Reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}) + + try: + reload(self.plugins[type][plugin]['module']) + + except Exception, e: + self.core.log.error(_("Error when reloading plugin: [%(type)s] %(name)s") % {'name': plugin, 'type': type}, e) + continue + + else: + reloaded.append((type, plugin)) + + #index creation + self.plugins[type] = self.parse(type) + setattr(self, "%sPlugins" % type, self.plugins[type]) + + if "account" in as_dict: #: accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return reloaded #: return a list of the plugins successfully reloaded + + + def reloadPlugin(self, type_plugin): + """ reload and reindex ONE plugin """ + return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/Remote.py b/pyload/manager/Remote.py new file mode 100644 index 000000000..910881164 --- /dev/null +++ b/pyload/manager/Remote.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from threading import Thread +from traceback import print_exc + +class BackendBase(Thread): + def __init__(self, manager): + Thread.__init__(self) + self.m = manager + self.core = manager.core + self.enabled = True + self.running = False + + def run(self): + self.running = True + try: + self.serve() + except Exception, e: + self.core.log.error(_("Remote backend error: %s") % e) + if self.core.debug: + print_exc() + finally: + self.running = False + + def setup(self, host, port): + pass + + def checkDeps(self): + return True + + def serve(self): + pass + + def shutdown(self): + pass + + def stop(self): + self.enabled = False# set flag and call shutdowm message, so thread can react + self.shutdown() + + +class RemoteManager(object): + available = [] + + def __init__(self, core): + self.core = core + self.backends = [] + + if self.core.remote: + self.available.append("ThriftBackend") +# else: +# self.available.append("SocketBackend") + + + def startBackends(self): + host = self.core.config["remote"]["listenaddr"] + port = self.core.config["remote"]["port"] + + for b in self.available: + klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) + backend = klass(self) + if not backend.checkDeps(): + continue + try: + backend.setup(host, port) + self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) + except Exception, e: + self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) + if self.core.debug: + print_exc() + else: + backend.start() + self.backends.append(backend) + + port += 1 diff --git a/pyload/manager/Thread.py b/pyload/manager/Thread.py new file mode 100644 index 000000000..2ace9c789 --- /dev/null +++ b/pyload/manager/Thread.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from os.path import exists, join +import re +from subprocess import Popen +from threading import Event, Lock +from time import sleep, time +from traceback import print_exc +from random import choice + +import pycurl + +from pyload.manager.thread.Decrypter import DecrypterThread +from pyload.manager.thread.Download import DownloadThread +from pyload.manager.thread.Info import InfoThread +from pyload.datatype.File import PyFile +from pyload.network.RequestFactory import getURL +from pyload.utils import freeSpace, lock + + +class ThreadManager(object): + """manages the download threads, assign jobs, reconnect etc""" + + + def __init__(self, core): + """Constructor""" + self.core = core + + self.threads = [] #: thread list + self.localThreads = [] #: addon+decrypter threads + + self.pause = True + + self.reconnecting = Event() + self.reconnecting.clear() + self.downloaded = 0 #number of files downloaded since last cleanup + + self.lock = Lock() + + # some operations require to fetch url info from hoster, so we caching them so it wont be done twice + # contains a timestamp and will be purged after timeout + self.infoCache = {} + + # pool of ids for online check + self.resultIDs = 0 + + # threads which are fetching hoster results + self.infoResults = {} + #timeout for cache purge + self.timestamp = 0 + + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + + for i in range(0, self.core.config.get("download", "max_downloads")): + self.createThread() + + + def createThread(self): + """create a download thread""" + + thread = DownloadThread(self) + self.threads.append(thread) + + def createInfoThread(self, data, pid): + """ + start a thread whichs fetches online status and other infos + data = [ .. () .. ] + """ + self.timestamp = time() + 5 * 60 + + InfoThread(self, data, pid) + + @lock + def createResultThread(self, data, add=False): + """ creates a thread to fetch online status, returns result id """ + self.timestamp = time() + 5 * 60 + + rid = self.resultIDs + self.resultIDs += 1 + + InfoThread(self, data, rid=rid, add=add) + + return rid + + + @lock + def getInfoResult(self, rid): + """returns result and clears it""" + self.timestamp = time() + 5 * 60 + + if rid in self.infoResults: + data = self.infoResults[rid] + self.infoResults[rid] = {} + return data + else: + return {} + + @lock + def setInfoResults(self, rid, result): + self.infoResults[rid].update(result) + + def getActiveFiles(self): + active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] + + for t in self.localThreads: + active.extend(t.getActiveFiles()) + + return active + + def processingIds(self): + """get a id list of all pyfiles processed""" + return [x.id for x in self.getActiveFiles()] + + + def work(self): + """run all task which have to be done (this is for repetivive call by core)""" + try: + self.tryReconnect() + except Exception, e: + self.core.log.error(_("Reconnect Failed: %s") % str(e) ) + self.reconnecting.clear() + if self.core.debug: + print_exc() + self.checkThreadCount() + + try: + self.assignJob() + except Exception, e: + self.core.log.warning("Assign job error", e) + if self.core.debug: + print_exc() + + sleep(0.5) + self.assignJob() + #it may be failed non critical so we try it again + + if (self.infoCache or self.infoResults) and self.timestamp < time(): + self.infoCache.clear() + self.infoResults.clear() + self.core.log.debug("Cleared Result cache") + + #-------------------------------------------------------------------------- + def tryReconnect(self): + """checks if reconnect needed""" + + if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): + return False + + active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] + + if not (0 < active.count(True) == len(active)): + return False + + if not exists(self.core.config['reconnect']['method']): + if exists(join(pypath, self.core.config['reconnect']['method'])): + self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) + else: + self.core.config["reconnect"]["activated"] = False + self.core.log.warning(_("Reconnect script not found!")) + return + + self.reconnecting.set() + + #Do reconnect + self.core.log.info(_("Starting reconnect")) + + while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: + sleep(0.25) + + ip = self.getIP() + + self.core.addonManager.beforeReconnecting(ip) + + self.core.log.debug("Old IP: %s" % ip) + + try: + reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) + except Exception: + self.core.log.warning(_("Failed executing reconnect script!")) + self.core.config["reconnect"]["activated"] = False + self.reconnecting.clear() + if self.core.debug: + print_exc() + return + + reconn.wait() + sleep(1) + ip = self.getIP() + self.core.addonManager.afterReconnecting(ip) + + self.core.log.info(_("Reconnected, new IP: %s") % ip) + + self.reconnecting.clear() + + def getIP(self): + """retrieve current ip""" + services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), + ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] + + ip = "" + for i in range(10): + try: + sv = choice(services) + ip = getURL(sv[0]) + ip = re.match(sv[1], ip).group(1) + break + except Exception: + ip = "" + sleep(1) + + return ip + + #-------------------------------------------------------------------------- + def checkThreadCount(self): + """checks if there are need for increasing or reducing thread count""" + + if len(self.threads) == self.core.config.get("download", "max_downloads"): + return True + elif len(self.threads) < self.core.config.get("download", "max_downloads"): + self.createThread() + else: + free = [x for x in self.threads if not x.active] + if free: + free[0].put("quit") + + + def cleanPycurl(self): + """ make a global curl cleanup (currently ununused) """ + if self.processingIds(): + return False + pycurl.global_cleanup() + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + self.downloaded = 0 + self.core.log.debug("Cleaned up pycurl") + return True + + #-------------------------------------------------------------------------- + def assignJob(self): + """assing a job to a thread if possible""" + + if self.pause or not self.core.api.isTimeDownload(): return + + #if self.downloaded > 20: + # if not self.cleanPyCurl(): return + + free = [x for x in self.threads if not x.active] + + inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) + inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) + onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] + + occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + + occ.sort() + occ = tuple(set(occ)) + job = self.core.files.getJob(occ) + if job: + try: + job.initPlugin() + except Exception, e: + self.core.log.critical(str(e)) + print_exc() + job.setStatus("failed") + job.error = str(e) + job.release() + return + + if job.plugin.__type__ == "hoster": + spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 + if spaceLeft < self.core.config["general"]["min_free_space"]: + self.core.log.warning(_("Not enough space left on device")) + self.pause = True + + if free and not self.pause: + thread = free[0] + #self.downloaded += 1 + + thread.put(job) + else: + #put job back + if occ not in self.core.files.jobCache: + self.core.files.jobCache[occ] = [] + self.core.files.jobCache[occ].append(job.id) + + #check for decrypt jobs + job = self.core.files.getDecryptJob() + if job: + job.initPlugin() + thread = DecrypterThread(self, job) + + + else: + thread = DecrypterThread(self, job) + + def getLimit(self, thread): + limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] + return int(limit) + + def cleanup(self): + """do global cleanup, should be called when finished with pycurl""" + pycurl.global_cleanup() diff --git a/pyload/manager/__init__.py b/pyload/manager/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/manager/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/manager/event/Scheduler.py b/pyload/manager/event/Scheduler.py new file mode 100644 index 000000000..fd428a956 --- /dev/null +++ b/pyload/manager/event/Scheduler.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from time import time +from heapq import heappop, heappush +from threading import Lock, Thread + +class AlreadyCalled(Exception): + pass + + +class Deferred(object): + def __init__(self): + self.call = [] + self.result = () + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args += tuple(cargs) + kwargs.update(ckwargs) + f(*args ** kwargs) + + +class Scheduler(object): + def __init__(self, core): + self.core = core + + self.queue = PriorityQueue() + + def addJob(self, t, call, args=[], kwargs={}, threaded=True): + d = Deferred() + t += time() + j = Job(t, call, args, kwargs, d, threaded) + self.queue.put((t, j)) + return d + + + def removeJob(self, d): + """ + :param d: defered object + :return: if job was deleted + """ + index = -1 + + for i, j in enumerate(self.queue): + if j[1].deferred == d: + index = i + + if index >= 0: + del self.queue[index] + return True + + return False + + def work(self): + while True: + t, j = self.queue.get() + if not j: + break + else: + if t <= time(): + j.start() + else: + self.queue.put((t, j)) + break + + +class Job(object): + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): + self.time = float(time) + self.call = call + self.args = args + self.kwargs = kwargs + self.deferred = deferred + self.threaded = threaded + + def run(self): + ret = self.call(*self.args, **self.kwargs) + if self.deferred is None: + return + else: + self.deferred.callback(ret) + + def start(self): + if self.threaded: + t = Thread(target=self.run) + t.setDaemon(True) + t.start() + else: + self.run() + + +class PriorityQueue(object): + """ a non blocking priority queue """ + + def __init__(self): + self.queue = [] + self.lock = Lock() + + def __iter__(self): + return iter(self.queue) + + def __delitem__(self, key): + del self.queue[key] + + def put(self, element): + self.lock.acquire() + heappush(self.queue, element) + self.lock.release() + + def get(self): + """ return element or None """ + self.lock.acquire() + try: + el = heappop(self.queue) + return el + except IndexError: + return None, None + finally: + self.lock.release() diff --git a/pyload/manager/event/__init__.py b/pyload/manager/event/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/manager/event/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/manager/thread/Addon.py b/pyload/manager/thread/Addon.py new file mode 100644 index 000000000..7feec227e --- /dev/null +++ b/pyload/manager/thread/Addon.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread + + +class AddonThread(PluginThread): + """thread for addons""" + + #-------------------------------------------------------------------------- + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) diff --git a/pyload/manager/thread/Decrypter.py b/pyload/manager/thread/Decrypter.py new file mode 100644 index 000000000..51544d1b9 --- /dev/null +++ b/pyload/manager/thread/Decrypter.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Retry + + +class DecrypterThread(PluginThread): + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + def getActiveFiles(self): + return [self.active] + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.log.info(_("Decrypting starts: %s") % pyfile.name) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + return + + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + return + + except Retry: + self.m.log.info(_("Retrying %s") % pyfile.name) + retry = True + return self.run() + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + return + + finally: + if not retry: + pyfile.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + if not retry: + pyfile.delete() diff --git a/pyload/manager/thread/Download.py b/pyload/manager/thread/Download.py new file mode 100644 index 000000000..c7d21a4ba --- /dev/null +++ b/pyload/manager/thread/Download.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.manager.thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() #: job queue + self.active = False + + self.start() + + #-------------------------------------------------------------------------- + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): + continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.addonManager.downloadPreparing(pyfile) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.addonManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except Exception: + pass + + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") diff --git a/pyload/manager/thread/Info.py b/pyload/manager/thread/Info.py new file mode 100644 index 000000000..4526a07ed --- /dev/null +++ b/pyload/manager/thread/Info.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatype.File import PyFile +from pyload.manager.thread.Plugin import PluginThread +from pyload.api import OnlineStatus + + +class InfoThread(PluginThread): + + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + # filter out container plugins + for name in self.m.core.pluginManager.containerPlugins: + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except Exception: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + self.m.infoResults[self.rid] = {} + + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: + result = [result] + + for res in result: + self.m.infoCache[res[3]] = res #: why don't assign res dict directly? + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/manager/thread/Plugin.py b/pyload/manager/thread/Plugin.py new file mode 100644 index 000000000..20d57c933 --- /dev/null +++ b/pyload/manager/thread/Plugin.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatype.File import PyFile +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from pyload.utils.packagetools import parseNames +from pyload.utils import safe_join +from pyload.api import OnlineStatus + +class PluginThread(Thread): + """abstract base class for thread types""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) + except Exception: + pass + + info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() diff --git a/pyload/manager/thread/Server.py b/pyload/manager/thread/Server.py new file mode 100644 index 000000000..f3f174e74 --- /dev/null +++ b/pyload/manager/thread/Server.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from os.path import exists + +import os +import threading +import logging + +core = None +setup = None +log = logging.getLogger("log") + +class WebServer(threading.Thread): + def __init__(self, pycore): + global core + threading.Thread.__init__(self) + self.core = pycore + core = pycore + self.running = True + self.server = pycore.config['webinterface']['server'] + self.https = pycore.config['webinterface']['https'] + self.cert = pycore.config["ssl"]["cert"] + self.key = pycore.config["ssl"]["key"] + self.host = pycore.config['webinterface']['host'] + self.port = pycore.config['webinterface']['port'] + + self.setDaemon(True) + + def run(self): + import pyload.webui as webinterface + global webinterface + + reset = False + + if self.https and (not exists(self.cert) or not exists(self.key)): + log.warning(_("SSL certificates not found.")) + self.https = False + + if self.server in ("lighttpd", "nginx"): + log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) + log.warning(_("You can use the threaded server which offers good performance and ssl,")) + log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) + log.warning(_("sample configs are located in the pyload/web/servers directory")) + reset = True + elif self.server == "fastcgi": + try: + import flup + except Exception: + log.warning(_("Can't use %(server)s, python-flup is not installed!") % { + "server": self.server}) + reset = True + + if reset or self.server == "lightweight": + if os.name != "nt": + try: + import bjoern + except Exception, e: + log.error(_("Error importing lightweight server: %s") % e) + log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) + log.warning(_("Copy the boern.so to the lib folder or use setup.py install")) + log.warning(_("Of course you need to be familiar with linux and know how to compile software")) + self.server = "builtin" + else: + self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) + self.core.config['webinterface']['server'] = "threaded" + self.server = "threaded" + + if self.server == "threaded": + self.start_threaded() + elif self.server == "fastcgi": + self.start_fcgi() + elif self.server == "lightweight": + self.start_lightweight() + else: + self.start_builtin() + + def start_builtin(self): + + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_simple(host=self.host, port=self.port) + + def start_threaded(self): + if self.https: + self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + else: + self.cert = "" + self.key = "" + self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + + webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + + def start_fcgi(self): + + self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_fcgi(host=self.host, port=self.port) + + + def start_lightweight(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_lightweight(host=self.host, port=self.port) + + def quit(self): + self.running = False diff --git a/pyload/manager/thread/__init__.py b/pyload/manager/thread/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/manager/thread/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/network/Browser.py b/pyload/network/Browser.py new file mode 100644 index 000000000..89341ff25 --- /dev/null +++ b/pyload/network/Browser.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +from logging import getLogger + +from pyload.network.HTTPRequest import HTTPRequest +from pyload.network.HTTPDownload import HTTPDownload + + +class Browser(object): + __slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl") + + def __init__(self, bucket=None, options={}): + self.log = getLogger("log") + + self.options = options #holds pycurl options + self.bucket = bucket + + self.cj = None # needs to be setted later + self._size = 0 + + self.renewHTTPRequest() + self.dl = None + + + def renewHTTPRequest(self): + if hasattr(self, "http"): self.http.close() + self.http = HTTPRequest(self.cj, self.options) + + def setLastURL(self, val): + self.http.lastURL = val + + # tunnel some attributes from HTTP Request to Browser + lastEffectiveURL = property(lambda self: self.http.lastEffectiveURL) + lastURL = property(lambda self: self.http.lastURL, setLastURL) + code = property(lambda self: self.http.code) + cookieJar = property(lambda self: self.cj) + + def setCookieJar(self, cj): + self.cj = cj + self.http.cj = cj + + @property + def speed(self): + if self.dl: + return self.dl.speed + return 0 + + @property + def size(self): + if self._size: + return self._size + if self.dl: + return self.dl.size + return 0 + + @property + def arrived(self): + if self.dl: + return self.dl.arrived + return 0 + + @property + def percent(self): + if not self.size: return 0 + return (self.arrived * 100) / self.size + + def clearCookies(self): + if self.cj: + self.cj.clear() + self.http.clearCookies() + + def clearReferer(self): + self.http.lastURL = None + + def abortDownloads(self): + self.http.abort = True + if self.dl: + self._size = self.dl.size + self.dl.abort = True + + def httpDownload(self, url, filename, get={}, post={}, ref=True, cookies=True, chunks=1, resume=False, + progressNotify=None, disposition=False): + """ this can also download ftp """ + self._size = 0 + self.dl = HTTPDownload(url, filename, get, post, self.lastEffectiveURL if ref else None, + self.cj if cookies else None, self.bucket, self.options, progressNotify, disposition) + name = self.dl.download(chunks, resume) + self._size = self.dl.size + + self.dl = None + + return name + + def load(self, *args, **kwargs): + """ retrieves page """ + return self.http.load(*args, **kwargs) + + def putHeader(self, name, value): + """ add a header to the request """ + self.http.putHeader(name, value) + + def addAuth(self, pwd): + """Adds user and pw for http auth + + :param pwd: string, user:password + """ + self.options["auth"] = pwd + self.renewHTTPRequest() #we need a new request + + def removeAuth(self): + if "auth" in self.options: del self.options["auth"] + self.renewHTTPRequest() + + def setOption(self, name, value): + """Adds an option to the request, see HTTPRequest for existing ones""" + self.options[name] = value + + def deleteOption(self, name): + if name in self.options: del self.options[name] + + def clearHeaders(self): + self.http.clearHeaders() + + def close(self): + """ cleanup """ + if hasattr(self, "http"): + self.http.close() + del self.http + if hasattr(self, "dl"): + del self.dl + if hasattr(self, "cj"): + del self.cj diff --git a/pyload/network/Bucket.py b/pyload/network/Bucket.py new file mode 100644 index 000000000..408a1e240 --- /dev/null +++ b/pyload/network/Bucket.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from time import time +from threading import Lock + +MIN_RATE = 10240 #: 10kb minimum rate + + +class Bucket(object): + + def __init__(self): + self.rate = 0 # bytes per second, maximum targeted throughput + self.tokens = 0 + self.timestamp = time() + self.lock = Lock() + + + def __nonzero__(self): + return False if self.rate < MIN_RATE else True + + + def setRate(self, rate): + self.lock.acquire() + self.rate = int(rate) + self.lock.release() + + + def consumed(self, amount): + """ return the time the process has to sleep, after it consumed a specified amount """ + if self.rate < MIN_RATE: + return 0 #: May become unresponsive otherwise + + self.lock.acquire() + self.calc_tokens() + self.tokens -= amount + + if self.tokens < 0: + time = -self.tokens/float(self.rate) + else: + time = 0 + + self.lock.release() + return time + + + def calc_tokens(self): + if self.tokens < self.rate: + now = time() + delta = self.rate * (now - self.timestamp) + self.tokens = min(self.rate, self.tokens + delta) + self.timestamp = now diff --git a/pyload/network/CookieJar.py b/pyload/network/CookieJar.py new file mode 100644 index 000000000..a2b302776 --- /dev/null +++ b/pyload/network/CookieJar.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +import Cookie + +from datetime import datetime, timedelta +from time import time + + +# monkey patch for 32 bit systems +def _getdate(future=0, weekdayname=Cookie._weekdayname, monthname=Cookie._monthname): + dt = datetime.now() + timedelta(seconds=int(future)) + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ + (weekdayname[dt.weekday()], dt.day, monthname[dt.month], dt.year, dt.hour, dt.minute, dt.second) + +Cookie._getdate = _getdate + + +class CookieJar(Cookie.SimpleCookie): + + def getCookie(self, name): + return self[name].value + + def setCookie(self, domain, name, value, path="/", exp=None, secure="FALSE"): + self[name] = value + self[name]["domain"] = domain + self[name]["path"] = path + + # Value of expires should be integer if possible + # otherwise the cookie won't be used + if not exp: + expires = time() + 3600 * 24 * 180 + else: + try: + expires = int(exp) + except ValueError: + expires = exp + + self[name]["expires"] = expires + + if secure == "TRUE": + self[name]["secure"] = secure diff --git a/pyload/network/HTTPChunk.py b/pyload/network/HTTPChunk.py new file mode 100644 index 000000000..41752b940 --- /dev/null +++ b/pyload/network/HTTPChunk.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from os import remove, stat, fsync +from os.path import exists +from time import sleep +from re import search +from pyload.utils import fs_encode +import codecs +import pycurl +import urllib + +from pyload.network.HTTPRequest import HTTPRequest + +class WrongFormat(Exception): + pass + + +class ChunkInfo(object): + def __init__(self, name): + self.name = unicode(name) + self.size = 0 + self.resume = False + self.chunks = [] + + def __repr__(self): + ret = "ChunkInfo: %s, %s\n" % (self.name, self.size) + for i, c in enumerate(self.chunks): + ret += "%s# %s\n" % (i, c[1]) + + return ret + + def setSize(self, size): + self.size = int(size) + + def addChunk(self, name, range): + self.chunks.append((name, range)) + + def clear(self): + self.chunks = [] + + def createChunks(self, chunks): + self.clear() + chunk_size = self.size / chunks + + current = 0 + for i in range(chunks): + end = self.size - 1 if (i == chunks - 1) else current + chunk_size + self.addChunk("%s.chunk%s" % (self.name, i), (current, end)) + current += chunk_size + 1 + + + def save(self): + fs_name = fs_encode("%s.chunks" % self.name) + fh = codecs.open(fs_name, "w", "utf_8") + fh.write("name:%s\n" % self.name) + fh.write("size:%s\n" % self.size) + for i, c in enumerate(self.chunks): + fh.write("#%d:\n" % i) + fh.write("\tname:%s\n" % c[0]) + fh.write("\trange:%i-%i\n" % c[1]) + fh.close() + + @staticmethod + def load(name): + fs_name = fs_encode("%s.chunks" % name) + if not exists(fs_name): + raise IOError() + fh = codecs.open(fs_name, "r", "utf_8") + name = fh.readline()[:-1] + size = fh.readline()[:-1] + if name.startswith("name:") and size.startswith("size:"): + name = name[5:] + size = size[5:] + else: + fh.close() + raise WrongFormat() + ci = ChunkInfo(name) + ci.loaded = True + ci.setSize(size) + while True: + if not fh.readline(): #skip line + break + name = fh.readline()[1:-1] + range = fh.readline()[1:-1] + if name.startswith("name:") and range.startswith("range:"): + name = name[5:] + range = range[6:].split("-") + else: + raise WrongFormat() + + ci.addChunk(name, (long(range[0]), long(range[1]))) + fh.close() + return ci + + def remove(self): + fs_name = fs_encode("%s.chunks" % self.name) + if exists(fs_name): remove(fs_name) + + def getCount(self): + return len(self.chunks) + + def getChunkName(self, index): + return self.chunks[index][0] + + def getChunkRange(self, index): + return self.chunks[index][1] + + +class HTTPChunk(HTTPRequest): + def __init__(self, id, parent, range=None, resume=False): + self.id = id + self.p = parent # HTTPDownload instance + self.range = range # tuple (start, end) + self.resume = resume + self.log = parent.log + + self.size = range[1] - range[0] if range else -1 + self.arrived = 0 + self.lastURL = self.p.referer + + self.c = pycurl.Curl() + + self.header = "" + self.headerParsed = False #indicates if the header has been processed + + self.fp = None #file handle + + self.initHandle() + self.setInterface(self.p.options) + + self.BOMChecked = False # check and remove byte order mark + + self.rep = None + + self.sleep = 0.000 + self.lastSize = 0 + + def __repr__(self): + return "" % (self.id, self.size, self.arrived) + + @property + def cj(self): + return self.p.cj + + def getHandle(self): + """ returns a Curl handle ready to use for perform/multiperform """ + + self.setRequestContext(self.p.url, self.p.get, self.p.post, self.p.referer, self.p.cj) + self.c.setopt(pycurl.WRITEFUNCTION, self.writeBody) + self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) + + # request all bytes, since some servers in russia seems to have a defect arihmetic unit + + fs_name = fs_encode(self.p.info.getChunkName(self.id)) + if self.resume: + self.fp = open(fs_name, "ab") + self.arrived = self.fp.tell() + if not self.arrived: + self.arrived = stat(fs_name).st_size + + if self.range: + #do nothing if chunk already finished + if self.arrived + self.range[0] >= self.range[1]: return None + + if self.id == len(self.p.info.chunks) - 1: #as last chunk dont set end range, so we get everything + range = "%i-" % (self.arrived + self.range[0]) + else: + range = "%i-%i" % (self.arrived + self.range[0], min(self.range[1] + 1, self.p.size - 1)) + + self.log.debug("Chunked resume with range %s" % range) + self.c.setopt(pycurl.RANGE, range) + else: + self.log.debug("Resume File from %i" % self.arrived) + self.c.setopt(pycurl.RESUME_FROM, self.arrived) + + else: + if self.range: + if self.id == len(self.p.info.chunks) - 1: # see above + range = "%i-" % self.range[0] + else: + range = "%i-%i" % (self.range[0], min(self.range[1] + 1, self.p.size - 1)) + + self.log.debug("Chunked with range %s" % range) + self.c.setopt(pycurl.RANGE, range) + + self.fp = open(fs_name, "wb") + + return self.c + + def writeHeader(self, buf): + self.header += buf + #@TODO forward headers?, this is possibly unneeeded, when we just parse valid 200 headers + # as first chunk, we will parse the headers + if not self.range and self.header.endswith("\r\n\r\n"): + self.parseHeader() + elif not self.range and buf.startswith("150") and "data connection" in buf.lower(): #: ftp file size parsing + size = search(r"(\d+) bytes", buf) + if size: + self.p.size = int(size.group(1)) + self.p.chunkSupport = True + + self.headerParsed = True + + def writeBody(self, buf): + #ignore BOM, it confuses unrar + if not self.BOMChecked: + if [ord(b) for b in buf[:3]] == [239, 187, 191]: + buf = buf[3:] + self.BOMChecked = True + + size = len(buf) + + self.arrived += size + + self.fp.write(buf) + + if self.p.bucket: + sleep(self.p.bucket.consumed(size)) + else: + # Avoid small buffers, increasing sleep time slowly if buffer size gets smaller + # otherwise reduce sleep time percentual (values are based on tests) + # So in general cpu time is saved without reducing bandwith too much + + if size < self.lastSize: + self.sleep += 0.002 + else: + self.sleep *= 0.7 + + self.lastSize = size + + sleep(self.sleep) + + if self.range and self.arrived > self.size: + return 0 #close if we have enough data + + + def parseHeader(self): + """parse data from recieved header""" + for orgline in self.decodeResponse(self.header).splitlines(): + line = orgline.strip().lower() + + if line.startswith("accept-ranges") and "bytes" in line: + self.p.chunkSupport = True + + if line.startswith("content-disposition") and ("filename=" in line or "filename*=" in line): + if "filename*=" in line: + # extended version according to RFC 6266 and RFC 5987. + encoding = line.partition("*=")[2].partition("''")[0] + name = orgline.partition("''")[2] + name = urllib.unquote(str(name)).decode(charEnc(encoding)) + else: + # basic version, US-ASCII only + name = orgline.partition("filename=")[2] + + name = name.replace('"', "").replace("'", "").replace(";", "").replace("/", "_").strip() + self.p.nameDisposition = name + + self.log.debug("Content-Disposition: %s" % name) + + if not self.resume and line.startswith("content-length"): + self.p.size = int(line.split(":")[1]) + + self.headerParsed = True + + def stop(self): + """The download will not proceed after next call of writeBody""" + self.range = [0, 0] + self.size = 0 + + def resetRange(self): + """ Reset the range, so the download will load all data available """ + self.range = None + + def setRange(self, range): + self.range = range + self.size = range[1] - range[0] + + def flushFile(self): + """ flush and close file """ + self.fp.flush() + fsync(self.fp.fileno()) #make sure everything was written to disk + self.fp.close() #needs to be closed, or merging chunks will fail + + def close(self): + """ closes everything, unusable after this """ + if self.fp: self.fp.close() + self.c.close() + if hasattr(self, "p"): del self.p + + +def charEnc(enc): + return {'utf-8' : "utf_8", + 'iso-8859-1': "latin_1"}.get(enc, "unknown") diff --git a/pyload/network/HTTPDownload.py b/pyload/network/HTTPDownload.py new file mode 100644 index 000000000..3b2bf26ca --- /dev/null +++ b/pyload/network/HTTPDownload.py @@ -0,0 +1,312 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from os import remove, fsync +from os.path import dirname +from time import sleep, time +from shutil import move +from logging import getLogger + +import pycurl + +from pyload.network.HTTPChunk import ChunkInfo, HTTPChunk +from pyload.network.HTTPRequest import BadHeader + +from pyload.plugin.Plugin import Abort +from pyload.utils import safe_join, fs_encode + + +class HTTPDownload(object): + """ loads a url http + ftp """ + + def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, + options={}, progress=None, disposition=False): + self.url = url + self.filename = filename #complete file destination, not only name + self.get = get + self.post = post + self.referer = referer + self.cj = cj #cookiejar if cookies are needed + self.bucket = bucket + self.options = options + self.disposition = disposition + # all arguments + + self.abort = False + self.size = 0 + self.nameDisposition = None #will be parsed from content disposition + + self.chunks = [] + + self.log = getLogger("log") + + try: + self.info = ChunkInfo.load(filename) + self.info.resume = True #resume is only possible with valid info file + self.size = self.info.size + self.infoSaved = True + except IOError: + self.info = ChunkInfo(filename) + + self.chunkSupport = True + self.m = pycurl.CurlMulti() + + #needed for speed calculation + self.lastArrived = [] + self.speeds = [] + self.lastSpeeds = [0, 0] + + self.progress = progress + + @property + def speed(self): + last = [sum(x) for x in self.lastSpeeds if x] + return (sum(self.speeds) + sum(last)) / (1 + len(last)) + + @property + def arrived(self): + return sum([c.arrived for c in self.chunks]) + + @property + def percent(self): + if not self.size: return 0 + return (self.arrived * 100) / self.size + + def _copyChunks(self): + init = fs_encode(self.info.getChunkName(0)) #initial chunk name + + if self.info.getCount() > 1: + fo = open(init, "rb+") #first chunkfile + for i in range(1, self.info.getCount()): + #input file + fo.seek( + self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks + fname = fs_encode("%s.chunk%d" % (self.filename, i)) + fi = open(fname, "rb") + buf = 32 * 1024 + while True: #copy in chunks, consumes less memory + data = fi.read(buf) + if not data: + break + fo.write(data) + fi.close() + if fo.tell() < self.info.getChunkRange(i)[1]: + fo.close() + remove(init) + self.info.remove() #there are probably invalid chunks + raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.") + remove(fname) #remove chunk + fo.close() + + if self.nameDisposition and self.disposition: + self.filename = safe_join(dirname(self.filename), self.nameDisposition) + + move(init, fs_encode(self.filename)) + self.info.remove() #remove info file + + def download(self, chunks=1, resume=False): + """ returns new filename or None """ + + chunks = max(1, chunks) + resume = self.info.resume and resume + + try: + self._download(chunks, resume) + except pycurl.error, e: + #code 33 - no resume + code = e.args[0] + if resume is True and code == 33: + # try again without resume + self.log.debug("Errno 33 -> Restart without resume") + + #remove old handles + for chunk in self.chunks: + self.closeChunk(chunk) + + return self._download(chunks, False) + else: + raise + finally: + self.close() + + if self.nameDisposition and self.disposition: return self.nameDisposition + return None + + def _download(self, chunks, resume): + if not resume: + self.info.clear() + self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry + self.info.save() + + self.chunks = [] + + init = HTTPChunk(0, self, None, resume) #initial chunk that will load complete file (if needed) + + self.chunks.append(init) + self.m.add_handle(init.getHandle()) + + lastFinishCheck = 0 + lastTimeCheck = 0 + chunksDone = set() # list of curl handles that are finished + chunksCreated = False + done = False + if self.info.getCount() is 0: # This is a resume, if we were chunked originally assume still can + self.chunkSupport = False + + while 1: + #need to create chunks + if not chunksCreated and self.chunkSupport and self.size: #will be setted later by first chunk + + if not resume: + self.info.setSize(self.size) + self.info.createChunks(chunks) + self.info.save() + + chunks = self.info.getCount() + + init.setRange(self.info.getChunkRange(0)) + + for i in range(1, chunks): + c = HTTPChunk(i, self, self.info.getChunkRange(i), resume) + + handle = c.getHandle() + if handle: + self.chunks.append(c) + self.m.add_handle(handle) + else: + #close immediatly + self.log.debug("Invalid curl handle -> closed") + c.close() + + chunksCreated = True + + while 1: + ret, num_handles = self.m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + t = time() + + # reduce these calls + while lastFinishCheck + 0.5 < t: + # list of failed curl handles + failed = [] + ex = None # save only last exception, we can only raise one anyway + + num_q, ok_list, err_list = self.m.info_read() + for c in ok_list: + chunk = self.findChunk(c) + try: # check if the header implies success, else add it to failed list + chunk.verifyHeader() + except BadHeader, e: + self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e))) + failed.append(chunk) + ex = e + else: + chunksDone.add(c) + + for c in err_list: + curl, errno, msg = c + chunk = self.findChunk(curl) + #test if chunk was finished + if errno != 23 or "0 !=" not in msg: + failed.append(chunk) + ex = pycurl.error(errno, msg) + self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(ex))) + continue + + try: # check if the header implies success, else add it to failed list + chunk.verifyHeader() + except BadHeader, e: + self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e))) + failed.append(chunk) + ex = e + else: + chunksDone.add(curl) + if not num_q: # no more infos to get + + # check if init is not finished so we reset download connections + # note that other chunks are closed and downloaded with init too + if failed and init not in failed and init.c not in chunksDone: + self.log.error(_("Download chunks failed, fallback to single connection | %s" % (str(ex)))) + + #list of chunks to clean and remove + to_clean = filter(lambda x: x is not init, self.chunks) + for chunk in to_clean: + self.closeChunk(chunk) + self.chunks.remove(chunk) + remove(fs_encode(self.info.getChunkName(chunk.id))) + + #let first chunk load the rest and update the info file + init.resetRange() + self.info.clear() + self.info.addChunk("%s.chunk0" % self.filename, (0, self.size)) + self.info.save() + elif failed: + raise ex + + lastFinishCheck = t + + if len(chunksDone) >= len(self.chunks): + if len(chunksDone) > len(self.chunks): + self.log.warning("Finished download chunks size incorrect, please report bug.") + done = True #all chunks loaded + + break + + if done: + break #all chunks loaded + + # calc speed once per second, averaging over 3 seconds + if lastTimeCheck + 1 < t: + diff = [c.arrived - (self.lastArrived[i] if len(self.lastArrived) > i else 0) for i, c in + enumerate(self.chunks)] + + self.lastSpeeds[1] = self.lastSpeeds[0] + self.lastSpeeds[0] = self.speeds + self.speeds = [float(a) / (t - lastTimeCheck) for a in diff] + self.lastArrived = [c.arrived for c in self.chunks] + lastTimeCheck = t + self.updateProgress() + + if self.abort: + raise Abort + + #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize + self.m.select(1) + + for chunk in self.chunks: + chunk.flushFile() #make sure downloads are written to disk + + self._copyChunks() + + def updateProgress(self): + if self.progress: + self.progress(self.percent) + + def findChunk(self, handle): + """ linear search to find a chunk (should be ok since chunk size is usually low) """ + for chunk in self.chunks: + if chunk.c == handle: return chunk + + def closeChunk(self, chunk): + try: + self.m.remove_handle(chunk.c) + except pycurl.error, e: + self.log.debug("Error removing chunk: %s" % str(e)) + finally: + chunk.close() + + def close(self): + """ cleanup """ + for chunk in self.chunks: + self.closeChunk(chunk) + + self.chunks = [] + if hasattr(self, "m"): + self.m.close() + del self.m + if hasattr(self, "cj"): + del self.cj + if hasattr(self, "info"): + del self.info diff --git a/pyload/network/HTTPRequest.py b/pyload/network/HTTPRequest.py new file mode 100644 index 000000000..eac03a365 --- /dev/null +++ b/pyload/network/HTTPRequest.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from __future__ import with_statement + +import pycurl + +from codecs import getincrementaldecoder, lookup, BOM_UTF8 +from urllib import quote, urlencode +from httplib import responses +from logging import getLogger +from cStringIO import StringIO + +from pyload.plugin.Plugin import Abort, Fail + +from pyload.utils import encode + + +def myquote(url): + return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]") + +def myurlencode(data): + data = dict(data) + return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems())) + +bad_headers = range(400, 404) + range(405, 418) + range(500, 506) + +class BadHeader(Exception): + def __init__(self, code, content=""): + Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + self.code = code + self.content = content + + +class HTTPRequest(object): + def __init__(self, cookies=None, options=None): + self.c = pycurl.Curl() + self.rep = StringIO() + + self.cj = cookies #cookiejar + + self.lastURL = None + self.lastEffectiveURL = None + self.abort = False + self.code = 0 # last http code + + self.header = "" + + self.headers = [] #temporary request header + + self.initHandle() + self.setInterface(options) + + self.c.setopt(pycurl.WRITEFUNCTION, self.write) + self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) + + self.log = getLogger("log") + + + def initHandle(self): + """ sets common options to curl handle """ + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.MAXREDIRS, 5) + self.c.setopt(pycurl.CONNECTTIMEOUT, 30) + self.c.setopt(pycurl.NOSIGNAL, 1) + self.c.setopt(pycurl.NOPROGRESS, 1) + if hasattr(pycurl, "AUTOREFERER"): + self.c.setopt(pycurl.AUTOREFERER, 1) + self.c.setopt(pycurl.SSL_VERIFYPEER, 0) + self.c.setopt(pycurl.LOW_SPEED_TIME, 60) + self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) + self.c.setopt(pycurl.USE_SSL, pycurl.CURLUSESSL_TRY) + + #self.c.setopt(pycurl.VERBOSE, 1) + + self.c.setopt(pycurl.USERAGENT, + "Mozilla/5.0 (Windows NT 6.1; Win64; x64;en; rv:5.0) Gecko/20110619 Firefox/5.0") + if pycurl.version_info()[7]: + self.c.setopt(pycurl.ENCODING, "gzip, deflate") + self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*", + "Accept-Language: en-US, en", + "Accept-Charset: ISO-8859-1, utf-8;q=0.7,*;q=0.7", + "Connection: keep-alive", + "Keep-Alive: 300", + "Expect:"]) + + def setInterface(self, options): + + interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] + + if interface and interface.lower() != "none": + self.c.setopt(pycurl.INTERFACE, str(interface)) + + if proxy: + if proxy["type"] == "socks4": + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) + elif proxy["type"] == "socks5": + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) + else: + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) + + self.c.setopt(pycurl.PROXY, str(proxy["address"])) + self.c.setopt(pycurl.PROXYPORT, proxy["port"]) + + if proxy["username"]: + self.c.setopt(pycurl.PROXYUSERPWD, str("%s:%s" % (proxy["username"], proxy["password"]))) + + if ipv6: + self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) + else: + self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + + if "auth" in options: + self.c.setopt(pycurl.USERPWD, str(options["auth"])) + + if "timeout" in options: + self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) + + + def addCookies(self): + """ put cookies from curl handle to cj """ + if self.cj: + self.cj.addCookies(self.c.getinfo(pycurl.INFO_COOKIELIST)) + + def getCookies(self): + """ add cookies from cj to curl handle """ + if self.cj: + for c in self.cj.getCookies(): + self.c.setopt(pycurl.COOKIELIST, c) + return + + def clearCookies(self): + self.c.setopt(pycurl.COOKIELIST, "") + + def setRequestContext(self, url, get, post, referer, cookies, multipart=False): + """ sets everything needed for the request """ + + url = myquote(url) + + if get: + get = urlencode(get) + url = "%s?%s" % (url, get) + + self.c.setopt(pycurl.URL, url) + self.c.lastUrl = url + + if post: + self.c.setopt(pycurl.POST, 1) + if not multipart: + if type(post) == unicode: + post = str(post) #unicode not allowed + elif type(post) == str: + pass + else: + post = myurlencode(post) + + self.c.setopt(pycurl.POSTFIELDS, post) + else: + post = [(x, encode(y)) for x, y in post.iteritems()] + self.c.setopt(pycurl.HTTPPOST, post) + else: + self.c.setopt(pycurl.POST, 0) + + if referer and self.lastURL: + self.c.setopt(pycurl.REFERER, str(self.lastURL)) + + if cookies: + self.c.setopt(pycurl.COOKIEFILE, "") + self.c.setopt(pycurl.COOKIEJAR, "") + self.getCookies() + + + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): + """ load and returns a given page """ + + self.setRequestContext(url, get, post, referer, cookies, multipart) + + self.header = "" + + self.c.setopt(pycurl.HTTPHEADER, self.headers) + + if post: + self.c.setopt(pycurl.POST, 1) + else: + self.c.setopt(pycurl.HTTPGET, 1) + + if not follow_location: + self.c.setopt(pycurl.FOLLOWLOCATION, 0) + + if just_header: + self.c.setopt(pycurl.NOBODY, 1) + + self.c.perform() + rep = self.header if just_header else self.getResponse() + + if not follow_location: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + + if just_header: + self.c.setopt(pycurl.NOBODY, 0) + + self.c.setopt(pycurl.POSTFIELDS, "") + self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) + self.code = self.verifyHeader() + + if save_cookies: + self.addCookies() + + if decode: + rep = self.decodeResponse(rep) + + return rep + + def verifyHeader(self): + """ raise an exceptions on bad headers """ + code = int(self.c.getinfo(pycurl.RESPONSE_CODE)) + if code in bad_headers: + #404 will NOT raise an exception + raise BadHeader(code, self.getResponse()) + return code + + def checkHeader(self): + """ check if header indicates failure""" + return int(self.c.getinfo(pycurl.RESPONSE_CODE)) not in bad_headers + + def getResponse(self): + """ retrieve response from string io """ + if self.rep is None: + return "" + else: + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value + + def decodeResponse(self, rep): + """ decode with correct encoding, relies on header """ + header = self.header.splitlines() + encoding = "utf8" # default encoding + + for line in header: + line = line.lower().replace(" ", "") + if not line.startswith("content-type:") or\ + ("text" not in line and "application" not in line): + continue + + none, delemiter, charset = line.rpartition("charset=") + if delemiter: + charset = charset.split(";") + if charset: + encoding = charset[0] + + try: + #self.log.debug("Decoded %s" % encoding ) + if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): + encoding = 'utf-8-sig' + + decoder = getincrementaldecoder(encoding)("replace") + rep = decoder.decode(rep, True) + + #TODO: html_unescape as default + + except LookupError: + self.log.debug("No Decoder foung for %s" % encoding) + + except Exception: + self.log.debug("Error when decoding string from %s." % encoding) + + return rep + + def write(self, buf): + """ writes response """ + if self.rep.tell() > 1000000 or self.abort: + rep = self.getResponse() + + if self.abort: + raise Abort + + with open("response.dump", "wb") as f: + f.write(rep) + raise Fail("Loaded url exceeded size limit") + else: + self.rep.write(buf) + + def writeHeader(self, buf): + """ writes header """ + self.header += buf + + def putHeader(self, name, value): + self.headers.append("%s: %s" % (name, value)) + + def clearHeaders(self): + self.headers = [] + + def close(self): + """ cleanup, unusable after this """ + self.rep.close() + if hasattr(self, "cj"): + del self.cj + if hasattr(self, "c"): + self.c.close() + del self.c diff --git a/pyload/network/JsEngine.py b/pyload/network/JsEngine.py new file mode 100644 index 000000000..2e98fa37d --- /dev/null +++ b/pyload/network/JsEngine.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +# @author: vuolter + +import subprocess +import sys + +from os import path +from urllib import quote + +from pyload.utils import encode, decode, uniqify + + +class JsEngine(object): + """ JS Engine superclass """ + + def __init__(self, core, engine=None): + self.core = core + self.engine = None #: Engine Instance + + if not engine: + engine = self.core.config.get("general", "jsengine") + + if engine != "auto" and self.set(engine) is False: + engine = "auto" + self.core.log.warning("JS Engine set to \"auto\" for safely") + + if engine == "auto": + for E in self.find(): + if self.set(E) is True: + break + else: + self.core.log.error("No JS Engine available") + + + @classmethod + def find(cls): + """ Check if there is any engine available """ + return [E for E in ENGINES if E.find()] + + + def get(self, engine=None): + """ Convert engine name (string) to relative JSE class (AbstractEngine extended) """ + if not engine: + JSE = self.engine + + elif isinstance(engine, basestring): + engine_name = engine.lower() + for E in ENGINES: + if E.__name == engine_name: #: doesn't check if E(NGINE) is available, just convert string to class + JSE = E + break + else: + raise ValueError("JSE") + + elif issubclass(engine, AbstractEngine): + JSE = engine + + else: + raise TypeError("engine") + + return JSE + + + def set(self, engine): + """ Set engine name (string) or JSE class (AbstractEngine extended) as default engine """ + if isinstance(engine, basestring): + return self.set(self.get(engine)) + + elif issubclass(engine, AbstractEngine) and engine.find(): + self.engine = engine + return True + + else: + return False + + + def eval(self, script, engine=None): #: engine can be a jse name """string""" or an AbstractEngine """class""" + JSE = self.get(engine) + + if not JSE: + raise TypeError("engine") + + script = encode(script) + + out, err = JSE.eval(script) + + results = [out] + + if self.core.config.get("general", "debug"): + if err: + self.core.log.debug(JSE.__name + ":", err) + + engines = self.find() + engines.remove(JSE) + for E in engines: + out, err = E.eval(script) + res = err or out + self.core.log.debug(E.__name + ":", res) + results.append(res) + + if len(results) > 1 and len(uniqify(results)) > 1: + self.core.log.warning("JS output of two or more engines mismatch") + + return results[0] + + +class AbstractEngine(object): + """ JSE base class """ + + __name = "" + + + def __init__(self): + self.setup() + self.available = self.find() + + + def setup(self): + pass + + + @classmethod + def find(cls): + """ Check if the engine is available """ + try: + __import__(cls.__name) + except Exception: + try: + out, err = cls().eval("print(23+19)") + except Exception: + res = False + else: + res = out == "42" + else: + res = True + + return res + + + def _eval(args): + if not self.available: + return None, "JS Engine \"%s\" not found" % self.__name + + try: + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=-1) + return map(lambda x: x.strip(), p.communicate()) + except Exception, e: + return None, e + + + def eval(script): + raise NotImplementedError + + +class Pyv8Engine(AbstractEngine): + + __name = "pyv8" + + + def eval(self, script): + if not self.available: + return None, "JS Engine \"%s\" not found" % self.__name + + try: + rt = PyV8.JSContext() + rt.enter() + res = rt.eval(script), None #@TODO: parse stderr + except Exception, e: + res = None, e + + return res + + +class CommonEngine(AbstractEngine): + + __name = "js" + + + def setup(self): + subprocess.Popen(["js", "-v"], bufsize=-1).communicate() + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = ["js", "-e", script] + return self._eval(args) + + +class NodeEngine(AbstractEngine): + + __name = "nodejs" + + + def setup(self): + subprocess.Popen(["node", "-v"], bufsize=-1).communicate() + + + def eval(self, script): + script = "console.log(eval(unescape('%s')))" % quote(script) + args = ["node", "-e", script] + return self._eval(args) + + +class RhinoEngine(AbstractEngine): + + __name = "rhino" + + + def setup(self): + jspath = [ + "/usr/share/java*/js.jar", + "js.jar", + path.join(pypath, "js.jar") + ] + for p in jspath: + if path.exists(p): + self.path = p + break + else: + self.path = "" + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = ["java", "-cp", self.path, "org.mozilla.javascript.tools.shell.Main", "-e", script] + res = decode(self._eval(args)) + try: + return res.encode("ISO-8859-1") + finally: + return res + + +class JscEngine(AbstractEngine): + + __name = "javascriptcore" + + + def setup(self): + jspath = "/System/Library/Frameworks/JavaScriptCore.framework/Resources/jsc" + self.path = jspath if path.exists(jspath) else "" + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = [self.path, "-e", script] + return self._eval(args) + + +#@NOTE: Priority ordered +ENGINES = [CommonEngine, Pyv8Engine, NodeEngine, RhinoEngine] + +if sys.platform == "darwin": + ENGINES.insert(JscEngine) diff --git a/pyload/network/RequestFactory.py b/pyload/network/RequestFactory.py new file mode 100644 index 000000000..579eafea7 --- /dev/null +++ b/pyload/network/RequestFactory.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from threading import Lock + +from pyload.network.Browser import Browser +from pyload.network.Bucket import Bucket +from pyload.network.HTTPRequest import HTTPRequest +from pyload.network.CookieJar import CookieJar + +from pyload.network.XDCCRequest import XDCCRequest + +class RequestFactory(object): + def __init__(self, core): + self.lock = Lock() + self.core = core + self.bucket = Bucket() + self.updateBucket() + self.cookiejars = {} + + def iface(self): + return self.core.config["download"]["interface"] + + def getRequest(self, pluginName, account=None, type="HTTP"): + self.lock.acquire() + + if type == "XDCC": + return XDCCRequest(proxies=self.getProxies()) + + req = Browser(self.bucket, self.getOptions()) + + if account: + cj = self.getCookieJar(pluginName, account) + req.setCookieJar(cj) + else: + req.setCookieJar(CookieJar(pluginName)) + + self.lock.release() + return req + + def getHTTPRequest(self, **kwargs): + """ returns a http request, dont forget to close it ! """ + options = self.getOptions() + options.update(kwargs) # submit kwargs as additional options + return HTTPRequest(CookieJar(None), options) + + def getURL(self, *args, **kwargs): + """ see HTTPRequest for argument list """ + cj = None + + if 'cookies' in kwargs: + if isinstance(kwargs['cookies'], CookieJar): + cj = kwargs['cookies'] + elif isinstance(kwargs['cookies'], list): + cj = CookieJar(None) + for cookie in kwargs['cookies']: + if isinstance(cookie, tuple) and len(cookie) == 3: + cj.setCookie(*cookie) + + h = HTTPRequest(cj, self.getOptions()) + try: + rep = h.load(*args, **kwargs) + finally: + h.close() + + return rep + + def getCookieJar(self, pluginName, account=None): + if (pluginName, account) in self.cookiejars: + return self.cookiejars[(pluginName, account)] + + cj = CookieJar(pluginName, account) + self.cookiejars[(pluginName, account)] = cj + return cj + + def getProxies(self): + """ returns a proxy list for the request classes """ + if not self.core.config["proxy"]["proxy"]: + return {} + else: + type = "http" + setting = self.core.config["proxy"]["type"].lower() + if setting == "socks4": type = "socks4" + elif setting == "socks5": type = "socks5" + + username = None + if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none": + username = self.core.config["proxy"]["username"] + + pw = None + if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none": + pw = self.core.config["proxy"]["password"] + + return { + "type": type, + "address": self.core.config["proxy"]["address"], + "port": self.core.config["proxy"]["port"], + "username": username, + "password": pw, + } + + def getOptions(self): + """returns options needed for pycurl""" + return {"interface": self.iface(), + "proxies": self.getProxies(), + "ipv6": self.core.config["download"]["ipv6"]} + + def updateBucket(self): + """ set values in the bucket according to settings""" + if not self.core.config["download"]["limit_speed"]: + self.bucket.setRate(-1) + else: + self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024) + +# needs pyreq in global namespace +def getURL(*args, **kwargs): + return pyreq.getURL(*args, **kwargs) + + +def getRequest(*args, **kwargs): + return pyreq.getHTTPRequest() diff --git a/pyload/network/XDCCRequest.py b/pyload/network/XDCCRequest.py new file mode 100644 index 000000000..c49f418c4 --- /dev/null +++ b/pyload/network/XDCCRequest.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# @author: jeix + +import socket +import re + +from os import remove +from os.path import exists + +from time import time + +import struct +from select import select + +from pyload.plugin.Plugin import Abort + + +class XDCCRequest(object): + def __init__(self, timeout=30, proxies={}): + + self.proxies = proxies + self.timeout = timeout + + self.filesize = 0 + self.recv = 0 + self.speed = 0 + + self.abort = False + + def createSocket(self): + # proxytype = None + # proxy = None + # if self.proxies.has_key("socks5"): + # proxytype = socks.PROXY_TYPE_SOCKS5 + # proxy = self.proxies["socks5"] + # elif self.proxies.has_key("socks4"): + # proxytype = socks.PROXY_TYPE_SOCKS4 + # proxy = self.proxies["socks4"] + # if proxytype: + # sock = socks.socksocket() + # t = _parse_proxy(proxy) + # sock.setproxy(proxytype, addr=t[3].split(":")[0], port=int(t[3].split(":")[1]), username=t[1], password=t[2]) + # else: + # sock = socket.socket() + # return sock + + return socket.socket() + + def download(self, ip, port, filename, irc, progress=None): + + ircbuffer = "" + lastUpdate = time() + cumRecvLen = 0 + + dccsock = self.createSocket() + + dccsock.settimeout(self.timeout) + dccsock.connect((ip, port)) + + if exists(filename): + i = 0 + nameParts = filename.rpartition(".") + while True: + newfilename = "%s-%d%s%s" % (nameParts[0], i, nameParts[1], nameParts[2]) + i += 1 + + if not exists(newfilename): + filename = newfilename + break + + fh = open(filename, "wb") + + # recv loop for dcc socket + while True: + if self.abort: + dccsock.close() + fh.close() + remove(filename) + raise Abort + + self._keepAlive(irc, ircbuffer) + + data = dccsock.recv(4096) + dataLen = len(data) + self.recv += dataLen + + cumRecvLen += dataLen + + now = time() + timespan = now - lastUpdate + if timespan > 1: + self.speed = cumRecvLen / timespan + cumRecvLen = 0 + lastUpdate = now + + if progress: + progress(self.percent) + + if not data: + break + + fh.write(data) + + # acknowledge data by sending number of recceived bytes + dccsock.send(struct.pack('!I', self.recv)) + + dccsock.close() + fh.close() + + return filename + + def _keepAlive(self, sock, *readbuffer): + fdset = select([sock], [], [], 0) + if sock not in fdset[0]: + return + + readbuffer += sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + line = line.rstrip() + first = line.split() + if first[0] == "PING": + sock.send("PONG %s\r\n" % first[1]) + + def abortDownloads(self): + self.abort = True + + @property + def size(self): + return self.filesize + + @property + def arrived(self): + return self.recv + + @property + def percent(self): + if not self.filesize: return 0 + return (self.recv * 100) / self.filesize + + def close(self): + pass diff --git a/pyload/network/__init__.py b/pyload/network/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/network/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/Account.py b/pyload/plugin/Account.py new file mode 100644 index 000000000..f8014908f --- /dev/null +++ b/pyload/plugin/Account.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- + +from random import choice +from time import time +from traceback import print_exc +from threading import RLock + +from pyload.plugin.Plugin import Base +from pyload.utils import compare_time, parseFileSize, lock + + +class WrongPassword(Exception): + pass + + +class Account(Base): + """ + Base class for every Account plugin. + Just overwrite `login` and cookies will be stored and account becomes accessible in\ + associated hoster plugin. Plugin should also provide `loadAccountInfo` + """ + __name__ = "Account" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Base account plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + #: after that time (in minutes) pyload will relogin the account + login_timeout = 10 * 60 + #: after that time (in minutes) account data will be reloaded + info_threshold = 10 * 60 + + + def __init__(self, manager, accounts): + Base.__init__(self, manager.core) + + self.manager = manager + self.accounts = {} + self.infos = {} #: cache for account information + self.lock = RLock() + self.timestamps = {} + + self.init() + + self.setAccounts(accounts) + + + def init(self): + pass + + + def login(self, user, data, req): + """login into account, the cookies will be saved so user can be recognized + + :param user: loginname + :param data: data dictionary + :param req: `Request` instance + """ + pass + + + @lock + def _login(self, user, data): + # set timestamp for login + self.timestamps[user] = time() + + req = self.getAccountRequest(user) + try: + self.login(user, data, req) + except WrongPassword: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user, + "msg": _("Wrong Password")}) + success = data['valid'] = False + except Exception, e: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user, + "msg": e}) + success = data['valid'] = False + if self.core.debug: + print_exc() + else: + success = True + finally: + if req: + req.close() + return success + + + def relogin(self, user): + req = self.getAccountRequest(user) + if req: + req.cj.clear() + req.close() + if user in self.infos: + del self.infos[user] #delete old information + + return self._login(user, self.accounts[user]) + + + def setAccounts(self, accounts): + self.accounts = accounts + for user, data in self.accounts.iteritems(): + self._login(user, data) + self.infos[user] = {} + + + def updateAccounts(self, user, password=None, options={}): + """ updates account and return true if anything changed """ + + if user in self.accounts: + self.accounts[user]['valid'] = True #do not remove or accounts will not login + if password: + self.accounts[user]['password'] = password + self.relogin(user) + return True + if options: + before = self.accounts[user]['options'] + self.accounts[user]['options'].update(options) + return self.accounts[user]['options'] != before + else: + self.accounts[user] = {"password": password, "options": options, "valid": True} + self._login(user, self.accounts[user]) + return True + + + def removeAccount(self, user): + if user in self.accounts: + del self.accounts[user] + if user in self.infos: + del self.infos[user] + if user in self.timestamps: + del self.timestamps[user] + + + @lock + def getAccountInfo(self, name, force=False): + """retrieve account infos for an user, do **not** overwrite this method!\\ + just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + + :param name: username + :param force: reloads cached account information + :return: dictionary with information + """ + data = Account.loadAccountInfo(self, name) + + if force or name not in self.infos: + self.logDebug("Get Account Info for %s" % name) + req = self.getAccountRequest(name) + + try: + infos = self.loadAccountInfo(name, req) + if not type(infos) == dict: + raise Exception("Wrong return format") + except Exception, e: + infos = {"error": str(e)} + print_exc() + + if req: + req.close() + + self.logDebug("Account Info: %s" % infos) + + infos['timestamp'] = time() + self.infos[name] = infos + elif "timestamp" in self.infos[name] and self.infos[name][ + "timestamp"] + self.info_threshold * 60 < time(): + self.logDebug("Reached timeout for account data") + self.scheduleRefresh(name) + + data.update(self.infos[name]) + return data + + + def isPremium(self, user): + info = self.getAccountInfo(user) + return info['premium'] + + + def loadAccountInfo(self, name, req=None): + """this should be overwritten in account plugin,\ + and retrieving account information for user + + :param name: + :param req: `Request` instance + :return: + """ + return {"validuntil" : None, #: -1 for unlimited + "login" : name, + # "password" : self.accounts[name]['password'], #: commented due security reason + "options" : self.accounts[name]['options'], + "valid" : self.accounts[name]['valid'], + "trafficleft": None, #: in bytes, -1 for unlimited + "maxtraffic" : None, + "premium" : None, + "timestamp" : 0, #: time this info was retrieved + "type" : self.__name__} + + + def getAllAccounts(self, force=False): + return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] + + + def getAccountRequest(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + req = self.core.requestFactory.getRequest(self.__name__, user) + return req + + + def getAccountCookies(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + cj = self.core.requestFactory.getCookieJar(self.__name__, user) + return cj + + + def getAccountData(self, user): + return self.accounts[user] + + + def selectAccount(self): + """ returns an valid account name and data""" + usable = [] + for user, data in self.accounts.iteritems(): + if not data['valid']: continue + + if "time" in data['options'] and data['options']['time']: + time_data = "" + try: + time_data = data['options']['time'][0] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + continue + except Exception: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if user in self.infos: + if "validuntil" in self.infos[user]: + if self.infos[user]['validuntil'] > 0 and time() > self.infos[user]['validuntil']: + continue + if "trafficleft" in self.infos[user]: + if self.infos[user]['trafficleft'] == 0: + continue + + usable.append((user, data)) + + if not usable: return None, None + return choice(usable) + + + def canUse(self): + return False if self.selectAccount() == (None, None) else True + + + def parseTraffic(self, value, unit=None): #: return bytes + if not unit and not isinstance(value, basestring): + unit = "KB" + return parseFileSize(value, unit) + + + def wrongPassword(self): + raise WrongPassword + + + def empty(self, user): + if user in self.infos: + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + + self.infos[user].update({"trafficleft": 0}) + self.scheduleRefresh(user, 30 * 60) + + + def expired(self, user): + if user in self.infos: + self.logWarning(_("Account %s is expired, checking again in 1h") % user) + + self.infos[user].update({"validuntil": time() - 1}) + self.scheduleRefresh(user, 60 * 60) + + + def scheduleRefresh(self, user, time=0, force=True): + """ add task to refresh account info to sheduler """ + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + + + @lock + def checkLogin(self, user): + """ checks if user is still logged in """ + if user in self.timestamps: + if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time(): + self.logDebug("Reached login timeout for %s" % user) + return self.relogin(user) + else: + return True + else: + return False diff --git a/pyload/plugin/Addon.py b/pyload/plugin/Addon.py new file mode 100644 index 000000000..3525d4e4d --- /dev/null +++ b/pyload/plugin/Addon.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- + +from traceback import print_exc + +from pyload.plugin.Plugin import Base +from pyload.utils import has_method + + +class Expose(object): + """ used for decoration to declare rpc services """ + + def __new__(cls, f, *args, **kwargs): + addonManager.addRPC(f.__module__, f.func_name, f.func_doc) + return f + + +def threaded(fn): + + def run(*args,**kwargs): + addonManager.startThread(fn, *args, **kwargs) + + return run + + +class Addon(Base): + __name__ = "Addon" + __type__ = "addon" + __version__ = "0.01" + + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base addon plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de"), + ("RaNaN", "RaNaN@pyload.org")] + + + #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + event_map = {} + + # Deprecated alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + event_list = [] #@NOTE: dont make duplicate entries in event_map + + + def __init__(self, core, manager): + Base.__init__(self, core) + + #: Provide information in dict here, usable by API `getInfo` + self.info = {} + + #: Callback of periodical job task, used by AddonManager + self.cb = None + self.interval = 60 + + #: `AddonManager` + self.manager = manager + + #register events + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.manager.addEvent(event, getattr(self,f)) + else: + self.manager.addEvent(event, getattr(self,funcs)) + + #delete for various reasons + self.event_map = None + + if self.event_list: + for f in self.event_list: + self.manager.addEvent(f, getattr(self,f)) + + self.event_list = None + + self.setup() + + # self.initPeriodical() + + + def initPeriodical(self, delay=0, threaded=False): + self.cb = self.core.scheduler.addJob(delay, self._periodical, args=[threaded], threaded=threaded) + + + def _periodical(self, threaded): + if self.interval < 0: + self.cb = None + return + + try: + self.periodical() + + except Exception, e: + self.logError(_("Error executing addon: %s") % e) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=threaded) + + + def __repr__(self): + return "" % self.__name__ + + + def setup(self): + """ more init stuff if needed """ + pass + + + def deactivate(self): + """ called when addon was deactivated """ + if has_method(self.__class__, "unload"): + self.unload() + + def unload(self): # Deprecated, use method deactivate() instead + pass + + + def isActivated(self): + """ checks if addon is activated""" + return self.core.config.getPlugin(self.__name__, "activated") + + + # Event methods - overwrite these if needed + def activate(self): + """ called when addon was activated """ + if has_method(self.__class__, "coreReady"): + self.coreReady() + + def coreReady(self): # Deprecated, use method activate() instead + pass + + + def exit(self): + """ called by core.shutdown just before pyLoad exit """ + if has_method(self.__class__, "coreExiting"): + self.coreExiting() + + def coreExiting(self): # Deprecated, use method exit() instead + pass + + + def downloadPreparing(self, pyfile): + pass + + + def downloadFinished(self, pyfile): + pass + + + def downloadFailed(self, pyfile): + pass + + + def packageFinished(self, pypack): + pass + + + def beforeReconnecting(self, ip): + pass + + + def afterReconnecting(self, ip): + pass + + + def periodical(self): + pass + + + def captchaTask(self, task): + """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ + pass + + + def captchaCorrect(self, task): + pass + + + def captchaInvalid(self, task): + pass diff --git a/pyload/plugin/Captcha.py b/pyload/plugin/Captcha.py new file mode 100644 index 000000000..36b60f026 --- /dev/null +++ b/pyload/plugin/Captcha.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Plugin import Base + + +#@TODO: Extend Plugin class; remove all `html` args +class Captcha(Base): + __name__ = "Captcha" + __type__ = "captcha" + __version__ = "0.25" + + __description__ = """Base captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, plugin): + self.plugin = plugin + self.key = None #: last key detected + super(CaptchaService, self).__init__(plugin.core) + + + def detect_key(self, html=None): + raise NotImplementedError + + + def challenge(self, key=None, html=None): + raise NotImplementedError + + + def result(self, server, challenge): + raise NotImplementedError diff --git a/pyload/plugin/Container.py b/pyload/plugin/Container.py new file mode 100644 index 000000000..bfc5713a7 --- /dev/null +++ b/pyload/plugin/Container.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from os import remove +from os.path import basename, exists + +from pyload.plugin.internal.Crypter import Crypter +from pyload.utils import safe_join + + +class Container(Crypter): + __name__ = "Container" + __type__ = "container" + __version__ = "0.01" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + def preprocessing(self, thread): + """prepare""" + + self.setup() + self.thread = thread + + self.loadToDisk() + + self.decrypt(self.pyfile) + self.deleteTmp() + + self.createPackages() + + + def loadToDisk(self): + """loads container to disk if its stored remotely and overwrite url, + or check existent on several places at disk""" + + if self.pyfile.url.startswith("http"): + self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] + content = self.load(self.pyfile.url) + self.pyfile.url = safe_join(self.core.config['general']['download_folder'], self.pyfile.name) + try: + with open(self.pyfile.url, "wb") as f: + f.write(content) + except IOError, e: + self.fail(str(e)) + + else: + self.pyfile.name = basename(self.pyfile.url) + if not exists(self.pyfile.url): + if exists(safe_join(pypath, self.pyfile.url)): + self.pyfile.url = safe_join(pypath, self.pyfile.url) + else: + self.fail(_("File not exists")) + + + def deleteTmp(self): + if self.pyfile.name.startswith("tmp_"): + remove(self.pyfile.url) diff --git a/pyload/plugin/Crypter.py b/pyload/plugin/Crypter.py new file mode 100644 index 000000000..f93ee254a --- /dev/null +++ b/pyload/plugin/Crypter.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +from urlparse import urlparse + +from pyload.plugin.Plugin import Plugin +from pyload.utils import decode, safe_filename + + +class Crypter(Plugin): + __name__ = "Crypter" + __type__ = "crypter" + __version__ = "0.05" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Base decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + html = None #: last html loaded + + + def __init__(self, pyfile): + #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) + self.packages = [] + + #: List of urls, pyLoad will generate packagenames + self.urls = [] + + Plugin.__init__(self, pyfile) + + + def process(self, pyfile): + """ main method """ + + self.decrypt(pyfile) + + if self.urls: + self.generatePackages() + + elif not self.packages: + self.error(_("No link extracted"), "decrypt") + + self.createPackages() + + + def decrypt(self, pyfile): + raise NotImplementedError + + + def generatePackages(self): + """ generate new packages from self.urls """ + + packages = map(lambda name, links: (name, links, None), self.core.api.generatePackages(self.urls).iteritems()) + self.packages.extend(packages) + + + def createPackages(self): + """ create new packages from self.packages """ + + package_folder = self.pyfile.package().folder + package_password = self.pyfile.package().password + package_queue = self.pyfile.package().queue + + folder_per_package = self.core.config['general']['folder_per_package'] + try: + use_subfolder = self.getConfig('use_subfolder') + except Exception: + use_subfolder = folder_per_package + try: + subfolder_per_package = self.getConfig('subfolder_per_package') + except Exception: + subfolder_per_package = True + + for pack in self.packages: + name, links, folder = pack + + self.logDebug("Parsed package: %s" % name, + "%d links" % len(links), + "Saved to folder: %s" % folder if folder else "Saved to download folder") + + links = map(decode, links) + + pid = self.core.api.addPackage(name, links, package_queue) + + if package_password: + self.core.api.setPackageData(pid, {"password": package_password}) + + setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #: Workaround to do not break API addPackage method + + if use_subfolder: + if not subfolder_per_package: + setFolder(package_folder) + self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + + elif not folder_per_package or name != folder: + if not folder: + folder = urlparse(name).path.split("/")[-1] + + setFolder(safe_filename(folder)) + self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + + elif folder_per_package: + setFolder(None) diff --git a/pyload/plugin/Extractor.py b/pyload/plugin/Extractor.py new file mode 100644 index 000000000..27573f860 --- /dev/null +++ b/pyload/plugin/Extractor.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +import os + +from pyload.datatype.File import PyFile +from pyload.plugin.Plugin import Base + + +class ArchiveError(Exception): + pass + + +class CRCError(Exception): + pass + + +class PasswordError(Exception): + pass + + +class Extractor(Base): + __name__ = "Extractor" + __version__ = "0.20" + + __description__ = """Base extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz", "immenz@gmx.net")] + + + EXTENSIONS = [] + VERSION = "" + + + @classmethod + def isArchive(cls, filename): + name = os.path.basename(filename).lower() + return any(name.endswith(ext) for ext in cls.EXTENSIONS) and not cls.isMultipart(filename) + + + @classmethod + def isMultipart(cls,filename): + return False + + + @classmethod + def isUsable(cls): + """ Check if system statisfy dependencies + :return: boolean + """ + return None + + + @classmethod + def getTargets(cls, files_ids): + """ Filter suited targets from list of filename id tuple list + :param files_ids: List of filepathes + :return: List of targets, id tuple list + """ + return [(fname, id, fout) for fname, id, fout in files_ids if cls.isArchive(fname)] + + + def __init__(self, manager, filename, out, + fullpath=True, + overwrite=False, + excludefiles=[], + renice=0, + delete=False, + keepbroken=False, + fid=None): + """ Initialize extractor for specific file """ + self.manager = manager + self.filename = filename + self.out = out + self.fullpath = fullpath + self.overwrite = overwrite + self.excludefiles = excludefiles + self.renice = renice + self.delete = delete + self.keepbroken = keepbroken + self.files = [] #: Store extracted files here + + pyfile = self.manager.core.files.getFile(fid) if fid else None + self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None + + + def init(self): + """ Initialize additional data structures """ + pass + + + def check(self): + """Check if password if needed. Raise ArchiveError if integrity is + questionable. + + :return: boolean + :raises ArchiveError + """ + raise PasswordError + + + def isPassword(self, password): + """ Check if the given password is/might be correct. + If it can not be decided at this point return true. + + :param password: + :return: boolean + """ + return None + + + def repair(self): + return None + + + def extract(self, password=None): + """Extract the archive. Raise specific errors in case of failure. + + :param progress: Progress function, call this to update status + :param password password to use + :raises PasswordError + :raises CRCError + :raises ArchiveError + :return: + """ + raise NotImplementedError + + + def getDeleteFiles(self): + """Return list of files to delete, do *not* delete them here. + + :return: List with paths of files to delete + """ + return [self.filename] + + + def list(self, password=None): + """Populate self.files at some point while extracting""" + return self.files diff --git a/pyload/plugin/Hook.py b/pyload/plugin/Hook.py new file mode 100644 index 000000000..58dda9d8b --- /dev/null +++ b/pyload/plugin/Hook.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Addon import Addon, threaded + + +class Hook(Addon): + __name__ = "Hook" + __type__ = "hook" + __version__ = "0.03" + + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] diff --git a/pyload/plugin/Hoster.py b/pyload/plugin/Hoster.py new file mode 100644 index 000000000..2d43ee845 --- /dev/null +++ b/pyload/plugin/Hoster.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Plugin import Plugin + + +def getInfo(self): + #result = [ .. (name, size, status, url) .. ] + return + + +class Hoster(Plugin): + __name__ = "Hoster" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] diff --git a/pyload/plugin/OCR.py b/pyload/plugin/OCR.py new file mode 100644 index 000000000..623f02865 --- /dev/null +++ b/pyload/plugin/OCR.py @@ -0,0 +1,319 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +try: + from PIL import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin + +except ImportError: + import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin + +import logging +import os +import subprocess +#import tempfile + +from pyload.plugin.Plugin import Base +from pyload.utils import safe_join + + +class OCR(Base): + __name__ = "OCR" + __type__ = "ocr" + __version__ = "0.11" + + __description__ = """OCR base plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + def __init__(self): + self.logger = logging.getLogger("log") + + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + + def deactivate(self): + """delete all tmp images""" + pass + + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() + " | " + popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") + try: + tmpTif = open(safe_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmpTif.close() + + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(safe_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + except IOError, e: + self.logError(e) + return + + self.logger.debug("save tiff") + self.image.save(tmpTif.name, 'TIFF') + + if os.name == "nt": + tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] + else: + tessparams = ["tesseract"] + + tessparams.extend([os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")] ) + + if subset and (digits or lowercase or uppercase): + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + with open(safe_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: + tmpSub.write("tessedit_char_whitelist ") + + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(os.path.abspath(tmpSub.name)) + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except Exception: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmpTif.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except Exception: + pass + + + def get_captcha(self, name): + raise NotImplementedError + + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: + continue + # No point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x - 1, y - 1] != 255: + count += 1 + if pixels[x - 1, y] != 255: + count += 1 + if pixels[x - 1, y + 1] != 255: + count += 1 + if pixels[x, y + 1] != 255: + count += 1 + if pixels[x + 1, y + 1] != 255: + count += 1 + if pixels[x + 1, y] != 255: + count += 1 + if pixels[x + 1, y - 1] != 255: + count += 1 + if pixels[x, y - 1] != 255: + count += 1 + except Exception: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: + pixels[x, y] = 255 + + self.pixels = pixels + + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in xrange(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: + bottomY = y + if y < topY: + topY = y + if x > lastX: + lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col is False and started is True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + + def correct(self, values, var=None): + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result diff --git a/pyload/plugin/Plugin.py b/pyload/plugin/Plugin.py new file mode 100644 index 000000000..0a9c647fb --- /dev/null +++ b/pyload/plugin/Plugin.py @@ -0,0 +1,753 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from time import time, sleep +from random import randint + +import os +from os import remove, makedirs, chmod, stat +from os.path import exists, join + +if os.name != "nt": + from os import chown + from pwd import getpwnam + from grp import getgrnam + +from itertools import islice +from traceback import print_exc +from urlparse import urlparse + +from pyload.utils import fs_decode, fs_encode, safe_filename, safe_join + + +def chunks(iterable, size): + it = iter(iterable) + item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) + + +class Abort(Exception): + """ raised when aborted """ + + +class Fail(Exception): + """ raised when failed """ + + +class Reconnect(Exception): + """ raised when reconnected """ + + +class Retry(Exception): + """ raised when start again from beginning """ + + +class SkipDownload(Exception): + """ raised when download should be skipped """ + + +class Base(object): + """ + A Base class with log/config/db methods *all* plugin types can use + """ + + def __init__(self, core): + #: Core instance + self.core = core + + + def _log(self, type, args): + msg = " | ".join([encode(a).strip() for a in args if a]) + logger = getattr(self.core.log, type) + logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper()))) + + + def logDebug(self, *args): + if self.core.debug: + return self._log("debug", args) + + + def logInfo(self, *args): + return self._log("info", args) + + + def logWarning(self, *args): + return self._log("warning", args) + + + def logError(self, *args): + return self._log("error", args) + + + def logCritical(self, *args): + return self._log("critical", args) + + + #: Deprecated method + def setConf(self, option, value): + """ see `setConfig` """ + self.setConfig(option, value) + + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.core.config.setPlugin(self.__name__, option, value) + + + #: Deprecated method + def getConf(self, option): + """ see `getConfig` """ + return self.getConfig(option) + + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.core.config.getPlugin(self.__name__, option) + + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + +class Plugin(Base): + """ + Base plugin for hoster/crypter. + Overwrite `process` / `decrypt` in your subclassed plugin. + """ + __name__ = "Plugin" + __type__ = "hoster" + __version__ = "0.07" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de")] + + + info = {} #: file info dict + + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + #: engage wan reconnection + self.wantReconnect = False + + #: enable simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + + #: chunk limit + self.chunkLimit = 1 + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + #: captcha reader instance + self.ocr = None + + #: account handler instance, see :py:class:`Account` + self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.canUse(): + self.account = None + + if self.account: + self.user, data = self.account.selectAccount() + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest(self.user) + self.chunkLimit = -1 # chunk limit, -1 for unlimited + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = True + self.multiDL = True #every hoster with account should provide multiple downloads + #: premium status + self.premium = self.account.isPremium(self.user) + else: + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + + #: js engine, see `JsEngine` + self.js = self.core.js + + #: captcha task + self.cTask = None + + self.html = None #@TODO: Move to hoster class in 0.4.10 + self.retries = 0 + + self.init() + + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.core.config['download']['chunks'] + return min(self.core.config['download']['chunks'], self.chunkLimit) + + + def __call__(self): + return self.__name__ + + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + self.account.checkLogin(self.user) + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check addon + + return True, 10 + + + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + + def setWait(self, seconds, reconnect=None): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + wait_time = int(seconds) + 1 + wait_until = time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) + + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.setReconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): + """ waits the time previously set """ + + pyfile = self.pyfile + + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) + + self.waiting = True + + status = pyfile.status + pyfile.setStatus("waiting") + + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()), + _("Reconnect: %s") % self.wantReconnect) + + if self.account: + self.logDebug("Ignore reconnection due account logged") + + while pyfile.waitUntil > time(): + if pyfile.abort: + self.abort() + + sleep(1) + else: + while pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + sleep(1) + + self.waiting = False + + pyfile.status = status + + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason) + + + def abort(self, reason=""): + """ abort and give reason """ + if reason: + self.pyfile.error = str(reason) + raise Abort + + + def error(self, reason="", type=""): + if not reason and not type: + type = "unknown" + + msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error") + msg += ": " + reason.strip() if reason else "" + msg += _(" | Plugin may be out of date") + + raise Fail(msg) + + + def offline(self, reason=""): + """ fail and indicate file is offline """ + if reason: + self.pyfile.error = str(reason) + raise Fail("offline") + + + def tempOffline(self, reason=""): + """ fail and indicates file ist temporary offline, the core may take consequences """ + if reason: + self.pyfile.error = str(reason) + raise Fail("temp. offline") + + + def retry(self, max_tries=5, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + self.error(reason or _("Max retries reached"), "retry") + + self.wait(wait_time, False) + + self.retries += 1 + raise Retry(reason) + + + def invalidCaptcha(self): + self.logError(_("Invalid captcha")) + if self.cTask: + self.cTask.invalid() + + + def correctCaptcha(self): + self.logInfo(_("Correct captcha")) + if self.cTask: + self.cTask.correct() + + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual', timeout=290): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + + with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: + self.abort() + + ocr = Ocr() + result = ocr.get_captcha(tmpCaptcha.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task, timeout) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + self.abort() + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + + result = task.result + self.logDebug("Received captcha result: %s" % result) + + if not self.core.debug: + try: + remove(tmpCaptcha.name) + except Exception: + pass + + return result + + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): + """Load content at url and returns it + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param just_header: If True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :param follow_location: If True follow location else not + :param save_cookies: If True saves received cookies else discard them + :return: Loaded content + """ + if self.pyfile.abort: + self.abort() + + if not url: + self.fail(_("No url given")) + + url = encode(url).strip() #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins + + if self.core.debug: + self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) + + if decode: + res = encode(res) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + framefile = safe_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + try: + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + with open(framefile, "wb") as f: + del frame #: delete the frame or it wont be cleaned + f.write(res) + except IOError, e: + self.logError(e) + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.strip().lower() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + if self.pyfile.abort: + self.abort() + + if not url: + self.fail(_("No url given")) + + url = encode(url).strip() + + if self.core.debug: + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.core.config['general']['download_folder'] + + location = safe_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + try: + makedirs(location, int(self.core.config['permission']['folder'], 8)) + + if self.core.config['permission']['change_dl'] and os.name != "nt": + uid = getpwnam(self.core.config['permission']['user'])[2] + gid = getgrnam(self.core.config['permission']['group'])[2] + chown(location, uid, gid) + + except Exception, e: + self.fail(e) + + # convert back to unicode + location = fs_decode(location) + name = safe_filename(self.pyfile.name) + + filename = join(location, name) + + self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if newname: + newname = urlparse(newname).path.split("/")[-1] + + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config['permission']['change_file']: + try: + chmod(fs_filename, int(self.core.config['permission']['file'], 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) + + if self.core.config['permission']['change_dl'] and os.name != "nt": + try: + uid = getpwnam(self.core.config['permission']['user'])[2] + gid = getgrnam(self.core.config['permission']['group'])[2] + chown(fs_filename, uid, gid) + + except Exception, e: + self.logWarning(_("Setting User and Group failed"), e) + + self.lastDownload = filename + return self.lastDownload + + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): + return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size if read_size else -1) + + #produces encoding errors, better log to other file in the future? + #self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: + """ + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.core.config['general']['download_folder'] + location = safe_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) + + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + + + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + + if hasattr(self, "req"): + self.req.close() + del self.req + + if hasattr(self, "thread"): + del self.thread + + if hasattr(self, "html"): + del self.html diff --git a/pyload/plugin/__init__.py b/pyload/plugin/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/account/AlldebridCom.py b/pyload/plugin/account/AlldebridCom.py new file mode 100644 index 000000000..7fc11e343 --- /dev/null +++ b/pyload/plugin/account/AlldebridCom.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +import re +import xml.dom.minidom as dom + +from time import time + +from BeautifulSoup import BeautifulSoup + +from pyload.plugin.Account import Account + + +class AlldebridCom(Account): + __name__ = "AlldebridCom" + __type__ = "account" + __version__ = "0.23" + + __description__ = """AllDebrid.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Andy Voigt", "spamsales@online.de")] + + + def loadAccountInfo(self, user, req): + data = self.getAccountData(user) + html = req.load("http://www.alldebrid.com/account/") + soup = BeautifulSoup(html) + + #Try to parse expiration date directly from the control panel page (better accuracy) + try: + time_text = soup.find('div', attrs={'class': 'remaining_time_text'}).strong.string + + self.logDebug("Account expires in: %s" % time_text) + + p = re.compile('\d+') + exp_data = p.findall(time_text) + exp_time = time() + int(exp_data[0]) * 24 * 60 * 60 + int( + exp_data[1]) * 60 * 60 + (int(exp_data[2]) - 1) * 60 + + #Get expiration date from API + except Exception: + data = self.getAccountData(user) + html = req.load("http://www.alldebrid.com/api.php", + get={'action': "info_user", 'login': user, 'pw': data['password']}) + + self.logDebug(html) + + xml = dom.parseString(html) + exp_time = time() + int(xml.getElementsByTagName("date")[0].childNodes[0].nodeValue) * 24 * 60 * 60 + + account_info = {"validuntil": exp_time, "trafficleft": -1} + return account_info + + + def login(self, user, data, req): + html = req.load("http://www.alldebrid.com/register/", + get={'action' : "login", + 'login_login' : user, + 'login_password': data['password']}, + decode=True) + + if "This login doesn't exist" in html \ + or "The password is not valid" in html \ + or "Invalid captcha" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/BackinNet.py b/pyload/plugin/account/BackinNet.py new file mode 100644 index 000000000..ac751c8fb --- /dev/null +++ b/pyload/plugin/account/BackinNet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class BackinNet(XFSAccount): + __name__ = "BackinNet" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Backin.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "backin.net" diff --git a/pyload/plugin/account/BillionuploadsCom.py b/pyload/plugin/account/BillionuploadsCom.py new file mode 100644 index 000000000..a3325c427 --- /dev/null +++ b/pyload/plugin/account/BillionuploadsCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class BillionuploadsCom(XFSAccount): + __name__ = "BillionuploadsCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Billionuploads.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "billionuploads.com" diff --git a/pyload/plugin/account/BitshareCom.py b/pyload/plugin/account/BitshareCom.py new file mode 100644 index 000000000..4774b9062 --- /dev/null +++ b/pyload/plugin/account/BitshareCom.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class BitshareCom(Account): + __name__ = "BitshareCom" + __type__ = "account" + __version__ = "0.13" + + __description__ = """Bitshare account plugin""" + __license__ = "GPLv3" + __authors__ = [("Paul King", "")] + + + def loadAccountInfo(self, user, req): + html = req.load("http://bitshare.com/mysettings.html") + + if "\"http://bitshare.com/myupgrade.html\">Free" in html: + return {"validuntil": -1, "trafficleft": -1, "premium": False} + + if not '' in html: + self.logWarning(_("Activate direct Download in your Bitshare Account")) + + return {"validuntil": -1, "trafficleft": -1, "premium": True} + + + def login(self, user, data, req): + html = req.load("http://bitshare.com/login.html", + post={"user": user, "password": data['password'], "submit": "Login"}, + cookies=True, + decode=True) + + if "login" in req.lastEffectiveURL: + self.wrongPassword() diff --git a/pyload/plugin/account/CatShareNet.py b/pyload/plugin/account/CatShareNet.py new file mode 100644 index 000000000..a604ebff1 --- /dev/null +++ b/pyload/plugin/account/CatShareNet.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import re + +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class CatShareNet(Account): + __name__ = "CatShareNet" + __type__ = "account" + __version__ = "0.05" + + __description__ = """CatShareNet account plugin""" + __license__ = "GPLv3" + __authors__ = [("prOq", "")] + + + PREMIUM_PATTERN = r'Konto:[\s\n]*Premium' + VALID_UNTIL_PATTERN = r'>Konto premium.*?(.*?)' + TRAFFIC_LEFT_PATTERN = r'([0-9.]+ [kMG]B)' + + + def loadAccountInfo(self, user, req): + premium = False + validuntil = -1 + trafficleft = -1 + + html = req.load("http://catshare.net/", decode=True) + + if re.search(self.PREMIUM_PATTERN, html): + premium = True + + try: + expiredate = re.search(self.VALID_UNTIL_PATTERN, html).group(1) + self.logDebug("Expire date: " + expiredate) + + validuntil = mktime(strptime(expiredate, "%Y-%m-%d %H:%M:%S")) + + except Exception: + pass + + try: + trafficleft = self.parseTraffic(re.search(self.TRAFFIC_LEFT_PATTERN, html).group(1)) + + except Exception: + pass + + return {'premium': premium, 'trafficleft': trafficleft, 'validuntil': validuntil} + + + def login(self, user, data, req): + html = req.load("http://catshare.net/login", + post={'user_email': user, + 'user_password': data['password'], + 'remindPassword': 0, + 'user[submit]': "Login"}, + decode=True) + + if not 'Wyloguj' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/CloudzillaTo.py b/pyload/plugin/account/CloudzillaTo.py new file mode 100644 index 000000000..a07621234 --- /dev/null +++ b/pyload/plugin/account/CloudzillaTo.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class CloudzillaTo(Account): + __name__ = "CloudzillaTo" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Cloudzilla.to account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + PREMIUM_PATTERN = r'

account type

\s*Premium Account' + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.cloudzilla.to/") + + premium = True if re.search(self.PREMIUM_PATTERN, html) else False + + return {'validuntil': -1, 'trafficleft': -1, 'premium': premium} + + + def login(self, user, data, req): + html = req.load("http://www.cloudzilla.to/", + post={'lusername': user, + 'lpassword': data['password'], + 'w' : "dologin"}, + decode=True) + + if "ERROR" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/CramitIn.py b/pyload/plugin/account/CramitIn.py new file mode 100644 index 000000000..21503f625 --- /dev/null +++ b/pyload/plugin/account/CramitIn.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class CramitIn(XFSAccount): + __name__ = "CramitIn" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Cramit.in account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "cramit.in" diff --git a/pyload/plugin/account/CzshareCom.py b/pyload/plugin/account/CzshareCom.py new file mode 100644 index 000000000..dfe78c21c --- /dev/null +++ b/pyload/plugin/account/CzshareCom.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +from time import mktime, strptime +import re + +from pyload.plugin.Account import Account + + +class CzshareCom(Account): + __name__ = "CzshareCom" + __type__ = "account" + __version__ = "0.18" + + __description__ = """Czshare.com account plugin, now Sdilej.cz""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + CREDIT_LEFT_PATTERN = r'\s*([\d ,]+) (KiB|MiB|GiB)\s*([^<]*)\s*' + + + def loadAccountInfo(self, user, req): + premium = False + validuntil = None + trafficleft = None + + html = req.load("http://sdilej.cz/prehled_kreditu/") + + try: + m = re.search(self.CREDIT_LEFT_PATTERN, html) + trafficleft = self.parseTraffic(m.group(1).replace(' ', '').replace(',', '.')) + m.group(2) + validuntil = mktime(strptime(m.group(3), '%d.%m.%y %H:%M')) + + except Exception, e: + self.logError(e) + + else: + premium = True + + return {'premium' : premium, + 'validuntil' : validuntil, + 'trafficleft': trafficleft} + + + def login(self, user, data, req): + html = req.load('https://sdilej.cz/index.php', + post={"Prihlasit": "Prihlasit", + "login-password": data['password'], + "login-name": user}, + decode=True) + + if '", html).group(1) + + validuntil = mktime(strptime(validuntil, "%Y-%m-%d %H:%M:%S")) + + return {"validuntil": validuntil, "trafficleft": -1} + + + def login(self, user, data, req): + html = req.load("https://dfiles.eu/de/login.php", get={"return": "/de/gold/payment.php"}, + post={"login": user, "password": data['password']}, + decode=True) + + if r'
Sie haben eine falsche Benutzername-Passwort-Kombination verwendet.
' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/DropboxCom.py b/pyload/plugin/account/DropboxCom.py new file mode 100644 index 000000000..862b7a8df --- /dev/null +++ b/pyload/plugin/account/DropboxCom.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class DropboxCom(SimpleHoster): + __name__ = "DropboxCom" + __type__ = "hoster" + __version__ = "0.04" + + __pattern__ = r'https?://(?:www\.)?dropbox\.com/.+' + + __description__ = """Dropbox.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'Dropbox - (?P<N>.+?)<' + SIZE_PATTERN = r' ·  (?P<S>[\d.,]+) (?P<U>[\w^_]+)' + + OFFLINE_PATTERN = r'<title>Dropbox - (404|Shared link error)<' + + COOKIES = [("dropbox.com", "lang", "en")] + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + self.resumeDownload = True + + + def handleFree(self, pyfile): + self.download(pyfile.url, get={'dl': "1"}) diff --git a/pyload/plugin/account/EasybytezCom.py b/pyload/plugin/account/EasybytezCom.py new file mode 100644 index 000000000..c7d717474 --- /dev/null +++ b/pyload/plugin/account/EasybytezCom.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class EasybytezCom(XFSAccount): + __name__ = "EasybytezCom" + __type__ = "account" + __version__ = "0.12" + + __description__ = """EasyBytez.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("guidobelix", "guidobelix@hotmail.it")] + + + HOSTER_DOMAIN = "easybytez.com" diff --git a/pyload/plugin/account/EuroshareEu.py b/pyload/plugin/account/EuroshareEu.py new file mode 100644 index 000000000..b37fc68fa --- /dev/null +++ b/pyload/plugin/account/EuroshareEu.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +from time import mktime, strptime +import re + +from pyload.plugin.Account import Account + + +class EuroshareEu(Account): + __name__ = "EuroshareEu" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Euroshare.eu account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + def loadAccountInfo(self, user, req): + self.relogin(user) + html = req.load("http://euroshare.eu/customer-zone/settings/") + + m = re.search('id="input_expire_date" value="(\d+\.\d+\.\d+ \d+:\d+)"', html) + if m is None: + premium, validuntil = False, -1 + else: + premium = True + validuntil = mktime(strptime(m.group(1), "%d.%m.%Y %H:%M")) + + return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} + + + def login(self, user, data, req): + html = req.load('http://euroshare.eu/customer-zone/login/', + post={"trvale": "1", + "login": user, + "password": data['password']}, + decode=True) + + if u">Nesprávne prihlasovacie meno alebo heslo" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/ExashareCom.py b/pyload/plugin/account/ExashareCom.py new file mode 100644 index 000000000..efd2587c0 --- /dev/null +++ b/pyload/plugin/account/ExashareCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class ExashareCom(XFSAccount): + __name__ = "ExashareCom" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Exashare.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "exashare.com" diff --git a/pyload/plugin/account/FastixRu.py b/pyload/plugin/account/FastixRu.py new file mode 100644 index 000000000..69f78c3d9 --- /dev/null +++ b/pyload/plugin/account/FastixRu.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class FastixRu(Account): + __name__ = "FastixRu" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Fastix account plugin""" + __license__ = "GPLv3" + __authors__ = [("Massimo Rosamilia", "max@spiritix.eu")] + + + def loadAccountInfo(self, user, req): + data = self.getAccountData(user) + html = json_loads(req.load("http://fastix.ru/api_v2/", get={'apikey': data['api'], 'sub': "getaccountdetails"})) + + points = html['points'] + kb = float(points) * 1024 ** 2 / 1000 + + if points > 0: + account_info = {"validuntil": -1, "trafficleft": kb} + else: + account_info = {"validuntil": None, "trafficleft": None, "premium": False} + return account_info + + + def login(self, user, data, req): + html = req.load("http://fastix.ru/api_v2/", + get={'sub': "get_apikey", 'email': user, 'password': data['password']}) + + api = json_loads(html) + api = api['apikey'] + + data['api'] = api + + if "error_code" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/FastshareCz.py b/pyload/plugin/account/FastshareCz.py new file mode 100644 index 000000000..8fe98438b --- /dev/null +++ b/pyload/plugin/account/FastshareCz.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account +from pyload.utils import parseFileSize + + +class FastshareCz(Account): + __name__ = "FastshareCz" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Fastshare.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + CREDIT_PATTERN = r'My account\s*\((.+?)\)' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = None + + html = req.load("http://www.fastshare.cz/user", decode=True) + + m = re.search(self.CREDIT_PATTERN, html) + if m: + trafficleft = self.parseTraffic(m.group(1)) + + if trafficleft: + premium = True + validuntil = -1 + else: + premium = False + + return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + req.cj.setCookie("fastshare.cz", "lang", "en") + + req.load('http://www.fastshare.cz/login') # Do not remove or it will not login + + html = req.load("http://www.fastshare.cz/sql.php", + post={'login': user, 'heslo': data['password']}, + decode=True) + + if ">Wrong username or password" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/File4SafeCom.py b/pyload/plugin/account/File4SafeCom.py new file mode 100644 index 000000000..c48956d38 --- /dev/null +++ b/pyload/plugin/account/File4SafeCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class File4SafeCom(XFSAccount): + __name__ = "File4SafeCom" + __type__ = "account" + __version__ = "0.05" + + __description__ = """File4Safe.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + HOSTER_DOMAIN = "file4safe.com" + + LOGIN_FAIL_PATTERN = r'input_login' diff --git a/pyload/plugin/account/FileParadoxIn.py b/pyload/plugin/account/FileParadoxIn.py new file mode 100644 index 000000000..02b923519 --- /dev/null +++ b/pyload/plugin/account/FileParadoxIn.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class FileParadoxIn(XFSAccount): + __name__ = "FileParadoxIn" + __type__ = "account" + __version__ = "0.02" + + __description__ = """FileParadox.in account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "fileparadox.in" diff --git a/pyload/plugin/account/FilecloudIo.py b/pyload/plugin/account/FilecloudIo.py new file mode 100644 index 000000000..6d2dcb92a --- /dev/null +++ b/pyload/plugin/account/FilecloudIo.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class FilecloudIo(Account): + __name__ = "FilecloudIo" + __type__ = "account" + __version__ = "0.04" + + __description__ = """FilecloudIo account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + # It looks like the first API request always fails, so we retry 5 times, it should work on the second try + for _i in xrange(5): + rep = req.load("https://secure.filecloud.io/api-fetch_apikey.api", + post={"username": user, "password": self.getAccountData(user)['password']}) + rep = json_loads(rep) + if rep['status'] == 'ok': + break + elif rep['status'] == 'error' and rep['message'] == 'no such user or wrong password': + self.logError(_("Wrong username or password")) + return {"valid": False, "premium": False} + else: + return {"premium": False} + + akey = rep['akey'] + self.accounts[user]['akey'] = akey # Saved for hoster plugin + rep = req.load("http://api.filecloud.io/api-fetch_account_details.api", + post={"akey": akey}) + rep = json_loads(rep) + + if rep['is_premium'] == 1: + return {"validuntil": float(rep['premium_until']), "trafficleft": -1} + else: + return {"premium": False} + + + def login(self, user, data, req): + req.cj.setCookie("secure.filecloud.io", "lang", "en") + html = req.load('https://secure.filecloud.io/user-login.html') + + if not hasattr(self, "form_data"): + self.form_data = {} + + self.form_data['username'] = user + self.form_data['password'] = data['password'] + + html = req.load('https://secure.filecloud.io/user-login_p.html', + post=self.form_data, + multipart=True) + + if "you have successfully logged in" not in html: + self.wrongPassword() diff --git a/pyload/plugin/account/FilefactoryCom.py b/pyload/plugin/account/FilefactoryCom.py new file mode 100644 index 000000000..3395b3f90 --- /dev/null +++ b/pyload/plugin/account/FilefactoryCom.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import re +from time import mktime, strptime + +from pycurl import REFERER + +from pyload.plugin.Account import Account + + +class FilefactoryCom(Account): + __name__ = "FilefactoryCom" + __type__ = "account" + __version__ = "0.15" + + __description__ = """Filefactory.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + VALID_UNTIL_PATTERN = r'Premium valid until: <strong>(?P<D>\d{1,2})\w{1,2} (?P<M>\w{3}), (?P<Y>\d{4})</strong>' + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.filefactory.com/account/") + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + premium = True + validuntil = re.sub(self.VALID_UNTIL_PATTERN, '\g<D> \g<M> \g<Y>', m.group(0)) + validuntil = mktime(strptime(validuntil, "%d %b %Y")) + else: + premium = False + validuntil = -1 + + return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} + + + def login(self, user, data, req): + req.http.c.setopt(REFERER, "http://www.filefactory.com/member/login.php") + + html = req.load("http://www.filefactory.com/member/signin.php", + post={"loginEmail" : user, + "loginPassword": data['password'], + "Submit" : "Sign In"}) + + if req.lastEffectiveURL != "http://www.filefactory.com/account/": + self.wrongPassword() diff --git a/pyload/plugin/account/FilejungleCom.py b/pyload/plugin/account/FilejungleCom.py new file mode 100644 index 000000000..2c476bffb --- /dev/null +++ b/pyload/plugin/account/FilejungleCom.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +import re +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class FilejungleCom(Account): + __name__ = "FilejungleCom" + __type__ = "account" + __version__ = "0.12" + + __description__ = """Filejungle.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + login_timeout = 60 + + URL = "http://filejungle.com/" + TRAFFIC_LEFT_PATTERN = r'"/extend_premium\.php">Until (\d+ \w+ \d+)<br' + LOGIN_FAILED_PATTERN = r'<span htmlfor="loginUser(Name|Password)" generated="true" class="fail_info">' + + + def loadAccountInfo(self, user, req): + html = req.load(self.URL + "dashboard.php") + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + premium = True + validuntil = mktime(strptime(m.group(1), "%d %b %Y")) + else: + premium = False + validuntil = -1 + + return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} + + + def login(self, user, data, req): + html = req.load(self.URL + "login.php", + post={"loginUserName": user, + "loginUserPassword": data['password'], + "loginFormSubmit": "Login", + "recaptcha_challenge_field": "", + "recaptcha_response_field": "", + "recaptcha_shortencode_field": ""}, + decode=True) + + if re.search(self.LOGIN_FAILED_PATTERN, html): + self.wrongPassword() diff --git a/pyload/plugin/account/FileomCom.py b/pyload/plugin/account/FileomCom.py new file mode 100644 index 000000000..36a11e411 --- /dev/null +++ b/pyload/plugin/account/FileomCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class FileomCom(XFSAccount): + __name__ = "FileomCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Fileom.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "fileom.com" diff --git a/pyload/plugin/account/FilerNet.py b/pyload/plugin/account/FilerNet.py new file mode 100644 index 000000000..f3cc42367 --- /dev/null +++ b/pyload/plugin/account/FilerNet.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from pyload.plugin.Account import Account + + +class FilerNet(Account): + __name__ = "FilerNet" + __type__ = "account" + __version__ = "0.04" + + __description__ = """Filer.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + TOKEN_PATTERN = r'_csrf_token" value="([^"]+)" />' + WALID_UNTIL_PATTERN = r'Der Premium-Zugang ist gültig bis (.+)\.\s*</td>' + TRAFFIC_PATTERN = r'Traffic</th>\s*<td>([^<]+)</td>' + FREE_PATTERN = r'Account Status</th>\s*<td>\s*Free' + + + def loadAccountInfo(self, user, req): + html = req.load("https://filer.net/profile") + + # Free user + if re.search(self.FREE_PATTERN, html): + return {"premium": False, "validuntil": None, "trafficleft": None} + + until = re.search(self.WALID_UNTIL_PATTERN, html) + traffic = re.search(self.TRAFFIC_PATTERN, html) + + if until and traffic: + validuntil = time.mktime(time.strptime(until.group(1), "%d.%m.%Y %H:%M:%S")) + trafficleft = self.parseTraffic(traffic.group(1)) + return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} + + else: + self.logError(_("Unable to retrieve account information")) + return {"premium": False, "validuntil": None, "trafficleft": None} + + + def login(self, user, data, req): + html = req.load("https://filer.net/login") + + token = re.search(self.TOKEN_PATTERN, html).group(1) + + html = req.load("https://filer.net/login_check", + post={"_username": user, + "_password": data['password'], + "_remember_me": "on", + "_csrf_token": token, + "_target_path": "https://filer.net/"}, + decode=True) + + if 'Logout' not in html: + self.wrongPassword() diff --git a/pyload/plugin/account/FilerioCom.py b/pyload/plugin/account/FilerioCom.py new file mode 100644 index 000000000..1d9f8744b --- /dev/null +++ b/pyload/plugin/account/FilerioCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class FilerioCom(XFSAccount): + __name__ = "FilerioCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """FileRio.in account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "filerio.in" diff --git a/pyload/plugin/account/FilesMailRu.py b/pyload/plugin/account/FilesMailRu.py new file mode 100644 index 000000000..e6afd0168 --- /dev/null +++ b/pyload/plugin/account/FilesMailRu.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class FilesMailRu(Account): + __name__ = "FilesMailRu" + __type__ = "account" + __version__ = "0.11" + + __description__ = """Filesmail.ru account plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + def loadAccountInfo(self, user, req): + return {"validuntil": None, "trafficleft": None} + + + def login(self, user, data, req): + user, domain = user.split("@") + + html = req.load("http://swa.mail.ru/cgi-bin/auth", + post={"Domain": domain, + "Login": user, + "Password": data['password'], + "Page": "http://files.mail.ru/"}, + cookies=True, + decode=True) + + if "Неверное имя пользователя или пароль" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/FileserveCom.py b/pyload/plugin/account/FileserveCom.py new file mode 100644 index 000000000..9aab88d2b --- /dev/null +++ b/pyload/plugin/account/FileserveCom.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +from time import mktime, strptime + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class FileserveCom(Account): + __name__ = "FileserveCom" + __type__ = "account" + __version__ = "0.20" + + __description__ = """Fileserve.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + def loadAccountInfo(self, user, req): + data = self.getAccountData(user) + + html = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], + "submit": "Submit+Query"}) + res = json_loads(html) + + if res['type'] == "premium": + validuntil = mktime(strptime(res['expireTime'], "%Y-%m-%d %H:%M:%S")) + return {"trafficleft": res['traffic'], "validuntil": validuntil} + else: + return {"premium": False, "trafficleft": None, "validuntil": None} + + + def login(self, user, data, req): + html = req.load("http://app.fileserve.com/api/login/", post={"username": user, "password": data['password'], + "submit": "Submit+Query"}) + res = json_loads(html) + + if not res['type']: + self.wrongPassword() + + #login at fileserv html + req.load("http://www.fileserve.com/login.php", + post={"loginUserName": user, "loginUserPassword": data['password'], "autoLogin": "checked", + "loginFormSubmit": "Login"}) diff --git a/pyload/plugin/account/FourSharedCom.py b/pyload/plugin/account/FourSharedCom.py new file mode 100644 index 000000000..127f9d58a --- /dev/null +++ b/pyload/plugin/account/FourSharedCom.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class FourSharedCom(Account): + __name__ = "FourSharedCom" + __type__ = "account" + __version__ = "0.04" + + __description__ = """FourShared.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + # Free mode only for now + return {"premium": False} + + + def login(self, user, data, req): + req.cj.setCookie("4shared.com", "4langcookie", "en") + + res = req.load("http://www.4shared.com/web/login", + post={'login' : user, + 'password' : data['password'], + 'remember' : "on", + '_remember': "on", + 'returnTo' : "http://www.4shared.com/account/home.jsp"}, + decode=True) + + if 'Please log in to access your 4shared account' in res: + self.wrongPassword() diff --git a/pyload/plugin/account/FreakshareCom.py b/pyload/plugin/account/FreakshareCom.py new file mode 100644 index 000000000..9c61ac513 --- /dev/null +++ b/pyload/plugin/account/FreakshareCom.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +import re + +from time import strptime, mktime + +from pyload.plugin.Account import Account + + +class FreakshareCom(Account): + __name__ = "FreakshareCom" + __type__ = "account" + __version__ = "0.13" + + __description__ = """Freakshare.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + def loadAccountInfo(self, user, req): + premium = False + validuntil = None + trafficleft = None + + html = req.load("http://freakshare.com/") + + try: + m = re.search(r'ltig bis:</td>\s*<td><b>([\d.:-]+)</b></td>', html, re.M) + validuntil = mktime(strptime(m.group(1).strip(), "%d.%m.%Y - %H:%M")) + + except Exception: + pass + + try: + m = re.search(r'Traffic verbleibend:</td>\s*<td>([^<]+)', html, re.M) + trafficleft = self.parseTraffic(m.group(1)) + + except Exception: + pass + + return {"premium": premium, "validuntil": validuntil, "trafficleft": trafficleft} + + + def login(self, user, data, req): + req.load("http://freakshare.com/index.php?language=EN") + + html = req.load("http://freakshare.com/login.html", + post={"submit": "Login", "user": user, "pass": data['password']}, + cookies=True, + decode=True) + + if ">Wrong Username or Password" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/FreeWayMe.py b/pyload/plugin/account/FreeWayMe.py new file mode 100644 index 000000000..dcd9d34cf --- /dev/null +++ b/pyload/plugin/account/FreeWayMe.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class FreeWayMe(Account): + __name__ = "FreeWayMe" + __type__ = "account" + __version__ = "0.13" + + __description__ = """FreeWayMe account plugin""" + __license__ = "GPLv3" + __authors__ = [("Nicolas Giese", "james@free-way.me")] + + + def loadAccountInfo(self, user, req): + status = self.getAccountStatus(user, req) + + self.logDebug(status) + + account_info = {"validuntil": -1, "premium": False} + if status['premium'] == "Free": + account_info['trafficleft'] = self.parseTraffic(status['guthaben'] + "MB") + elif status['premium'] == "Spender": + account_info['trafficleft'] = -1 + elif status['premium'] == "Flatrate": + account_info = {"validuntil": float(status['Flatrate']), + "trafficleft": -1, + "premium": True} + + return account_info + + + def login(self, user, data, req): + status = self.getAccountStatus(user, req) + + # Check if user and password are valid + if not status: + self.wrongPassword() + + + def getAccountStatus(self, user, req): + answer = req.load("https://www.free-way.me/ajax/jd.php", + get={"id": 4, "user": user, "pass": self.getAccountData(user)['password']}) + + self.logDebug("Login: %s" % answer) + + if answer == "Invalid login": + self.wrongPassword() + + return json_loads(answer) diff --git a/pyload/plugin/account/FshareVn.py b/pyload/plugin/account/FshareVn.py new file mode 100644 index 000000000..282a17751 --- /dev/null +++ b/pyload/plugin/account/FshareVn.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import re + +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class FshareVn(Account): + __name__ = "FshareVn" + __type__ = "account" + __version__ = "0.09" + + __description__ = """Fshare.vn account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + VALID_UNTIL_PATTERN = ur'<dt>Thời hạn dùng:</dt>\s*<dd>([^<]+)</dd>' + LIFETIME_PATTERN = ur'<dt>Lần đăng nhập trước:</dt>\s*<dd>[^<]+</dd>' + TRAFFIC_LEFT_PATTERN = ur'<dt>Tổng Dung Lượng Tài Khoản</dt>\s*<dd[^>]*>([\d.]+) ([kKMG])B</dd>' + DIRECT_DOWNLOAD_PATTERN = ur'<input type="checkbox"\s*([^=>]*)[^>]*/>Kích hoạt download trực tiếp</dt>' + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.fshare.vn/account_info.php", decode=True) + + if re.search(self.LIFETIME_PATTERN, html): + self.logDebug("Lifetime membership detected") + trafficleft = self.getTrafficLeft() + return {"validuntil": -1, "trafficleft": trafficleft, "premium": True} + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + premium = True + validuntil = mktime(strptime(m.group(1), '%I:%M:%S %p %d-%m-%Y')) + trafficleft = self.getTrafficLeft() + else: + premium = False + validuntil = None + trafficleft = None + + return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + html = req.load("https://www.fshare.vn/login.php", + post={'LoginForm[email]' : user, + 'LoginForm[password]' : data['password'], + 'LoginForm[rememberMe]': 1, + 'yt0' : "Login"}, + referer=True, + decode=True) + + if not re.search(r'<img\s+alt="VIP"', html): + self.wrongPassword() + + + def getTrafficLeft(self): + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + return self.parseTraffic(m.group(1) + m.group(2)) if m else 0 diff --git a/pyload/plugin/account/Ftp.py b/pyload/plugin/account/Ftp.py new file mode 100644 index 000000000..67cde2cdd --- /dev/null +++ b/pyload/plugin/account/Ftp.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class Ftp(Account): + __name__ = "Ftp" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Ftp dummy account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + login_timeout = -1 #: Unlimited + info_threshold = -1 #: Unlimited diff --git a/pyload/plugin/account/HellshareCz.py b/pyload/plugin/account/HellshareCz.py new file mode 100644 index 000000000..94467b375 --- /dev/null +++ b/pyload/plugin/account/HellshareCz.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from pyload.plugin.Account import Account + + +class HellshareCz(Account): + __name__ = "HellshareCz" + __type__ = "account" + __version__ = "0.16" + + __description__ = """Hellshare.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + CREDIT_LEFT_PATTERN = r'<div class="credit-link">\s*<table>\s*<tr>\s*<th>(\d+|\d\d\.\d\d\.)</th>' + + + def loadAccountInfo(self, user, req): + self.relogin(user) + html = req.load("http://www.hellshare.com/") + + m = re.search(self.CREDIT_LEFT_PATTERN, html) + if m is None: + trafficleft = None + validuntil = None + premium = False + else: + credit = m.group(1) + premium = True + try: + if "." in credit: + #Time-based account + vt = [int(x) for x in credit.split('.')[:2]] + lt = time.localtime() + year = lt.tm_year + int(vt[1] < lt.tm_mon or (vt[1] == lt.tm_mon and vt[0] < lt.tm_mday)) + validuntil = time.mktime(time.strptime("%s%d 23:59:59" % (credit, year), "%d.%m.%Y %H:%M:%S")) + trafficleft = -1 + else: + #Traffic-based account + trafficleft = self.parseTraffic(credit + "MB") + validuntil = -1 + except Exception, e: + self.logError(_("Unable to parse credit info"), e) + validuntil = -1 + trafficleft = -1 + + return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + html = req.load('http://www.hellshare.com/', decode=True) + if req.lastEffectiveURL != 'http://www.hellshare.com/': + #Switch to English + self.logDebug("Switch lang - URL: %s" % req.lastEffectiveURL) + + json = req.load("%s?do=locRouter-show" % req.lastEffectiveURL) + hash = re.search(r"(\-\-[0-9a-f]+\-)", json).group(1) + + self.logDebug("Switch lang - HASH: %s" % hash) + + html = req.load('http://www.hellshare.com/%s/' % hash, decode=True) + + if re.search(self.CREDIT_LEFT_PATTERN, html): + self.logDebug("Already logged in") + return + + html = req.load('http://www.hellshare.com/login?do=loginForm-submit', + post={"login": "Log in", + "password": data['password'], + "username": user, + "perm_login": "on"}, + decode=True) + + if "<p>You input a wrong user name or wrong password</p>" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/Http.py b/pyload/plugin/account/Http.py new file mode 100644 index 000000000..2571ef712 --- /dev/null +++ b/pyload/plugin/account/Http.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class Http(Account): + __name__ = "Http" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Http dummy account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + login_timeout = -1 #: Unlimited + info_threshold = -1 #: Unlimited diff --git a/pyload/plugin/account/HugefilesNet.py b/pyload/plugin/account/HugefilesNet.py new file mode 100644 index 000000000..eb383fb17 --- /dev/null +++ b/pyload/plugin/account/HugefilesNet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class HugefilesNet(XFSAccount): + __name__ = "HugefilesNet" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Hugefiles.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "hugefiles.net" diff --git a/pyload/plugin/account/HundredEightyUploadCom.py b/pyload/plugin/account/HundredEightyUploadCom.py new file mode 100644 index 000000000..72185a4bb --- /dev/null +++ b/pyload/plugin/account/HundredEightyUploadCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class HundredEightyUploadCom(XFSAccount): + __name__ = "HundredEightyUploadCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """180upload.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "180upload.com" diff --git a/pyload/plugin/account/JunkyvideoCom.py b/pyload/plugin/account/JunkyvideoCom.py new file mode 100644 index 000000000..3380b8dc7 --- /dev/null +++ b/pyload/plugin/account/JunkyvideoCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class JunkyvideoCom(XFSAccount): + __name__ = "JunkyvideoCom" + __type__ = "account" + __version__ = "0.01" + + __description__ = """Junkyvideo.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "junkyvideo.com" diff --git a/pyload/plugin/account/JunocloudMe.py b/pyload/plugin/account/JunocloudMe.py new file mode 100644 index 000000000..0ffa92eb6 --- /dev/null +++ b/pyload/plugin/account/JunocloudMe.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class JunocloudMe(XFSAccount): + __name__ = "JunocloudMe" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Junocloud.me account plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it")] + + + HOSTER_DOMAIN = "junocloud.me" diff --git a/pyload/plugin/account/Keep2ShareCc.py b/pyload/plugin/account/Keep2ShareCc.py new file mode 100644 index 000000000..7ed15dc62 --- /dev/null +++ b/pyload/plugin/account/Keep2ShareCc.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import re + +from time import gmtime, mktime, strptime + +from pyload.plugin.Account import Account + + +class Keep2shareCc(Account): + __name__ = "Keep2shareCc" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Keep2share.cc account plugin""" + __license__ = "GPLv3" + __authors__ = [("aeronaut", "aeronaut@pianoguy.de")] + + + VALID_UNTIL_PATTERN = r'Premium expires: <b>(.+?)</b>' + TRAFFIC_LEFT_PATTERN = r'Available traffic \(today\):<b><a href="/user/statistic.html">(.+?)</a>' + + LOGIN_FAIL_PATTERN = r'Please fix the following input errors' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = None + + html = req.load("http://keep2share.cc/site/profile.html", decode=True) + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%Y.%m.%d")) + + except Exception, e: + self.logError(e) + + else: + if validuntil > mktime(gmtime()): + premium = True + else: + premium = False + validuntil = None + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + trafficleft = self.parseTraffic(m.group(1)) + + except Exception, e: + self.logError(e) + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + + + def login(self, user, data, req): + req.cj.setCookie("keep2share.cc", "lang", "en") + + html = req.load("http://keep2share.cc/login.html", + post={'LoginForm[username]': user, 'LoginForm[password]': data['password']}) + + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.wrongPassword() diff --git a/pyload/plugin/account/LetitbitNet.py b/pyload/plugin/account/LetitbitNet.py new file mode 100644 index 000000000..d0f08d0bb --- /dev/null +++ b/pyload/plugin/account/LetitbitNet.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +# from pyload.utils import json_loads, json_dumps + + +class LetitbitNet(Account): + __name__ = "LetitbitNet" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Letitbit.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + ## DISABLED BECAUSE IT GET 'key exausted' EVEN IF VALID ## + # api_key = self.getAccountData(user)['password'] + # json_data = [api_key, ['key/info']] + # api_rep = req.load('http://api.letitbit.net/json', post={'r': json_dumps(json_data)}) + # self.logDebug("API Key Info: " + api_rep) + # api_rep = json_loads(api_rep) + # + # if api_rep['status'] == 'FAIL': + # self.logWarning(api_rep['data']) + # return {'valid': False, 'premium': False} + + return {"premium": True} + + + def login(self, user, data, req): + # API_KEY is the username and the PREMIUM_KEY is the password + self.logInfo(_("You must use your API KEY as username and the PREMIUM KEY as password")) diff --git a/pyload/plugin/account/LinestorageCom.py b/pyload/plugin/account/LinestorageCom.py new file mode 100644 index 000000000..7a5d63a47 --- /dev/null +++ b/pyload/plugin/account/LinestorageCom.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class LinestorageCom(XFSAccount): + __name__ = "LinestorageCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Linestorage.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "linestorage.com" + HOSTER_URL = "http://linestorage.com/" diff --git a/pyload/plugin/account/LinksnappyCom.py b/pyload/plugin/account/LinksnappyCom.py new file mode 100644 index 000000000..0b1176ee9 --- /dev/null +++ b/pyload/plugin/account/LinksnappyCom.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +from hashlib import md5 + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class LinksnappyCom(Account): + __name__ = "LinksnappyCom" + __type__ = "account" + __version__ = "0.05" + __description__ = """Linksnappy.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + data = self.getAccountData(user) + r = req.load('http://gen.linksnappy.com/lseAPI.php', + get={'act': 'USERDETAILS', 'username': user, 'password': md5(data['password']).hexdigest()}) + + self.logDebug("JSON data: " + r) + + j = json_loads(r) + + if j['error']: + return {"premium": False} + + validuntil = j['return']['expire'] + + if validuntil == 'lifetime': + validuntil = -1 + + elif validuntil == 'expired': + return {"premium": False} + + else: + validuntil = float(validuntil) + + if 'trafficleft' not in j['return'] or isinstance(j['return']['trafficleft'], str): + trafficleft = -1 + else: + trafficleft = self.parseTraffic("%d MB" % j['return']['trafficleft']) + + return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} + + + def login(self, user, data, req): + r = req.load("http://gen.linksnappy.com/lseAPI.php", + get={'act' : 'USERDETAILS', + 'username': user, + 'password': md5(data['password']).hexdigest()}, + decode=True) + + if 'Invalid Account Details' in r: + self.wrongPassword() diff --git a/pyload/plugin/account/MegaDebridEu.py b/pyload/plugin/account/MegaDebridEu.py new file mode 100644 index 000000000..c2e64bcc7 --- /dev/null +++ b/pyload/plugin/account/MegaDebridEu.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class MegaDebridEu(Account): + __name__ = "MegaDebridEu" + __type__ = "account" + __version__ = "0.20" + + __description__ = """mega-debrid.eu account plugin""" + __license__ = "GPLv3" + __authors__ = [("D.Ducatel", "dducatel@je-geek.fr")] + + + # Define the base URL of MegaDebrid api + API_URL = "https://www.mega-debrid.eu/api.php" + + + def loadAccountInfo(self, user, req): + data = self.getAccountData(user) + jsonResponse = req.load(self.API_URL, + get={'action': 'connectUser', 'login': user, 'password': data['password']}) + res = json_loads(jsonResponse) + + if res['response_code'] == "ok": + return {"premium": True, "validuntil": float(res['vip_end']), "status": True} + else: + self.logError(res) + return {"status": False, "premium": False} + + + def login(self, user, data, req): + jsonResponse = req.load(self.API_URL, + get={'action': 'connectUser', 'login': user, 'password': data['password']}) + res = json_loads(jsonResponse) + if res['response_code'] != "ok": + self.wrongPassword() diff --git a/pyload/plugin/account/MegaRapidCz.py b/pyload/plugin/account/MegaRapidCz.py new file mode 100644 index 000000000..5596fd623 --- /dev/null +++ b/pyload/plugin/account/MegaRapidCz.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import re + +from time import mktime, strptime +from pyload.plugin.Account import Account + + +class MegaRapidCz(Account): + __name__ = "MegaRapidCz" + __type__ = "account" + __version__ = "0.35" + + __description__ = """MegaRapid.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("MikyWoW", "mikywow@seznam.cz"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + login_timeout = 60 + + LIMITDL_PATTERN = ur'<td>Max. počet paralelních stahování: </td><td>(\d+)' + VALID_UNTIL_PATTERN = ur'<td>Paušální stahování aktivní. Vyprší </td><td><strong>(.*?)</strong>' + TRAFFIC_LEFT_PATTERN = r'<tr><td>Kredit</td><td>(.*?) GiB' + + + def loadAccountInfo(self, user, req): + htmll = req.load("http://megarapid.cz/mujucet/", decode=True) + + m = re.search(self.LIMITDL_PATTERN, htmll) + if m: + data = self.getAccountData(user) + data['options']['limitDL'] = [int(m.group(1))] + + m = re.search(self.VALID_UNTIL_PATTERN, htmll) + if m: + validuntil = mktime(strptime(m.group(1), "%d.%m.%Y - %H:%M")) + return {"premium": True, "trafficleft": -1, "validuntil": validuntil} + + m = re.search(self.TRAFFIC_LEFT_PATTERN, htmll) + if m: + trafficleft = float(m.group(1)) * (1 << 20) + return {"premium": True, "trafficleft": trafficleft, "validuntil": -1} + + return {"premium": False, "trafficleft": None, "validuntil": None} + + + def login(self, user, data, req): + html = req.load("http://megarapid.cz/prihlaseni/", decode=True) + + if "Heslo:" in html: + start = html.index('id="inp_hash" name="hash" value="') + html = html[start + 33:] + hashes = html[0:32] + html = req.load("http://megarapid.cz/prihlaseni/", + post={"hash": hashes, + "login": user, + "pass1": data['password'], + "remember": 0, + "sbmt": u"Přihlásit"}) diff --git a/pyload/plugin/account/MegasharesCom.py b/pyload/plugin/account/MegasharesCom.py new file mode 100644 index 000000000..53b854f65 --- /dev/null +++ b/pyload/plugin/account/MegasharesCom.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +import re +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class MegasharesCom(Account): + __name__ = "MegasharesCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Megashares.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + VALID_UNTIL_PATTERN = r'<p class="premium_info_box">Period Ends: (\w{3} \d{1,2}, \d{4})</p>' + + + def loadAccountInfo(self, user, req): + #self.relogin(user) + html = req.load("http://d01.megashares.com/myms.php", decode=True) + + premium = False if '>Premium Upgrade<' in html else True + + validuntil = trafficleft = -1 + try: + timestr = re.search(self.VALID_UNTIL_PATTERN, html).group(1) + self.logDebug(timestr) + validuntil = mktime(strptime(timestr, "%b %d, %Y")) + except Exception, e: + self.logError(e) + + return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} + + + def login(self, user, data, req): + html = req.load('http://d01.megashares.com/myms_login.php', + post={"httpref" : "", + "myms_login" : "Login", + "mymslogin_name": user, + "mymspassword" : data['password']}, + decode=True) + + if not '<span class="b ml">%s</span>' % user in html: + self.wrongPassword() diff --git a/pyload/plugin/account/MovReelCom.py b/pyload/plugin/account/MovReelCom.py new file mode 100644 index 000000000..4d2855de1 --- /dev/null +++ b/pyload/plugin/account/MovReelCom.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class MovReelCom(XFSAccount): + __name__ = "MovReelCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Movreel.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("t4skforce", "t4skforce1337[AT]gmail[DOT]com")] + + + login_timeout = 60 + info_threshold = 30 + + HOSTER_DOMAIN = "movreel.com" diff --git a/pyload/plugin/account/MultihostersCom.py b/pyload/plugin/account/MultihostersCom.py new file mode 100644 index 000000000..3f5d90c4e --- /dev/null +++ b/pyload/plugin/account/MultihostersCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.account.ZeveraCom import ZeveraCom + + +class MultihostersCom(ZeveraCom): + __name__ = "MultihostersCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Multihosters.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("tjeh", "tjeh@gmx.net")] + + + HOSTER_DOMAIN = "multihosters.com" diff --git a/pyload/plugin/account/MultishareCz.py b/pyload/plugin/account/MultishareCz.py new file mode 100644 index 000000000..9b4f4447b --- /dev/null +++ b/pyload/plugin/account/MultishareCz.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class MultishareCz(Account): + __name__ = "MultishareCz" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Multishare.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + TRAFFIC_LEFT_PATTERN = r'<span class="profil-zvyrazneni">Kredit:</span>\s*<strong>(?P<S>[\d.,]+) (?P<U>[\w^_]+)</strong>' + ACCOUNT_INFO_PATTERN = r'<input type="hidden" id="(u_ID|u_hash)" name="[^"]*" value="([^"]+)">' + + + def loadAccountInfo(self, user, req): + #self.relogin(user) + html = req.load("http://www.multishare.cz/profil/", decode=True) + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + trafficleft = self.parseTraffic(m.group('S') + m.group('U')) if m else 0 + self.premium = True if trafficleft else False + + html = req.load("http://www.multishare.cz/", decode=True) + mms_info = dict(re.findall(self.ACCOUNT_INFO_PATTERN, html)) + + return dict(mms_info, **{"validuntil": -1, "trafficleft": trafficleft}) + + + def login(self, user, data, req): + html = req.load('http://www.multishare.cz/html/prihlaseni_process.php', + post={"akce" : "Přihlásit", + "heslo": data['password'], + "jmeno": user}, + decode=True) + + if '<div class="akce-chyba akce">' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/MyfastfileCom.py b/pyload/plugin/account/MyfastfileCom.py new file mode 100644 index 000000000..838a1eefd --- /dev/null +++ b/pyload/plugin/account/MyfastfileCom.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from time import time + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class MyfastfileCom(Account): + __name__ = "MyfastfileCom" + __type__ = "account" + __version__ = "0.04" + + __description__ = """Myfastfile.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + if 'days_left' in self.json_data: + validuntil = time() + self.json_data['days_left'] * 24 * 60 * 60 + return {"premium": True, "validuntil": validuntil, "trafficleft": -1} + else: + self.logError(_("Unable to get account information")) + + + def login(self, user, data, req): + # Password to use is the API-Password written in http://myfastfile.com/myaccount + html = req.load("http://myfastfile.com/api.php", + get={"user": user, "pass": data['password']}) + + self.logDebug("JSON data: " + html) + + self.json_data = json_loads(html) + if self.json_data['status'] != 'ok': + self.logError(_('Invalid login. The password to use is the API-Password you find in your "My Account" page')) + self.wrongPassword() diff --git a/pyload/plugin/account/NetloadIn.py b/pyload/plugin/account/NetloadIn.py new file mode 100644 index 000000000..68982e828 --- /dev/null +++ b/pyload/plugin/account/NetloadIn.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +import re +from time import time + +from pyload.plugin.Account import Account + + +class NetloadIn(Account): + __name__ = "NetloadIn" + __type__ = "account" + __version__ = "0.23" + + __description__ = """Netload.in account plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("CryNickSystems", "webmaster@pcProfil.de")] + + + def loadAccountInfo(self, user, req): + html = req.load("http://netload.in/index.php", get={'id': 2, 'lang': "de"}) + left = r'>(\d+) (Tag|Tage), (\d+) Stunden<' + left = re.search(left, html) + if left: + validuntil = time() + int(left.group(1)) * 24 * 60 * 60 + int(left.group(3)) * 60 * 60 + trafficleft = -1 + premium = True + else: + validuntil = None + premium = False + trafficleft = None + return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + html = req.load("http://netload.in/index.php", + post={"txtuser" : user, + "txtpass" : data['password'], + "txtcheck": "login", + "txtlogin": "Login"}, + cookies=True, + decode=True) + if "password or it might be invalid!" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/NoPremiumPl.py b/pyload/plugin/account/NoPremiumPl.py new file mode 100644 index 000000000..929e8f7d9 --- /dev/null +++ b/pyload/plugin/account/NoPremiumPl.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +import hashlib + +from datetime import datetime +from time import mktime + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class NoPremiumPl(Account): + __name__ = "NoPremiumPl" + __version__ = "0.01" + __type__ = "account" + __description__ = "NoPremium.pl account plugin" + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@nopremium.pl")] + + _api_url = "http://crypt.nopremium.pl" + + _api_query = { + "site": "nopremium", + "username": "", + "password": "", + "output": "json", + "loc": "1", + "info": "1" + } + + _req = None + _usr = None + _pwd = None + + def loadAccountInfo(self, name, req): + self._req = req + try: + result = json_loads(self.runAuthQuery()) + except Exception: + # todo: return or let it be thrown? + return + + premium = False + valid_untill = -1 + + if "expire" in result.keys() and result["expire"]: + premium = True + valid_untill = mktime(datetime.fromtimestamp(int(result["expire"])).timetuple()) + traffic_left = result["balance"] * 1024 + + return ({ + "validuntil": valid_untill, + "trafficleft": traffic_left, + "premium": premium + }) + + def login(self, user, data, req): + self._usr = user + self._pwd = hashlib.sha1(hashlib.md5(data["password"]).hexdigest()).hexdigest() + self._req = req + + try: + response = json_loads(self.runAuthQuery()) + except Exception: + self.wrongPassword() + + if "errno" in response.keys(): + self.wrongPassword() + data['usr'] = self._usr + data['pwd'] = self._pwd + + def createAuthQuery(self): + query = self._api_query + query["username"] = self._usr + query["password"] = self._pwd + + return query + + def runAuthQuery(self): + data = self._req.load(self._api_url, post=self.createAuthQuery()) + + return data \ No newline at end of file diff --git a/pyload/plugin/account/NosuploadCom.py b/pyload/plugin/account/NosuploadCom.py new file mode 100644 index 000000000..7fc8b49de --- /dev/null +++ b/pyload/plugin/account/NosuploadCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class NosuploadCom(XFSAccount): + __name__ = "NosuploadCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Nosupload.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "nosupload.com" diff --git a/pyload/plugin/account/NovafileCom.py b/pyload/plugin/account/NovafileCom.py new file mode 100644 index 000000000..71a7dc2dc --- /dev/null +++ b/pyload/plugin/account/NovafileCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class NovafileCom(XFSAccount): + __name__ = "NovafileCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Novafile.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "novafile.com" diff --git a/pyload/plugin/account/NowVideoSx.py b/pyload/plugin/account/NowVideoSx.py new file mode 100644 index 000000000..8359e0410 --- /dev/null +++ b/pyload/plugin/account/NowVideoSx.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +import re + +from time import gmtime, mktime, strptime + +from pyload.plugin.Account import Account + + +class NowVideoSx(Account): + __name__ = "NowVideoSx" + __type__ = "account" + __version__ = "0.03" + + __description__ = """NowVideo.at account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + VALID_UNTIL_PATTERN = r'>Your premium membership expires on: (.+?)<' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = -1 + premium = None + + html = req.load("http://www.nowvideo.sx/premium.php") + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%Y-%b-%d")) + + except Exception, e: + self.logError(e) + + else: + if validuntil > mktime(gmtime()): + premium = True + else: + premium = False + validuntil = -1 + + return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + html = req.load("http://www.nowvideo.sx/login.php", + post={'user': user, 'pass': data['password']}, + decode=True) + + if re.search(r'>Log In<', html): + self.wrongPassword() diff --git a/pyload/plugin/account/OboomCom.py b/pyload/plugin/account/OboomCom.py new file mode 100644 index 000000000..012fb42c3 --- /dev/null +++ b/pyload/plugin/account/OboomCom.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import time + +from beaker.crypto.pbkdf2 import PBKDF2 + +from pyload.utils import json_loads +from pyload.plugin.Account import Account + + +class OboomCom(Account): + __name__ = "OboomCom" + __type__ = "account" + __version__ = "0.23" + + __description__ = """Oboom.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("stanley", "stanley.foerster@gmail.com")] + + + def loadAccountData(self, user, req): + passwd = self.getAccountData(user)['password'] + salt = passwd[::-1] + pbkdf2 = PBKDF2(passwd, salt, 1000).hexread(16) + + result = json_loads(req.load("https://www.oboom.com/1/login", get={"auth": user, "pass": pbkdf2})) + + if not result[0] == 200: + self.logWarning(_("Failed to log in: %s") % result[1]) + self.wrongPassword() + + return result[1] + + + def loadAccountInfo(self, name, req): + accountData = self.loadAccountData(name, req) + + userData = accountData['user'] + + if userData['premium'] == "null": + premium = False + else: + premium = True + + if userData['premium_unix'] == "null": + validUntil = -1 + else: + validUntil = float(userData['premium_unix']) + + traffic = userData['traffic'] + + trafficLeft = traffic['current'] / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + maxTraffic = traffic['max'] / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + + session = accountData['session'] + + return {'premium' : premium, + 'validuntil' : validUntil, + 'trafficleft': trafficLeft, + 'maxtraffic' : maxTraffic, + 'session' : session} + + + def login(self, user, data, req): + self.loadAccountData(user, req) diff --git a/pyload/plugin/account/OneFichierCom.py b/pyload/plugin/account/OneFichierCom.py new file mode 100644 index 000000000..65ec841c4 --- /dev/null +++ b/pyload/plugin/account/OneFichierCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import re + +from time import strptime, mktime + +from pycurl import REFERER + +from pyload.plugin.Account import Account + + +class OneFichierCom(Account): + __name__ = "OneFichierCom" + __type__ = "account" + __version__ = "0.12" + + __description__ = """1fichier.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Elrick69", "elrick69[AT]rocketmail[DOT]com"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + VALID_UNTIL_PATTERN = r'Your Premium Status will end the (\d+/\d+/\d+)' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = -1 + premium = None + + html = req.load("https://1fichier.com/console/abo.pl") + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1) + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%d/%m/%Y")) + except Exception, e: + self.logError(e) + else: + premium = True + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} + + + def login(self, user, data, req): + req.http.c.setopt(REFERER, "https://1fichier.com/login.pl?lg=en") + + html = req.load("https://1fichier.com/login.pl?lg=en", + post={'mail' : user, + 'pass' : data['password'], + 'It' : "on", + 'purge' : "off", + 'valider': "Send"}, + decode=True) + + if '>Invalid email address' in html or '>Invalid password' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/OverLoadMe.py b/pyload/plugin/account/OverLoadMe.py new file mode 100644 index 000000000..d945dd7bd --- /dev/null +++ b/pyload/plugin/account/OverLoadMe.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class OverLoadMe(Account): + __name__ = "OverLoadMe" + __type__ = "account" + __version__ = "0.04" + + __description__ = """Over-Load.me account plugin""" + __license__ = "GPLv3" + __authors__ = [("marley", "marley@over-load.me")] + + + def loadAccountInfo(self, user, req): + https = "https" if self.getConfig("ssl") else "http" + data = self.getAccountData(user) + html = req.load(https + "://api.over-load.me/account.php", + get={'user': user, + 'auth': data['password']}).strip() + + data = json_loads(html) + self.logDebug(data) + + # Check for premium + if data['membership'] == "Free": + return {'premium': False, 'validuntil': None, 'trafficleft': None} + else: + return {'premium': True, 'validuntil': data['expirationunix'], 'trafficleft': -1} + + + def login(self, user, data, req): + https = "https" if self.getConfig("ssl") else "http" + jsondata = req.load(https + "://api.over-load.me/account.php", + get={'user': user, + 'auth': data['password']}).strip() + + data = json_loads(jsondata) + + if data['err'] == 1: + self.wrongPassword() diff --git a/pyload/plugin/account/PremiumTo.py b/pyload/plugin/account/PremiumTo.py new file mode 100644 index 000000000..04bbc10d5 --- /dev/null +++ b/pyload/plugin/account/PremiumTo.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class PremiumTo(Account): + __name__ = "PremiumTo" + __type__ = "account" + __version__ = "0.08" + + __description__ = """Premium.to account plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + + def loadAccountInfo(self, user, req): + traffic = req.load("http://premium.to/api/straffic.php", + get={'username': self.username, 'password': self.password}) + + if "wrong username" not in traffic: + trafficleft = sum(map(float, traffic.split(';'))) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + return {'premium': True, 'trafficleft': trafficleft, 'validuntil': -1} + else: + return {'premium': False, 'trafficleft': None, 'validuntil': None} + + + def login(self, user, data, req): + self.username = user + self.password = data['password'] + authcode = req.load("http://premium.to/api/getauthcode.php", + get={'username': user, 'password': self.password}, + decode=True) + + if "wrong username" in authcode: + self.wrongPassword() diff --git a/pyload/plugin/account/PremiumizeMe.py b/pyload/plugin/account/PremiumizeMe.py new file mode 100644 index 000000000..3cd15ce23 --- /dev/null +++ b/pyload/plugin/account/PremiumizeMe.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + +from pyload.utils import json_loads + + +class PremiumizeMe(Account): + __name__ = "PremiumizeMe" + __type__ = "account" + __version__ = "0.13" + + __description__ = """Premiumize.me account plugin""" + __license__ = "GPLv3" + __authors__ = [("Florian Franzen", "FlorianFranzen@gmail.com")] + + + def loadAccountInfo(self, user, req): + # Get user data from premiumize.me + status = self.getAccountStatus(user, req) + self.logDebug(status) + + # Parse account info + account_info = {"validuntil": float(status['result']['expires']), + "trafficleft": max(0, status['result']['trafficleft_bytes'] / 1024)} #@TODO: Remove `/ 1024` in 0.4.10 + + if status['result']['type'] == 'free': + account_info['premium'] = False + + return account_info + + + def login(self, user, data, req): + # Get user data from premiumize.me + status = self.getAccountStatus(user, req) + + # Check if user and password are valid + if status['status'] != 200: + self.wrongPassword() + + + def getAccountStatus(self, user, req): + # Use premiumize.me API v1 (see https://secure.premiumize.me/?show=api) + # to retrieve account info and return the parsed json answer + answer = req.load("https://api.premiumize.me/pm-api/v1.php", + get={'method' : "accountstatus", + 'params[login]': user, + 'params[pass]' : self.getAccountData(user)['password']}) + return json_loads(answer) diff --git a/pyload/plugin/account/PutdriveCom.py b/pyload/plugin/account/PutdriveCom.py new file mode 100644 index 000000000..b30fb6565 --- /dev/null +++ b/pyload/plugin/account/PutdriveCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.account.ZeveraCom import ZeveraCom + + +class PutdriveCom(ZeveraCom): + __name__ = "PutdriveCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Putdrive.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "putdrive.com" diff --git a/pyload/plugin/account/QuickshareCz.py b/pyload/plugin/account/QuickshareCz.py new file mode 100644 index 000000000..2bcde1c9d --- /dev/null +++ b/pyload/plugin/account/QuickshareCz.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class QuickshareCz(Account): + __name__ = "QuickshareCz" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Quickshare.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + TRAFFIC_LEFT_PATTERN = r'Stav kreditu: <strong>(.+?)</strong>' + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.quickshare.cz/premium", decode=True) + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + trafficleft = self.parseTraffic(m.group(1)) + premium = True if trafficleft else False + else: + trafficleft = None + premium = False + + return {"validuntil": -1, "trafficleft": trafficleft, "premium": premium} + + + def login(self, user, data, req): + html = req.load('http://www.quickshare.cz/html/prihlaseni_process.php', + post={"akce": u'Přihlásit', + "heslo": data['password'], + "jmeno": user}, + decode=True) + + if u'>Takový uživatel neexistuje.<' in html or u'>Špatné heslo.<' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/RPNetBiz.py b/pyload/plugin/account/RPNetBiz.py new file mode 100644 index 000000000..e0b35b68c --- /dev/null +++ b/pyload/plugin/account/RPNetBiz.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class RPNetBiz(Account): + __name__ = "RPNetBiz" + __type__ = "account" + __version__ = "0.12" + + __description__ = """RPNet.biz account plugin""" + __license__ = "GPLv3" + __authors__ = [("Dman", "dmanugm@gmail.com")] + + + def loadAccountInfo(self, user, req): + # Get account information from rpnet.biz + res = self.getAccountStatus(user, req) + try: + if res['accountInfo']['isPremium']: + # Parse account info. Change the trafficleft later to support per host info. + account_info = {"validuntil": float(res['accountInfo']['premiumExpiry']), + "trafficleft": -1, "premium": True} + else: + account_info = {"validuntil": None, "trafficleft": None, "premium": False} + + except KeyError: + #handle wrong password exception + account_info = {"validuntil": None, "trafficleft": None, "premium": False} + + return account_info + + + def login(self, user, data, req): + # Get account information from rpnet.biz + res = self.getAccountStatus(user, req) + + # If we have an error in the res, we have wrong login information + if 'error' in res: + self.wrongPassword() + + + def getAccountStatus(self, user, req): + # Using the rpnet API, check if valid premium account + res = req.load("https://premium.rpnet.biz/client_api.php", + get={"username": user, "password": self.getAccountData(user)['password'], + "action": "showAccountInformation"}) + self.logDebug("JSON data: %s" % res) + + return json_loads(res) diff --git a/pyload/plugin/account/RapideoPl.py b/pyload/plugin/account/RapideoPl.py new file mode 100644 index 000000000..b4b74fab5 --- /dev/null +++ b/pyload/plugin/account/RapideoPl.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +import hashlib + +from datetime import datetime +from time import mktime + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class RapideoPl(Account): + __name__ = "RapideoPl" + __version__ = "0.01" + __type__ = "account" + __description__ = "Rapideo.pl account plugin" + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@rapideo.pl")] + + _api_url = "http://enc.rapideo.pl" + + _api_query = { + "site": "newrd", + "username": "", + "password": "", + "output": "json", + "loc": "1", + "info": "1" + } + + _req = None + _usr = None + _pwd = None + + def loadAccountInfo(self, name, req): + self._req = req + try: + result = json_loads(self.runAuthQuery()) + except Exception: + # todo: return or let it be thrown? + return + + premium = False + valid_untill = -1 + if "expire" in result.keys() and result["expire"]: + premium = True + valid_untill = mktime(datetime.fromtimestamp(int(result["expire"])).timetuple()) + + traffic_left = result["balance"] + + return ({ + "validuntil": valid_untill, + "trafficleft": traffic_left, + "premium": premium + }) + + def login(self, user, data, req): + self._usr = user + self._pwd = hashlib.md5(data["password"]).hexdigest() + self._req = req + try: + response = json_loads(self.runAuthQuery()) + except Exception: + self.wrongPassword() + + if "errno" in response.keys(): + self.wrongPassword() + data['usr'] = self._usr + data['pwd'] = self._pwd + + def createAuthQuery(self): + query = self._api_query + query["username"] = self._usr + query["password"] = self._pwd + + return query + + def runAuthQuery(self): + data = self._req.load(self._api_url, post=self.createAuthQuery()) + + return data \ No newline at end of file diff --git a/pyload/plugin/account/RapidfileshareNet.py b/pyload/plugin/account/RapidfileshareNet.py new file mode 100644 index 000000000..ec0bf8db4 --- /dev/null +++ b/pyload/plugin/account/RapidfileshareNet.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class RapidfileshareNet(XFSAccount): + __name__ = "RapidfileshareNet" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Rapidfileshare.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it")] + + + HOSTER_DOMAIN = "rapidfileshare.net" + + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:</TD><TD><label for="name">\s*(?P<S>[\d.,]+)\s*(?:(?P<U>[\w^_]+))?' diff --git a/pyload/plugin/account/RapidgatorNet.py b/pyload/plugin/account/RapidgatorNet.py new file mode 100644 index 000000000..7643f07d2 --- /dev/null +++ b/pyload/plugin/account/RapidgatorNet.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class RapidgatorNet(Account): + __name__ = "RapidgatorNet" + __type__ = "account" + __version__ = "0.09" + + __description__ = """Rapidgator.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + API_URL = "http://rapidgator.net/api/user" + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = False + sid = None + + try: + sid = self.getAccountData(user).get('sid') + assert sid + + html = req.load("%s/info" % self.API_URL, get={'sid': sid}) + + self.logDebug("API:USERINFO", html) + + json = json_loads(html) + + if json['response_status'] == 200: + if "reset_in" in json['response']: + self.scheduleRefresh(user, json['response']['reset_in']) + + validuntil = json['response']['expire_date'] + trafficleft = float(json['response']['traffic_left']) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + premium = True + else: + self.logError(json['response_details']) + + except Exception, e: + self.logError(e) + + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium, + 'sid' : sid} + + + def login(self, user, data, req): + try: + html = req.load('%s/login' % self.API_URL, post={"username": user, "password": data['password']}) + + self.logDebug("API:LOGIN", html) + + json = json_loads(html) + + if json['response_status'] == 200: + data['sid'] = str(json['response']['session_id']) + return + else: + self.logError(json['response_details']) + + except Exception, e: + self.logError(e) + + self.wrongPassword() diff --git a/pyload/plugin/account/RapiduNet.py b/pyload/plugin/account/RapiduNet.py new file mode 100644 index 000000000..5c165f486 --- /dev/null +++ b/pyload/plugin/account/RapiduNet.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +import re + +from time import time + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class RapiduNet(Account): + __name__ = "RapiduNet" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Rapidu.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("prOq", None), + ("Walter Purcaro", "vuolter@gmail.com")] + + + PREMIUM_PATTERN = r'>Account: <b>Premium' + + VALID_UNTIL_PATTERN = r'>Account: <b>\w+ \((\d+)' + + TRAFFIC_LEFT_PATTERN = r'class="tipsyS"><b>(.+?)<' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = -1 + premium = False + + html = req.load("https://rapidu.net/", decode=True) + + if re.search(self.PREMIUM_PATTERN, html): + premium = True + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + validuntil = time() + (86400 * int(m.group(1))) + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + trafficleft = self.parseTraffic(m.group(1)) + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + + + def login(self, user, data, req): + req.load("https://rapidu.net/ajax.php", + get={'a': "getChangeLang"}, + post={'_go' : "", + 'lang': "en"}) + + json = json_loads(req.load("https://rapidu.net/ajax.php", + get={'a': "getUserLogin"}, + post={'_go' : "", + 'login' : user, + 'pass' : data['password'], + 'remember': "1"})) + + self.logDebug(json) + + if not json['message'] == "success": + self.wrongPassword() diff --git a/pyload/plugin/account/RarefileNet.py b/pyload/plugin/account/RarefileNet.py new file mode 100644 index 000000000..1dc93681c --- /dev/null +++ b/pyload/plugin/account/RarefileNet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class RarefileNet(XFSAccount): + __name__ = "RarefileNet" + __type__ = "account" + __version__ = "0.04" + + __description__ = """RareFile.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "rarefile.net" diff --git a/pyload/plugin/account/RealdebridCom.py b/pyload/plugin/account/RealdebridCom.py new file mode 100644 index 000000000..07ff70496 --- /dev/null +++ b/pyload/plugin/account/RealdebridCom.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +import xml.dom.minidom as dom + +from pyload.plugin.Account import Account + + +class RealdebridCom(Account): + __name__ = "RealdebridCom" + __type__ = "account" + __version__ = "0.45" + + __description__ = """Real-Debrid.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Devirex Hazzard", "naibaf_11@yahoo.de")] + + + def loadAccountInfo(self, user, req): + if self.pin_code: + return {"premium": False} + html = req.load("https://real-debrid.com/api/account.php") + xml = dom.parseString(html) + account_info = {"validuntil": float(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue), + "trafficleft": -1} + + return account_info + + + def login(self, user, data, req): + self.pin_code = False + html = req.load("https://real-debrid.com/ajax/login.php", + get={"user": user, "pass": data['password']}, + decode=True) + + if "Your login informations are incorrect" in html: + self.wrongPassword() + + elif "PIN Code required" in html: + self.logWarning(_("PIN code required. Please login to https://real-debrid.com using the PIN or disable the double authentication in your control panel on https://real-debrid.com")) + self.pin_code = True diff --git a/pyload/plugin/account/RehostTo.py b/pyload/plugin/account/RehostTo.py new file mode 100644 index 000000000..d62e1918a --- /dev/null +++ b/pyload/plugin/account/RehostTo.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class RehostTo(Account): + __name__ = "RehostTo" + __type__ = "account" + __version__ = "0.16" + + __description__ = """Rehost.to account plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + def loadAccountInfo(self, user, req): + premium = False + trafficleft = None + validuntil = -1 + session = "" + + html = req.load("http://rehost.to/api.php", + get={'cmd' : "login", 'user': user, + 'pass': self.getAccountData(user)['password']}) + try: + session = html.split(",")[1].split("=")[1] + + html = req.load("http://rehost.to/api.php", + get={'cmd': "get_premium_credits", 'long_ses': session}) + + if html.strip() == "0,0" or "ERROR" in html: + self.logDebug(html) + else: + traffic, valid = html.split(",") + + premium = True + trafficleft = self.parseTraffic(traffic + "MB") + validuntil = float(valid) + + finally: + return {'premium' : premium, + 'trafficleft': trafficleft, + 'validuntil' : validuntil, + 'session' : session} + + + def login(self, user, data, req): + html = req.load("http://rehost.to/api.php", + get={'cmd': "login", 'user': user, 'pass': data['password']}, + decode=True) + + if "ERROR" in html: + self.logDebug(html) + self.wrongPassword() diff --git a/pyload/plugin/account/RyushareCom.py b/pyload/plugin/account/RyushareCom.py new file mode 100644 index 000000000..466d971f6 --- /dev/null +++ b/pyload/plugin/account/RyushareCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class RyushareCom(XFSAccount): + __name__ = "RyushareCom" + __type__ = "account" + __version__ = "0.06" + + __description__ = """Ryushare.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "ryushare.com" diff --git a/pyload/plugin/account/SafesharingEu.py b/pyload/plugin/account/SafesharingEu.py new file mode 100644 index 000000000..f5cbf050e --- /dev/null +++ b/pyload/plugin/account/SafesharingEu.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class SafesharingEu(XFSAccount): + __name__ = "SafesharingEu" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Safesharing.eu account plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it")] + + + HOSTER_DOMAIN = "safesharing.eu" diff --git a/pyload/plugin/account/SecureUploadEu.py b/pyload/plugin/account/SecureUploadEu.py new file mode 100644 index 000000000..bb47bcba3 --- /dev/null +++ b/pyload/plugin/account/SecureUploadEu.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class SecureUploadEu(XFSAccount): + __name__ = "SecureUploadEu" + __type__ = "account" + __version__ = "0.02" + + __description__ = """SecureUpload.eu account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "secureupload.eu" diff --git a/pyload/plugin/account/SendmywayCom.py b/pyload/plugin/account/SendmywayCom.py new file mode 100644 index 000000000..d64658de3 --- /dev/null +++ b/pyload/plugin/account/SendmywayCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class SendmywayCom(XFSAccount): + __name__ = "SendmywayCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Sendmyway.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "sendmyway.com" diff --git a/pyload/plugin/account/ShareonlineBiz.py b/pyload/plugin/account/ShareonlineBiz.py new file mode 100644 index 000000000..28bc3b9bc --- /dev/null +++ b/pyload/plugin/account/ShareonlineBiz.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class ShareonlineBiz(Account): + __name__ = "ShareonlineBiz" + __type__ = "account" + __version__ = "0.31" + + __description__ = """Share-online.biz account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def api_response(self, user, req): + return req.load("http://api.share-online.biz/cgi-bin", + get={'q' : "userdetails", + 'aux' : "traffic", + 'username': user, + 'password': self.getAccountData(user)['password']}) + + + def loadAccountInfo(self, user, req): + premium = False + validuntil = None + trafficleft = -1 + maxtraffic = 100 * 1024 * 1024 * 1024 #: 100 GB + + api = {} + for line in self.api_response(user, req).splitlines(): + if "=" in line: + key, value = line.split("=") + api[key] = value + + self.logDebug(api) + + if api['a'].lower() != "not_available": + req.cj.setCookie("share-online.biz", 'a', api['a']) + + premium = api['group'] in ("Premium", "PrePaid") + + validuntil = float(api['expire_date']) + + traffic = float(api['traffic_1d'].split(";")[0]) + maxtraffic = max(maxtraffic, traffic) + trafficleft = maxtraffic - traffic + + maxtraffic /= 1024 #@TODO: Remove `/ 1024` in 0.4.10 + trafficleft /= 1024 #@TODO: Remove `/ 1024` in 0.4.10 + + return {'premium' : premium, + 'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'maxtraffic' : maxtraffic} + + + def login(self, user, data, req): + html = self.api_response(user, req) + err = re.search(r'\*\*(.+?)\*\*', html) + if err: + self.logError(err.group(1)) + self.wrongPassword() diff --git a/pyload/plugin/account/SimplyPremiumCom.py b/pyload/plugin/account/SimplyPremiumCom.py new file mode 100644 index 000000000..298ad8d59 --- /dev/null +++ b/pyload/plugin/account/SimplyPremiumCom.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.Account import Account + + +class SimplyPremiumCom(Account): + __name__ = "SimplyPremiumCom" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Simply-Premium.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("EvolutionClip", "evolutionclip@live.de")] + + + def loadAccountInfo(self, user, req): + premium = False + validuntil = -1 + trafficleft = None + + json_data = req.load('http://www.simply-premium.com/api/user.php?format=json') + + self.logDebug("JSON data: %s" % json_data) + + json_data = json_loads(json_data) + + if 'vip' in json_data['result'] and json_data['result']['vip']: + premium = True + + if 'timeend' in json_data['result'] and json_data['result']['timeend']: + validuntil = float(json_data['result']['timeend']) + + if 'remain_traffic' in json_data['result'] and json_data['result']['remain_traffic']: + trafficleft = float(json_data['result']['remain_traffic']) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + + return {"premium": premium, "validuntil": validuntil, "trafficleft": trafficleft} + + + def login(self, user, data, req): + req.cj.setCookie("simply-premium.com", "lang", "EN") + + html = req.load("http://www.simply-premium.com/login.php", + post={'key': user} if not data['password'] else {'login_name': user, 'login_pass': data['password']}, + decode=True) + + if 'logout' not in html: + self.wrongPassword() diff --git a/pyload/plugin/account/SimplydebridCom.py b/pyload/plugin/account/SimplydebridCom.py new file mode 100644 index 000000000..a826e44c7 --- /dev/null +++ b/pyload/plugin/account/SimplydebridCom.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class SimplydebridCom(Account): + __name__ = "SimplydebridCom" + __type__ = "account" + __version__ = "0.11" + + __description__ = """Simply-Debrid.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + def loadAccountInfo(self, user, req): + get_data = {'login': 2, 'u': self.loginname, 'p': self.password} + res = req.load("http://simply-debrid.com/api.php", get=get_data, decode=True) + data = [x.strip() for x in res.split(";")] + if str(data[0]) != "1": + return {"premium": False} + else: + return {"trafficleft": -1, "validuntil": mktime(strptime(str(data[2]), "%d/%m/%Y"))} + + + def login(self, user, data, req): + self.loginname = user + self.password = data['password'] + get_data = {'login': 1, 'u': self.loginname, 'p': self.password} + + res = req.load("http://simply-debrid.com/api.php", get=get_data, decode=True) + if res != "02: loggin success": + self.wrongPassword() diff --git a/pyload/plugin/account/SmoozedCom.py b/pyload/plugin/account/SmoozedCom.py new file mode 100644 index 000000000..27249c7c2 --- /dev/null +++ b/pyload/plugin/account/SmoozedCom.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import hashlib + +from beaker.crypto.pbkdf2 import PBKDF2 +from time import time + +from pyload.utils import json_loads +from pyload.plugin.Account import Account + + +class SmoozedCom(Account): + __name__ = "SmoozedCom" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Smoozed.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("", "")] + + + def loadAccountInfo(self, user, req): + # Get user data from premiumize.me + status = self.getAccountStatus(user, req) + + self.logDebug(status) + + if status['state'] != 'ok': + info = {'validuntil' : None, + 'trafficleft': None, + 'premium' : False} + else: + # Parse account info + info = {'validuntil' : float(status["data"]["user"]["user_premium"]), + 'trafficleft': max(0, status["data"]["traffic"][1] - status["data"]["traffic"][0]), + 'session' : status["data"]["session_key"], + 'hosters' : [hoster["name"] for hoster in status["data"]["hoster"]]} + + if info['validuntil'] < time(): + info['premium'] = False + else: + info['premium'] = True + + return info + + + def login(self, user, data, req): + # Get user data from premiumize.me + status = self.getAccountStatus(user, req) + + # Check if user and password are valid + if status['state'] != 'ok': + self.wrongPassword() + + + def getAccountStatus(self, user, req): + password = self.getAccountData(user)['password'] + salt = hashlib.sha256(password).hexdigest() + encrypted = PBKDF2(password, salt, iterations=1000).hexread(32) + + return json_loads(req.load("http://www2.smoozed.com/api/login", + get={'auth': user, 'password': encrypted})) diff --git a/pyload/plugin/account/StahnuTo.py b/pyload/plugin/account/StahnuTo.py new file mode 100644 index 000000000..ed8df3b77 --- /dev/null +++ b/pyload/plugin/account/StahnuTo.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class StahnuTo(Account): + __name__ = "StahnuTo" + __type__ = "account" + __version__ = "0.05" + + __description__ = """StahnuTo account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.stahnu.to/") + + m = re.search(r'>VIP: (\d+.*)<', html) + trafficleft = self.parseTraffic(m.group(1)) if m else 0 + + return {"premium": trafficleft > 512, "trafficleft": trafficleft, "validuntil": -1} + + + def login(self, user, data, req): + html = req.load("http://www.stahnu.to/login.php", + post={"username": user, + "password": data['password'], + "submit": "Login"}, + decode=True) + + if not '<a href="logout.php">' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/StreamcloudEu.py b/pyload/plugin/account/StreamcloudEu.py new file mode 100644 index 000000000..3ac74fbd0 --- /dev/null +++ b/pyload/plugin/account/StreamcloudEu.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class StreamcloudEu(XFSAccount): + __name__ = "StreamcloudEu" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Streamcloud.eu account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "streamcloud.eu" diff --git a/pyload/plugin/account/TurbobitNet.py b/pyload/plugin/account/TurbobitNet.py new file mode 100644 index 000000000..e3c07da2a --- /dev/null +++ b/pyload/plugin/account/TurbobitNet.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +import re +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class TurbobitNet(Account): + __name__ = "TurbobitNet" + __type__ = "account" + __version__ = "0.02" + + __description__ = """TurbobitNet account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + def loadAccountInfo(self, user, req): + html = req.load("http://turbobit.net") + + m = re.search(r'<u>Turbo Access</u> to ([\d.]+)', html) + if m: + premium = True + validuntil = mktime(strptime(m.group(1), "%d.%m.%Y")) + else: + premium = False + validuntil = -1 + + return {"premium": premium, "trafficleft": -1, "validuntil": validuntil} + + + def login(self, user, data, req): + req.cj.setCookie("turbobit.net", "user_lang", "en") + + html = req.load("http://turbobit.net/user/login", + post={"user[login]": user, + "user[pass]": data['password'], + "user[submit]": "Login"}, + decode=True) + + if not '<div class="menu-item user-name">' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/TusfilesNet.py b/pyload/plugin/account/TusfilesNet.py new file mode 100644 index 000000000..84e9ef9c6 --- /dev/null +++ b/pyload/plugin/account/TusfilesNet.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +import re + +from time import mktime, strptime, gmtime + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class TusfilesNet(XFSAccount): + __name__ = "TusfilesNet" + __type__ = "account" + __version__ = "0.06" + + __description__ = """Tusfile.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it")] + + + HOSTER_DOMAIN = "tusfiles.net" + + VALID_UNTIL_PATTERN = r'<span class="label label-default">([^<]+)</span>' + TRAFFIC_LEFT_PATTERN = r'<td><img src="//www\.tusfiles\.net/i/icon/meter\.png" alt=""/></td>\n<td> (?P<S>[\d.,]+)' diff --git a/pyload/plugin/account/UlozTo.py b/pyload/plugin/account/UlozTo.py new file mode 100644 index 000000000..1570419b0 --- /dev/null +++ b/pyload/plugin/account/UlozTo.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.Account import Account + + +class UlozTo(Account): + __name__ = "UlozTo" + __type__ = "account" + __version__ = "0.10" + + __description__ = """Uloz.to account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("pulpe", "")] + + + TRAFFIC_LEFT_PATTERN = r'<li class="menu-kredit"><a .*?title="[^"]*?GB = ([\d.]+) MB"' + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.ulozto.net/", decode=True) + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + + trafficleft = float(m.group(1).replace(' ', '').replace(',', '.')) * 1000 * 1.048 if m else 0 + premium = True if trafficleft else False + + return {'validuntil': -1, 'trafficleft': trafficleft, 'premium': premium} + + + def login(self, user, data, req): + login_page = req.load('http://www.ulozto.net/?do=web-login', decode=True) + action = re.findall('<form action="(.+?)"', login_page)[1].replace('&', '&') + token = re.search('_token_" value="(.+?)"', login_page).group(1) + + html = req.load(urljoin("http://www.ulozto.net/", action), + post={'_token_' : token, + 'do' : "loginForm-submit", + 'login' : u"Přihlásit", + 'password': data['password'], + 'username': user, + 'remember': "on"}, + decode=True) + + if '<div class="flash error">' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/UnrestrictLi.py b/pyload/plugin/account/UnrestrictLi.py new file mode 100644 index 000000000..d8d7789bb --- /dev/null +++ b/pyload/plugin/account/UnrestrictLi.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account +from pyload.utils import json_loads + + +class UnrestrictLi(Account): + __name__ = "UnrestrictLi" + __type__ = "account" + __version__ = "0.05" + + __description__ = """Unrestrict.li account plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def loadAccountInfo(self, user, req): + json_data = req.load('http://unrestrict.li/api/jdownloader/user.php?format=json') + self.logDebug("JSON data: " + json_data) + json_data = json_loads(json_data) + + if 'vip' in json_data['result'] and json_data['result']['vip'] == 0: + return {"premium": False} + + validuntil = json_data['result']['expires'] + trafficleft = float(json_data['result']['traffic'] / 1024) #@TODO: Remove `/ 1024` in 0.4.10 + + return {"premium": True, "validuntil": validuntil, "trafficleft": trafficleft} + + + def login(self, user, data, req): + req.cj.setCookie("unrestrict.li", "lang", "EN") + html = req.load("https://unrestrict.li/sign_in", decode=True) + + if 'solvemedia' in html: + self.logError(_("A Captcha is required. Go to http://unrestrict.li/sign_in and login, then retry")) + return + + post_data = {"username": user, "password": data['password'], + "remember_me": "remember", "signin": "Sign in"} + html = req.load("https://unrestrict.li/sign_in", post=post_data, decode=True) + + if 'sign_out' not in html: + self.wrongPassword() diff --git a/pyload/plugin/account/UploadableCh.py b/pyload/plugin/account/UploadableCh.py new file mode 100644 index 000000000..9406118cd --- /dev/null +++ b/pyload/plugin/account/UploadableCh.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Account import Account + + +class UploadableCh(Account): + __name__ = "UploadableCh" + __type__ = "account" + __version__ = "0.03" + + __description__ = """Uploadable.ch account plugin""" + __license__ = "GPLv3" + __authors__ = [("Sasch", "gsasch@gmail.com")] + + + def loadAccountInfo(self, user, req): + html = req.load("http://www.uploadable.ch/login.php") + + premium = '<a href="/logout.php"' in html + trafficleft = -1 if premium else None + + return {'validuntil': None, 'trafficleft': trafficleft, 'premium': premium} #@TODO: validuntil + + + def login(self, user, data, req): + html = req.load("http://www.uploadable.ch/login.php", + post={'userName' : user, + 'userPassword' : data["password"], + 'autoLogin' : "1", + 'action__login': "normalLogin"}, + decode=True) + + if "Login failed" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/UploadcCom.py b/pyload/plugin/account/UploadcCom.py new file mode 100644 index 000000000..66863c456 --- /dev/null +++ b/pyload/plugin/account/UploadcCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class UploadcCom(XFSAccount): + __name__ = "UploadcCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """Uploadc.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "uploadc.com" diff --git a/pyload/plugin/account/UploadedTo.py b/pyload/plugin/account/UploadedTo.py new file mode 100644 index 000000000..3b0d957a5 --- /dev/null +++ b/pyload/plugin/account/UploadedTo.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import re +from time import time + +from pyload.plugin.Account import Account + + +class UploadedTo(Account): + __name__ = "UploadedTo" + __type__ = "account" + __version__ = "0.30" + + __description__ = """Uploaded.to account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + PREMIUM_PATTERN = r'<em>Premium</em>' + VALID_UNTIL_PATTERN = r'<td>Duration:</td>\s*<th>(.+?)<' + TRAFFIC_LEFT_PATTERN = r'<b class="cB">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = None + + html = req.load("http://uploaded.net/me") + + premium = True if re.search(self.PREMIUM_PATTERN, html) else False + + m = re.search(self.VALID_UNTIL_PATTERN, html, re.M) + if m: + expiredate = m.group(1).lower().strip() + + if expiredate == "unlimited": + validuntil = -1 + else: + m = re.findall(r'(\d+) (week|day|hour)', expiredate) + if m: + validuntil = time() + for n, u in m: + validuntil += float(n) * 60 * 60 * {'week': 168, 'day': 24, 'hour': 1}[u] + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + traffic = m.groupdict() + size = traffic['S'].replace('.', '') + unit = traffic['U'].lower() + + if unit.startswith('t'): #@NOTE: Remove in 0.4.10 + trafficleft = float(size.replace(',', '.')) / 1024 + trafficleft *= 1 << 40 + else: + trafficleft = self.parseTraffic(size + unit) + + return {'validuntil' : validuntil, + 'trafficleft': trafficleft, + 'premium' : premium} + + + def login(self, user, data, req): + # req.cj.setCookie("uploaded.net", "lang", "en") + + html = req.load("http://uploaded.net/io/login", + post={'id': user, 'pw': data['password'], '_': ""}, + decode=True) + + if '"err"' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/UploadheroCom.py b/pyload/plugin/account/UploadheroCom.py new file mode 100644 index 000000000..c73fc30f5 --- /dev/null +++ b/pyload/plugin/account/UploadheroCom.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +import re +import datetime +import time + +from pyload.plugin.Account import Account + + +class UploadheroCom(Account): + __name__ = "UploadheroCom" + __type__ = "account" + __version__ = "0.21" + + __description__ = """Uploadhero.co account plugin""" + __license__ = "GPLv3" + __authors__ = [("mcmyst", "mcmyst@hotmail.fr")] + + + def loadAccountInfo(self, user, req): + premium_pattern = re.compile('Il vous reste <span class="bleu">(\d+)</span> jours premium') + + data = self.getAccountData(user) + html = req.load("http://uploadhero.co/my-account") + + if premium_pattern.search(html): + end_date = datetime.date.today() + datetime.timedelta(days=int(premium_pattern.search(html).group(1))) + end_date = time.mktime(future.timetuple()) + account_info = {"validuntil": end_date, "trafficleft": -1, "premium": True} + else: + account_info = {"validuntil": -1, "trafficleft": -1, "premium": False} + + return account_info + + + def login(self, user, data, req): + html = req.load("http://uploadhero.co/lib/connexion.php", + post={"pseudo_login": user, "password_login": data['password']}, + decode=True) + + if "mot de passe invalide" in html: + self.wrongPassword() diff --git a/pyload/plugin/account/UploadingCom.py b/pyload/plugin/account/UploadingCom.py new file mode 100644 index 000000000..6d54469e8 --- /dev/null +++ b/pyload/plugin/account/UploadingCom.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import re + +from time import time, strptime, mktime + +from pyload.plugin.Account import Account +from pyload.plugin.internal.SimpleHoster import set_cookies + + +class UploadingCom(Account): + __name__ = "UploadingCom" + __type__ = "account" + __version__ = "0.11" + + __description__ = """Uploading.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + PREMIUM_PATTERN = r'UPGRADE TO PREMIUM' + VALID_UNTIL_PATTERN = r'Valid Until:(.+?)<' + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = None + + html = req.load("http://uploading.com/") + + premium = False if re.search(self.PREMIUM_PATTERN, html) else True + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%b %d, %Y")) + + except Exception, e: + self.logError(e) + + else: + if validuntil > mktime(gmtime()): + premium = True + else: + premium = False + validuntil = None + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + + + def login(self, user, data, req): + set_cookies([("uploading.com", "lang", "1"), + ("uploading.com", "language", "1"), + ("uploading.com", "setlang", "en"), + ("uploading.com", "_lang", "en")] + + req.load("http://uploading.com/") + req.load("http://uploading.com/general/login_form/?JsHttpRequest=%s-xml" % long(time() * 1000), + post={'email': user, 'password': data['password'], 'remember': "on"}) diff --git a/pyload/plugin/account/UptoboxCom.py b/pyload/plugin/account/UptoboxCom.py new file mode 100644 index 000000000..f7cb7a82e --- /dev/null +++ b/pyload/plugin/account/UptoboxCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class UptoboxCom(XFSAccount): + __name__ = "UptoboxCom" + __type__ = "account" + __version__ = "0.08" + + __description__ = """DDLStorage.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "uptobox.com" + HOSTER_URL = "https://uptobox.com/" + LOGIN_URL = "https://login.uptobox.com/" diff --git a/pyload/plugin/account/VidPlayNet.py b/pyload/plugin/account/VidPlayNet.py new file mode 100644 index 000000000..390520a00 --- /dev/null +++ b/pyload/plugin/account/VidPlayNet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class VidPlayNet(XFSAccount): + __name__ = "VidPlayNet" + __type__ = "account" + __version__ = "0.02" + + __description__ = """VidPlay.net account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "vidplay.net" diff --git a/pyload/plugin/account/WebshareCz.py b/pyload/plugin/account/WebshareCz.py new file mode 100644 index 000000000..efa2a0045 --- /dev/null +++ b/pyload/plugin/account/WebshareCz.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import re + +from hashlib import md5, sha1 +from passlib.hash import md5_crypt +from time import mktime, strptime, time + +from pyload.plugin.Account import Account + + +class WebshareCz(Account): + __name__ = "WebshareCz" + __type__ = "account" + __version__ = "0.07" + + __description__ = """Webshare.cz account plugin""" + __license__ = "GPLv3" + __authors__ = [("rush", "radek.senfeld@gmail.com")] + + + VALID_UNTIL_PATTERN = r'<vip_until>(.+)</vip_until>' + + TRAFFIC_LEFT_PATTERN = r'<bytes>(.+)</bytes>' + + + def loadAccountInfo(self, user, req): + html = req.load("https://webshare.cz/api/user_data/", + post={'wst': self.infos['wst']}, + decode=True) + + self.logDebug("Response: " + html) + + expiredate = re.search(self.VALID_UNTIL_PATTERN, html).group(1) + self.logDebug("Expire date: " + expiredate) + + validuntil = mktime(strptime(expiredate, "%Y-%m-%d %H:%M:%S")) + trafficleft = self.parseTraffic(re.search(self.TRAFFIC_LEFT_PATTERN, html).group(1)) + premium = validuntil > time() + + return {'validuntil': validuntil, 'trafficleft': -1, 'premium': premium} + + + def login(self, user, data, req): + salt = req.load("https://webshare.cz/api/salt/", + post={'username_or_email': user, + 'wst' : ""}, + decode=True) + + if "<status>OK</status>" not in salt: + self.wrongPassword() + + salt = re.search('<salt>(.+)</salt>', salt).group(1) + password = sha1(md5_crypt.encrypt(data["password"], salt=salt)).hexdigest() + digest = md5(user + ":Webshare:" + password).hexdigest() + + login = req.load("https://webshare.cz/api/login/", + post={'digest' : digest, + 'keep_logged_in' : 1, + 'password' : password, + 'username_or_email': user, + 'wst' : ""}, + decode=True) + + if "<status>OK</status>" not in login: + self.wrongPassword() + + self.infos['wst'] = re.search('<token>(.+)</token>', login).group(1) diff --git a/pyload/plugin/account/XFileSharingPro.py b/pyload/plugin/account/XFileSharingPro.py new file mode 100644 index 000000000..216af5385 --- /dev/null +++ b/pyload/plugin/account/XFileSharingPro.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSAccount import XFSAccount + + +class XFileSharingPro(XFSAccount): + __name__ = "XFileSharingPro" + __type__ = "account" + __version__ = "0.06" + + __description__ = """XFileSharingPro multi-purpose account plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + + + def init(self): + if self.HOSTER_DOMAIN: + return super(XFileSharingPro, self).init() + + + def loadAccountInfo(self, user, req): + return super(XFileSharingPro if self.HOSTER_DOMAIN else XFSAccount, self).loadAccountInfo(user, req) + + + def login(self, user, data, req): + if self.HOSTER_DOMAIN: + try: + return super(XFileSharingPro, self).login(user, data, req) + except Exception: + self.HOSTER_URL = self.HOSTER_URL.replace("www.", "") + return super(XFileSharingPro, self).login(user, data, req) diff --git a/pyload/plugin/account/YibaishiwuCom.py b/pyload/plugin/account/YibaishiwuCom.py new file mode 100644 index 000000000..150b0d931 --- /dev/null +++ b/pyload/plugin/account/YibaishiwuCom.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Account import Account + + +class YibaishiwuCom(Account): + __name__ = "YibaishiwuCom" + __type__ = "account" + __version__ = "0.02" + + __description__ = """115.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + ACCOUNT_INFO_PATTERN = r'var USER_PERMISSION = {(.*?)}' + + + def loadAccountInfo(self, user, req): + #self.relogin(user) + html = req.load("http://115.com/", decode=True) + + m = re.search(self.ACCOUNT_INFO_PATTERN, html, re.S) + premium = True if m and 'is_vip: 1' in m.group(1) else False + validuntil = trafficleft = (-1 if m else 0) + return dict({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) + + + def login(self, user, data, req): + html = req.load("http://passport.115.com/?ac=login", + post={"back": "http://www.115.com/", + "goto": "http://115.com/", + "login[account]": user, + "login[passwd]": data['password']}, + decode=True) + + if not 'var USER_PERMISSION = {' in html: + self.wrongPassword() diff --git a/pyload/plugin/account/ZeveraCom.py b/pyload/plugin/account/ZeveraCom.py new file mode 100644 index 000000000..db23170f3 --- /dev/null +++ b/pyload/plugin/account/ZeveraCom.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +from time import mktime, strptime + +from pyload.plugin.Account import Account + + +class ZeveraCom(Account): + __name__ = "ZeveraCom" + __type__ = "account" + __version__ = "0.26" + + __description__ = """Zevera.com account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = "zevera.com" + + + def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 + self.init() + return super(ZeveraCom, self).__init__(manager, accounts) + + + def init(self): + if not self.HOSTER_DOMAIN: + self.logError(_("Missing HOSTER_DOMAIN")) + + if not hasattr(self, "API_URL"): + self.API_URL = "http://api.%s/jDownloader.ashx" % (self.HOSTER_DOMAIN or "") + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + premium = False + + api = self.api_response(req) + + if "No trafic" not in api and api['endsubscriptiondate'] != "Expired!": + validuntil = mktime(strptime(api['endsubscriptiondate'], "%Y/%m/%d %H:%M:%S")) + trafficleft = float(api['availabletodaytraffic']) * 1024 if api['orondaytrafficlimit'] != '0' else -1 + premium = True + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + + + def login(self, user, data, req): + self.user = user + self.password = data['password'] + + if self.api_response(req) == "No trafic": + self.wrongPassword() + + + def api_response(self, req, just_header=False, **kwargs): + get_data = {'cmd' : "accountinfo", + 'login': self.user, + 'pass' : self.password} + + get_data.update(kwargs) + + res = req.load(self.API_URL, + get=get_data, + just_header=just_header, + decode=True) + + self.logDebug(res) + + if ':' in res: + if not just_header: + res = res.replace(',', '\n') + return dict((y.strip().lower(), z.strip()) for (y, z) in + [x.split(':', 1) for x in res.splitlines() if ':' in x]) + else: + return res diff --git a/pyload/plugin/account/__init__.py b/pyload/plugin/account/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/account/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/addon/AndroidPhoneNotify.py b/pyload/plugin/addon/AndroidPhoneNotify.py new file mode 100644 index 000000000..2b4f8fcca --- /dev/null +++ b/pyload/plugin/addon/AndroidPhoneNotify.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +from time import time + +from pyload.network.RequestFactory import getURL +from pyload.plugin.Addon import Addon + + +class AndroidPhoneNotify(Addon): + __name__ = "AndroidPhoneNotify" + __type__ = "addon" + __version__ = "0.05" + + __config__ = [("apikey" , "str" , "API key" , "" ), + ("notifycaptcha" , "bool", "Notify captcha request" , True ), + ("notifypackage" , "bool", "Notify package finished" , True ), + ("notifyprocessed", "bool", "Notify processed packages status" , True ), + ("timeout" , "int" , "Timeout between captchas in seconds" , 5 ), + ("force" , "bool", "Send notifications if client is connected", False)] + + __description__ = """Send push notifications to your Android Phone using notifymyandroid.com""" + __license__ = "GPLv3" + __authors__ = [("Steven Kosyra", "steven.kosyra@gmail.com"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + event_list = ["allDownloadsProcessed"] + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.last_notify = 0 + + + def newCaptchaTask(self, task): + if not self.getConfig("notifycaptcha"): + return False + + if time() - self.last_notify < self.getConf("timeout"): + return False + + self.notify(_("Captcha"), _("New request waiting user input")) + + + def packageFinished(self, pypack): + if self.getConfig("notifypackage"): + self.notify(_("Package finished"), pypack.name) + + + def allDownloadsProcessed(self): + if not self.getConfig("notifyprocessed"): + return False + + if any(True for pdata in self.core.api.getQueue() if pdata.linksdone < pdata.linkstotal): + self.notify(_("Package failed"), _("One or more packages was not completed successfully")) + else: + self.notify(_("All packages finished")) + + + def notify(self, event, msg=""): + apikey = self.getConfig("apikey") + + if not apikey: + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + getURL("http://www.notifymyandroid.com/publicapi/notify", + get={'apikey' : apikey, + 'application': "pyLoad", + 'event' : event, + 'description': msg}) + + self.last_notify = time() diff --git a/pyload/plugin/addon/Checksum.py b/pyload/plugin/addon/Checksum.py new file mode 100644 index 000000000..35be60773 --- /dev/null +++ b/pyload/plugin/addon/Checksum.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import hashlib +import re +import zlib + +from os import remove +from os.path import getsize, isfile, splitext + +from pyload.plugin.Addon import Addon +from pyload.utils import safe_join, fs_encode + + +def computeChecksum(local_file, algorithm): + if algorithm in getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")): + h = getattr(hashlib, algorithm)() + + with open(local_file, 'rb') as f: + for chunk in iter(lambda: f.read(128 * h.block_size), ''): + h.update(chunk) + + return h.hexdigest() + + elif algorithm in ("adler32", "crc32"): + hf = getattr(zlib, algorithm) + last = 0 + + with open(local_file, 'rb') as f: + for chunk in iter(lambda: f.read(8192), ''): + last = hf(chunk, last) + + return "%x" % last + + else: + return None + + +class Checksum(Addon): + __name__ = "Checksum" + __type__ = "addon" + __version__ = "0.16" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("check_checksum", "bool" , "Check checksum? (If False only size will be verified)", True ), + ("check_action" , "fail;retry;nothing", "What to do if check fails?" , "retry"), + ("max_tries" , "int" , "Number of retries" , 2 ), + ("retry_action" , "fail;nothing" , "What to do if all retries fail?" , "fail" ), + ("wait_time" , "int" , "Time to wait before each retry (seconds)" , 1 )] + + __description__ = """Verify downloaded file size and checksum""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("stickell", "l.stickell@yahoo.it")] + + + methods = {'sfv' : 'crc32', + 'crc' : 'crc32', + 'hash': 'md5'} + regexps = {'sfv' : r'^(?P<NAME>[^;].+)\s+(?P<HASH>[0-9A-Fa-f]{8})$', + 'md5' : r'^(?P<NAME>[0-9A-Fa-f]{32}) (?P<FILE>.+)$', + 'crc' : r'filename=(?P<NAME>.+)\nsize=(?P<SIZE>\d+)\ncrc32=(?P<HASH>[0-9A-Fa-f]{8})$', + 'default': r'^(?P<HASH>[0-9A-Fa-f]+)\s+\*?(?P<NAME>.+)$'} + + + def activate(self): + if not self.getConfig("check_checksum"): + self.logInfo(_("Checksum validation is disabled in plugin configuration")) + + + def setup(self): + self.algorithms = sorted( + getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")), reverse=True) + self.algorithms.extend(["crc32", "adler32"]) + self.formats = self.algorithms + ["sfv", "crc", "hash"] + + + def downloadFinished(self, pyfile): + """ + Compute checksum for the downloaded file and compare it with the hash provided by the hoster. + pyfile.plugin.check_data should be a dictionary which can contain: + a) if known, the exact filesize in bytes (e.g. "size": 123456789) + b) hexadecimal hash string with algorithm name as key (e.g. "md5": "d76505d0869f9f928a17d42d66326307") + """ + if hasattr(pyfile.plugin, "check_data") and isinstance(pyfile.plugin.check_data, dict): + data = pyfile.plugin.check_data.copy() + + elif hasattr(pyfile.plugin, "api_data") and isinstance(pyfile.plugin.api_data, dict): + data = pyfile.plugin.api_data.copy() + + elif hasattr(pyfile.plugin, "info") and isinstance(pyfile.plugin.info, dict): + data = pyfile.plugin.info.copy() + data.pop('size', None) #@NOTE: Don't check file size until a similary matcher will be implemented + + else: + return + + self.logDebug(data) + + if not pyfile.plugin.lastDownload: + self.checkFailed(pyfile, None, "No file downloaded") + + local_file = fs_encode(pyfile.plugin.lastDownload) + #download_folder = self.config['general']['download_folder'] + #local_file = fs_encode(safe_join(download_folder, pyfile.package().folder, pyfile.name)) + + if not isfile(local_file): + self.checkFailed(pyfile, None, "File does not exist") + + # validate file size + if "size" in data: + api_size = int(data['size']) + file_size = getsize(local_file) + + if api_size != file_size: + self.logWarning(_("File %s has incorrect size: %d B (%d expected)") % (pyfile.name, file_size, api_size)) + self.checkFailed(pyfile, local_file, "Incorrect file size") + + data.pop('size', None) + + # validate checksum + if data and self.getConfig("check_checksum"): + + if not 'md5' in data: + for type in ("checksum", "hashsum", "hash"): + if type in data: + data['md5'] = data[type] #@NOTE: What happens if it's not an md5 hash? + break + + for key in self.algorithms: + if key in data: + checksum = computeChecksum(local_file, key.replace("-", "").lower()) + if checksum: + if checksum == data[key].lower(): + self.logInfo(_('File integrity of "%s" verified by %s checksum (%s)') % + (pyfile.name, key.upper(), checksum)) + break + else: + self.logWarning(_("%s checksum for file %s does not match (%s != %s)") % + (key.upper(), pyfile.name, checksum, data[key])) + self.checkFailed(pyfile, local_file, "Checksums do not match") + else: + self.logWarning(_("Unsupported hashing algorithm"), key.upper()) + else: + self.logWarning(_("Unable to validate checksum for file: ") + pyfile.name) + + + def checkFailed(self, pyfile, local_file, msg): + check_action = self.getConfig("check_action") + if check_action == "retry": + max_tries = self.getConfig("max_tries") + retry_action = self.getConfig("retry_action") + if pyfile.plugin.retries < max_tries: + if local_file: + remove(local_file) + pyfile.plugin.retry(max_tries, self.getConfig("wait_time"), msg) + elif retry_action == "nothing": + return + elif check_action == "nothing": + return + pyfile.plugin.fail(reason=msg) + + + def packageFinished(self, pypack): + download_folder = safe_join(self.config['general']['download_folder'], pypack.folder, "") + + for link in pypack.getChildren().itervalues(): + file_type = splitext(link['name'])[1][1:].lower() + + if file_type not in self.formats: + continue + + hash_file = fs_encode(safe_join(download_folder, link['name'])) + if not isfile(hash_file): + self.logWarning(_("File not found"), link['name']) + continue + + with open(hash_file) as f: + text = f.read() + + for m in re.finditer(self.regexps.get(file_type, self.regexps['default']), text): + data = m.groupdict() + self.logDebug(link['name'], data) + + local_file = fs_encode(safe_join(download_folder, data['NAME'])) + algorithm = self.methods.get(file_type, file_type) + checksum = computeChecksum(local_file, algorithm) + if checksum == data['HASH']: + self.logInfo(_('File integrity of "%s" verified by %s checksum (%s)') % + (data['NAME'], algorithm, checksum)) + else: + self.logWarning(_("%s checksum for file %s does not match (%s != %s)") % + (algorithm, data['NAME'], checksum, data['HASH'])) diff --git a/pyload/plugin/addon/ClickAndLoad.py b/pyload/plugin/addon/ClickAndLoad.py new file mode 100644 index 000000000..cd71e9972 --- /dev/null +++ b/pyload/plugin/addon/ClickAndLoad.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +import socket + +from threading import Lock + +from pyload.plugin.Addon import Addon, threaded + + +def forward(source, destination): + try: + bufsize = 1024 + bufdata = source.recv(bufsize) + while bufdata: + destination.sendall(bufdata) + bufdata = source.recv(bufsize) + finally: + destination.shutdown(socket.SHUT_WR) + + +#: create_connection wrapper for python 2.5 socket module +def create_connection(address, timeout=object(), source_address=None): + if hasattr(socket, 'create_connection'): + if type(timeout) == object: + timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + return socket.create_connection(address, timeout, source_address) + + else: + host, port = address + err = None + for res in getaddrinfo(host, port, 0, SOCK_STREAM): + af, socktype, proto, canonname, sa = res + sock = None + try: + sock = socket(af, socktype, proto) + if type(timeout) != object: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + return sock + + except socket.error, _: + err = _ + if sock is not None: + sock.close() + + if err is not None: + raise err + else: + raise socket.error("getaddrinfo returns an empty list") + + +class ClickAndLoad(Addon): + __name__ = "ClickAndLoad" + __type__ = "addon" + __version__ = "0.35" + + __config__ = [("activated", "bool", "Activated" , True), + ("port" , "int" , "Port" , 9666), + ("extern" , "bool", "Listen on the public network interface", True)] + + __description__ = """Click'N'Load addon plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.de"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def activate(self): + if not self.config['webinterface']['activated']: + return + + ip = socket.gethostbyname(socket.gethostname()) if self.getConfig("extern") else "127.0.0.1" + webport = int(self.config['webinterface']['port']) + cnlport = self.getConfig('port') + + self.proxy(ip, webport, cnlport) + + + @threaded + def proxy(self, ip, webport, cnlport): + self.manager.startThread(self._server, ip, webport, cnlport) + lock = Lock() + lock.acquire() + lock.acquire() + + + def _server(self, ip, webport, cnlport, thread): + try: + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind((ip, cnlport)) + server_socket.listen(5) + + while True: + client_socket = server_socket.accept()[0] + dock_socket = create_connection(("127.0.0.1", webport)) + + self.manager.startThread(forward, dock_socket, client_socket) + self.manager.startThread(forward, client_socket, dock_socket) + + except socket.error, e: + self.logDebug(e) + self._server(ip, webport, cnlport, thread) + + except Exception, e: + self.logError(e) + + try: + client_socket.close() + dock_socket.close() + except Exception: + pass + + try: + server_socket.close() + except Exception: + pass diff --git a/pyload/plugin/addon/DeleteFinished.py b/pyload/plugin/addon/DeleteFinished.py new file mode 100644 index 000000000..59f2e3321 --- /dev/null +++ b/pyload/plugin/addon/DeleteFinished.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +from pyload.database import style +from pyload.plugin.Addon import Addon + + +class DeleteFinished(Addon): + __name__ = "DeleteFinished" + __type__ = "addon" + __version__ = "1.11" + + __config__ = [('interval' , 'int' , 'Delete every (hours)' , '72' ), + ('deloffline', 'bool', 'Delete packages with offline links', 'False')] + + __description__ = """Automatically delete all finished packages from queue""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + + + ## overwritten methods ## + def periodical(self): + if not self.info['sleep']: + deloffline = self.getConfig('deloffline') + mode = '0,1,4' if deloffline else '0,4' + msg = _('delete all finished packages in queue list (%s packages with offline links)') + self.logInfo(msg % (_('including') if deloffline else _('excluding'))) + self.deleteFinished(mode) + self.info['sleep'] = True + self.addEvent('packageFinished', self.wakeup) + + + def pluginConfigChanged(self, plugin, name, value): + if name == "interval" and value != self.interval: + self.interval = value * 3600 + self.initPeriodical() + + + def deactivate(self): + self.removeEvent('packageFinished', self.wakeup) + + + def activate(self): + self.info = {'sleep': True} + interval = self.getConfig('interval') + self.pluginConfigChanged(self.__name__, 'interval', interval) + self.addEvent('packageFinished', self.wakeup) + + + ## own methods ## + @style.queue + def deleteFinished(self, mode): + self.c.execute('DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE package=packages.id AND status NOT IN (%s))' % mode) + self.c.execute('DELETE FROM links WHERE NOT EXISTS(SELECT 1 FROM packages WHERE id=links.package)') + + + def wakeup(self, pypack): + self.removeEvent('packageFinished', self.wakeup) + self.info['sleep'] = False + + + ## event managing ## + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.m.events: + if func in self.m.events[event]: + self.logDebug("Function already registered", func) + else: + self.m.events[event].append(func) + else: + self.m.events[event] = [func] + + + def setup(self): + self.interval = 0 + self.m = self.manager + self.removeEvent = self.m.removeEvent diff --git a/pyload/plugin/addon/DownloadScheduler.py b/pyload/plugin/addon/DownloadScheduler.py new file mode 100644 index 000000000..e5e25e389 --- /dev/null +++ b/pyload/plugin/addon/DownloadScheduler.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +import re + +from time import localtime + +from pyload.plugin.Addon import Addon + + +class DownloadScheduler(Addon): + __name__ = "DownloadScheduler" + __type__ = "addon" + __version__ = "0.22" + + __config__ = [("timetable", "str" , "List time periods as hh:mm full or number(kB/s)" , "0:00 full, 7:00 250, 10:00 0, 17:00 150"), + ("abort" , "bool", "Abort active downloads when start period with speed 0", False )] + + __description__ = """Download Scheduler""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + def setup(self): + self.cb = None #: callback to scheduler job; will be by removed AddonManager when addon unloaded + + + def activate(self): + self.updateSchedule() + + + def updateSchedule(self, schedule=None): + if schedule is None: + schedule = self.getConfig("timetable") + + schedule = re.findall("(\d{1,2}):(\d{2})[\s]*(-?\d+)", + schedule.lower().replace("full", "-1").replace("none", "0")) + if not schedule: + self.logError(_("Invalid schedule")) + return + + t0 = localtime() + now = (t0.tm_hour, t0.tm_min, t0.tm_sec, "X") + schedule = sorted([(int(x[0]), int(x[1]), 0, int(x[2])) for x in schedule] + [now]) + + self.logDebug("Schedule", schedule) + + for i, v in enumerate(schedule): + if v[3] == "X": + last, next = schedule[i - 1], schedule[(i + 1) % len(schedule)] + self.logDebug("Now/Last/Next", now, last, next) + + self.setDownloadSpeed(last[3]) + + next_time = (((24 + next[0] - now[0]) * 60 + next[1] - now[1]) * 60 + next[2] - now[2]) % 86400 + self.core.scheduler.removeJob(self.cb) + self.cb = self.core.scheduler.addJob(next_time, self.updateSchedule, threaded=False) + + + def setDownloadSpeed(self, speed): + if speed == 0: + abort = self.getConfig("abort") + self.logInfo(_("Stopping download server. (Running downloads will %sbe aborted.)") % '' if abort else _('not ')) + self.core.api.pauseServer() + if abort: + self.core.api.stopAllDownloads() + else: + self.core.api.unpauseServer() + + if speed > 0: + self.logInfo(_("Setting download speed to %d kB/s") % speed) + self.core.api.setConfigValue("download", "limit_speed", 1) + self.core.api.setConfigValue("download", "max_speed", speed) + else: + self.logInfo(_("Setting download speed to FULL")) + self.core.api.setConfigValue("download", "limit_speed", 0) + self.core.api.setConfigValue("download", "max_speed", -1) diff --git a/pyload/plugin/addon/ExternalScripts.py b/pyload/plugin/addon/ExternalScripts.py new file mode 100644 index 000000000..5aebf2338 --- /dev/null +++ b/pyload/plugin/addon/ExternalScripts.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- + +import os +import subprocess + +from itertools import chain + +from pyload.plugin.Addon import Addon +from pyload.utils import safe_join + + +class ExternalScripts(Addon): + __name__ = "ExternalScripts" + __type__ = "addon" + __version__ = "0.29" + + __config__ = [("activated", "bool", "Activated" , True ), + ("wait" , "bool", "Wait script ending", False)] + + __description__ = """Run external scripts""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de"), + ("RaNaN", "ranan@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + event_map = {'archive-extracted' : "archive_extracted", + 'package-extracted' : "package_extracted", + 'all_archives-extracted' : "all_archives_extracted", + 'all_archives-processed' : "all_archives_processed", + 'all_downloads-finished' : "allDownloadsFinished", + 'all_downloads-processed': "allDownloadsProcessed"} + + + def setup(self): + self.scripts = {} + + folders = ["download_preparing", "download_finished", "all_downloads_finished", "all_downloads_processed", + "before_reconnect", "after_reconnect", + "package_finished", "package_extracted", + "archive_extracted", "all_archives_extracted", "all_archives_processed", + # deprecated folders + "unrar_finished", "all_dls_finished", "all_dls_processed"] + + for folder in folders: + self.scripts[folder] = [] + + self.initPluginType(folder, os.path.join(pypath, 'scripts', folder)) + self.initPluginType(folder, os.path.join('scripts', folder)) + + for script_type, names in self.scripts.iteritems(): + if names: + self.logInfo(_("Installed scripts for"), script_type, ", ".join(map(os.path.basename, names))) + + + def initPluginType(self, folder, path): + if not os.path.exists(path): + try: + os.makedirs(path) + + except Exception: + self.logDebug("Script folder %s not created" % folder) + return + + for f in os.listdir(path): + if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): + continue + + if not os.access(os.path.join(path, f), os.X_OK): + self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) + + self.scripts[folder].append(os.path.join(path, f)) + + + def callScript(self, script, *args): + try: + cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args] + + self.logDebug("Executing", os.path.abspath(script), " ".join(cmd)) + + p = subprocess.Popen(cmd, bufsize=-1) #@NOTE: output goes to pyload + if self.getConfig('wait'): + p.communicate() + + except Exception, e: + self.logError(_("Error in %(script)s: %(error)s") % {"script": os.path.basename(script), "error": e}) + + + def downloadPreparing(self, pyfile): + for script in self.scripts['download_preparing']: + self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id) + + + def downloadFinished(self, pyfile): + download_folder = self.config['general']['download_folder'] + for script in self.scripts['download_finished']: + filename = safe_join(download_folder, pyfile.package().folder, pyfile.name) + self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, filename, pyfile.id) + + + def packageFinished(self, pypack): + download_folder = self.config['general']['download_folder'] + for script in self.scripts['package_finished']: + folder = safe_join(download_folder, pypack.folder) + self.callScript(script, pypack.name, folder, pypack.password, pypack.id) + + + def beforeReconnecting(self, ip): + for script in self.scripts['before_reconnect']: + self.callScript(script, ip) + + + def afterReconnecting(self, ip): + for script in self.scripts['after_reconnect']: + self.callScript(script, ip) + + + def archive_extracted(self, pyfile, folder, filename, files): + for script in self.scripts['archive_extracted']: + self.callScript(script, folder, filename, files) + for script in self.scripts['unrar_finished']: #: deprecated + self.callScript(script, folder, filename) + + + def package_extracted(self, pypack): + download_folder = self.config['general']['download_folder'] + for script in self.scripts['package_extracted']: + folder = safe_join(download_folder, pypack.folder) + self.callScript(script, pypack.name, folder, pypack.password, pypack.id) + + + def all_archives_extracted(self): + for script in self.scripts['all_archives_extracted']: + self.callScript(script) + + + def all_archives_processed(self): + for script in self.scripts['all_archives_processed']: + self.callScript(script) + + + def allDownloadsFinished(self): + for script in chain(self.scripts['all_downloads_finished'], self.scripts['all_dls_finished']): + self.callScript(script) + + + def allDownloadsProcessed(self): + for script in chain(self.scripts['all_downloads_processed'], self.scripts['all_dls_processed']): + self.callScript(script) diff --git a/pyload/plugin/addon/ExtractArchive.py b/pyload/plugin/addon/ExtractArchive.py new file mode 100644 index 000000000..951434e3e --- /dev/null +++ b/pyload/plugin/addon/ExtractArchive.py @@ -0,0 +1,504 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import os +import sys + +from copy import copy +from traceback import print_exc + +# monkey patch bug in python 2.6 and lower +# http://bugs.python.org/issue6122 , http://bugs.python.org/issue1236 , http://bugs.python.org/issue1731717 +if sys.version_info < (2, 7) and os.name != "nt": + import errno + + from subprocess import Popen + + def _eintr_retry_call(func, *args): + while True: + try: + return func(*args) + + except OSError, e: + if e.errno == errno.EINTR: + continue + raise + + + # unsued timeout option for older python version + def wait(self, timeout=0): + """Wait for child process to terminate. Returns returncode + attribute.""" + if self.returncode is None: + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 + self._handle_exitstatus(sts) + return self.returncode + + Popen.wait = wait + +if os.name != "nt": + from grp import getgrnam + from pwd import getpwnam + +from pyload.plugin.Addon import Addon, threaded, Expose +from pyload.plugin.internal.Extractor import ArchiveError, CRCError, PasswordError +from pyload.plugin.internal.SimpleHoster import replace_patterns +from pyload.utils import fs_encode, safe_join, uniqify + + +class ArchiveQueue(object): + + def __init__(self, plugin, storage): + self.plugin = plugin + self.storage = storage + + + def get(self): + try: + return [int(pid) for pid in self.plugin.getStorage("ExtractArchive:%s" % self.storage, "").decode('base64').split()] + except Exception: + return [] + + + def set(self, value): + if isinstance(value, list): + item = str(value)[1:-1].replace(' ', '').replace(',', ' ') + else: + item = str(value).strip() + return self.plugin.setStorage("ExtractArchive:%s" % self.storage, item.encode('base64')[:-1]) + + + def delete(self): + return self.plugin.delStorage("ExtractArchive:%s" % self.storage) + + + def add(self, item): + queue = self.get() + if item not in queue: + return self.set(queue + [item]) + else: + return True + + + def remove(self, item): + queue = self.get() + try: + queue.remove(item) + except ValueError: + pass + if queue == []: + return self.delete() + return self.set(queue) + + + +class ExtractArchive(Addon): + __name__ = "ExtractArchive" + __type__ = "addon" + __version__ = "1.29" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("fullpath" , "bool" , "Extract with full paths" , True ), + ("overwrite" , "bool" , "Overwrite files" , False ), + ("keepbroken" , "bool" , "Try to extract broken archives" , False ), + ("repair" , "bool" , "Repair broken archives" , True ), + ("usepasswordfile" , "bool" , "Use password file" , True ), + ("passwordfile" , "file" , "Password file" , "archive_password.txt" ), + ("delete" , "bool" , "Delete archive when successfully extracted", False ), + ("subfolder" , "bool" , "Create subfolder for each package" , False ), + ("destination" , "folder", "Extract files to folder" , "" ), + ("extensions" , "str" , "Extract the following extensions" , "7z,bz2,bzip2,gz,gzip,lha,lzh,lzma,rar,tar,taz,tbz,tbz2,tgz,xar,xz,z,zip"), + ("excludefiles" , "str" , "Don't extract the following files" , "*.nfo,*.DS_Store,index.dat,thumb.db" ), + ("recursive" , "bool" , "Extract archives in archives" , True ), + ("waitall" , "bool" , "Wait for all downloads to be finished" , False ), + ("renice" , "int" , "CPU priority" , 0 )] + + __description__ = """Extract different kind of archives""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz" , "immenz@gmx.net" )] + + + event_list = ["allDownloadsProcessed"] + + NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")] + + + def setup(self): + self.queue = ArchiveQueue(self, "Queue") + self.failed = ArchiveQueue(self, "Failed") + + self.interval = 60 + self.extracting = False + self.extractors = [] + self.passwords = [] + + + def activate(self): + # self.extracting = False + + for p in ("UnRar", "SevenZip", "UnZip"): + try: + module = self.core.pluginManager.loadModule("internal", p) + klass = getattr(module, p) + if klass.isUsable(): + self.extractors.append(klass) + + except OSError, e: + if e.errno == 2: + self.logInfo(_("No %s installed") % p) + else: + self.logWarning(_("Could not activate: %s") % p, e) + if self.core.debug: + print_exc() + + except Exception, e: + self.logWarning(_("Could not activate: %s") % p, e) + if self.core.debug: + print_exc() + + if self.extractors: + self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors)) + + if self.getConfig("waitall"): + self.extractPackage(*self.queue.get()) #: Resume unfinished extractions + else: + super(ExtractArchive, self).initPeriodical() + + else: + self.logInfo(_("No Extract plugins activated")) + + + def periodical(self): + if not self.extracting: + self.extractPackage(*self.queue.get()) + + + @Expose + def extractPackage(self, *ids): + """ Extract packages with given id""" + self.manager.startThread(self.extract, ids) + + + def packageFinished(self, pypack): + self.queue.add(pypack.id) + + + @threaded + def allDownloadsProcessed(self, thread): + if self.extract(self.queue.get(), thread): #@NOTE: check only if all gone fine, no failed reporting for now + self.manager.dispatchEvent("all_archives_extracted") + + self.manager.dispatchEvent("all_archives_processed") + + + def extract(self, ids, thread=None): + if not ids: + return False + + self.extracting = True + + processed = [] + extracted = [] + failed = [] + + toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') + + destination = self.getConfig("destination") + subfolder = self.getConfig("subfolder") + fullpath = self.getConfig("fullpath") + overwrite = self.getConfig("overwrite") + renice = self.getConfig("renice") + recursive = self.getConfig("recursive") + delete = self.getConfig("delete") + keepbroken = self.getConfig("keepbroken") + + extensions = [x.lstrip('.').lower() for x in toList(self.getConfig("extensions"))] + excludefiles = toList(self.getConfig("excludefiles")) + + if extensions: + self.logDebug("Use for extensions: %s" % "|.".join(extensions)) + + # reload from txt file + self.reloadPasswords() + + # dl folder + dl = self.config['general']['download_folder'] + + #iterate packages -> extractors -> targets + for pid in ids: + pypack = self.core.files.getPackage(pid) + + if not pypack: + continue + + self.logInfo(_("Check package: %s") % pypack.name) + + # determine output folder + out = safe_join(dl, pypack.folder, destination, "") #: force trailing slash + + if subfolder: + out = safe_join(out, pypack.folder) + + if not os.path.exists(out): + os.makedirs(out) + + matched = False + success = True + files_ids = [(safe_join(dl, pypack.folder, pylink['name']), pylink['id'], out) for pylink in pypack.getChildren().itervalues()] + + # check as long there are unseen files + while files_ids: + new_files_ids = [] + + if extensions: + files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ + if filter(lambda ext: fname.lower().endswith(ext), extensions)] + + for Extractor in self.extractors: + targets = Extractor.getTargets(files_ids) + if targets: + self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets)) + matched = True + + for fname, fid, fout in targets: + name = os.path.basename(fname) + + if not os.path.exists(fname): + self.logDebug(name, "File not found") + continue + + self.logInfo(name, _("Extract to: %s") % fout) + try: + archive = Extractor(self, + fname, + fout, + fullpath, + overwrite, + excludefiles, + renice, + delete, + keepbroken, + fid) + archive.init() + + new_files = self._extract(archive, fid, pypack.password, thread) + + except Exception, e: + self.logError(name, e) + success = False + continue + + files_ids.remove((fname, fid, fout)) # don't let other extractors spam log + self.logDebug("Extracted files: %s" % new_files) + self.setPermissions(new_files) + + for filename in new_files: + file = fs_encode(safe_join(os.path.dirname(archive.filename), filename)) + if not os.path.exists(file): + self.logDebug("New file %s does not exists" % filename) + continue + + if recursive and os.path.isfile(file): + new_files_ids.append((filename, fid, os.path.dirname(filename))) # append as new target + + files_ids = new_files_ids # also check extracted files + + if matched: + if success: + extracted.append(pid) + self.manager.dispatchEvent("package_extracted", pypack) + else: + failed.append(pid) + self.manager.dispatchEvent("package_extract_failed", pypack) + + self.failed.add(pid) + else: + self.logInfo(_("No files found to extract")) + + if not matched or not success and subfolder: + try: + os.rmdir(out) + + except OSError: + pass + + self.queue.remove(pid) + + self.extracting = False + return True if not failed else False + + + def _extract(self, archive, fid, password, thread): + pyfile = self.core.files.getFile(fid) + name = os.path.basename(archive.filename) + + thread.addActive(pyfile) + pyfile.setStatus("processing") + + encrypted = False + try: + try: + archive.check() + + except CRCError, e: + self.logDebug(name, e) + self.logInfo(name, _("Header protected")) + + if self.getConfig("repair"): + self.logWarning(name, _("Repairing...")) + + pyfile.setCustomStatus(_("repairing")) + pyfile.setProgress(0) + + repaired = archive.repair() + + pyfile.setProgress(100) + + if not repaired and not self.getConfig("keepbroken"): + raise CRCError("Archive damaged") + + except PasswordError: + self.logInfo(name, _("Password protected")) + encrypted = True + + except ArchiveError, e: + raise ArchiveError(e) + + self.logDebug("Password: %s" % (password or "No provided")) + + pyfile.setCustomStatus(_("extracting")) + pyfile.setProgress(0) + + if not encrypted or not self.getConfig("usepasswordfile"): + archive.extract(password) + else: + for pw in filter(None, uniqify([password] + self.getPasswords(False))): + try: + self.logDebug("Try password: %s" % pw) + + ispw = archive.isPassword(pw) + if ispw or ispw is None: + archive.extract(pw) + self.addPassword(pw) + break + + except PasswordError: + self.logDebug("Password was wrong") + else: + raise PasswordError + + pyfile.setProgress(100) + pyfile.setCustomStatus(_("finalizing")) + + if self.core.debug: + self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles())) + + if self.getConfig("delete"): + files = archive.getDeleteFiles() + self.logInfo(_("Deleting %s files") % len(files)) + for f in files: + file = fs_encode(f) + if os.path.exists(file): + os.remove(file) + else: + self.logDebug("%s does not exists" % f) + + self.logInfo(name, _("Extracting finished")) + + extracted_files = archive.files or archive.list() + self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files) + + return extracted_files + + except PasswordError: + self.logError(name, _("Wrong password" if password else "No password found")) + + except CRCError, e: + self.logError(name, _("CRC mismatch"), e) + + except ArchiveError, e: + self.logError(name, _("Archive error"), e) + + except Exception, e: + self.logError(name, _("Unknown error"), e) + if self.core.debug: + print_exc() + + finally: + pyfile.finishIfDone() + + self.manager.dispatchEvent("archive_extract_failed", pyfile) + + raise Exception(_("Extract failed")) + + + @Expose + def getPasswords(self, reload=True): + """ List of saved passwords """ + if reload: + self.reloadPasswords() + + return self.passwords + + + def reloadPasswords(self): + try: + passwords = [] + + file = fs_encode(self.getConfig("passwordfile")) + with open(file) as f: + for pw in f.read().splitlines(): + passwords.append(pw) + + except IOError, e: + self.logError(e) + + else: + self.passwords = passwords + + + @Expose + def addPassword(self, password): + """ Adds a password to saved list""" + try: + self.passwords = uniqify([password] + self.passwords) + + file = fs_encode(self.getConfig("passwordfile")) + with open(file, "wb") as f: + for pw in self.passwords: + f.write(pw + '\n') + + except IOError, e: + self.logError(e) + + + def setPermissions(self, files): + for f in files: + if not os.path.exists(f): + continue + + try: + if self.config['permission']['change_file']: + if os.path.isfile(f): + os.chmod(f, int(self.config['permission']['file'], 8)) + + elif os.path.isdir(f): + os.chmod(f, int(self.config['permission']['folder'], 8)) + + if self.config['permission']['change_dl'] and os.name != "nt": + uid = getpwnam(self.config['permission']['user'])[2] + gid = getgrnam(self.config['permission']['group'])[2] + os.chown(f, uid, gid) + + except Exception, e: + self.logWarning(_("Setting User and Group failed"), e) diff --git a/pyload/plugin/addon/HotFolder.py b/pyload/plugin/addon/HotFolder.py new file mode 100644 index 000000000..eb607ac7e --- /dev/null +++ b/pyload/plugin/addon/HotFolder.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import time + +from os import listdir, makedirs +from os.path import exists, isfile, join +from shutil import move + +from pyload.plugin.Addon import Addon +from pyload.utils import fs_encode, safe_join + + +class HotFolder(Addon): + __name__ = "HotFolder" + __type__ = "addon" + __version__ = "0.12" + + __config__ = [("folder" , "str" , "Folder to observe" , "container"), + ("watch_file", "bool", "Observe link file" , False ), + ("keep" , "bool", "Keep added containers", True ), + ("file" , "str" , "Link file" , "links.txt")] + + __description__ = """Observe folder and file for changes and add container and links""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.de")] + + + def setup(self): + self.interval = 10 + + + def activate(self): + self.initPeriodical() + + + def periodical(self): + folder = fs_encode(self.getConfig("folder")) + + try: + if not exists(join(folder, "finished")): + makedirs(join(folder, "finished")) + + if self.getConfig("watch_file"): + file = fs_encode(self.getConfig("file")) + with open(file, "a+") as f: + content = f.read().strip() + + if content: + name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y")) + + with open(safe_join(folder, "finished", name), "wb") as f: + f.write(content) + + self.core.api.addPackage(f.name, [f.name], 1) + + for f in listdir(folder): + path = join(folder, f) + + if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): + continue + + newpath = join(folder, "finished", f if self.getConfig("keep") else "tmp_" + f) + move(path, newpath) + + self.logInfo(_("Added %s from HotFolder") % f) + self.core.api.addPackage(f, [newpath], 1) + + except IOError, e: + self.logError(e) diff --git a/pyload/plugin/addon/IRCInterface.py b/pyload/plugin/addon/IRCInterface.py new file mode 100644 index 000000000..86d9ea688 --- /dev/null +++ b/pyload/plugin/addon/IRCInterface.py @@ -0,0 +1,431 @@ +# -*- coding: utf-8 -*- + +import re +import socket +import ssl +import time + +from pycurl import FORM_FILE +from select import select +from threading import Thread +from time import sleep +from traceback import print_exc + +from pyload.api import PackageDoesNotExists, FileDoesNotExists +from pyload.network.RequestFactory import getURL +from pyload.plugin.Addon import Addon +from pyload.utils import formatSize + + +class IRCInterface(Thread, Addon): + __name__ = "IRCInterface" + __type__ = "addon" + __version__ = "0.13" + + __config__ = [("host" , "str" , "IRC-Server Address" , "Enter your server here!"), + ("port" , "int" , "IRC-Server Port" , 6667 ), + ("ident" , "str" , "Clients ident" , "pyload-irc" ), + ("realname" , "str" , "Realname" , "pyload-irc" ), + ("ssl" , "bool", "Use SSL" , False ), + ("nick" , "str" , "Nickname the Client will take" , "pyLoad-IRC" ), + ("owner" , "str" , "Nickname the Client will accept commands from", "Enter your nick here!" ), + ("info_file", "bool", "Inform about every file finished" , False ), + ("info_pack", "bool", "Inform about every package finished" , True ), + ("captcha" , "bool", "Send captcha requests" , True )] + + __description__ = """Connect to irc and let owner perform different tasks""" + __license__ = "GPLv3" + __authors__ = [("Jeix", "Jeix@hasnomail.com")] + + + def __init__(self, core, manager): + Thread.__init__(self) + Addon.__init__(self, core, manager) + self.setDaemon(True) + + + def activate(self): + self.abort = False + self.more = [] + self.new_package = {} + + self.start() + + + def packageFinished(self, pypack): + try: + if self.getConfig("info_pack"): + self.response(_("Package finished: %s") % pypack.name) + except Exception: + pass + + + def downloadFinished(self, pyfile): + try: + if self.getConfig("info_file"): + self.response( + _("Download finished: %(name)s @ %(plugin)s ") % {"name": pyfile.name, "plugin": pyfile.pluginname}) + except Exception: + pass + + + def captchaTask(self, task): + if self.getConfig("captcha") and task.isTextual(): + task.handler.append(self) + task.setWaiting(60) + + html = getURL("http://www.freeimagehosting.net/upload.php", + post={"attached": (FORM_FILE, task.captchaFile)}, multipart=True) + + url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", html).group(1) + self.response(_("New Captcha Request: %s") % url) + self.response(_("Answer with 'c %s text on the captcha'") % task.id) + + + def run(self): + # connect to IRC etc. + self.sock = socket.socket() + host = self.getConfig("host") + self.sock.connect((host, self.getConfig("port"))) + + if self.getConfig("ssl"): + self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE) #@TODO: support custom certificate + + nick = self.getConfig("nick") + self.sock.send("NICK %s\r\n" % nick) + self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick)) + for t in self.getConfig("owner").split(): + if t.strip().startswith("#"): + self.sock.send("JOIN %s\r\n" % t.strip()) + self.logInfo(_("Connected to"), host) + self.logInfo(_("Switching to listening mode!")) + try: + self.main_loop() + + except IRCError, ex: + self.sock.send("QUIT :byebye\r\n") + print_exc() + self.sock.close() + + + def main_loop(self): + readbuffer = "" + while True: + sleep(1) + fdset = select([self.sock], [], [], 0) + if self.sock not in fdset[0]: + continue + + if self.abort: + raise IRCError("quit") + + readbuffer += self.sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + line = line.rstrip() + first = line.split() + + if first[0] == "PING": + self.sock.send("PONG %s\r\n" % first[1]) + + if first[0] == "ERROR": + raise IRCError(line) + + msg = line.split(None, 3) + if len(msg) < 4: + continue + + msg = { + "origin": msg[0][1:], + "action": msg[1], + "target": msg[2], + "text": msg[3][1:] + } + + self.handle_events(msg) + + + def handle_events(self, msg): + if not msg['origin'].split("!", 1)[0] in self.getConfig("owner").split(): + return + + if msg['target'].split("!", 1)[0] != self.getConfig("nick"): + return + + if msg['action'] != "PRIVMSG": + return + + # HANDLE CTCP ANTI FLOOD/BOT PROTECTION + if msg['text'] == "\x01VERSION\x01": + self.logDebug("Sending CTCP VERSION") + self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) + return + elif msg['text'] == "\x01TIME\x01": + self.logDebug("Sending CTCP TIME") + self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) + return + elif msg['text'] == "\x01LAG\x01": + self.logDebug("Received CTCP LAG") #: don't know how to answer + return + + trigger = "pass" + args = None + + try: + temp = msg['text'].split() + trigger = temp[0] + if len(temp) > 1: + args = temp[1:] + except Exception: + pass + + handler = getattr(self, "event_%s" % trigger, self.event_pass) + try: + res = handler(args) + for line in res: + self.response(line, msg['origin']) + except Exception, e: + self.logError(e) + + + def response(self, msg, origin=""): + if origin == "": + for t in self.getConfig("owner").split(): + self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg)) + else: + self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg)) + + + #### Events + + def event_pass(self, args): + return [] + + + def event_status(self, args): + downloads = self.core.api.statusDownloads() + if not downloads: + return ["INFO: There are no active downloads currently."] + + temp_progress = "" + lines = ["ID - Name - Status - Speed - ETA - Progress"] + for data in downloads: + + if data.status == 5: + temp_progress = data.format_wait + else: + temp_progress = "%d%% (%s)" % (data.percent, data.format_size) + + lines.append("#%d - %s - %s - %s - %s - %s" % + ( + data.fid, + data.name, + data.statusmsg, + "%s/s" % formatSize(data.speed), + "%s" % data.format_eta, + temp_progress + )) + return lines + + + def event_queue(self, args): + ps = self.core.api.getQueueData() + + if not ps: + return ["INFO: There are no packages in queue."] + + lines = [] + for pack in ps: + lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links))) + + return lines + + + def event_collector(self, args): + ps = self.core.api.getCollectorData() + if not ps: + return ["INFO: No packages in collector!"] + + lines = [] + for pack in ps: + lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.links))) + + return lines + + + def event_info(self, args): + if not args: + return ["ERROR: Use info like this: info <id>"] + + info = None + try: + info = self.core.api.getFileData(int(args[0])) + + except FileDoesNotExists: + return ["ERROR: Link doesn't exists."] + + return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.statusmsg, info.plugin)] + + + def event_packinfo(self, args): + if not args: + return ["ERROR: Use packinfo like this: packinfo <id>"] + + lines = [] + pack = None + try: + pack = self.core.api.getPackageData(int(args[0])) + + except PackageDoesNotExists: + return ["ERROR: Package doesn't exists."] + + id = args[0] + + self.more = [] + + lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links))) + for pyfile in pack.links: + self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size, + pyfile.statusmsg, pyfile.plugin)) + + if len(self.more) < 6: + lines.extend(self.more) + self.more = [] + else: + lines.extend(self.more[:6]) + self.more = self.more[6:] + lines.append("%d more links do display." % len(self.more)) + + return lines + + + def event_more(self, args): + if not self.more: + return ["No more information to display."] + + lines = self.more[:6] + self.more = self.more[6:] + lines.append("%d more links do display." % len(self.more)) + + return lines + + + def event_start(self, args): + self.core.api.unpauseServer() + return ["INFO: Starting downloads."] + + + def event_stop(self, args): + self.core.api.pauseServer() + return ["INFO: No new downloads will be started."] + + + def event_add(self, args): + if len(args) < 2: + return ['ERROR: Add links like this: "add <packagename|id> links". ', + "This will add the link <link> to to the package <package> / the package with id <id>!"] + + pack = args[0].strip() + links = [x.strip() for x in args[1:]] + + count_added = 0 + count_failed = 0 + try: + id = int(pack) + pack = self.core.api.getPackageData(id) + if not pack: + return ["ERROR: Package doesn't exists."] + + #TODO add links + + return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack['name'], id)] + + except Exception: + # create new package + id = self.core.api.addPackage(pack, links, 1) + return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))] + + + def event_del(self, args): + if len(args) < 2: + return ["ERROR: Use del command like this: del -p|-l <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] + + if args[0] == "-p": + ret = self.core.api.deletePackages(map(int, args[1:])) + return ["INFO: Deleted %d packages!" % len(args[1:])] + + elif args[0] == "-l": + ret = self.core.api.delLinks(map(int, args[1:])) + return ["INFO: Deleted %d links!" % len(args[1:])] + + else: + return ["ERROR: Use del command like this: del <-p|-l> <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] + + + def event_push(self, args): + if not args: + return ["ERROR: Push package to queue like this: push <package id>"] + + id = int(args[0]) + try: + info = self.core.api.getPackageInfo(id) + except PackageDoesNotExists: + return ["ERROR: Package #%d does not exist." % id] + + self.core.api.pushToQueue(id) + return ["INFO: Pushed package #%d to queue." % id] + + + def event_pull(self, args): + if not args: + return ["ERROR: Pull package from queue like this: pull <package id>."] + + id = int(args[0]) + if not self.core.api.getPackageData(id): + return ["ERROR: Package #%d does not exist." % id] + + self.core.api.pullFromQueue(id) + return ["INFO: Pulled package #%d from queue to collector." % id] + + + def event_c(self, args): + """ captcha answer """ + if not args: + return ["ERROR: Captcha ID missing."] + + task = self.core.captchaManager.getTaskByID(args[0]) + if not task: + return ["ERROR: Captcha Task with ID %s does not exists." % args[0]] + + task.setResult(" ".join(args[1:])) + return ["INFO: Result %s saved." % " ".join(args[1:])] + + + def event_help(self, args): + lines = ["The following commands are available:", + "add <package|packid> <links> [...] Adds link to package. (creates new package if it does not exist)", + "queue Shows all packages in the queue", + "collector Shows all packages in collector", + "del -p|-l <id> [...] Deletes all packages|links with the ids specified", + "info <id> Shows info of the link with id <id>", + "packinfo <id> Shows info of the package with id <id>", + "more Shows more info when the result was truncated", + "start Starts all downloads", + "stop Stops the download (but not abort active downloads)", + "push <id> Push package to queue", + "pull <id> Pull package from queue", + "status Show general download status", + "help Shows this help message"] + return lines + + +class IRCError(Exception): + + def __init__(self, value): + self.value = value + + + def __str__(self): + return repr(self.value) diff --git a/pyload/plugin/addon/JustPremium.py b/pyload/plugin/addon/JustPremium.py new file mode 100644 index 000000000..d3c4d8eff --- /dev/null +++ b/pyload/plugin/addon/JustPremium.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Addon import Addon + + +class JustPremium(Addon): + __name__ = "JustPremium" + __type__ = "addon" + __version__ = "0.21" + + __config__ = [("excluded", "str", "Exclude hosters (comma separated)", "")] + + __description__ = """Remove all not premium links from urls added""" + __license__ = "GPLv3" + __authors__ = [("mazleu", "mazleica@gmail.com"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("immenz", "immenz@gmx.net")] + + + event_list = ["linksAdded"] + + + def linksAdded(self, links, pid): + hosterdict = self.core.pluginManager.hosterPlugins + linkdict = self.core.api.checkURLs(links) + + premiumplugins = set(account.type for account in self.core.api.getAccounts(False) \ + if account.valid and account.premium) + multihosters = set(hoster for hoster in self.core.pluginManager.hosterPlugins \ + if 'new_name' in hosterdict[hoster] \ + and hosterdict[hoster]['new_name'] in premiumplugins) + + #: Found at least one hoster with account or multihoster + if not any(True for pluginname in linkdict if pluginname in premiumplugins | multihosters): + return + + excluded = map(lambda domain: "".join(part.capitalize() for part in re.split(r'(\.|\d+)', domain) if part != '.'), + self.getConfig('excluded').replace(' ', '').replace(',', '|').replace(';', '|').split('|')) + + for pluginname in set(linkdict.keys()) - (premiumplugins | multihosters).union(excluded): + self.logInfo(_("Remove links of plugin: %s") % pluginname) + for link in linkdict[pluginname]: + self.logDebug("Remove link: %s" % link) + links.remove(link) diff --git a/pyload/plugin/addon/MergeFiles.py b/pyload/plugin/addon/MergeFiles.py new file mode 100644 index 000000000..11e869aee --- /dev/null +++ b/pyload/plugin/addon/MergeFiles.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import os +import re + +from traceback import print_exc + +from pyload.plugin.Addon import Addon, threaded +from pyload.utils import safe_join + + +class MergeFiles(Addon): + __name__ = "MergeFiles" + __type__ = "addon" + __version__ = "0.14" + + __config__ = [("activated", "bool", "Activated", True)] + + __description__ = """Merges parts splitted with hjsplit""" + __license__ = "GPLv3" + __authors__ = [("and9000", "me@has-no-mail.com")] + + + BUFFER_SIZE = 4096 + + + def setup(self): + pass + + + @threaded + def packageFinished(self, pack): + files = {} + fid_dict = {} + for fid, data in pack.getChildren().iteritems(): + if re.search("\.\d{3}$", data['name']): + if data['name'][:-4] not in files: + files[data['name'][:-4]] = [] + files[data['name'][:-4]].append(data['name']) + files[data['name'][:-4]].sort() + fid_dict[data['name']] = fid + + download_folder = self.config['general']['download_folder'] + + if self.config['general']['folder_per_package']: + download_folder = safe_join(download_folder, pack.folder) + + for name, file_list in files.iteritems(): + self.logInfo(_("Starting merging of"), name) + + final_file = open(safe_join(download_folder, name), "wb") + for splitted_file in file_list: + self.logDebug("Merging part", splitted_file) + + pyfile = self.core.files.getFile(fid_dict[splitted_file]) + + pyfile.setStatus("processing") + + try: + with open(safe_join(download_folder, splitted_file), "rb") as s_file: + size_written = 0 + s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file))) + + while True: + f_buffer = s_file.read(self.BUFFER_SIZE) + if f_buffer: + final_file.write(f_buffer) + size_written += self.BUFFER_SIZE + pyfile.setProgress((size_written * 100) / s_file_size) + else: + break + + self.logDebug("Finished merging part", splitted_file) + + except Exception, e: + print_exc() + + finally: + pyfile.setProgress(100) + pyfile.setStatus("finished") + pyfile.release() + + self.logInfo(_("Finished merging of"), name) diff --git a/pyload/plugin/addon/MultiHome.py b/pyload/plugin/addon/MultiHome.py new file mode 100644 index 000000000..521749fc8 --- /dev/null +++ b/pyload/plugin/addon/MultiHome.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from time import time + +from pyload.plugin.Addon import Addon + + +class MultiHome(Addon): + __name__ = "MultiHome" + __type__ = "addon" + __version__ = "0.12" + + __config__ = [("interfaces", "str", "Interfaces", "None")] + + __description__ = """Ip address changer""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + def setup(self): + self.register = {} + self.interfaces = [] + self.parseInterfaces(self.getConfig("interfaces").split(";")) + if not self.interfaces: + self.parseInterfaces([self.config['download']['interface']]) + self.setConfig("interfaces", self.toConfig()) + + + def toConfig(self): + return ";".join(i.adress for i in self.interfaces) + + + def parseInterfaces(self, interfaces): + for interface in interfaces: + if not interface or str(interface).lower() == "none": + continue + self.interfaces.append(Interface(interface)) + + + def activate(self): + requestFactory = self.core.requestFactory + oldGetRequest = requestFactory.getRequest + + def getRequest(pluginName, account=None): + iface = self.bestInterface(pluginName, account) + if iface: + iface.useFor(pluginName, account) + requestFactory.iface = lambda: iface.adress + self.logDebug("Using address", iface.adress) + return oldGetRequest(pluginName, account) + + requestFactory.getRequest = getRequest + + + def bestInterface(self, pluginName, account): + best = None + for interface in self.interfaces: + if not best or interface.lastPluginAccess(pluginName, account) < best.lastPluginAccess(pluginName, account): + best = interface + return best + + +class Interface(object): + + def __init__(self, adress): + self.adress = adress + self.history = {} + + + def lastPluginAccess(self, pluginName, account): + if (pluginName, account) in self.history: + return self.history[(pluginName, account)] + return 0 + + + def useFor(self, pluginName, account): + self.history[(pluginName, account)] = time() + + + def __repr__(self): + return "<Interface - %s>" % self.adress diff --git a/pyload/plugin/addon/RestartFailed.py b/pyload/plugin/addon/RestartFailed.py new file mode 100644 index 000000000..2fe5f13bf --- /dev/null +++ b/pyload/plugin/addon/RestartFailed.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Addon import Addon + + +class RestartFailed(Addon): + __name__ = "RestartFailed" + __type__ = "addon" + __version__ = "1.57" + + __config__ = [("activated", "bool", "Activated" , True), + ("interval" , "int" , "Check interval in minutes", 90 )] + + __description__ = """Periodically restart all failed downloads in queue""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + + MIN_INTERVAL = 15 * 60 #: 15m minimum check interval (value is in seconds) + + + def pluginConfigChanged(self, plugin, name, value): + if name == "interval": + interval = value * 60 + if self.MIN_INTERVAL <= interval != self.interval: + self.core.scheduler.removeJob(self.cb) + self.interval = interval + self.initPeriodical() + else: + self.logDebug("Invalid interval value, kept current") + + + def periodical(self): + self.logDebug(_("Restart failed downloads")) + self.core.api.restartFailed() + + + def setup(self): + self.interval = 0 + + + def activate(self): + self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) diff --git a/pyload/plugin/addon/RestartSlow.py b/pyload/plugin/addon/RestartSlow.py new file mode 100644 index 000000000..332047da7 --- /dev/null +++ b/pyload/plugin/addon/RestartSlow.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +import pycurl + +from pyload.plugin.Addon import Addon + + +class RestartSlow(Addon): + __name__ = "RestartSlow" + __type__ = "addon" + __version__ = "0.04" + + __config__ = [("free_limit" , "int" , "Transfer speed threshold in kilobytes" , 100 ), + ("free_time" , "int" , "Sample interval in minutes" , 5 ), + ("premium_limit", "int" , "Transfer speed threshold for premium download in kilobytes", 300 ), + ("premium_time" , "int" , "Sample interval for premium download in minutes" , 2 ), + ("safe_mode" , "bool", "Don't restart if download is not resumable" , True)] + + __description__ = """Restart slow downloads""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + event_map = {'download-start': "downloadStarts"} + + + def setup(self): + self.info = {'chunk': {}} + + + def periodical(self): + if not self.pyfile.plugin.req.dl: + return + + if self.getConfig("safe_mode") and not self.pyfile.plugin.resumeDownload: + time = 30 + limit = 5 + else: + type = "premium" if self.pyfile.plugin.premium else "free" + time = max(30, self.getConfig("%s_time" % type) * 60) + limit = max(5, self.getConfig("%s_limit" % type) * 1024) + + chunks = [chunk for chunk in self.pyfile.plugin.req.dl.chunks \ + if chunk.id not in self.info['chunk'] or self.info['chunk'][chunk.id] is not (time, limit)] + + for chunk in chunks: + chunk.c.setopt(pycurl.LOW_SPEED_TIME , time) + chunk.c.setopt(pycurl.LOW_SPEED_LIMIT, limit) + + self.info['chunk'][chunk.id] = (time, limit) + + + def downloadStarts(self, pyfile, url, filename): + if self.cb or (self.getConfig("safe_mode") and not pyfile.plugin.resumeDownload): + return + self.pyfile = pyfile + self.initPeriodical() diff --git a/pyload/plugin/addon/SkipRev.py b/pyload/plugin/addon/SkipRev.py new file mode 100644 index 000000000..efc96cb7b --- /dev/null +++ b/pyload/plugin/addon/SkipRev.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +from types import MethodType +from urllib import unquote +from urlparse import urlparse + +from pyload.datatype.File import PyFile +from pyload.plugin.Addon import Addon +from pyload.plugin.Plugin import SkipDownload + + +def _setup(self): + self.pyfile.plugin._setup() + if self.pyfile.hasStatus("skipped"): + raise SkipDownload(self.pyfile.statusname or self.pyfile.pluginname) + + +class SkipRev(Addon): + __name__ = "SkipRev" + __type__ = "addon" + __version__ = "0.25" + + __config__ = [("tokeep", "int", "Number of rev files to keep for package (-1 to auto)", -1)] + + __description__ = """Skip files ending with extension rev""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def _pyname(self, pyfile): + if hasattr(pyfile.pluginmodule, "getInfo"): + return getattr(pyfile.pluginmodule, "getInfo")([pyfile.url]).next()[0] + else: + self.logWarning("Unable to grab file name") + return urlparse(unquote(pyfile.url)).path.split('/')[-1] + + + def _pyfile(self, link): + return PyFile(self.core.files, + link.fid, + link.url, + link.name, + link.size, + link.status, + link.error, + link.plugin, + link.packageID, + link.order) + + + def downloadPreparing(self, pyfile): + if pyfile.statusname is "unskipped" or not self._pyname(pyfile).endswith(".rev"): + return + + tokeep = self.getConfig("tokeep") + + if tokeep: + status_list = (1, 4, 8, 9, 14) if tokeep < 0 else (1, 3, 4, 8, 9, 14) + + queued = [True for link in self.core.api.getPackageData(pyfile.package().id).links \ + if link.name.endswith(".rev") and link.status not in status_list].count(True) + + if not queued or queued < tokeep: #: keep one rev at least in auto mode + return + + pyfile.setCustomStatus("SkipRev", "skipped") + pyfile.plugin._setup = pyfile.plugin.setup + pyfile.plugin.setup = MethodType(_setup, pyfile.plugin) #: work-around: inject status checker inside the preprocessing routine of the plugin + + + def downloadFailed(self, pyfile): + #: Check if pyfile is still "failed", + # maybe might has been restarted in meantime + if pyfile.status != 8: + return + + tokeep = self.getConfig("tokeep") + + if not tokeep: + return + + for link in self.core.api.getPackageData(pyfile.package().id).links: + if link.status is 4 and link.name.endswith(".rev"): + pylink = self._pyfile(link) + + if tokeep > -1 or pyfile.name.endswith(".rev"): + pylink.setStatus("queued") + else: + pylink.setCustomStatus("unskipped", "queued") + + self.core.files.save() + pylink.release() + return diff --git a/pyload/plugin/addon/UnSkipOnFail.py b/pyload/plugin/addon/UnSkipOnFail.py new file mode 100644 index 000000000..7d787d1ed --- /dev/null +++ b/pyload/plugin/addon/UnSkipOnFail.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +from pyload.datatype.File import PyFile +from pyload.plugin.Addon import Addon + + +class UnSkipOnFail(Addon): + __name__ = "UnSkipOnFail" + __type__ = "addon" + __version__ = "0.05" + + __config__ = [("activated", "bool", "Activated", True)] + + __description__ = """Queue skipped duplicates when download fails""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def downloadFailed(self, pyfile): + #: Check if pyfile is still "failed", + # maybe might has been restarted in meantime + if pyfile.status != 8: + return + + msg = _("Looking for skipped duplicates of: %s (pid:%s)") + self.logInfo(msg % (pyfile.name, pyfile.package().id)) + + dup = self.findDuplicate(pyfile) + if dup: + self.logInfo(_("Queue found duplicate: %s (pid:%s)") % (dup.name, dup.packageID)) + + #: Change status of "link" to "new_status". + # "link" has to be a valid FileData object, + # "new_status" has to be a valid status name + # (i.e. "queued" for this Plugin) + # It creates a temporary PyFile object using + # "link" data, changes its status, and tells + # the core.files-manager to save its data. + pylink = _pyfile(link) + + pylink.setCustomStatus("UnSkipOnFail", "queued") + + self.core.files.save() + pylink.release() + + else: + self.logInfo(_("No duplicates found")) + + + def findDuplicate(self, pyfile): + """ Search all packages for duplicate links to "pyfile". + Duplicates are links that would overwrite "pyfile". + To test on duplicity the package-folder and link-name + of twolinks are compared (link.name). + So this method returns a list of all links with equal + package-folders and filenames as "pyfile", but except + the data for "pyfile" iotselöf. + It does MOT check the link's status. + """ + queue = self.core.api.getQueue() #: get packages (w/o files, as most file data is useless here) + + for package in queue: + #: check if package-folder equals pyfile's package folder + if package.folder != pyfile.package().folder: + continue + + #: now get packaged data w/ files/links + pdata = self.core.api.getPackageData(package.pid) + for link in pdata.links: + #: check if link is "skipped" + if link.status != 4: + continue + + #: check if link name collides with pdata's name + #: AND at last check if it is not pyfile itself + if link.name == pyfile.name and link.fid != pyfile.id: + return link + + + def _pyfile(self, link): + return PyFile(self.core.files, + link.fid, + link.url, + link.name, + link.size, + link.status, + link.error, + link.plugin, + link.packageID, + link.order) diff --git a/pyload/plugin/addon/UpdateManager.py b/pyload/plugin/addon/UpdateManager.py new file mode 100644 index 000000000..cf138f64a --- /dev/null +++ b/pyload/plugin/addon/UpdateManager.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import sys + +from operator import itemgetter +from os import path, remove, stat + +from pyload.network.RequestFactory import getURL +from pyload.plugin.Addon import Expose, Addon, threaded +from pyload.utils import safe_join + + +class UpdateManager(Addon): + __name__ = "UpdateManager" + __type__ = "addon" + __version__ = "0.43" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"), + ("interval" , "int" , "Check interval in hours" , 8 ), + ("autorestart" , "bool" , "Automatically restart pyLoad when required" , True ), + ("reloadplugins", "bool" , "Monitor plugins for code changes in debug mode", True ), + ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , False )] + + __description__ = """Check for updates""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + + SERVER_URL = "http://updatemanager.pyload.org" + VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + MIN_INTERVAL = 3 * 60 * 60 #: 3h minimum check interval (value is in seconds) + + + def pluginConfigChanged(self, plugin, name, value): + if name == "interval": + interval = value * 60 * 60 + if self.MIN_INTERVAL <= interval != self.interval: + self.core.scheduler.removeJob(self.cb) + self.interval = interval + self.initPeriodical() + else: + self.logDebug("Invalid interval value, kept current") + + elif name == "reloadplugins": + if self.cb2: + self.core.scheduler.removeJob(self.cb2) + if value is True and self.core.debug: + self.periodical2() + + + def activate(self): + self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) + x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins")) + self.core.scheduler.addJob(10, x, threaded=False) + + + def deactivate(self): + self.pluginConfigChanged(self.__name__, "reloadplugins", False) + + + def setup(self): + self.cb2 = None + self.interval = 0 + self.updating = False + self.info = {'pyload': False, 'version': None, 'plugins': False} + self.mtimes = {} #: store modification time for each plugin + + + def periodical2(self): + if not self.updating: + self.autoreloadPlugins() + + self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False) + + + @Expose + def autoreloadPlugins(self): + """ reload and reindex all modified plugins """ + modules = filter( + lambda m: m and (m.__name__.startswith("pyload.plugin.") or + m.__name__.startswith("userplugins.")) and + m.__name__.count(".") >= 2, sys.modules.itervalues() + ) + + reloads = [] + + for m in modules: + root, type, name = m.__name__.rsplit(".", 2) + id = (type, name) + if type in self.core.pluginManager.plugins: + f = m.__file__.replace(".pyc", ".py") + if not path.isfile(f): + continue + + mtime = stat(f).st_mtime + + if id not in self.mtimes: + self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: + reloads.append(id) + self.mtimes[id] = mtime + + return True if self.core.pluginManager.reloadPlugins(reloads) else False + + + def periodical(self): + if self.info['pyload'] or self.getConfig("nodebugupdate") and self.core.debug: + return + + self.updateThread() + + + def server_request(self): + try: + return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() + except Exception: + self.logWarning(_("Unable to contact server to get updates")) + + + @threaded + def updateThread(self): + self.updating = True + + status = self.update(onlyplugin=self.getConfig("mode") == "plugins only") + + if status is 2 and self.getConfig("autorestart"): + self.core.api.restart() + else: + self.updating = False + + + @Expose + def updatePlugins(self): + """ simple wrapper for calling plugin update quickly """ + return self.update(onlyplugin=True) + + + @Expose + def update(self, onlyplugin=False): + """ check for updates """ + data = self.server_request() + + if not data: + exitcode = 0 + + elif data[0] == "None": + self.logInfo(_("No new pyLoad version available")) + updates = data[1:] + exitcode = self._updatePlugins(updates) + + elif onlyplugin: + exitcode = 0 + + else: + newversion = data[0] + self.logInfo(_("*** New pyLoad Version %s available ***") % newversion) + self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) + exitcode = 3 + self.info['pyload'] = True + self.info['version'] = newversion + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available + + + def _updatePlugins(self, updates): + """ check for plugin updates """ + + if self.info['plugins']: + return False #: plugins were already updated + + exitcode = 0 + updated = [] + + url = updates[0] + schema = updates[1].split('|') + + if "BLACKLIST" in updates: + blacklist = updates[updates.index('BLACKLIST') + 1:] + updates = updates[2:updates.index('BLACKLIST')] + else: + blacklist = None + updates = updates[2:] + + upgradable = [dict(zip(schema, x.split('|'))) for x in updates] + blacklisted = [(x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]) for x in blacklist] if blacklist else [] + + if blacklist: + # Protect UpdateManager from self-removing + try: + blacklisted.remove(("addon", "UpdateManager")) + except Exception: + pass + + for t, n in blacklisted: + for idx, plugin in enumerate(upgradable): + if n == plugin['name'] and t == plugin['type']: + upgradable.pop(idx) + break + + for t, n in self.removePlugins(sorted(blacklisted)): + self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % { + 'type': t, + 'name': n, + }) + + for plugin in sorted(upgradable, key=itemgetter("type", "name")): + filename = plugin['name'] + type = plugin['type'] + version = plugin['version'] + + if filename.endswith(".pyc"): + name = filename[:filename.find("_")] + else: + name = filename.replace(".py", "") + + plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + + oldver = float(plugins[name]['version']) if name in plugins else None + newver = float(version) + + if not oldver: + msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" + elif newver > oldver: + msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" + else: + continue + + self.logInfo(_(msg) % {'type' : type, + 'name' : name, + 'oldver': oldver, + 'newver': newver}) + try: + content = getURL(url % plugin) + m = self.VERSION.search(content) + + if m and m.group(2) == version: + with open(safe_join("userplugins", prefix, filename), "wb") as f: + f.write(content) + + updated.append((prefix, name)) + else: + raise Exception, _("Version mismatch") + + except Exception, e: + self.logError(_("Error updating plugin: %s") % filename, str(e)) + + if updated: + reloaded = self.core.pluginManager.reloadPlugins(updated) + if reloaded: + self.logInfo(_("Plugins updated and reloaded")) + exitcode = 1 + else: + self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***")) + self.info['plugins'] = True + exitcode = 2 + else: + self.logInfo(_("No plugin updates available")) + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required + + + @Expose + def removePlugins(self, type_plugins): + """ delete plugins from disk """ + + if not type_plugins: + return + + self.logDebug("Requested deletion of plugins: %s" % type_plugins) + + removed = [] + + for type, name in type_plugins: + err = False + file = name + ".py" + + for root in ("userplugins", path.join(pypath, "pyload", "plugins")): + + filename = safe_join(root, type, file) + try: + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + filename += "c" + if path.isfile(filename): + try: + if type == "addon": + self.manager.deactivateAddon(name) + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + if not err: + id = (type, name) + removed.append(id) + + return removed #: return a list of the plugins successfully removed diff --git a/pyload/plugin/addon/WindowsPhoneNotify.py b/pyload/plugin/addon/WindowsPhoneNotify.py new file mode 100644 index 000000000..b9710c2f0 --- /dev/null +++ b/pyload/plugin/addon/WindowsPhoneNotify.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +import httplib + +from time import time + +from pyload.plugin.Addon import Addon + + +class WindowsPhoneNotify(Addon): + __name__ = "WindowsPhoneNotify" + __type__ = "addon" + __version__ = "0.07" + + __config__ = [("id" , "str" , "Push ID" , "" ), + ("url" , "str" , "Push url" , "" ), + ("notifycaptcha" , "bool", "Notify captcha request" , True ), + ("notifypackage" , "bool", "Notify package finished" , True ), + ("notifyprocessed", "bool", "Notify processed packages status" , True ), + ("timeout" , "int" , "Timeout between captchas in seconds" , 5 ), + ("force" , "bool", "Send notifications if client is connected", False)] + + __description__ = """Send push notifications to Windows Phone""" + __license__ = "GPLv3" + __authors__ = [("Andy Voigt", "phone-support@hotmail.de"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + event_list = ["allDownloadsProcessed"] + + + def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.last_notify = 0 + + + def newCaptchaTask(self, task): + if not self.getConfig("notifycaptcha"): + return False + + if time() - self.last_notify < self.getConf("timeout"): + return False + + self.notify(_("Captcha"), _("New request waiting user input")) + + + def packageFinished(self, pypack): + if self.getConfig("notifypackage"): + self.notify(_("Package finished"), pypack.name) + + + def allDownloadsProcessed(self): + if not self.getConfig("notifyprocessed"): + return False + + if any(True for pdata in self.core.api.getQueue() if pdata.linksdone < pdata.linkstotal): + self.notify(_("Package failed"), _("One or more packages was not completed successfully")) + else: + self.notify(_("All packages finished")) + + + def getXmlData(self, msg): + return ("<?xml version='1.0' encoding='utf-8'?> <wp:Notification xmlns:wp='WPNotification'> " + "<wp:Toast> <wp:Text1>pyLoad</wp:Text1> <wp:Text2>%s</wp:Text2> " + "</wp:Toast> </wp:Notification>" % msg) + + + def notify(self, event, msg=""): + id = self.getConfig("id") + url = self.getConfig("url") + + if not id or not url: + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + request = self.getXmlData("%s: %s" % (event, msg) if msg else event) + webservice = httplib.HTTP(url) + + webservice.putrequest("POST", id) + webservice.putheader("Host", url) + webservice.putheader("Content-type", "text/xml") + webservice.putheader("X-NotificationClass", "2") + webservice.putheader("X-WindowsPhone-Target", "toast") + webservice.putheader("Content-length", "%d" % len(request)) + webservice.endheaders() + webservice.send(request) + webservice.close() + + self.last_notify = time() diff --git a/pyload/plugin/addon/XMPPInterface.py b/pyload/plugin/addon/XMPPInterface.py new file mode 100644 index 000000000..77a49af6f --- /dev/null +++ b/pyload/plugin/addon/XMPPInterface.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- + +from pyxmpp import streamtls +from pyxmpp.all import JID, Message +from pyxmpp.interface import implements +from pyxmpp.interfaces import * +from pyxmpp.jabber.client import JabberClient + +from pyload.plugin.addon.IRCInterface import IRCInterface + + +class XMPPInterface(IRCInterface, JabberClient): + __name__ = "XMPPInterface" + __type__ = "addon" + __version__ = "0.11" + + __config__ = [("jid" , "str" , "Jabber ID" , "user@exmaple-jabber-server.org" ), + ("pw" , "str" , "Password" , "" ), + ("tls" , "bool", "Use TLS" , False ), + ("owners" , "str" , "List of JIDs accepting commands from", "me@icq-gateway.org;some@msn-gateway.org"), + ("info_file", "bool", "Inform about every file finished" , False ), + ("info_pack", "bool", "Inform about every package finished" , True ), + ("captcha" , "bool", "Send captcha requests" , True )] + + __description__ = """Connect to jabber and let owner perform different tasks""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + implements(IMessageHandlersProvider) + + + def __init__(self, core, manager): + IRCInterface.__init__(self, core, manager) + + self.jid = JID(self.getConfig("jid")) + password = self.getConfig("pw") + + # if bare JID is provided add a resource -- it is required + if not self.jid.resource: + self.jid = JID(self.jid.node, self.jid.domain, "pyLoad") + + if self.getConfig("tls"): + tls_settings = streamtls.TLSSettings(require=True, verify_peer=False) + auth = ("sasl:PLAIN", "sasl:DIGEST-MD5") + else: + tls_settings = None + auth = ("sasl:DIGEST-MD5", "digest") + + # setup client with provided connection information + # and identity data + JabberClient.__init__(self, self.jid, password, + disco_name="pyLoad XMPP Client", disco_type="bot", + tls_settings=tls_settings, auth_methods=auth) + + self.interface_providers = [ + VersionHandler(self), + self, + ] + + + def activate(self): + self.new_package = {} + + self.start() + + + def packageFinished(self, pypack): + try: + if self.getConfig("info_pack"): + self.announce(_("Package finished: %s") % pypack.name) + except Exception: + pass + + + def downloadFinished(self, pyfile): + try: + if self.getConfig("info_file"): + self.announce( + _("Download finished: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.pluginname}) + except Exception: + pass + + + def run(self): + # connect to IRC etc. + self.connect() + try: + self.loop() + except Exception, ex: + self.logError(ex) + + + def stream_state_changed(self, state, arg): + """This one is called when the state of stream connecting the component + to a server changes. This will usually be used to let the user + know what is going on.""" + self.logDebug("*** State changed: %s %r ***" % (state, arg)) + + + def disconnected(self): + self.logDebug("Client was disconnected") + + + def stream_closed(self, stream): + self.logDebug("Stream was closed", stream) + + + def stream_error(self, err): + self.logDebug("Stream Error", err) + + + def get_message_handlers(self): + """Return list of (message_type, message_handler) tuples. + + The handlers returned will be called when matching message is received + in a client session.""" + return [("normal", self.message)] + + + def message(self, stanza): + """Message handler for the component.""" + subject = stanza.get_subject() + body = stanza.get_body() + t = stanza.get_type() + self.logDebug("Message from %s received." % unicode(stanza.get_from())) + self.logDebug("Body: %s Subject: %s Type: %s" % (body, subject, t)) + + if t == "headline": + # 'headline' messages should never be replied to + return True + if subject: + subject = u"Re: " + subject + + to_jid = stanza.get_from() + from_jid = stanza.get_to() + + #j = JID() + to_name = to_jid.as_utf8() + from_name = from_jid.as_utf8() + + names = self.getConfig("owners").split(";") + + if to_name in names or to_jid.node + "@" + to_jid.domain in names: + messages = [] + + trigger = "pass" + args = None + + try: + temp = body.split() + trigger = temp[0] + if len(temp) > 1: + args = temp[1:] + except Exception: + pass + + handler = getattr(self, "event_%s" % trigger, self.event_pass) + try: + res = handler(args) + for line in res: + m = Message( + to_jid=to_jid, + from_jid=from_jid, + stanza_type=stanza.get_type(), + subject=subject, + body=line) + + messages.append(m) + except Exception, e: + self.logError(e) + + return messages + + else: + return True + + + def response(self, msg, origin=""): + return self.announce(msg) + + + def announce(self, message): + """ send message to all owners""" + for user in self.getConfig("owners").split(";"): + self.logDebug("Send message to", user) + + to_jid = JID(user) + + m = Message(from_jid=self.jid, + to_jid=to_jid, + stanza_type="chat", + body=message) + + stream = self.get_stream() + if not stream: + self.connect() + stream = self.get_stream() + + stream.send(m) + + + def beforeReconnecting(self, ip): + self.disconnect() + + + def afterReconnecting(self, ip): + self.connect() + + +class VersionHandler(object): + """Provides handler for a version query. + + This class will answer version query and announce 'jabber:iq:version' namespace + in the client's disco#info results.""" + + implements(IIqHandlersProvider, IFeaturesProvider) + + + def __init__(self, client): + """Just remember who created this.""" + self.client = client + + + def get_features(self): + """Return namespace which should the client include in its reply to a + disco#info query.""" + return ["jabber:iq:version"] + + + def get_iq_get_handlers(self): + """Return list of tuples (element_name, namespace, handler) describing + handlers of <iq type='get'/> stanzas""" + return [("query", "jabber:iq:version", self.get_version)] + + + def get_iq_set_handlers(self): + """Return empty list, as this class provides no <iq type='set'/> stanza handler.""" + return [] + + + def get_version(self, iq): + """Handler for jabber:iq:version queries. + + jabber:iq:version queries are not supported directly by PyXMPP, so the + XML node is accessed directly through the libxml2 API. This should be + used very carefully!""" + iq = iq.make_result_response() + q = iq.new_query("jabber:iq:version") + q.newTextChild(q.ns(), "name", "Echo component") + q.newTextChild(q.ns(), "version", "1.0") + return iq diff --git a/pyload/plugin/addon/__init__.py b/pyload/plugin/addon/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/addon/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/captcha/AdYouLike.py b/pyload/plugin/captcha/AdYouLike.py new file mode 100644 index 000000000..0ff1a799b --- /dev/null +++ b/pyload/plugin/captcha/AdYouLike.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Captcha import Captcha +from pyload.utils import json_loads + + +class AdYouLike(Captcha): + __name__ = "AdYouLike" + __version__ = "0.05" + + __description__ = """AdYouLike captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' + CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("AdYouLike html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.AYL_PATTERN, html) + n = re.search(self.CALLBACK_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) + self.logDebug("Ayl|callback: %s | %s" % self.key) + return self.key #: key is the tuple(ayl, callback) + else: + self.logDebug("Ayl or callback not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("AdYouLike key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + ayl, callback = key + + # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + ayl = json_loads(ayl) + + html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + get={'key' : ayl['adyoulike']['key'], + 'env' : ayl['all']['env'], + 'callback': callback}) + try: + challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) + + except AttributeError: + errmsg = _("AdYouLike challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(ayl, challenge), challenge + + + def result(self, server, challenge): + # Adyoulike.g._jsonp_5579316662423138 + # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, + # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], + # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", + # "iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\" + # height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256, + # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, + # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, + # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + + if isinstance(server, basestring): + server = json_loads(server) + + if isinstance(challenge, basestring): + challenge = json_loads(challenge) + + try: + instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] + result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() + + except AttributeError: + errmsg = _("AdYouLike result not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + result = {'_ayl_captcha_engine' : "adyoulike", + '_ayl_env' : server['all']['env'], + '_ayl_tid' : challenge['tid'], + '_ayl_token_challenge': challenge['token'], + '_ayl_response' : response} + + self.logDebug("Result: %s" % result) + + return result diff --git a/pyload/plugin/captcha/AdsCaptcha.py b/pyload/plugin/captcha/AdsCaptcha.py new file mode 100644 index 000000000..75852e36d --- /dev/null +++ b/pyload/plugin/captcha/AdsCaptcha.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +import re + +from random import random + +from pyload.plugin.Captcha import Captcha + + +class AdsCaptcha(Captcha): + __name__ = "AdsCaptcha" + __version__ = "0.08" + + __description__ = """AdsCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)' + PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("AdsCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.PUBLICKEY_PATTERN, html) + n = re.search(self.CAPTCHAID_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) + self.logDebug("Key|id: %s | %s" % self.key) + return self.key + else: + self.logDebug("Key or id not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("AdsCaptcha key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + PublicKey, CaptchaId = key + + html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", + get={'CaptchaId': CaptchaId, + 'PublicKey': PublicKey}) + try: + challenge = re.search("challenge: '(.+?)',", html).group(1) + server = re.search("server: '(.+?)',", html).group(1) + + except AttributeError: + errmsg = _("AdsCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(server, challenge), challenge + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, + get={'cid': challenge, 'dummy': random()}, + cookies=True, + imgtype="jpg") + + self.logDebug("Result: %s" % result) + + return result diff --git a/pyload/plugin/captcha/ReCaptcha.py b/pyload/plugin/captcha/ReCaptcha.py new file mode 100644 index 000000000..295491bd8 --- /dev/null +++ b/pyload/plugin/captcha/ReCaptcha.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from base64 import b64encode +from random import randint +from urlparse import urljoin, urlparse + +from pyload.plugin.Captcha import Captcha + + +class ReCaptcha(Captcha): + __name__ = "ReCaptcha" + __version__ = "0.14" + + __description__ = """ReCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("zapp-brannigan", "fuerst.reinje@web.de")] + + + KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' + KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("ReCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) + if m: + self.key = m.group(1).strip() + self.logDebug("Key: %s" % self.key) + return self.key + else: + self.logDebug("Key not found") + return None + + + def challenge(self, key=None, html=None, version=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("ReCaptcha key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + if version in (1, 2): + return getattr(self, "_challenge_v%s" % version)(key) + + elif not html and hasattr(self.plugin, "html") and self.plugin.html: + version = 2 if re.search(self.KEY_V2_PATTERN, self.plugin.html) else 1 + return self.challenge(key, self.plugin.html, version) + + else: + errmsg = _("ReCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + + def _challenge_v1(self, key): + html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", + get={'k': key}) + try: + challenge = re.search("challenge : '(.+?)',", html).group(1) + server = re.search("server : '(.+?)',", html).group(1) + + except AttributeError: + errmsg = _("ReCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.logDebug("Challenge: %s" % challenge) + + return self.result(server, challenge), challenge + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha("%simage" % server, + get={'c': challenge}, + cookies=True, + forceUser=True, + imgtype="jpg") + + self.logDebug("Result: %s" % result) + + return result + + + def _collectApiInfo(self): + html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") + a = re.search(r'po.src = \'(.*?)\';', html).group(1) + vers = a.split("/")[5] + + self.logDebug("API version: %s" %vers) + + language = a.split("__")[1].split(".")[0] + + self.logDebug("API language: %s" % language) + + html = self.plugin.req.load("https://apis.google.com/js/api.js") + b = re.search(r'"h":"(.*?)","', html).group(1) + jsh = b.decode('unicode-escape') + + self.logDebug("API jsh-string: %s" % jsh) + + return vers, language, jsh + + + def _prepareTimeAndRpc(self): + self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") + + millis = int(round(time.time() * 1000)) + + self.logDebug("Time: %s" % millis) + + rand = randint(1, 99999999) + a = "0.%s" % str(rand * 2147483647) + rpc = int(100000000 * float(a)) + + self.logDebug("Rpc-token: %s" % rpc) + + return millis, rpc + + + def _challenge_v2(self, key, parent=None): + if parent is None: + try: + parent = urljoin("http://", urlparse(self.plugin.pyfile.url).netloc) + + except Exception: + parent = "" + + botguardstring = "!A" + vers, language, jsh = self._collectApiInfo() + millis, rpc = self._prepareTimeAndRpc() + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", + get={'k' : key, + 'hl' : language, + 'v' : vers, + 'usegapi' : "1", + 'jsh' : "%s#id=IO_%s" % (jsh, millis), + 'parent' : parent, + 'pfname' : "", + 'rpctoken': rpc}) + + token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) + self.logDebug("Token #1: %s" % token1.group(1)) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", + get={'c' : token1.group(1), + 'hl' : language, + 'v' : vers, + 'bg' : botguardstring, + 'k' : key, + 'usegapi': "1", + 'jsh' : jsh}).decode('unicode-escape') + + token2 = re.search(r'"finput","(.*?)",', html) + self.logDebug("Token #2: %s" % token2.group(1)) + + token3 = re.search(r'."asconf".\s,".*?".\s,"(.*?)".', html) + self.logDebug("Token #3: %s" % token3.group(1)) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/reload", + post={'k' : key, + 'c' : token2.group(1), + 'reason': "fi", + 'fbg' : token3.group(1)}) + + token4 = re.search(r'"rresp","(.*?)",', html) + self.logDebug("Token #4: %s" % token4.group(1)) + + millis_captcha_loading = int(round(time.time() * 1000)) + captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", + get={'c':token4.group(1), 'k':key}, + cookies=True, + forceUser=True) + response = b64encode('{"response":"%s"}' % captcha_response) + + self.logDebug("Result: %s" % response) + + timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading + timeToSolveMore = timeToSolve + int(float("0." + str(randint(1, 99999999))) * 500) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", + post={'k' : key, + 'c' : token4.group(1), + 'response': response, + 't' : timeToSolve, + 'ct' : timeToSolveMore, + 'bg' : botguardstring}) + + token5 = re.search(r'"uvresp","(.*?)",', html) + self.logDebug("Token #5: %s" % token5.group(1)) + + result = token5.group(1) + + return result, None diff --git a/pyload/plugin/captcha/SolveMedia.py b/pyload/plugin/captcha/SolveMedia.py new file mode 100644 index 000000000..af45101b4 --- /dev/null +++ b/pyload/plugin/captcha/SolveMedia.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Captcha import Captcha + + +class SolveMedia(Captcha): + __name__ = "SolveMedia" + __version__ = "0.12" + + __description__ = """SolveMedia captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("SolveMedia html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_PATTERN, html) + if m: + self.key = m.group(1).strip() + self.logDebug("Key: %s" % self.key) + return self.key + else: + self.logDebug("Key not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("SolveMedia key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", + get={'k': key}) + try: + challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="([^"]+)">', + html).group(1) + server = "http://api.solvemedia.com/papi/media" + + except AttributeError: + errmsg = _("SolveMedia challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.logDebug("Challenge: %s" % challenge) + + result = self.result(server, challenge) + + try: + magic = re.search(r'name="magic" value="(.+?)"', html).group(1) + + except AttributeError: + self.logDebug("Magic code not found") + + else: + if not self._verify(key, magic, result, challenge): + self.logDebug("Captcha code was invalid") + + return result, challenge + + + def _verify(self, key, magic, result, challenge, ref=None): #@TODO: Clean up + if ref is None: + try: + ref = self.plugin.pyfile.url + + except Exception: + ref = "" + + html = self.plugin.req.load("http://api.solvemedia.com/papi/verify.noscript", + post={'adcopy_response' : result, + 'k' : key, + 'l' : "en", + 't' : "img", + 's' : "standard", + 'magic' : magic, + 'adcopy_challenge' : challenge, + 'ref' : ref}) + try: + html = self.plugin.req.load(re.search(r'URL=(.+?)">', html).group(1)) + gibberish = re.search(r'id=gibberish>(.+?)</textarea>', html).group(1) + + except Exception: + return False + + else: + return True + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha(server, + get={'c': challenge}, + cookies=True, + imgtype="gif") + + self.logDebug("Result: %s" % result) + + return result diff --git a/pyload/plugin/captcha/__init__.py b/pyload/plugin/captcha/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/captcha/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/container/CCF.py b/pyload/plugin/container/CCF.py new file mode 100644 index 000000000..311eb2006 --- /dev/null +++ b/pyload/plugin/container/CCF.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from urllib2 import build_opener + +from MultipartPostHandler import MultipartPostHandler + +from pyload.plugin.Container import Container +from pyload.utils import fs_encode, safe_join + + +class CCF(Container): + __name__ = "CCF" + __type__ = "container" + __version__ = "0.23" + + __pattern__ = r'.+\.ccf$' + + __description__ = """CCF container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Willnix", "Willnix@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def decrypt(self, pyfile): + file = fs_encode(pyfile.url.strip()) + opener = build_opener(MultipartPostHandler) + + dlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php', + {'src' : "ccf", + 'filename': "test.ccf", + 'upload' : open(file, "rb")}).read() + + download_folder = self.config['general']['download_folder'] + dlc_file = safe_join(download_folder, "tmp_%s.dlc" % pyfile.name) + + try: + dlc = re.search(r'<dlc>(.+)</dlc>', dlc_content, re.S).group(1).decode('base64') + + except AttributeError: + self.fail(_("Container is corrupted")) + + with open(dlc_file, "w") as tempdlc: + tempdlc.write(dlc) + + self.urls = [dlc_file] diff --git a/pyload/plugin/container/DLC.py b/pyload/plugin/container/DLC.py new file mode 100644 index 000000000..8b8a0199b --- /dev/null +++ b/pyload/plugin/container/DLC.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import xml.dom.minidom + +from Crypto.Cipher import AES + +from pyload.plugin.Container import Container +from pyload.utils import decode, fs_encode + + +class DLC(Container): + __name__ = "DLC" + __type__ = "container" + __version__ = "0.24" + + __pattern__ = r'.+\.dlc$' + + __description__ = """DLC container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de"), + ("Schnusch", "Schnusch@users.noreply.github.com"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + KEY = "cb99b5cbc24db398" + IV = "9bc24cb995cb8db3" + API_URL = "http://service.jdownloader.org/dlcrypt/service.php?srcType=dlc&destType=pylo&data=%s" + + + def decrypt(self, pyfile): + file = fs_encode(pyfile.url.strip()) + with open(file) as dlc: + data = dlc.read().strip() + + data += '=' * (-len(data) % 4) + + dlc_key = data[-88:] + dlc_data = data[:-88].decode('base64') + dlc_content = self.load(self.API_URL % dlc_key) + + try: + rc = re.search(r'<rc>(.+)</rc>', dlc_content, re.S).group(1).decode('base64') + + except AttributeError: + self.fail(_("Container is corrupted")) + + cipher = AES.new(self.KEY, AES.MODE_CBC, self.IV).decrypt(rc) + + self.data = AES.new(cipher, AES.MODE_CBC, cipher).decrypt(dlc_data).decode('base64') + self.packages = [(entry[0] if entry[0] else pyfile.name, entry[1], entry[0] if entry[0] else pyfile.name) \ + for entry in self.getPackages()] + + + def getPackages(self): + root = xml.dom.minidom.parseString(self.data).documentElement + content = root.getElementsByTagName("content")[0] + return self.parsePackages(content) + + + def parsePackages(self, startNode): + return [(decode(node.getAttribute("name")).decode('base64'), self.parseLinks(node)) \ + for node in startNode.getElementsByTagName("package")] + + + def parseLinks(self, startNode): + return [node.getElementsByTagName("url")[0].firstChild.data.decode('base64') \ + for node in startNode.getElementsByTagName("file")] diff --git a/pyload/plugin/container/RSDF.py b/pyload/plugin/container/RSDF.py new file mode 100644 index 000000000..c4b743d14 --- /dev/null +++ b/pyload/plugin/container/RSDF.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import binascii +import re + +from Crypto.Cipher import AES + +from pyload.plugin.Container import Container +from pyload.utils import fs_encode + + +class RSDF(Container): + __name__ = "RSDF" + __type__ = "container" + __version__ = "0.27" + + __pattern__ = r'.+\.rsdf$' + + __description__ = """RSDF container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + KEY = "8C35192D964DC3182C6F84F3252239EB4A320D2500000000" + IV = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + + + def decrypt(self, pyfile): + KEY = binascii.unhexlify(self.KEY) + IV = AES.new(Key, AES.MODE_ECB).encrypt(binascii.unhexlify(self.IV)) + + cipher = AES.new(KEY, AES.MODE_CFB, IV) + + try: + file = fs_encode(pyfile.url.strip()) + with open(file, 'r') as rsdf: + data = rsdf.read() + + except IOError, e: + self.fail(e) + + if re.search(r"<title>404 - Not Found", data): + return + + for link in binascii.unhexlify(''.join(data.split())).splitlines(): + if link: + link = cipher.decrypt(link.decode('base64')).replace('CCF: ', '') + self.urls.append(link) diff --git a/pyload/plugin/container/TXT.py b/pyload/plugin/container/TXT.py new file mode 100644 index 000000000..2bf5bf2aa --- /dev/null +++ b/pyload/plugin/container/TXT.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import codecs + +from pyload.plugin.Container import Container +from pyload.utils import fs_encode + + +class TXT(Container): + __name__ = "TXT" + __type__ = "container" + __version__ = "0.15" + + __pattern__ = r'.+\.(txt|text)$' + __config__ = [("flush" , "bool" , "Flush list after adding", False ), + ("encoding", "string", "File encoding" , "utf-8")] + + __description__ = """Read link lists in plain text formats""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("jeix", "jeix@hasnomail.com")] + + + def decrypt(self, pyfile): + try: + encoding = codecs.lookup(self.getConfig("encoding")).name + + except Exception: + encoding = "utf-8" + + file = fs_encode(pyfile.url.strip()) + txt = codecs.open(file, 'r', encoding) + curPack = "Parsed links from %s" % pyfile.name + packages = {curPack:[],} + + for link in txt.readlines(): + link = link.strip() + + if not link: + continue + + if link.startswith(";"): + continue + + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + + packages[curPack].append(link) + + txt.close() + + # empty packages fix + for key, value in packages.iteritems(): + if not value: + packages.pop(key, None) + + if self.getConfig("flush"): + try: + txt = open(file, 'wb') + txt.close() + + except IOError: + self.logWarning(_("Failed to flush list")) + + for name, links in packages.iteritems(): + self.packages.append((name, links, name)) diff --git a/pyload/plugin/container/__init__.py b/pyload/plugin/container/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/container/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/crypter/BitshareCom.py b/pyload/plugin/crypter/BitshareCom.py new file mode 100644 index 000000000..524307127 --- /dev/null +++ b/pyload/plugin/crypter/BitshareCom.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class BitshareCom(SimpleCrypter): + __name__ = "BitshareCom" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?bitshare\.com/\?d=\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Bitshare.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + LINK_PATTERN = r'.+' + NAME_PATTERN = r'View public folder "(?P.+)"' diff --git a/pyload/plugin/crypter/C1NeonCom.py b/pyload/plugin/crypter/C1NeonCom.py new file mode 100644 index 000000000..dc1555b46 --- /dev/null +++ b/pyload/plugin/crypter/C1NeonCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class C1NeonCom(DeadCrypter): + __name__ = "C1NeonCom" + __type__ = "crypter" + __version__ = "0.05" + + __pattern__ = r'http://(?:www\.)?c1neon\.com/.+' + __config__ = [] + + __description__ = """C1neon.com decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("godofdream", "soilfiction@gmail.com")] diff --git a/pyload/plugin/crypter/ChipDe.py b/pyload/plugin/crypter/ChipDe.py new file mode 100644 index 000000000..2f47236e8 --- /dev/null +++ b/pyload/plugin/crypter/ChipDe.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import re +from pyload.plugin.Crypter import Crypter + + +class ChipDe(Crypter): + __name__ = "ChipDe" + __type__ = "crypter" + __version__ = "0.10" + + __pattern__ = r'http://(?:www\.)?chip\.de/video/.+\.html' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Chip.de decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("4Christopher", "4Christopher@gmx.de")] + + + def decrypt(self, pyfile): + self.html = self.load(pyfile.url) + try: + f = re.search(r'"(http://video\.chip\.de/.+)"', self.html) + except Exception: + self.fail(_("Failed to find the URL")) + else: + self.urls = [f.group(1)] + self.logDebug("The file URL is %s" % self.urls[0]) diff --git a/pyload/plugin/crypter/CloudzillaTo.py b/pyload/plugin/crypter/CloudzillaTo.py new file mode 100644 index 000000000..340892136 --- /dev/null +++ b/pyload/plugin/crypter/CloudzillaTo.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class CloudzillaTo(SimpleHoster): + __name__ = "CloudzillaTo" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?cloudzilla\.to/share/folder/(?P[\w^_]+)' + + __description__ = """Cloudzilla.to folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + INFO_PATTERN = r'File not found...<' + + LINK_PATTERN = r'' + + PASSWORD_PATTERN = r'
' + + + def checkErrors(self): + m = re.search(self.PASSWORD_PATTERN, self.html) + if m: + self.html = self.load(self.pyfile.url, get={'key': self.getPassword()}) + + if re.search(self.PASSWORD_PATTERN, self.html): + self.retry(reason="Wrong password") diff --git a/pyload/plugin/crypter/CrockoCom.py b/pyload/plugin/crypter/CrockoCom.py new file mode 100644 index 000000000..c93f4afab --- /dev/null +++ b/pyload/plugin/crypter/CrockoCom.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class CrockoCom(SimpleCrypter): + __name__ = "CrockoCom" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?crocko\.com/f/.+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Crocko.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + LINK_PATTERN = r'download' diff --git a/pyload/plugin/crypter/CryptItCom.py b/pyload/plugin/crypter/CryptItCom.py new file mode 100644 index 000000000..0d72cba61 --- /dev/null +++ b/pyload/plugin/crypter/CryptItCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class CryptItCom(DeadCrypter): + __name__ = "CryptItCom" + __type__ = "crypter" + __version__ = "0.11" + + __pattern__ = r'http://(?:www\.)?crypt-it\.com/(s|e|d|c)/\w+' + __config__ = [] + + __description__ = """Crypt-it.com decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de")] diff --git a/pyload/plugin/crypter/CzshareCom.py b/pyload/plugin/crypter/CzshareCom.py new file mode 100644 index 000000000..e527d683f --- /dev/null +++ b/pyload/plugin/crypter/CzshareCom.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re +from pyload.plugin.Crypter import Crypter + + +class CzshareCom(Crypter): + __name__ = "CzshareCom" + __type__ = "crypter" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?(czshare|sdilej)\.(com|cz)/folders/.+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Czshare.com folder decrypter plugin, now Sdilej.cz""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + FOLDER_PATTERN = r'\s*\s*(.*?)
' + LINK_PATTERN = r'info' + + + def decrypt(self, pyfile): + html = self.load(pyfile.url) + + m = re.search(self.FOLDER_PATTERN, html, re.S) + if m is None: + self.error(_("FOLDER_PATTERN not found")) + + self.urls.extend(re.findall(self.LINK_PATTERN, m.group(1))) diff --git a/pyload/plugin/crypter/DDLMusicOrg.py b/pyload/plugin/crypter/DDLMusicOrg.py new file mode 100644 index 000000000..2b6738799 --- /dev/null +++ b/pyload/plugin/crypter/DDLMusicOrg.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from pyload.plugin.Crypter import Crypter + + +class DDLMusicOrg(Crypter): + __name__ = "DDLMusicOrg" + __type__ = "crypter" + __version__ = "0.30" + + __pattern__ = r'http://(?:www\.)?ddl-music\.org/captcha/ddlm_cr\d\.php\?\d+\?\d+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Ddl-music.org decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] + + + def setup(self): + self.multiDL = False + + + def decrypt(self, pyfile): + html = self.load(pyfile.url, cookies=True) + + if re.search(r"Wer dies nicht rechnen kann", html) is not None: + self.offline() + + math = re.search(r"(\d+) ([+-]) (\d+) =\s+", htmlwithlink) + if m: + self.urls = [m.group(1)] + else: + self.retry() diff --git a/pyload/plugin/crypter/DailymotionBatch.py b/pyload/plugin/crypter/DailymotionBatch.py new file mode 100644 index 000000000..c66c7c829 --- /dev/null +++ b/pyload/plugin/crypter/DailymotionBatch.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.utils import json_loads +from pyload.plugin.Crypter import Crypter +from pyload.utils import safe_join + + +class DailymotionBatch(Crypter): + __name__ = "DailymotionBatch" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?dailymotion\.com/((playlists/)?(?Pplaylist|user)/)?(?P[\w^_]+)(?(TYPE)|#)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Dailymotion.com channel & playlist decrypter""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def api_response(self, ref, req=None): + url = urljoin("https://api.dailymotion.com/", ref) + html = self.load(url, get=req) + return json_loads(html) + + + def getPlaylistInfo(self, id): + ref = "playlist/" + id + req = {"fields": "name,owner.screenname"} + playlist = self.api_response(ref, req) + + if "error" in playlist: + return + + name = playlist['name'] + owner = playlist['owner.screenname'] + return name, owner + + + def _getPlaylists(self, user_id, page=1): + ref = "user/%s/playlists" % user_id + req = {"fields": "id", "page": page, "limit": 100} + user = self.api_response(ref, req) + + if "error" in user: + return + + for playlist in user['list']: + yield playlist['id'] + + if user['has_more']: + for item in self._getPlaylists(user_id, page + 1): + yield item + + + def getPlaylists(self, user_id): + return [(id,) + self.getPlaylistInfo(id) for id in self._getPlaylists(user_id)] + + + def _getVideos(self, id, page=1): + ref = "playlist/%s/videos" % id + req = {"fields": "url", "page": page, "limit": 100} + playlist = self.api_response(ref, req) + + if "error" in playlist: + return + + for video in playlist['list']: + yield video['url'] + + if playlist['has_more']: + for item in self._getVideos(id, page + 1): + yield item + + + def getVideos(self, playlist_id): + return list(self._getVideos(playlist_id))[::-1] + + + def decrypt(self, pyfile): + m = re.match(self.__pattern__, pyfile.url) + m_id = m.group('ID') + m_type = m.group('TYPE') + + if m_type == "playlist": + self.logDebug("Url recognized as Playlist") + p_info = self.getPlaylistInfo(m_id) + playlists = [(m_id,) + p_info] if p_info else None + else: + self.logDebug("Url recognized as Channel") + playlists = self.getPlaylists(m_id) + self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), m_id)) + + if not playlists: + self.fail(_("No playlist available")) + + for p_id, p_name, p_owner in playlists: + p_videos = self.getVideos(p_id) + p_folder = safe_join(self.config['general']['download_folder'], p_owner, p_name) + self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name)) + self.packages.append((p_name, p_videos, p_folder)) #: folder is NOT recognized by pyload 0.4.9! diff --git a/pyload/plugin/crypter/DataHu.py b/pyload/plugin/crypter/DataHu.py new file mode 100644 index 000000000..f69d6ee3e --- /dev/null +++ b/pyload/plugin/crypter/DataHu.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class DataHu(SimpleCrypter): + __name__ = "DataHu" + __type__ = "crypter" + __version__ = "0.06" + + __pattern__ = r'http://(?:www\.)?data\.hu/dir/\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Data.hu folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("crash", ""), + ("stickell", "l.stickell@yahoo.it")] + + + LINK_PATTERN = r'\1' + NAME_PATTERN = ur'(?P<N>.+) Let\xf6lt\xe9se' + + + def prepare(self): + super(DataHu, self).prepare() + + if u'K\xe9rlek add meg a jelsz\xf3t' in self.html: # Password protected + password = self.getPassword() + if not password: + self.fail(_("Password required")) + + self.logDebug("The folder is password protected', 'Using password: " + password) + + self.html = self.load(self.pyfile.url, post={'mappa_pass': password}, decode=True) + + if u'Hib\xe1s jelsz\xf3' in self.html: # Wrong password + self.fail(_("Wrong password")) diff --git a/pyload/plugin/crypter/DdlstorageCom.py b/pyload/plugin/crypter/DdlstorageCom.py new file mode 100644 index 000000000..f3744e15d --- /dev/null +++ b/pyload/plugin/crypter/DdlstorageCom.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class DdlstorageCom(DeadCrypter): + __name__ = "DdlstorageCom" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?ddlstorage\.com/folder/\w+' + __config__ = [] + + __description__ = """DDLStorage.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("godofdream", "soilfiction@gmail.com"), + ("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/crypter/DepositfilesCom.py b/pyload/plugin/crypter/DepositfilesCom.py new file mode 100644 index 000000000..24fa9134a --- /dev/null +++ b/pyload/plugin/crypter/DepositfilesCom.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class DepositfilesCom(SimpleCrypter): + __name__ = "DepositfilesCom" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?depositfiles\.com/folders/\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Depositfiles.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + LINK_PATTERN = r'
]*>\s*' diff --git a/pyload/plugin/crypter/Dereferer.py b/pyload/plugin/crypter/Dereferer.py new file mode 100644 index 000000000..138282d02 --- /dev/null +++ b/pyload/plugin/crypter/Dereferer.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Crypter import Crypter + + +class Dereferer(SimpleDereferer): + __name__ = "Dereferer" + __type__ = "crypter" + __version__ = "0.11" + + __pattern__ = r'https?://([^/]+)/.*?(?P(ht|f)tps?(://|%3A%2F%2F).+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Crypter for dereferers""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/crypter/DevhostSt.py b/pyload/plugin/crypter/DevhostSt.py new file mode 100644 index 000000000..4d1a2ae09 --- /dev/null +++ b/pyload/plugin/crypter/DevhostSt.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://d-h.st/users/shine/?fld_id=37263#files + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class DevhostSt(SimpleCrypter): + __name__ = "DevhostSt" + __type__ = "crypter" + __version__ = "0.04" + + __pattern__ = r'http://(?:www\.)?d-h\.st/users/(?P\w+)(/\?fld_id=(?P\d+))?' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """d-h.st folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + LINK_PATTERN = r'(?:/> |;">)Back to \w+<)' + OFFLINE_PATTERN = r'"/cHP">test\.png<' + + + def getFileInfo(self): + if re.search(self.OFFLINE_PATTERN, self.html): + self.offline() + + try: + id = re.match(self.__pattern__, self.pyfile.url).group('ID') + if id == "0": + raise + + p = r'href="(.+?)">Back to \w+<' + m = re.search(p, self.html) + html = self.load(urljoin("http://d-h.st", m.group(1)), + cookies=False) + + p = '\?fld_id=%s.*?">(.+?)<' % id + m = re.search(p, html) + name = folder = m.group(1) + + except Exception, e: + self.logDebug(e) + name = folder = re.match(self.__pattern__, self.pyfile.url).group('USER') + + return {'name': name, 'folder': folder} diff --git a/pyload/plugin/crypter/DlProtectCom.py b/pyload/plugin/crypter/DlProtectCom.py new file mode 100644 index 000000000..a5e104f70 --- /dev/null +++ b/pyload/plugin/crypter/DlProtectCom.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import re + +from base64 import urlsafe_b64encode +from time import time + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class DlProtectCom(SimpleCrypter): + __name__ = "DlProtectCom" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?dl-protect\.com/((en|fr)/)?\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Dl-protect.com decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + COOKIES = [("dl-protect.com", "l", "en")] + + OFFLINE_PATTERN = r'Unfortunately, the link you are looking for is not found' + + + def getLinks(self): + # Direct link with redirect + if not re.match(r"https?://(?:www\.)?dl-protect\.com/.+", self.req.http.lastEffectiveURL): + return [self.req.http.lastEffectiveURL] + + post_req = {'key' : re.search(r'name="key" value="(.+?)"', self.html).group(1), + 'submitform': ""} + + if "Please click on continue to see the content" in self.html: + post_req['submitform'] = "Continue" + self.wait(2) + + else: + mstime = int(round(time() * 1000)) + b64time = "_" + urlsafe_b64encode(str(mstime)).replace("=", "%3D") + + post_req.update({'i' : b64time, + 'submitform': "Decrypt+link"}) + + if "Password :" in self.html: + post_req['pwd'] = self.getPassword() + + if "Security Code" in self.html: + captcha_id = re.search(r'/captcha\.php\?uid=(.+?)"', self.html).group(1) + captcha_url = "http://www.dl-protect.com/captcha.php?uid=" + captcha_id + captcha_code = self.decryptCaptcha(captcha_url, imgtype="gif") + + post_req['secure'] = captcha_code + + self.html = self.load(self.pyfile.url, post=post_req) + + for errmsg in ("The password is incorrect", "The security code is incorrect"): + if errmsg in self.html: + self.fail(_(errmsg[1:])) + + return re.findall(r'', self.html) diff --git a/pyload/plugin/crypter/DontKnowMe.py b/pyload/plugin/crypter/DontKnowMe.py new file mode 100644 index 000000000..1e83ab853 --- /dev/null +++ b/pyload/plugin/crypter/DontKnowMe.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Crypter import Crypter + + +class DontKnowMe(SimpleDereferer): + __name__ = "DontKnowMe" + __type__ = "crypter" + __version__ = "0.11" + + __pattern__ = r'http://(?:www\.)?dontknow\.me/at/\?(?P.+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """DontKnow.me decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("selaux", "")] diff --git a/pyload/plugin/crypter/DuckCryptInfo.py b/pyload/plugin/crypter/DuckCryptInfo.py new file mode 100644 index 000000000..55681fd5e --- /dev/null +++ b/pyload/plugin/crypter/DuckCryptInfo.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import re + +from BeautifulSoup import BeautifulSoup + +from pyload.plugin.Crypter import Crypter + + +class DuckCryptInfo(Crypter): + __name__ = "DuckCryptInfo" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?duckcrypt\.info/(folder|wait|link)/(\w+)/?(\w*)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """DuckCrypt.info decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("godofdream", "soilfiction@gmail.com")] + + + TIMER_PATTERN = r'(.*)' + + + def decrypt(self, pyfile): + url = pyfile.url + + m = re.match(self.__pattern__, url) + if m is None: + self.fail(_("Weird error in link")) + if str(m.group(1)) == "link": + self.handleLink(url) + else: + self.handleFolder(m) + + + def handleFolder(self, m): + html = self.load("http://duckcrypt.info/ajax/auth.php?hash=" + str(m.group(2))) + m = re.match(self.__pattern__, html) + self.logDebug("Redirectet to " + str(m.group(0))) + html = self.load(str(m.group(0))) + soup = BeautifulSoup(html) + cryptlinks = soup.findAll("div", attrs={"class": "folderbox"}) + self.logDebug("Redirectet to " + str(cryptlinks)) + if not cryptlinks: + self.error(_("No link found")) + for clink in cryptlinks: + if clink.find("a"): + self.handleLink(clink.find("a")['href']) + + + def handleLink(self, url): + html = self.load(url) + soup = BeautifulSoup(html) + self.urls = [soup.find("iframe")['src']] + if not self.urls: + self.logInfo(_("No link found")) diff --git a/pyload/plugin/crypter/DuploadOrg.py b/pyload/plugin/crypter/DuploadOrg.py new file mode 100644 index 000000000..f92eb2ede --- /dev/null +++ b/pyload/plugin/crypter/DuploadOrg.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class DuploadOrg(DeadCrypter): + __name__ = "DuploadOrg" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?dupload\.org/folder/\d+' + __config__ = [] + + __description__ = """Dupload.org folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/crypter/EasybytezCom.py b/pyload/plugin/crypter/EasybytezCom.py new file mode 100644 index 000000000..c5c7b0193 --- /dev/null +++ b/pyload/plugin/crypter/EasybytezCom.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSCrypter import XFSCrypter + + +class EasybytezCom(XFSCrypter): + __name__ = "EasybytezCom" + __type__ = "crypter" + __version__ = "0.10" + + __pattern__ = r'http://(?:www\.)?easybytez\.com/users/\d+/\d+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Easybytez.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + LOGIN_ACCOUNT = True diff --git a/pyload/plugin/crypter/EmbeduploadCom.py b/pyload/plugin/crypter/EmbeduploadCom.py new file mode 100644 index 000000000..c94e7e106 --- /dev/null +++ b/pyload/plugin/crypter/EmbeduploadCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import re +from pyload.plugin.Crypter import Crypter +from pyload.network.HTTPRequest import BadHeader + + +class EmbeduploadCom(Crypter): + __name__ = "EmbeduploadCom" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?embedupload\.com/\?d=.+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True), + ("preferedHoster", "str", "Prefered hoster list (bar-separated)", "embedupload"), + ("ignoredHoster", "str", "Ignored hoster list (bar-separated)", "")] + + __description__ = """EmbedUpload.com decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + LINK_PATTERN = r'
]*>\s*' + + + def decrypt(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + tmp_links = [] + + m = re.findall(self.LINK_PATTERN, self.html) + if m: + prefered_set = set(self.getConfig("preferedHoster").split('|')) + prefered_set = map(lambda s: s.lower().split('.')[0], prefered_set) + + self.logDebug("PF: %s" % prefered_set) + + tmp_links.extend(x[1] for x in m if x[0] in prefered_set) + self.urls = self.getLocation(tmp_links) + + if not self.urls: + ignored_set = set(self.getConfig("ignoredHoster").split('|')) + ignored_set = map(lambda s: s.lower().split('.')[0], ignored_set) + + self.logDebug("IG: %s" % ignored_set) + + tmp_links.extend(x[1] for x in m if x[0] not in ignored_set) + self.urls = self.getLocation(tmp_links) + + + def getLocation(self, tmp_links): + new_links = [] + for link in tmp_links: + try: + header = self.load(link, just_header=True) + if 'location' in header: + new_links.append(header['location']) + except BadHeader: + pass + return new_links diff --git a/pyload/plugin/crypter/FilebeerInfo.py b/pyload/plugin/crypter/FilebeerInfo.py new file mode 100644 index 000000000..e2e0e1e3c --- /dev/null +++ b/pyload/plugin/crypter/FilebeerInfo.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class FilebeerInfo(DeadCrypter): + __name__ = "FilebeerInfo" + __type__ = "crypter" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?filebeer\.info/\d*~f\w+' + __config__ = [] + + __description__ = """Filebeer.info folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/crypter/FilecloudIo.py b/pyload/plugin/crypter/FilecloudIo.py new file mode 100644 index 000000000..f4c967a07 --- /dev/null +++ b/pyload/plugin/crypter/FilecloudIo.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class FilecloudIo(SimpleCrypter): + __name__ = "FilecloudIo" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?(filecloud\.io|ifile\.it)/_\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Filecloud.io folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + LINK_PATTERN = r'href="(http://filecloud\.io/\w+)" title' + NAME_PATTERN = r'>(?P.+?) - filecloud\.io<' diff --git a/pyload/plugin/crypter/FilecryptCc.py b/pyload/plugin/crypter/FilecryptCc.py new file mode 100644 index 000000000..816a2d365 --- /dev/null +++ b/pyload/plugin/crypter/FilecryptCc.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://filecrypt.cc/Container/64E039F859.html + +import binascii +import re + +from Crypto.Cipher import AES +from urlparse import urljoin + +from pyload.plugin.Crypter import Crypter +from pyload.plugin.internal.CaptchaService import ReCaptcha + + +class FilecryptCc(Crypter): + __name__ = "FilecryptCc" + __type__ = "crypter" + __version__ = "0.11" + + __pattern__ = r'https?://(?:www\.)?filecrypt\.cc/Container/\w+' + + __description__ = """Filecrypt.cc decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + # URL_REPLACEMENTS = [(r'.html$', ""), (r'$', ".html")] #@TODO: Extend SimpleCrypter + + DLC_LINK_PATTERN = r'
.*?bgcolor="#EFEFEF">(?P.*?)', flags=re.I | re.S) + PATTERN_DL_LINK_PAGE = re.compile(r'"(dl_links_\d+_\d+\.html)"', flags=re.I) + PATTERN_REDIRECT_LINKS = re.compile(r'value="(http://sexuria\.com/out\.php\?id=\d+\&part=\d+\&link=\d+)" readonly', flags=re.I) + + + def decrypt(self, pyfile): + # Init + self.pyfile = pyfile + self.package = pyfile.package() + + # Get package links + package_name, self.links, folder_name, package_pwd = self.decryptLinks(self.pyfile.url) + self.packages = [(package_name, self.links, folder_name)] + + + def decryptLinks(self, url): + linklist = [] + name = self.package.name + folder = self.package.folder + password = None + + if re.match(self.PATTERN_SUPPORTED_MAIN, url): + # Processing main page + html = self.load(url) + links = re.findall(self.PATTERN_DL_LINK_PAGE, html) + for link in links: + linklist.append("http://sexuria.com/v1/" + link) + + elif re.match(self.PATTERN_SUPPORTED_REDIRECT, url): + # Processing direct redirect link (out.php), redirecting to main page + id = re.search(self.PATTERN_SUPPORTED_REDIRECT, url).group('ID') + if id: + linklist.append("http://sexuria.com/v1/Pornos_Kostenlos_liebe_%s.html" % id) + + elif re.match(self.PATTERN_SUPPORTED_CRYPT, url): + # Extract info from main file + id = re.search(self.PATTERN_SUPPORTED_CRYPT, url).group('ID') + html = self.load("http://sexuria.com/v1/Pornos_Kostenlos_info_%s.html" % id, decode=True) + + title = re.search(self.PATTERN_TITLE, html).group('TITLE').strip() + if title: + name = folder = title + self.logDebug("Package info found, name [%s] and folder [%s]" % (name, folder)) + + pwd = re.search(self.PATTERN_PASSWORD, html).group('PWD') + if pwd: + password = pwd.strip() + self.logDebug("Password info [%s] found" % password) + + # Process link (dl_link) + html = self.load(url) + links = re.findall(self.PATTERN_REDIRECT_LINKS, html) + if len(links) == 0: + self.LogError("Broken for link %s" % link) + else: + for link in links: + link = link.replace("http://sexuria.com/", "http://www.sexuria.com/") + finallink = self.load(link, just_header=True)['location'] + if not finallink or "sexuria.com/" in finallink: + self.LogError("Broken for link %s" % link) + else: + linklist.append(finallink) + + # Debug log + self.logDebug("%d supported links" % len(linklist)) + for i, link in enumerate(linklist): + self.logDebug("Supported link %d, %s" % (i + 1, link)) + + return name, linklist, folder, password diff --git a/pyload/plugin/crypter/ShareLinksBiz.py b/pyload/plugin/crypter/ShareLinksBiz.py new file mode 100644 index 000000000..1328e86aa --- /dev/null +++ b/pyload/plugin/crypter/ShareLinksBiz.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- + +import binascii +import re + +from Crypto.Cipher import AES +from pyload.plugin.Crypter import Crypter + + +class ShareLinksBiz(Crypter): + __name__ = "ShareLinksBiz" + __type__ = "crypter" + __version__ = "1.14" + + __pattern__ = r'http://(?:www\.)?(share-links|s2l)\.biz/(?P_?\w+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Share-Links.biz decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("fragonib", "fragonib[AT]yahoo[DOT]es")] + + + def setup(self): + self.baseUrl = None + self.fileId = None + self.package = None + self.captcha = False + + + def decrypt(self, pyfile): + # Init + self.initFile(pyfile) + + # Request package + url = self.baseUrl + '/' + self.fileId + self.html = self.load(url, decode=True) + + # Unblock server (load all images) + self.unblockServer() + + # Check for protection + if self.isPasswordProtected(): + self.unlockPasswordProtection() + self.handleErrors() + + if self.isCaptchaProtected(): + self.captcha = True + self.unlockCaptchaProtection() + self.handleErrors() + + # Extract package links + package_links = [] + package_links.extend(self.handleWebLinks()) + package_links.extend(self.handleContainers()) + package_links.extend(self.handleCNL2()) + package_links = set(package_links) + + # Get package info + package_name, package_folder = self.getPackageInfo() + + # Pack + self.packages = [(package_name, package_links, package_folder)] + + + def initFile(self, pyfile): + url = pyfile.url + if 's2l.biz' in url: + url = self.load(url, just_header=True)['location'] + self.baseUrl = "http://www.%s.biz" % re.match(self.__pattern__, url).group(1) + self.fileId = re.match(self.__pattern__, url).group('ID') + self.package = pyfile.package() + + + def isOnline(self): + if "No usable content was found" in self.html: + self.logDebug("File not found") + return False + return True + + + def isPasswordProtected(self): + if re.search(r'''''', self.html): + self.logDebug("Links are protected") + return True + return False + + + def isCaptchaProtected(self): + if '= x1 and x <= x2) and (y >= y1 and y <= y2): + return href + + + def handleErrors(self): + if "The inserted password was wrong" in self.html: + self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry") + self.fail(_("Incorrect password, please set right password on 'Edit package' form and retry")) + + if self.captcha: + if "Your choice was wrong" in self.html: + self.invalidCaptcha() + self.retry(wait_time=5) + else: + self.correctCaptcha() + + + def getPackageInfo(self): + name = folder = None + + # Extract from web package header + title_re = r'

(.*)

' + m = re.search(title_re, self.html, re.S) + if m is not None: + title = m.group(1).strip() + if 'unnamed' not in title: + name = folder = title + self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) + + # Fallback to defaults + if not name or not folder: + name = self.package.name + folder = self.package.folder + self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) + + # Return package info + return name, folder + + + def handleWebLinks(self): + package_links = [] + self.logDebug("Handling Web links") + + #@TODO: Gather paginated web links + pattern = r'javascript:_get\(\'(.*?)\', \d+, \'\'\)' + ids = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Web links" % len(ids)) + for i, ID in enumerate(ids): + try: + self.logDebug("Decrypting Web link %d, [%s]" % (i + 1, ID)) + + dwLink = self.baseUrl + "/get/lnk/" + ID + res = self.load(dwLink) + + code = re.search(r'frm/(\d+)', res).group(1) + fwLink = self.baseUrl + "/get/frm/" + code + res = self.load(fwLink) + + jscode = re.search(r'', res, re.S).group(1) + jscode = self.js.eval("f = %s" % jscode) + jslauncher = "window=''; parent={frames:{Main:{location:{href:''}}},location:''}; %s; parent.frames.Main.location.href" + + dlLink = self.js.eval(jslauncher % jscode) + + self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) + + package_links.append(dlLink) + except Exception, detail: + self.logDebug("Error decrypting Web link [%s], %s" % (ID, detail)) + return package_links + + + def handleContainers(self): + package_links = [] + self.logDebug("Handling Container links") + + pattern = r'javascript:_get\(\'(.*?)\', 0, \'(rsdf|ccf|dlc)\'\)' + containersLinks = re.findall(pattern, self.html) + self.logDebug("Decrypting %d Container links" % len(containersLinks)) + for containerLink in containersLinks: + link = "%s/get/%s/%s" % (self.baseUrl, containerLink[1], containerLink[0]) + package_links.append(link) + return package_links + + + def handleCNL2(self): + package_links = [] + self.logDebug("Handling CNL2 links") + + if '/lib/cnl2/ClicknLoad.swf' in self.html: + try: + (crypted, jk) = self._getCipherParams() + package_links.extend(self._getLinks(crypted, jk)) + except Exception: + self.fail(_("Unable to decrypt CNL2 links")) + return package_links + + + def _getCipherParams(self): + # Request CNL2 + code = re.search(r'ClicknLoad.swf\?code=(.*?)"', self.html).group(1) + url = "%s/get/cnl2/%s" % (self.baseUrl, code) + res = self.load(url) + params = res.split(";;") + + # Get jk + strlist = list(params[1].decode('base64')) + jk = ''.join(strlist[::-1]) + + # Get crypted + strlist = list(params[2].decode('base64')) + crypted = ''.join(strlist[::-1]) + + # Log and return + return crypted, jk + + + def _getLinks(self, crypted, jk): + # Get key + jreturn = self.js.eval("%s f()" % jk) + self.logDebug("JsEngine returns value [%s]" % jreturn) + key = binascii.unhexlify(jreturn) + + # Decrypt + Key = key + IV = key + obj = AES.new(Key, AES.MODE_CBC, IV) + text = obj.decrypt(crypted.decode('base64')) + + # Extract links + text = text.replace("\x00", "").replace("\r", "") + links = filter(bool, text.split('\n')) + + # Log and return + self.logDebug("Block has %d links" % len(links)) + return links diff --git a/pyload/plugin/crypter/SharingmatrixCom.py b/pyload/plugin/crypter/SharingmatrixCom.py new file mode 100644 index 000000000..7db09b246 --- /dev/null +++ b/pyload/plugin/crypter/SharingmatrixCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class SharingmatrixCom(DeadCrypter): + __name__ = "SharingmatrixCom" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?sharingmatrix\.com/folder/\w+' + + __description__ = """Sharingmatrix.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/crypter/SpeedLoadOrg.py b/pyload/plugin/crypter/SpeedLoadOrg.py new file mode 100644 index 000000000..95c8864dc --- /dev/null +++ b/pyload/plugin/crypter/SpeedLoadOrg.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class SpeedLoadOrg(DeadCrypter): + __name__ = "SpeedLoadOrg" + __type__ = "crypter" + __version__ = "0.30" + + __pattern__ = r'http://(?:www\.)?speedload\.org/(\d+~f$|folder/\d+/)' + __config__ = [] + + __description__ = """Speedload decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/crypter/StealthTo.py b/pyload/plugin/crypter/StealthTo.py new file mode 100644 index 000000000..23747be1b --- /dev/null +++ b/pyload/plugin/crypter/StealthTo.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class StealthTo(DeadCrypter): + __name__ = "StealthTo" + __type__ = "crypter" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?stealth\.to/folder/.+' + __config__ = [] + + __description__ = """Stealth.to decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org")] diff --git a/pyload/plugin/crypter/TnyCz.py b/pyload/plugin/crypter/TnyCz.py new file mode 100644 index 000000000..d73f6de63 --- /dev/null +++ b/pyload/plugin/crypter/TnyCz.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + +import re + + +class TnyCz(SimpleCrypter): + __name__ = "TnyCz" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?tny\.cz/\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Tny.cz decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'(?P<N>.+) - .+' + + + def getLinks(self): + m = re.search(r'
', self.html) + return re.findall(".+", self.load(m.group(1), decode=True)) if m else None diff --git a/pyload/plugin/crypter/TrailerzoneInfo.py b/pyload/plugin/crypter/TrailerzoneInfo.py new file mode 100644 index 000000000..06894054b --- /dev/null +++ b/pyload/plugin/crypter/TrailerzoneInfo.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class TrailerzoneInfo(DeadCrypter): + __name__ = "TrailerzoneInfo" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?trailerzone\.info/.+' + __config__ = [] + + __description__ = """TrailerZone.info decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("godofdream", "soilfiction@gmail.com")] diff --git a/pyload/plugin/crypter/TurbobitNet.py b/pyload/plugin/crypter/TurbobitNet.py new file mode 100644 index 000000000..d6b6d7b38 --- /dev/null +++ b/pyload/plugin/crypter/TurbobitNet.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter +from pyload.utils import json_loads + + +class TurbobitNet(SimpleCrypter): + __name__ = "TurbobitNet" + __type__ = "crypter" + __version__ = "0.05" + + __pattern__ = r'http://(?:www\.)?turbobit\.net/download/folder/(?P\w+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Turbobit.net folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'src=\'/js/lib/grid/icon/folder.png\'> (?P.+?)' + + + def _getLinks(self, id, page=1): + gridFile = self.load("http://turbobit.net/downloadfolder/gridFile", + get={"rootId": id, "rows": 200, "page": page}, decode=True) + grid = json_loads(gridFile) + + if grid['rows']: + for i in grid['rows']: + yield i['id'] + for id in self._getLinks(id, page + 1): + yield id + else: + return + + + def getLinks(self): + id = re.match(self.__pattern__, self.pyfile.url).group('ID') + fixurl = lambda id: "http://turbobit.net/%s.html" % id + return map(fixurl, self._getLinks(id)) diff --git a/pyload/plugin/crypter/TusfilesNet.py b/pyload/plugin/crypter/TusfilesNet.py new file mode 100644 index 000000000..39d5dee9d --- /dev/null +++ b/pyload/plugin/crypter/TusfilesNet.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +import math +import re +from urlparse import urljoin + +from pyload.plugin.internal.XFSCrypter import XFSCrypter + + +class TusfilesNet(XFSCrypter): + __name__ = "TusfilesNet" + __type__ = "crypter" + __version__ = "0.08" + + __pattern__ = r'https?://(?:www\.)?tusfiles\.net/go/(?P\w+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Tusfiles.net folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("stickell", "l.stickell@yahoo.it")] + + + PAGES_PATTERN = r'>\((\d+) \w+\)<' + + URL_REPLACEMENTS = [(__pattern__ + ".*", r'https://www.tusfiles.net/go/\g/')] + + + def loadPage(self, page_n): + return self.load(urljoin(self.pyfile.url, str(page_n)), decode=True) + + + def handlePages(self, pyfile): + pages = re.search(self.PAGES_PATTERN, self.html) + if pages: + pages = int(math.ceil(int(pages.group('pages')) / 25.0)) + else: + return + + for p in xrange(2, pages + 1): + self.html = self.loadPage(p) + self.links += self.getLinks() diff --git a/pyload/plugin/crypter/UlozTo.py b/pyload/plugin/crypter/UlozTo.py new file mode 100644 index 000000000..81fbee172 --- /dev/null +++ b/pyload/plugin/crypter/UlozTo.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import re +from pyload.plugin.Crypter import Crypter + + +class UlozTo(Crypter): + __name__ = "UlozTo" + __type__ = "crypter" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj\.cz|zachowajto\.pl)/(m|soubory)/.+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Uloz.to folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + FOLDER_PATTERN = r'
    (.*?)
' + LINK_PATTERN = r'
[^<]+' + NEXT_PAGE_PATTERN = r'' + + + def decrypt(self, pyfile): + html = self.load(pyfile.url) + + new_links = [] + for i in xrange(1, 100): + self.logInfo(_("Fetching links from page %i") % i) + m = re.search(self.FOLDER_PATTERN, html, re.S) + if m is None: + self.error(_("FOLDER_PATTERN not found")) + + new_links.extend(re.findall(self.LINK_PATTERN, m.group(1))) + m = re.search(self.NEXT_PAGE_PATTERN, html) + if m: + html = self.load("http://ulozto.net/" + m.group(1)) + else: + break + else: + self.logInfo(_("Limit of 99 pages reached, aborting")) + + if new_links: + self.urls = [map(lambda s: "http://ulozto.net/%s" % s, new_links)] diff --git a/pyload/plugin/crypter/UploadableCh.py b/pyload/plugin/crypter/UploadableCh.py new file mode 100644 index 000000000..f5fb0c3bc --- /dev/null +++ b/pyload/plugin/crypter/UploadableCh.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class UploadableCh(SimpleCrypter): + __name__ = "UploadableCh" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?uploadable\.ch/list/\w+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Uploadable.ch folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + LINK_PATTERN = r'"(.+?)" class="icon_zipfile">' + NAME_PATTERN = r'
 (?P.+?)
' + OFFLINE_PATTERN = r'We are sorry... The URL you entered cannot be found on the server.' + TEMP_OFFLINE_PATTERN = r'
' diff --git a/pyload/plugin/crypter/UploadedTo.py b/pyload/plugin/crypter/UploadedTo.py new file mode 100644 index 000000000..282f82be7 --- /dev/null +++ b/pyload/plugin/crypter/UploadedTo.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class UploadedTo(SimpleCrypter): + __name__ = "UploadedTo" + __type__ = "crypter" + __version__ = "0.42" + + __pattern__ = r'http://(?:www\.)?(uploaded|ul)\.(to|net)/(f|folder|list)/(?P\w+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """UploadedTo decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + PLAIN_PATTERN = r'(?P.+?)<' + + + def getLinks(self): + m = re.search(self.PLAIN_PATTERN, self.html) + if m is None: + self.error(_("PLAIN_PATTERN not found")) + + plain_link = urljoin("http://uploaded.net/", m.group(1)) + return self.load(plain_link).split('\n')[:-1] diff --git a/pyload/plugin/crypter/WiiReloadedOrg.py b/pyload/plugin/crypter/WiiReloadedOrg.py new file mode 100644 index 000000000..4190cb340 --- /dev/null +++ b/pyload/plugin/crypter/WiiReloadedOrg.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class WiiReloadedOrg(DeadCrypter): + __name__ = "WiiReloadedOrg" + __type__ = "crypter" + __version__ = "0.11" + + __pattern__ = r'http://(?:www\.)?wii-reloaded\.org/protect/get\.php\?i=.+' + __config__ = [] + + __description__ = """Wii-Reloaded.org decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("hzpz", "")] diff --git a/pyload/plugin/crypter/WuploadCom.py b/pyload/plugin/crypter/WuploadCom.py new file mode 100644 index 000000000..fb00cb136 --- /dev/null +++ b/pyload/plugin/crypter/WuploadCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadCrypter import DeadCrypter + + +class WuploadCom(DeadCrypter): + __name__ = "WuploadCom" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?wupload\.com/folder/\w+' + + __description__ = """Wupload.com folder decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/crypter/XFileSharingPro.py b/pyload/plugin/crypter/XFileSharingPro.py new file mode 100644 index 000000000..3dc2e8f70 --- /dev/null +++ b/pyload/plugin/crypter/XFileSharingPro.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.XFSCrypter import XFSCrypter + + +class XFileSharingPro(XFSCrypter): + __name__ = "XFileSharingPro" + __type__ = "crypter" + __version__ = "0.04" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """XFileSharingPro dummy folder decrypter plugin for hook""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def _log(self, type, args): + msg = " | ".join(str(a).strip() for a in args if a) + logger = getattr(self.log, type) + logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + + + def init(self): + super(XFileSharingPro, self).init() + + self.__pattern__ = self.core.pluginManager.crypterPlugins[self.__name__]['pattern'] + + self.HOSTER_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() + self.HOSTER_NAME = "".join(part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.') + + if self.HOSTER_NAME[0].isdigit(): + self.HOSTER_NAME = 'X' + self.HOSTER_NAME + + account = self.core.accountManager.getAccountPlugin(self.HOSTER_NAME) + + if account and account.canUse(): + self.account = account + + elif self.account: + self.account.HOSTER_DOMAIN = self.HOSTER_DOMAIN + + else: + return + + self.user, data = self.account.selectAccount() + self.req = self.account.getAccountRequest(self.user) + self.premium = self.account.isPremium(self.user) diff --git a/pyload/plugin/crypter/XupPl.py b/pyload/plugin/crypter/XupPl.py new file mode 100644 index 000000000..9d4d27b61 --- /dev/null +++ b/pyload/plugin/crypter/XupPl.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Crypter import Crypter + + +class XupPl(Crypter): + __name__ = "XupPl" + __type__ = "crypter" + __version__ = "0.10" + + __pattern__ = r'https?://(?:[^/]*\.)?xup\.pl/.+' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Xup.pl decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("z00nx", "z00nx0@gmail.com")] + + + def decrypt(self, pyfile): + header = self.load(pyfile.url, just_header=True) + if 'location' in header: + self.urls = [header['location']] + else: + self.fail(_("Unable to find link")) diff --git a/pyload/plugin/crypter/YoutubeBatch.py b/pyload/plugin/crypter/YoutubeBatch.py new file mode 100644 index 000000000..5e4269fd2 --- /dev/null +++ b/pyload/plugin/crypter/YoutubeBatch.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.utils import json_loads +from pyload.plugin.Crypter import Crypter +from pyload.utils import safe_join + + +class YoutubeBatch(Crypter): + __name__ = "YoutubeBatch" + __type__ = "crypter" + __version__ = "1.01" + + __pattern__ = r'https?://(?:www\.|m\.)?youtube\.com/(?Puser|playlist|view_play_list)(/|.*?[?&](?:list|p)=)(?P[\w-]+)' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True), + ("likes", "bool", "Grab user (channel) liked videos", False), + ("favorites", "bool", "Grab user (channel) favorite videos", False), + ("uploads", "bool", "Grab channel unplaylisted videos", True)] + + __description__ = """Youtube.com channel & playlist decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + API_KEY = "AIzaSyCKnWLNlkX-L4oD1aEzqqhRw1zczeD6_k0" + + + def api_response(self, ref, req): + req.update({"key": self.API_KEY}) + url = urljoin("https://www.googleapis.com/youtube/v3/", ref) + html = self.load(url, get=req) + return json_loads(html) + + + def getChannel(self, user): + channels = self.api_response("channels", {"part": "id,snippet,contentDetails", "forUsername": user, "maxResults": "50"}) + if channels['items']: + channel = channels['items'][0] + return {"id": channel['id'], + "title": channel['snippet']['title'], + "relatedPlaylists": channel['contentDetails']['relatedPlaylists'], + "user": user} # One lone channel for user? + + + def getPlaylist(self, p_id): + playlists = self.api_response("playlists", {"part": "snippet", "id": p_id}) + if playlists['items']: + playlist = playlists['items'][0] + return {"id": p_id, + "title": playlist['snippet']['title'], + "channelId": playlist['snippet']['channelId'], + "channelTitle": playlist['snippet']['channelTitle']} + + + def _getPlaylists(self, id, token=None): + req = {"part": "id", "maxResults": "50", "channelId": id} + if token: + req.update({"pageToken": token}) + + playlists = self.api_response("playlists", req) + + for playlist in playlists['items']: + yield playlist['id'] + + if "nextPageToken" in playlists: + for item in self._getPlaylists(id, playlists['nextPageToken']): + yield item + + + def getPlaylists(self, ch_id): + return map(self.getPlaylist, self._getPlaylists(ch_id)) + + + def _getVideosId(self, id, token=None): + req = {"part": "contentDetails", "maxResults": "50", "playlistId": id} + if token: + req.update({"pageToken": token}) + + playlist = self.api_response("playlistItems", req) + + for item in playlist['items']: + yield item['contentDetails']['videoId'] + + if "nextPageToken" in playlist: + for item in self._getVideosId(id, playlist['nextPageToken']): + yield item + + + def getVideosId(self, p_id): + return list(self._getVideosId(p_id)) + + + def decrypt(self, pyfile): + m = re.match(self.__pattern__, pyfile.url) + m_id = m.group('ID') + m_type = m.group('TYPE') + + if m_type == "user": + self.logDebug("Url recognized as Channel") + user = m_id + channel = self.getChannel(user) + + if channel: + playlists = self.getPlaylists(channel['id']) + self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), channel['title'])) + + relatedplaylist = {p_name: self.getPlaylist(p_id) for p_name, p_id in channel['relatedPlaylists'].iteritems()} + self.logDebug("Channel's related playlists found = %s" % relatedplaylist.keys()) + + relatedplaylist['uploads']['title'] = "Unplaylisted videos" + relatedplaylist['uploads']['checkDups'] = True #: checkDups flag + + for p_name, p_data in relatedplaylist.iteritems(): + if self.getConfig(p_name): + p_data['title'] += " of " + user + playlists.append(p_data) + else: + playlists = [] + else: + self.logDebug("Url recognized as Playlist") + playlists = [self.getPlaylist(m_id)] + + if not playlists: + self.fail(_("No playlist available")) + + addedvideos = [] + urlize = lambda x: "https://www.youtube.com/watch?v=" + x + for p in playlists: + p_name = p['title'] + p_videos = self.getVideosId(p['id']) + p_folder = safe_join(self.config['general']['download_folder'], p['channelTitle'], p_name) + self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name)) + + if not p_videos: + continue + elif "checkDups" in p: + p_urls = [urlize(v_id) for v_id in p_videos if v_id not in addedvideos] + self.logDebug("%s video\s available on playlist \"%s\" after duplicates cleanup" % (len(p_urls), p_name)) + else: + p_urls = map(urlize, p_videos) + + self.packages.append((p_name, p_urls, p_folder)) #: folder is NOT recognized by pyload 0.4.9! + + addedvideos.extend(p_videos) diff --git a/pyload/plugin/crypter/__init__.py b/pyload/plugin/crypter/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/crypter/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/extractor/SevenZip.py b/pyload/plugin/extractor/SevenZip.py new file mode 100644 index 000000000..4b23c6ff2 --- /dev/null +++ b/pyload/plugin/extractor/SevenZip.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +import os +import re + +from subprocess import Popen, PIPE + +from pyload.plugin.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice +from pyload.utils import fs_encode, safe_join + + +class SevenZip(UnRar): + __name__ = "SevenZip" + __version__ = "0.08" + + __description__ = """7-Zip extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("Michael Nowak", ""), + ("Walter Purcaro", "vuolter@gmail.com")] + + + CMD = "7z" + VERSION = "" + + EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2", + ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab", + ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha", + ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt", + ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr", + ".squashfs", ".cramfs", ".scap"] + + + #@NOTE: there are some more uncovered 7z formats + re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') + re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I) + re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I) + re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) + + + @classmethod + def isUsable(cls): + if os.name == "nt": + cls.CMD = os.path.join(pypath, "7z.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out,err = p.communicate() + else: + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + + cls.VERSION = cls.re_version.search(out).group(1) + + return True + + + def check(self): + file = fs_encode(self.filename) + + p = self.call_cmd("t", file) + out, err = p.communicate() + + if p.returncode > 1: + raise CRCError(err) + + p = self.call_cmd("l", "-slt", file) + out, err = p.communicate() + + if p.returncode > 1: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + # check if output or error macthes the 'wrong password'-Regexp + if self.re_wrongpwd.search(out): + raise PasswordError + + # check if output matches 'Encrypted = +' + if self.re_wrongcrc.search(out): + raise CRCError(_("Header protected")) + + + def isPassword(self, password): + p = self.call_cmd("l", fs_encode(self.filename), password=password) + p.communicate() + return p.returncode == 0 + + + def repair(self): + return False + + + def extract(self, password=None): + command = "x" if self.fullpath else "e" + + p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password) + + renice(p.pid, self.renice) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError(err) + + else: #: raise error if anything is on stderr + raise ArchiveError(err) + + if p.returncode > 1: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + self.files = self.list(password) + + + def list(self, password=None): + command = "l" if self.fullpath else "l" + + p = self.call_cmd(command, fs_encode(self.filename), password=password) + out, err = p.communicate() + + if "Can not open" in err: + raise ArchiveError(_("Cannot open file")) + + if p.returncode > 1: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + result = set() + for groups in self.re_filelist.findall(out): + f = groups[-1].strip() + result.add(safe_join(self.out, f)) + + return list(result) + + + def call_cmd(self, command, *xargs, **kwargs): + args = [] + + #overwrite flag + if self.overwrite: + args.append("-y") + + #set a password + if "password" in kwargs and kwargs["password"]: + args.append("-p'%s'" % kwargs["password"]) + else: + args.append("-p-") + + #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue + call = [self.CMD, command] + args + list(xargs) + + self.manager.logDebug(" ".join(call)) + + p = Popen(call, stdout=PIPE, stderr=PIPE) + return p diff --git a/pyload/plugin/extractor/UnRar.py b/pyload/plugin/extractor/UnRar.py new file mode 100644 index 000000000..8a3985678 --- /dev/null +++ b/pyload/plugin/extractor/UnRar.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- + +import os +import re + +from glob import glob +from string import digits +from subprocess import Popen, PIPE + +from pyload.plugin.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError +from pyload.utils import decode, fs_encode, safe_join + + +def renice(pid, value): + if value and os.name != "nt": + try: + Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) + + except Exception: + pass + + +class UnRar(Extractor): + __name__ = "UnRar" + __version__ = "1.13" + + __description__ = """Rar extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("Immenz", "immenz@gmx.net"),] + + + CMD = "unrar" + VERSION = "" + + EXTENSIONS = [".rar"] + + + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I) + + re_filefixed = re.compile(r'Building (.+)') + re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) + + re_wrongpwd = re.compile(r'password', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + + re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I) + + + @classmethod + def isUsable(cls): + if os.name == "nt": + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + else: + try: + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + + except OSError: #: fallback to rar + cls.CMD = "rar" + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + + cls.VERSION = cls.re_version.search(out).group(1) + + return True + + + @classmethod + def isMultipart(cls,filename): + multipart = cls.re_multipart.search(filename) + if multipart: + # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive + return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True + + return False + + + def check(self): + p = self.call_cmd("l", "-v", fs_encode(self.filename)) + out, err = p.communicate() + + if self.re_wrongpwd.search(err): + raise PasswordError + + if self.re_wrongcrc.search(err): + raise CRCError(err) + + # output only used to check if passworded files are present + for attr in self.re_filelist.findall(out): + if attr[0].startswith("*"): + raise PasswordError + + + def isPassword(self, password): + # at this point we can only verify header protected files + p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) + out, err = p.communicate() + return False if self.re_wrongpwd.search(err) else True + + + def repair(self): + p = self.call_cmd("rc", fs_encode(self.filename)) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: + p = self.call_cmd("r", fs_encode(self.filename)) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: + return False + else: + dir = os.path.dirname(filename) + name = re_filefixed.search(out).group(1) + + self.filename = os.path.join(dir, name) + + return True + + + def _progress(self, process): + s = "" + while True: + c = process.stdout.read(1) + # quit loop on eof + if not c: + break + # reading a percentage sign -> set progress and restart + if c == '%': + self.notifyProgress(int(s)) + s = "" + # not reading a digit -> therefore restart + elif c not in digits: + s = "" + # add digit to progressstring + else: + s += c + + + def extract(self, password=None): + command = "x" if self.fullpath else "e" + + p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) + + renice(p.pid, self.renice) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError(err) + + else: #: raise error if anything is on stderr + raise ArchiveError(err) + + if p.returncode: + raise ArchiveError(_("Process return code: %d") % p.returncode) + + self.files = self.list(password) + + + def getDeleteFiles(self): + dir, name = os.path.split(self.filename) + + # actually extracted file + files = [self.filename] + + # eventually Multipart Files + files.extend(safe_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) + + return files + + + def list(self, password=None): + command = "vb" if self.fullpath else "lb" + + p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) + out, err = p.communicate() + + if "Cannot open" in err: + raise ArchiveError(_("Cannot open file")) + + if err.strip(): #: only log error at this point + self.manager.logError(err.strip()) + + result = set() + if not self.fullpath and self.VERSION.startswith('5'): + # NOTE: Unrar 5 always list full path + for f in decode(out).splitlines(): + f = safe_join(self.out, os.path.basename(f.strip())) + if os.path.isfile(f): + result.add(safe_join(self.out, os.path.basename(f))) + else: + for f in decode(out).splitlines(): + f = f.strip() + result.add(safe_join(self.out, f)) + + return list(result) + + + def call_cmd(self, command, *xargs, **kwargs): + args = [] + + # overwrite flag + if self.overwrite: + args.append("-o+") + else: + args.append("-o-") + if self.delete: + args.append("-or") + + for word in self.excludefiles: + args.append("-x'%s'" % word.strip()) + + # assume yes on all queries + args.append("-y") + + # set a password + if "password" in kwargs and kwargs['password']: + args.append("-p%s" % kwargs['password']) + else: + args.append("-p-") + + if self.keepbroken: + args.append("-kb") + + # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue + call = [self.CMD, command] + args + list(xargs) + + self.manager.logDebug(" ".join(call)) + + p = Popen(call, stdout=PIPE, stderr=PIPE) + return p diff --git a/pyload/plugin/extractor/UnZip.py b/pyload/plugin/extractor/UnZip.py new file mode 100644 index 000000000..cb6621f99 --- /dev/null +++ b/pyload/plugin/extractor/UnZip.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import os +import sys +import zipfile + +from pyload.plugin.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError +from pyload.utils import fs_encode + + +class UnZip(Extractor): + __name__ = "UnZip" + __version__ = "1.10" + + __description__ = """Zip extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + EXTENSIONS = [".zip", ".zip64"] + VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) + + + @classmethod + def isUsable(cls): + return sys.version_info[:2] >= (2, 6) + + + def list(self, password=None): + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + z.setpassword(password) + return z.namelist() + + + def check(self): + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + badfile = z.testzip() + + if badfile: + raise CRCError(badfile) + else: + raise PasswordError + + + def extract(self, password=None): + try: + with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + z.setpassword(password) + + badfile = z.testzip() + + if badfile: + raise CRCError(badfile) + else: + z.extractall(self.out) + + except (zipfile.BadZipfile, zipfile.LargeZipFile), e: + raise ArchiveError(e) + + except RuntimeError, e: + if "encrypted" in e: + raise PasswordError + else: + raise ArchiveError(e) + else: + self.files = z.namelist() diff --git a/pyload/plugin/extractor/__init__.py b/pyload/plugin/extractor/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/extractor/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/hook/AlldebridCom.py b/pyload/plugin/hook/AlldebridCom.py new file mode 100644 index 000000000..6d3e5db48 --- /dev/null +++ b/pyload/plugin/hook/AlldebridCom.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class AlldebridCom(MultiHook): + __name__ = "AlldebridCom" + __type__ = "hook" + __version__ = "0.16" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 ), + ("ssl" , "bool" , "Use HTTPS" , True )] + + __description__ = """Alldebrid.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Andy Voigt", "spamsales@online.de")] + + + def getHosters(self): + https = "https" if self.getConfig("ssl") else "http" + html = self.getURL(https + "://www.alldebrid.com/api.php", get={'action': "get_host"}).replace("\"", "").strip() + + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/pyload/plugin/hook/BypassCaptcha.py b/pyload/plugin/hook/BypassCaptcha.py new file mode 100644 index 000000000..7b7e156bb --- /dev/null +++ b/pyload/plugin/hook/BypassCaptcha.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- + +from pycurl import FORM_FILE, LOW_SPEED_TIME + +from pyload.network.HTTPRequest import BadHeader +from pyload.network.RequestFactory import getURL, getRequest +from pyload.plugin.Hook import Hook, threaded + + +class BypassCaptchaException(Exception): + + def __init__(self, err): + self.err = err + + + def getCode(self): + return self.err + + + def __str__(self): + return "" % self.err + + + def __repr__(self): + return "" % self.err + + +class BypassCaptcha(Hook): + __name__ = "BypassCaptcha" + __type__ = "hook" + __version__ = "0.06" + + __config__ = [("force", "bool", "Force BC even if client is connected", False), + ("passkey", "password", "Passkey", "")] + + __description__ = """Send captchas to BypassCaptcha.com""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Godofdream", "soilfcition@gmail.com"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + PYLOAD_KEY = "4f771155b640970d5607f919a615bdefc67e7d32" + + SUBMIT_URL = "http://bypasscaptcha.com/upload.php" + RESPOND_URL = "http://bypasscaptcha.com/check_value.php" + GETCREDITS_URL = "http://bypasscaptcha.com/ex_left.php" + + + def getCredits(self): + res = getURL(self.GETCREDITS_URL, post={"key": self.getConfig("passkey")}) + + data = dict(x.split(' ', 1) for x in res.splitlines()) + return int(data['Left']) + + + def submit(self, captcha, captchaType="file", match=None): + req = getRequest() + + #raise timeout threshold + req.c.setopt(LOW_SPEED_TIME, 80) + + try: + res = req.load(self.SUBMIT_URL, + post={'vendor_key': self.PYLOAD_KEY, + 'key': self.getConfig("passkey"), + 'gen_task_id': "1", + 'file': (FORM_FILE, captcha)}, + multipart=True) + finally: + req.close() + + data = dict(x.split(' ', 1) for x in res.splitlines()) + if not data or "Value" not in data: + raise BypassCaptchaException(res) + + result = data['Value'] + ticket = data['TaskId'] + self.logDebug("Result %s : %s" % (ticket, result)) + + return ticket, result + + + def respond(self, ticket, success): + try: + res = getURL(self.RESPOND_URL, post={"task_id": ticket, "key": self.getConfig("passkey"), + "cv": 1 if success else 0}) + except BadHeader, e: + self.logError(_("Could not send response"), e) + + + def captchaTask(self, task): + if "service" in task.data: + return False + + if not task.isTextual(): + return False + + if not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + if self.getCredits() > 0: + task.handler.append(self) + task.data['service'] = self.__name__ + task.setWaiting(100) + self._processCaptcha(task) + + else: + self.logInfo(_("Your %s account has not enough credits") % self.__name__) + + + def captchaCorrect(self, task): + if task.data['service'] == self.__name__ and "ticket" in task.data: + self.respond(task.data['ticket'], True) + + + def captchaInvalid(self, task): + if task.data['service'] == self.__name__ and "ticket" in task.data: + self.respond(task.data['ticket'], False) + + + @threaded + def _processCaptcha(self, task): + c = task.captchaFile + try: + ticket, result = self.submit(c) + except BypassCaptchaException, e: + task.error = e.getCode() + return + + task.data['ticket'] = ticket + task.setResult(result) diff --git a/pyload/plugin/hook/Captcha9Kw.py b/pyload/plugin/hook/Captcha9Kw.py new file mode 100644 index 000000000..dd3fb1ba1 --- /dev/null +++ b/pyload/plugin/hook/Captcha9Kw.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from base64 import b64encode +from time import sleep + +from pyload.network.HTTPRequest import BadHeader +from pyload.network.RequestFactory import getURL + +from pyload.plugin.Hook import Hook, threaded + + +class Captcha9kw(Hook): + __name__ = "Captcha9Kw" + __type__ = "hook" + __version__ = "0.28" + + __config__ = [("ssl" , "bool" , "Use HTTPS" , True ), + ("force" , "bool" , "Force captcha resolving even if client is connected" , True ), + ("confirm" , "bool" , "Confirm Captcha (cost +6 credits)" , False ), + ("captchaperhour", "int" , "Captcha per hour" , "9999" ), + ("captchapermin" , "int" , "Captcha per minute" , "9999" ), + ("prio" , "int" , "Priority (max 10)(cost +0 -> +10 credits)" , "0" ), + ("queue" , "int" , "Max. Queue (max 999)" , "50" ), + ("hoster_options", "string" , "Hoster options (format: pluginname:prio=1:selfsolfe=1:confirm=1:timeout=900|...)", "ShareonlineBiz:prio=0:timeout=999 | UploadedTo:prio=0:timeout=999"), + ("selfsolve" , "bool" , "Selfsolve (manually solve your captcha in your 9kw client if active)" , "0" ), + ("passkey" , "password", "API key" , "" ), + ("timeout" , "int" , "Timeout in seconds (min 60, max 3999)" , "900" )] + + __description__ = """Send captchas to 9kw.eu""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + API_URL = "http://www.9kw.eu/index.cgi" + + + def activate(self): + if self.getConfig("ssl"): + self.API_URL = self.API_URL.replace("http://", "https://") + + + def getCredits(self): + res = getURL(self.API_URL, + get={'apikey': self.getConfig("passkey"), + 'pyload': "1", + 'source': "pyload", + 'action': "usercaptchaguthaben"}) + + if res.isdigit(): + self.logInfo(_("%s credits left") % res) + credits = self.info['credits'] = int(res) + return credits + else: + self.logError(res) + return 0 + + + @threaded + def _processCaptcha(self, task): + try: + with open(task.captchaFile, 'rb') as f: + data = f.read() + + except IOError, e: + self.logError(e) + return + + pluginname = re.search(r'_([^_]*)_\d+.\w+', task.captchaFile).group(1) + option = {'min' : 2, + 'max' : 50, + 'phrase' : 0, + 'numeric' : 0, + 'case_sensitive': 0, + 'math' : 0, + 'prio' : min(max(self.getConfig("prio"), 0), 10), + 'confirm' : self.getConfig("confirm"), + 'timeout' : min(max(self.getConfig("timeout"), 300), 3999), + 'selfsolve' : self.getConfig("selfsolve"), + 'cph' : self.getConfig("captchaperhour"), + 'cpm' : self.getConfig("captchapermin")} + + for opt in str(self.getConfig("hoster_options").split('|')): + + details = map(str.strip, opt.split(':')) + + if not details or details[0].lower() != pluginname.lower(): + continue + + for d in details: + hosteroption = d.split("=") + + if len(hosteroption) < 2 or not hosteroption[1].isdigit(): + continue + + o = hosteroption[0].lower() + if o in option: + option[o] = hosteroption[1] + + break + + post_data = {'apikey' : self.getConfig("passkey"), + 'prio' : option['prio'], + 'confirm' : option['confirm'], + 'maxtimeout' : option['timeout'], + 'selfsolve' : option['selfsolve'], + 'captchaperhour': option['cph'], + 'captchapermin' : option['cpm'], + 'case-sensitive': option['case_sensitive'], + 'min_len' : option['min'], + 'max_len' : option['max'], + 'phrase' : option['phrase'], + 'numeric' : option['numeric'], + 'math' : option['math'], + 'oldsource' : pluginname, + 'pyload' : "1", + 'source' : "pyload", + 'base64' : "1", + 'mouse' : 1 if task.isPositional() else 0, + 'file-upload-01': b64encode(data), + 'action' : "usercaptchaupload"} + + for _i in xrange(5): + try: + res = getURL(self.API_URL, post=post_data) + except BadHeader, e: + sleep(3) + else: + if res and res.isdigit(): + break + else: + self.logError(_("Bad upload: %s") % res) + return + + self.logDebug(_("NewCaptchaID ticket: %s") % res, task.captchaFile) + + task.data["ticket"] = res + + for _i in xrange(int(self.getConfig("timeout") / 5)): + result = getURL(self.API_URL, + get={'apikey': self.getConfig("passkey"), + 'id' : res, + 'pyload': "1", + 'info' : "1", + 'source': "pyload", + 'action': "usercaptchacorrectdata"}) + + if not result or result == "NO DATA": + sleep(5) + else: + break + else: + self.logDebug("Could not send request: %s" % res) + result = None + + self.logInfo(_("Captcha result for ticket %s: %s") % (res, result)) + + task.setResult(result) + + + def captchaTask(self, task): + if not task.isTextual() and not task.isPositional(): + return + + if not self.getConfig("passkey"): + return + + if self.core.isClientConnected() and not self.getConfig("force"): + return + + credits = self.getCredits() + + if not credits: + self.logError(_("Your captcha 9kw.eu account has not enough credits")) + return + + queue = min(self.getConfig("queue"), 999) + timeout = min(max(self.getConfig("timeout"), 300), 3999) + pluginname = re.search(r'_([^_]*)_\d+.\w+', task.captchaFile).group(1) + + for _i in xrange(5): + servercheck = getURL("http://www.9kw.eu/grafik/servercheck.txt") + if queue < re.search(r'queue=(\d+)', servercheck).group(1): + break + + sleep(10) + else: + self.fail(_("Too many captchas in queue")) + + for opt in str(self.getConfig("hoster_options").split('|')): + details = map(str.strip, opt.split(':')) + + if not details or details[0].lower() != pluginname.lower(): + continue + + for d in details: + hosteroption = d.split("=") + + if len(hosteroption) > 1 \ + and hosteroption[0].lower() == 'timeout' \ + and hosteroption[1].isdigit(): + timeout = int(hosteroption[1]) + + break + + task.handler.append(self) + + task.setWaiting(timeout) + + self._processCaptcha(task) + + + def _captchaResponse(self, task, correct): + type = "correct" if correct else "refund" + + if 'ticket' not in task.data: + self.logDebug("No CaptchaID for %s request (task: %s)" % (type, task)) + return + + passkey = self.getConfig("passkey") + + for _i in xrange(3): + res = getURL(self.API_URL, + get={'action' : "usercaptchacorrectback", + 'apikey' : passkey, + 'api_key': passkey, + 'correct': "1" if correct else "2", + 'pyload' : "1", + 'source' : "pyload", + 'id' : task.data["ticket"]}) + + self.logDebug("Request %s: %s" % (type, res)) + + if res == "OK": + break + + sleep(5) + else: + self.logDebug("Could not send %s request: %s" % (type, res)) + + + def captchaCorrect(self, task): + self._captchaResponse(task, True) + + + def captchaInvalid(self, task): + self._captchaResponse(task, False) diff --git a/pyload/plugin/hook/CaptchaBrotherhood.py b/pyload/plugin/hook/CaptchaBrotherhood.py new file mode 100644 index 000000000..554d102ec --- /dev/null +++ b/pyload/plugin/hook/CaptchaBrotherhood.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import StringIO +import pycurl + +try: + from PIL import Image +except ImportError: + import Image + +from time import sleep +from urllib import urlencode + +from pyload.network.RequestFactory import getURL, getRequest +from pyload.plugin.Hook import Hook, threaded + + +class CaptchaBrotherhoodException(Exception): + + def __init__(self, err): + self.err = err + + + def getCode(self): + return self.err + + + def __str__(self): + return "" % self.err + + + def __repr__(self): + return "" % self.err + + +class CaptchaBrotherhood(Hook): + __name__ = "CaptchaBrotherhood" + __type__ = "hook" + __version__ = "0.08" + + __config__ = [("username", "str", "Username", ""), + ("force", "bool", "Force CT even if client is connected", False), + ("passkey", "password", "Password", "")] + + __description__ = """Send captchas to CaptchaBrotherhood.com""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + API_URL = "http://www.captchabrotherhood.com/" + + + def getCredits(self): + res = getURL(self.API_URL + "askCredits.aspx", + get={"username": self.getConfig("username"), "password": self.getConfig("passkey")}) + if not res.startswith("OK"): + raise CaptchaBrotherhoodException(res) + else: + credits = int(res[3:]) + self.logInfo(_("%d credits left") % credits) + self.info['credits'] = credits + return credits + + + def submit(self, captcha, captchaType="file", match=None): + try: + img = Image.open(captcha) + output = StringIO.StringIO() + self.logDebug("CAPTCHA IMAGE", img, img.format, img.mode) + if img.format in ("GIF", "JPEG"): + img.save(output, img.format) + else: + if img.mode != "RGB": + img = img.convert("RGB") + img.save(output, "JPEG") + data = output.getvalue() + output.close() + except Exception, e: + raise CaptchaBrotherhoodException("Reading or converting captcha image failed: %s" % e) + + req = getRequest() + + url = "%ssendNewCaptcha.aspx?%s" % (self.API_URL, + urlencode({'username' : self.getConfig("username"), + 'password' : self.getConfig("passkey"), + 'captchaSource': "pyLoad", + 'timeout' : "80"})) + + req.c.setopt(pycurl.URL, url) + req.c.setopt(pycurl.POST, 1) + req.c.setopt(pycurl.POSTFIELDS, data) + req.c.setopt(pycurl.HTTPHEADER, ["Content-Type: text/html"]) + + try: + req.c.perform() + res = req.getResponse() + except Exception, e: + raise CaptchaBrotherhoodException("Submit captcha image failed") + + req.close() + + if not res.startswith("OK"): + raise CaptchaBrotherhoodException(res[1]) + + ticket = res[3:] + + for _i in xrange(15): + sleep(5) + res = self.api_response("askCaptchaResult", ticket) + if res.startswith("OK-answered"): + return ticket, res[12:] + + raise CaptchaBrotherhoodException("No solution received in time") + + + def api_response(self, api, ticket): + res = getURL("%s%s.aspx" % (self.API_URL, api), + get={"username": self.getConfig("username"), + "password": self.getConfig("passkey"), + "captchaID": ticket}) + if not res.startswith("OK"): + raise CaptchaBrotherhoodException("Unknown response: %s" % res) + + return res + + + def captchaTask(self, task): + if "service" in task.data: + return False + + if not task.isTextual(): + return False + + if not self.getConfig("username") or not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + if self.getCredits() > 10: + task.handler.append(self) + task.data['service'] = self.__name__ + task.setWaiting(100) + self._processCaptcha(task) + else: + self.logInfo(_("Your CaptchaBrotherhood Account has not enough credits")) + + + def captchaInvalid(self, task): + if task.data['service'] == self.__name__ and "ticket" in task.data: + res = self.api_response("complainCaptcha", task.data['ticket']) + + + @threaded + def _processCaptcha(self, task): + c = task.captchaFile + try: + ticket, result = self.submit(c) + except CaptchaBrotherhoodException, e: + task.error = e.getCode() + return + + task.data['ticket'] = ticket + task.setResult(result) diff --git a/pyload/plugin/hook/DeathByCaptcha.py b/pyload/plugin/hook/DeathByCaptcha.py new file mode 100644 index 000000000..e0108963b --- /dev/null +++ b/pyload/plugin/hook/DeathByCaptcha.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from base64 import b64encode +from pycurl import FORM_FILE, HTTPHEADER +from time import sleep + +from pyload.utils import json_loads +from pyload.network.HTTPRequest import BadHeader +from pyload.network.RequestFactory import getRequest +from pyload.plugin.Hook import Hook, threaded + + +class DeathByCaptchaException(Exception): + DBC_ERRORS = {'not-logged-in': 'Access denied, check your credentials', + 'invalid-credentials': 'Access denied, check your credentials', + 'banned': 'Access denied, account is suspended', + 'insufficient-funds': 'Insufficient account balance to decrypt CAPTCHA', + 'invalid-captcha': 'CAPTCHA is not a valid image', + 'service-overload': 'CAPTCHA was rejected due to service overload, try again later', + 'invalid-request': 'Invalid request', + 'timed-out': 'No CAPTCHA solution received in time'} + + + def __init__(self, err): + self.err = err + + + def getCode(self): + return self.err + + + def getDesc(self): + if self.err in self.DBC_ERRORS.keys(): + return self.DBC_ERRORS[self.err] + else: + return self.err + + + def __str__(self): + return "" % self.err + + + def __repr__(self): + return "" % self.err + + +class DeathByCaptcha(Hook): + __name__ = "DeathByCaptcha" + __type__ = "hook" + __version__ = "0.06" + + __config__ = [("username", "str", "Username", ""), + ("passkey", "password", "Password", ""), + ("force", "bool", "Force DBC even if client is connected", False)] + + __description__ = """Send captchas to DeathByCaptcha.com""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + API_URL = "http://api.dbcapi.me/api/" + + + def api_response(self, api="captcha", post=False, multipart=False): + req = getRequest() + req.c.setopt(HTTPHEADER, ["Accept: application/json", "User-Agent: pyLoad %s" % self.core.version]) + + if post: + if not isinstance(post, dict): + post = {} + post.update({"username": self.getConfig("username"), + "password": self.getConfig("passkey")}) + + res = None + try: + json = req.load("%s%s" % (self.API_URL, api), + post=post, + multipart=multipart) + self.logDebug(json) + res = json_loads(json) + + if "error" in res: + raise DeathByCaptchaException(res['error']) + elif "status" not in res: + raise DeathByCaptchaException(str(res)) + + except BadHeader, e: + if 403 == e.code: + raise DeathByCaptchaException('not-logged-in') + elif 413 == e.code: + raise DeathByCaptchaException('invalid-captcha') + elif 503 == e.code: + raise DeathByCaptchaException('service-overload') + elif e.code in (400, 405): + raise DeathByCaptchaException('invalid-request') + else: + raise + + finally: + req.close() + + return res + + + def getCredits(self): + res = self.api_response("user", True) + + if 'is_banned' in res and res['is_banned']: + raise DeathByCaptchaException('banned') + elif 'balance' in res and 'rate' in res: + self.info.update(res) + else: + raise DeathByCaptchaException(res) + + + def getStatus(self): + res = self.api_response("status", False) + + if 'is_service_overloaded' in res and res['is_service_overloaded']: + raise DeathByCaptchaException('service-overload') + + + def submit(self, captcha, captchaType="file", match=None): + #@NOTE: Workaround multipart-post bug in HTTPRequest.py + if re.match("^\w*$", self.getConfig("passkey")): + multipart = True + data = (FORM_FILE, captcha) + else: + multipart = False + with open(captcha, 'rb') as f: + data = f.read() + data = "base64:" + b64encode(data) + + res = self.api_response("captcha", {"captchafile": data}, multipart) + + if "captcha" not in res: + raise DeathByCaptchaException(res) + ticket = res['captcha'] + + for _i in xrange(24): + sleep(5) + res = self.api_response("captcha/%d" % ticket, False) + if res['text'] and res['is_correct']: + break + else: + raise DeathByCaptchaException('timed-out') + + result = res['text'] + self.logDebug("Result %s : %s" % (ticket, result)) + + return ticket, result + + + def captchaTask(self, task): + if "service" in task.data: + return False + + if not task.isTextual(): + return False + + if not self.getConfig("username") or not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + try: + self.getStatus() + self.getCredits() + except DeathByCaptchaException, e: + self.logError(e.getDesc()) + return False + + balance, rate = self.info['balance'], self.info['rate'] + self.logInfo(_("Account balance"), + _("US$%.3f (%d captchas left at %.2f cents each)") % (balance / 100, + balance // rate, rate)) + + if balance > rate: + task.handler.append(self) + task.data['service'] = self.__name__ + task.setWaiting(180) + self._processCaptcha(task) + + + def captchaInvalid(self, task): + if task.data['service'] == self.__name__ and "ticket" in task.data: + try: + res = self.api_response("captcha/%d/report" % task.data['ticket'], True) + + except DeathByCaptchaException, e: + self.logError(e.getDesc()) + + except Exception, e: + self.logError(e) + + + @threaded + def _processCaptcha(self, task): + c = task.captchaFile + try: + ticket, result = self.submit(c) + except DeathByCaptchaException, e: + task.error = e.getCode() + self.logError(e.getDesc()) + return + + task.data['ticket'] = ticket + task.setResult(result) diff --git a/pyload/plugin/hook/DebridItaliaCom.py b/pyload/plugin/hook/DebridItaliaCom.py new file mode 100644 index 000000000..e2f766d86 --- /dev/null +++ b/pyload/plugin/hook/DebridItaliaCom.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHook import MultiHook + + +class DebridItaliaCom(MultiHook): + __name__ = "DebridItaliaCom" + __type__ = "hook" + __version__ = "0.12" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Debriditalia.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def getHosters(self): + return self.getURL("http://debriditalia.com/api.php", get={'hosts': ""}).replace('"', '').split(',') diff --git a/pyload/plugin/hook/EasybytezCom.py b/pyload/plugin/hook/EasybytezCom.py new file mode 100644 index 000000000..1163bbf46 --- /dev/null +++ b/pyload/plugin/hook/EasybytezCom.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHook import MultiHook + + +class EasybytezCom(MultiHook): + __name__ = "EasybytezCom" + __type__ = "hook" + __version__ = "0.07" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """EasyBytez.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + def getHosters(self): + user, data = self.account.selectAccount() + + req = self.account.getAccountRequest(user) + html = req.load("http://www.easybytez.com") + + return re.search(r'\s*Supported sites:(.*)', html).group(1).split(',') diff --git a/pyload/plugin/hook/ExpertDecoders.py b/pyload/plugin/hook/ExpertDecoders.py new file mode 100644 index 000000000..8fbc88c80 --- /dev/null +++ b/pyload/plugin/hook/ExpertDecoders.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +from base64 import b64encode +from pycurl import LOW_SPEED_TIME +from uuid import uuid4 + +from pyload.network.HTTPRequest import BadHeader +from pyload.network.RequestFactory import getURL, getRequest +from pyload.plugin.Hook import Hook, threaded + + +class ExpertDecoders(Hook): + __name__ = "ExpertDecoders" + __type__ = "hook" + __version__ = "0.04" + + __config__ = [("force", "bool", "Force CT even if client is connected", False), + ("passkey", "password", "Access key", "")] + + __description__ = """Send captchas to expertdecoders.com""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + API_URL = "http://www.fasttypers.org/imagepost.ashx" + + + def getCredits(self): + res = getURL(self.API_URL, post={"key": self.getConfig("passkey"), "action": "balance"}) + + if res.isdigit(): + self.logInfo(_("%s credits left") % res) + self.info['credits'] = credits = int(res) + return credits + else: + self.logError(res) + return 0 + + + @threaded + def _processCaptcha(self, task): + task.data['ticket'] = ticket = uuid4() + result = None + + with open(task.captchaFile, 'rb') as f: + data = f.read() + + req = getRequest() + #raise timeout threshold + req.c.setopt(LOW_SPEED_TIME, 80) + + try: + result = req.load(self.API_URL, + post={'action' : "upload", + 'key' : self.getConfig("passkey"), + 'file' : b64encode(data), + 'gen_task_id': ticket}) + finally: + req.close() + + self.logDebug("Result %s : %s" % (ticket, result)) + task.setResult(result) + + + def captchaTask(self, task): + if not task.isTextual(): + return False + + if not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + if self.getCredits() > 0: + task.handler.append(self) + task.setWaiting(100) + self._processCaptcha(task) + + else: + self.logInfo(_("Your ExpertDecoders Account has not enough credits")) + + + def captchaInvalid(self, task): + if "ticket" in task.data: + + try: + res = getURL(self.API_URL, + post={'action': "refund", 'key': self.getConfig("passkey"), 'gen_task_id': task.data['ticket']}) + self.logInfo(_("Request refund"), res) + + except BadHeader, e: + self.logError(_("Could not send refund request"), e) diff --git a/pyload/plugin/hook/FastixRu.py b/pyload/plugin/hook/FastixRu.py new file mode 100644 index 000000000..8bc3b3f16 --- /dev/null +++ b/pyload/plugin/hook/FastixRu.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class FastixRu(MultiHook): + __name__ = "FastixRu" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Fastix.ru hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Massimo Rosamilia", "max@spiritix.eu")] + + + def getHosters(self): + html = self.getURL("http://fastix.ru/api_v2", + get={'apikey': "5182964c3f8f9a7f0b00000a_kelmFB4n1IrnCDYuIFn2y", + 'sub' : "allowed_sources"}) + host_list = json_loads(html) + host_list = host_list['allow'] + return host_list diff --git a/pyload/plugin/hook/FreeWayMe.py b/pyload/plugin/hook/FreeWayMe.py new file mode 100644 index 000000000..711be37a8 --- /dev/null +++ b/pyload/plugin/hook/FreeWayMe.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class FreeWayMe(MultiHook): + __name__ = "FreeWayMe" + __type__ = "hook" + __version__ = "0.14" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """FreeWay.me hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Nicolas Giese", "james@free-way.me")] + + + def getHosters(self): + hostis = self.getURL("https://www.free-way.me/ajax/jd.php", get={'id': 3}).replace("\"", "").strip() + self.logDebug("Hosters", hostis) + return [x.strip() for x in hostis.split(",") if x.strip()] diff --git a/pyload/plugin/hook/ImageTyperz.py b/pyload/plugin/hook/ImageTyperz.py new file mode 100644 index 000000000..a9d7326de --- /dev/null +++ b/pyload/plugin/hook/ImageTyperz.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from base64 import b64encode +from pycurl import FORM_FILE, LOW_SPEED_TIME + +from pyload.network.RequestFactory import getURL, getRequest +from pyload.plugin.Hook import Hook, threaded + + +class ImageTyperzException(Exception): + + def __init__(self, err): + self.err = err + + + def getCode(self): + return self.err + + + def __str__(self): + return "" % self.err + + + def __repr__(self): + return "" % self.err + + +class ImageTyperz(Hook): + __name__ = "ImageTyperz" + __type__ = "hook" + __version__ = "0.06" + + __config__ = [("username", "str", "Username", ""), + ("passkey", "password", "Password", ""), + ("force", "bool", "Force IT even if client is connected", False)] + + __description__ = """Send captchas to ImageTyperz.com""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + SUBMIT_URL = "http://captchatypers.com/Forms/UploadFileAndGetTextNEW.ashx" + RESPOND_URL = "http://captchatypers.com/Forms/SetBadImage.ashx" + GETCREDITS_URL = "http://captchatypers.com/Forms/RequestBalance.ashx" + + + def getCredits(self): + res = getURL(self.GETCREDITS_URL, + post={'action': "REQUESTBALANCE", + 'username': self.getConfig("username"), + 'password': self.getConfig("passkey")}) + + if res.startswith('ERROR'): + raise ImageTyperzException(res) + + try: + balance = float(res) + except Exception: + raise ImageTyperzException("Invalid response") + + self.logInfo(_("Account balance: $%s left") % res) + return balance + + + def submit(self, captcha, captchaType="file", match=None): + req = getRequest() + #raise timeout threshold + req.c.setopt(LOW_SPEED_TIME, 80) + + try: + #@NOTE: Workaround multipart-post bug in HTTPRequest.py + if re.match("^\w*$", self.getConfig("passkey")): + multipart = True + data = (FORM_FILE, captcha) + else: + multipart = False + with open(captcha, 'rb') as f: + data = f.read() + data = b64encode(data) + + res = req.load(self.SUBMIT_URL, + post={'action': "UPLOADCAPTCHA", + 'username': self.getConfig("username"), + 'password': self.getConfig("passkey"), "file": data}, + multipart=multipart) + finally: + req.close() + + if res.startswith("ERROR"): + raise ImageTyperzException(res) + else: + data = res.split('|') + if len(data) == 2: + ticket, result = data + else: + raise ImageTyperzException("Unknown response: %s" % res) + + return ticket, result + + + def captchaTask(self, task): + if "service" in task.data: + return False + + if not task.isTextual(): + return False + + if not self.getConfig("username") or not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + if self.getCredits() > 0: + task.handler.append(self) + task.data['service'] = self.__name__ + task.setWaiting(100) + self._processCaptcha(task) + + else: + self.logInfo(_("Your %s account has not enough credits") % self.__name__) + + + def captchaInvalid(self, task): + if task.data['service'] == self.__name__ and "ticket" in task.data: + res = getURL(self.RESPOND_URL, + post={'action': "SETBADIMAGE", + 'username': self.getConfig("username"), + 'password': self.getConfig("passkey"), + 'imageid': task.data['ticket']}) + + if res == "SUCCESS": + self.logInfo(_("Bad captcha solution received, requested refund")) + else: + self.logError(_("Bad captcha solution received, refund request failed"), res) + + + @threaded + def _processCaptcha(self, task): + c = task.captchaFile + try: + ticket, result = self.submit(c) + except ImageTyperzException, e: + task.error = e.getCode() + return + + task.data['ticket'] = ticket + task.setResult(result) diff --git a/pyload/plugin/hook/LinkdecrypterCom.py b/pyload/plugin/hook/LinkdecrypterCom.py new file mode 100644 index 000000000..efdf77065 --- /dev/null +++ b/pyload/plugin/hook/LinkdecrypterCom.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHook import MultiHook + + +class LinkdecrypterCom(MultiHook): + __name__ = "LinkdecrypterCom" + __type__ = "hook" + __version__ = "1.02" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Linkdecrypter.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def getCrypters(self): + return re.search(r'>Supported\(\d+\): (.[\w.\-, ]+)', + self.getURL("http://linkdecrypter.com/").replace("(g)", "")).group(1).split(', ') diff --git a/pyload/plugin/hook/LinksnappyCom.py b/pyload/plugin/hook/LinksnappyCom.py new file mode 100644 index 000000000..dc0c47496 --- /dev/null +++ b/pyload/plugin/hook/LinksnappyCom.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class LinksnappyCom(MultiHook): + __name__ = "LinksnappyCom" + __type__ = "hook" + __version__ = "0.04" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Linksnappy.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def getHosters(self): + json_data = self.getURL("http://gen.linksnappy.com/lseAPI.php", get={'act': "FILEHOSTS"}) + json_data = json_loads(json_data) + + return json_data['return'].keys() diff --git a/pyload/plugin/hook/MegaDebridEu.py b/pyload/plugin/hook/MegaDebridEu.py new file mode 100644 index 000000000..7e86d4d49 --- /dev/null +++ b/pyload/plugin/hook/MegaDebridEu.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class MegaDebridEu(MultiHook): + __name__ = "MegaDebridEu" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Mega-debrid.eu hook plugin""" + __license__ = "GPLv3" + __authors__ = [("D.Ducatel", "dducatel@je-geek.fr")] + + + def getHosters(self): + reponse = self.getURL("http://www.mega-debrid.eu/api.php", get={'action': "getHosters"}) + json_data = json_loads(reponse) + + if json_data['response_code'] == "ok": + host_list = [element[0] for element in json_data['hosters']] + else: + self.logError(_("Unable to retrieve hoster list")) + host_list = list() + + return host_list diff --git a/pyload/plugin/hook/MultihostersCom.py b/pyload/plugin/hook/MultihostersCom.py new file mode 100644 index 000000000..7b92089a1 --- /dev/null +++ b/pyload/plugin/hook/MultihostersCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.hook.ZeveraCom import ZeveraCom + + +class MultihostersCom(ZeveraCom): + __name__ = "MultihostersCom" + __type__ = "hook" + __version__ = "0.02" + + __config__ = [("mode" , "all;listed;unlisted", "Use for plugins (if supported)" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed", "bool" , "Revert to standard download if download fails", False), + ("interval" , "int" , "Reload interval in hours (0 to disable)" , 12 )] + + __description__ = """Multihosters.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("tjeh", "tjeh@gmx.net")] diff --git a/pyload/plugin/hook/MultishareCz.py b/pyload/plugin/hook/MultishareCz.py new file mode 100644 index 000000000..56f30fb39 --- /dev/null +++ b/pyload/plugin/hook/MultishareCz.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHook import MultiHook + + +class MultishareCz(MultiHook): + __name__ = "MultishareCz" + __type__ = "hook" + __version__ = "0.07" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """MultiShare.cz hook plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_PATTERN = r']*?alt="([^"]+)">\s*[^>]*?alt="OK"' + + + def getHosters(self): + html = self.getURL("http://www.multishare.cz/monitoring/") + return re.findall(self.HOSTER_PATTERN, html) diff --git a/pyload/plugin/hook/MyfastfileCom.py b/pyload/plugin/hook/MyfastfileCom.py new file mode 100644 index 000000000..097a54b60 --- /dev/null +++ b/pyload/plugin/hook/MyfastfileCom.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook +from pyload.utils import json_loads + + +class MyfastfileCom(MultiHook): + __name__ = "MyfastfileCom" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Myfastfile.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def getHosters(self): + json_data = self.getURL("http://myfastfile.com/api.php", get={'hosts': ""}, decode=True) + self.logDebug("JSON data", json_data) + json_data = json_loads(json_data) + + return json_data['hosts'] diff --git a/pyload/plugin/hook/NoPremiumPl.py b/pyload/plugin/hook/NoPremiumPl.py new file mode 100644 index 000000000..da7967154 --- /dev/null +++ b/pyload/plugin/hook/NoPremiumPl.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class NoPremiumPl(MultiHook): + __name__ = "NoPremiumPl" + __type__ = "hook" + __version__ = "0.03" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """NoPremium.pl hook plugin""" + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@nopremium.pl")] + + + def getHosters(self): + hostings = json_loads(self.getURL("https://www.nopremium.pl/clipboard.php?json=3").strip()) + hostings_domains = [domain for row in hostings for domain in row["domains"] if row["sdownload"] == "0"] + + self.logDebug(hostings_domains) + + return hostings_domains diff --git a/pyload/plugin/hook/OverLoadMe.py b/pyload/plugin/hook/OverLoadMe.py new file mode 100644 index 000000000..e4602019f --- /dev/null +++ b/pyload/plugin/hook/OverLoadMe.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class OverLoadMe(MultiHook): + __name__ = "OverLoadMe" + __type__ = "hook" + __version__ = "0.04" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 ), + ("ssl" , "bool" , "Use HTTPS" , True )] + + __description__ = """Over-Load.me hook plugin""" + __license__ = "GPLv3" + __authors__ = [("marley", "marley@over-load.me")] + + + def getHosters(self): + https = "https" if self.getConfig("ssl") else "http" + html = self.getURL(https + "://api.over-load.me/hoster.php", + get={'auth': "0001-cb1f24dadb3aa487bda5afd3b76298935329be7700cd7-5329be77-00cf-1ca0135f"}).replace("\"", "").strip() + self.logDebug("Hosterlist", html) + + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/pyload/plugin/hook/PremiumTo.py b/pyload/plugin/hook/PremiumTo.py new file mode 100644 index 000000000..58d753cec --- /dev/null +++ b/pyload/plugin/hook/PremiumTo.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class PremiumTo(MultiHook): + __name__ = "PremiumTo" + __type__ = "hook" + __version__ = "0.08" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Premium.to hook plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] + + + def getHosters(self): + html = self.getURL("http://premium.to/api/hosters.php", + get={'username': self.account.username, 'password': self.account.password}) + return [x.strip() for x in html.replace("\"", "").split(";")] diff --git a/pyload/plugin/hook/PremiumizeMe.py b/pyload/plugin/hook/PremiumizeMe.py new file mode 100644 index 000000000..c5f1588ec --- /dev/null +++ b/pyload/plugin/hook/PremiumizeMe.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class PremiumizeMe(MultiHook): + __name__ = "PremiumizeMe" + __type__ = "hook" + __version__ = "0.17" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Premiumize.me hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Florian Franzen", "FlorianFranzen@gmail.com")] + + + def getHosters(self): + # Get account data + user, data = self.account.selectAccount() + + # Get supported hosters list from premiumize.me using the + # json API v1 (see https://secure.premiumize.me/?show=api) + answer = self.getURL("https://api.premiumize.me/pm-api/v1.php", + get={'method': "hosterlist", 'params[login]': user, 'params[pass]': data['password']}) + data = json_loads(answer) + + # If account is not valid thera are no hosters available + if data['status'] != 200: + return [] + + # Extract hosters from json file + return data['result']['hosterlist'] diff --git a/pyload/plugin/hook/PutdriveCom.py b/pyload/plugin/hook/PutdriveCom.py new file mode 100644 index 000000000..85e2f541d --- /dev/null +++ b/pyload/plugin/hook/PutdriveCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.hook.ZeveraCom import ZeveraCom + + +class PutdriveCom(ZeveraCom): + __name__ = "PutdriveCom" + __type__ = "hook" + __version__ = "0.01" + + __config__ = [("mode" , "all;listed;unlisted", "Use for plugins (if supported)" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed", "bool" , "Revert to standard download if download fails", False), + ("interval" , "int" , "Reload interval in hours (0 to disable)" , 12 )] + + __description__ = """Putdrive.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] diff --git a/pyload/plugin/hook/RPNetBiz.py b/pyload/plugin/hook/RPNetBiz.py new file mode 100644 index 000000000..75fe0e39b --- /dev/null +++ b/pyload/plugin/hook/RPNetBiz.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class RPNetBiz(MultiHook): + __name__ = "RPNetBiz" + __type__ = "hook" + __version__ = "0.14" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """RPNet.biz hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Dman", "dmanugm@gmail.com")] + + + def getHosters(self): + # Get account data + user, data = self.account.selectAccount() + + res = self.getURL("https://premium.rpnet.biz/client_api.php", + get={'username': user, 'password': data['password'], 'action': "showHosterList"}) + hoster_list = json_loads(res) + + # If account is not valid thera are no hosters available + if 'error' in hoster_list: + return [] + + # Extract hosters from json file + return hoster_list['hosters'] diff --git a/pyload/plugin/hook/RapideoPl.py b/pyload/plugin/hook/RapideoPl.py new file mode 100644 index 000000000..12d53830b --- /dev/null +++ b/pyload/plugin/hook/RapideoPl.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class RapideoPl(MultiHook): + __name__ = "RapideoPl" + __type__ = "hook" + __version__ = "0.03" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Rapideo.pl hook plugin""" + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@rapideo.pl")] + + + def getHosters(self): + hostings = json_loads(self.getURL("https://www.rapideo.pl/clipboard.php?json=3").strip()) + hostings_domains = [domain for row in hostings for domain in row["domains"] if row["sdownload"] == "0"] + + self.logDebug(hostings_domains) + + return hostings_domains diff --git a/pyload/plugin/hook/RealdebridCom.py b/pyload/plugin/hook/RealdebridCom.py new file mode 100644 index 000000000..a8eb03f83 --- /dev/null +++ b/pyload/plugin/hook/RealdebridCom.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class RealdebridCom(MultiHook): + __name__ = "RealdebridCom" + __type__ = "hook" + __version__ = "0.46" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 ), + ("ssl" , "bool" , "Use HTTPS" , True )] + + __description__ = """Real-Debrid.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Devirex Hazzard", "naibaf_11@yahoo.de")] + + + def getHosters(self): + https = "https" if self.getConfig("ssl") else "http" + html = self.getURL(https + "://real-debrid.com/api/hosters.php").replace("\"", "").strip() + + return [x.strip() for x in html.split(",") if x.strip()] diff --git a/pyload/plugin/hook/RehostTo.py b/pyload/plugin/hook/RehostTo.py new file mode 100644 index 000000000..cda80417c --- /dev/null +++ b/pyload/plugin/hook/RehostTo.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class RehostTo(MultiHook): + __name__ = "RehostTo" + __type__ = "hook" + __version__ = "0.50" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Rehost.to hook plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + def getHosters(self): + user, data = self.account.selectAccount() + html = self.getURL("http://rehost.to/api.php", + get={'cmd' : "get_supported_och_dl", + 'long_ses': self.account.getAccountInfo(user)['session']}) + return [x.strip() for x in html.replace("\"", "").split(",")] diff --git a/pyload/plugin/hook/SimplyPremiumCom.py b/pyload/plugin/hook/SimplyPremiumCom.py new file mode 100644 index 000000000..28d9af41f --- /dev/null +++ b/pyload/plugin/hook/SimplyPremiumCom.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class SimplyPremiumCom(MultiHook): + __name__ = "SimplyPremiumCom" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Simply-Premium.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("EvolutionClip", "evolutionclip@live.de")] + + + def getHosters(self): + json_data = self.getURL("http://www.simply-premium.com/api/hosts.php", get={'format': "json", 'online': 1}) + json_data = json_loads(json_data) + + host_list = [element['regex'] for element in json_data['result']] + + return host_list diff --git a/pyload/plugin/hook/SimplydebridCom.py b/pyload/plugin/hook/SimplydebridCom.py new file mode 100644 index 000000000..011b346b8 --- /dev/null +++ b/pyload/plugin/hook/SimplydebridCom.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class SimplydebridCom(MultiHook): + __name__ = "SimplydebridCom" + __type__ = "hook" + __version__ = "0.04" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Simply-Debrid.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + def getHosters(self): + html = self.getURL("http://simply-debrid.com/api.php", get={'list': 1}) + return [x.strip() for x in html.rstrip(';').replace("\"", "").split(";")] diff --git a/pyload/plugin/hook/SmoozedCom.py b/pyload/plugin/hook/SmoozedCom.py new file mode 100644 index 000000000..3f5aab63d --- /dev/null +++ b/pyload/plugin/hook/SmoozedCom.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class SmoozedCom(MultiHook): + __name__ = "SmoozedCom" + __type__ = "hook" + __version__ = "0.03" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Smoozed.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("", "")] + + + def getHosters(self): + user, data = self.account.selectAccount() + return self.account.getAccountInfo(user)["hosters"] diff --git a/pyload/plugin/hook/UnrestrictLi.py b/pyload/plugin/hook/UnrestrictLi.py new file mode 100644 index 000000000..0c4984042 --- /dev/null +++ b/pyload/plugin/hook/UnrestrictLi.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHook import MultiHook + + +class UnrestrictLi(MultiHook): + __name__ = "UnrestrictLi" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 ), + ("history" , "bool" , "Delete History" , False)] + + __description__ = """Unrestrict.li hook plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + def getHosters(self): + json_data = self.getURL("http://unrestrict.li/api/jdownloader/hosts.php", get={'format': "json"}) + json_data = json_loads(json_data) + + return [element['host'] for element in json_data['result']] diff --git a/pyload/plugin/hook/XFileSharingPro.py b/pyload/plugin/hook/XFileSharingPro.py new file mode 100644 index 000000000..e3f816502 --- /dev/null +++ b/pyload/plugin/hook/XFileSharingPro.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Hook import Hook + + +class XFileSharingPro(Hook): + __name__ = "XFileSharingPro" + __type__ = "hook" + __version__ = "0.31" + + __config__ = [("activated" , "bool", "Activated" , True ), + ("use_hoster_list" , "bool", "Load listed hosters only" , False), + ("use_crypter_list", "bool", "Load listed crypters only" , False), + ("use_builtin_list", "bool", "Load built-in plugin list" , True ), + ("hoster_list" , "str" , "Hoster list (comma separated)" , "" ), + ("crypter_list" , "str" , "Crypter list (comma separated)", "" )] + + __description__ = """Load XFileSharingPro based hosters and crypter which don't need a own plugin to run""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + regexp = {'hoster' : (r'https?://(?:www\.)?(?P[\w.^_]+(?:\.[a-zA-Z]{2,})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)', + r'https?://(?:[^/]+\.)?(?P%s)/(?:embed-)?\w+'), + 'crypter': (r'https?://(?:www\.)?(?P[\w.^_]+(?:\.[a-zA-Z]{2,})(?:\:\d+)?)/(?:user|folder)s?/\w+', + r'https?://(?:[^/]+\.)?(?P%s)/(?:user|folder)s?/\w+')} + + HOSTER_BUILTIN = [#WORKING HOSTERS: + "180upload.com", "backin.net", "eyesfile.ca", "file4safe.com", "fileband.com", "filedwon.com", + "fileparadox.in", "filevice.com", "hostingbulk.com", "junkyvideo.com", "linestorage.com", "ravishare.com", + "ryushare.com", "salefiles.com", "sendmyway.com", "sharesix.com", "thefile.me", "verzend.be", "xvidstage.com", + #NOT TESTED: + "101shared.com", "4upfiles.com", "filemaze.ws", "filenuke.com", "linkzhost.com", "mightyupload.com", + "rockdizfile.com", "sharebeast.com", "sharerepo.com", "shareswift.com", "uploadbaz.com", "uploadc.com", + "vidbull.com", "zalaa.com", "zomgupload.com", + #NOT WORKING: + "amonshare.com", "banicrazy.info", "boosterking.com", "host4desi.com", "laoupload.com", "rd-fs.com"] + CRYPTER_BUILTIN = ["junocloud.me", "rapidfileshare.net"] + + + # def pluginConfigChanged(self.__name__, plugin, name, value): + # self.loadPattern() + + + def activate(self): + self.loadPattern() + + + def loadPattern(self): + use_builtin_list = self.getConfig('use_builtin_list') + + for type in ("hoster", "crypter"): + every_plugin = not self.getConfig("use_%s_list" % type) + + if every_plugin: + self.logInfo(_("Handling any %s I can!") % type) + pattern = self.regexp[type][0] + else: + plugins = self.getConfig('%s_list' % type) + plugin_set = set(plugins.replace(' ', '').replace('\\', '').replace('|', ',').replace(';', ',').lower().split(',')) + + if use_builtin_list: + plugin_set |= set(x.lower() for x in getattr(self, "%s_BUILTIN" % type.upper())) + + plugin_set -= set(('', u'')) + + if not plugin_set: + self.logInfo(_("No %s to handle") % type) + self._unload(type) + return + + match_list = '|'.join(sorted(plugin_set)) + + len_match_list = len(plugin_set) + self.logInfo(_("Handling %d %s%s: %s") % (len_match_list, + type, + "" if len_match_list == 1 else "s", + match_list.replace('|', ', '))) + + pattern = self.regexp[type][1] % match_list.replace('.', '\.') + + dict = self.core.pluginManager.plugins[type]["XFileSharingPro"] + dict['pattern'] = pattern + dict['re'] = re.compile(pattern) + + self.logDebug("Loaded %s pattern: %s" % (type, pattern)) + + + def _unload(self, type): + dict = self.core.pluginManager.plugins[type]["XFileSharingPro"] + dict['pattern'] = r'^unmatchable$' + dict['re'] = re.compile(dict['pattern']) + + + def deactivate(self): + # self.unloadHoster("BasePlugin") + for type in ("hoster", "crypter"): + self._unload(type, "XFileSharingPro") + + + # def downloadFailed(self, pyfile): + # if pyfile.pluginname == "BasePlugin" \ + # and pyfile.hasStatus("failed") \ + # and not self.getConfig("use_hoster_list") \ + # and self.unloadHoster("BasePlugin"): + # self.logDebug("Unloaded XFileSharingPro from BasePlugin") + # pyfile.setStatus("queued") diff --git a/pyload/plugin/hook/ZeveraCom.py b/pyload/plugin/hook/ZeveraCom.py new file mode 100644 index 000000000..a60c33bd4 --- /dev/null +++ b/pyload/plugin/hook/ZeveraCom.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.MultiHook import MultiHook + + +class ZeveraCom(MultiHook): + __name__ = "ZeveraCom" + __type__ = "hook" + __version__ = "0.05" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Zevera.com hook plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def getHosters(self): + html = self.account.api_response(pyreq.getHTTPRequest(timeout=120), cmd="gethosters") + return [x.strip() for x in html.split(",")] diff --git a/pyload/plugin/hook/__init__.py b/pyload/plugin/hook/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/hook/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/hoster/AlldebridCom.py b/pyload/plugin/hoster/AlldebridCom.py new file mode 100644 index 000000000..8ab98bc6f --- /dev/null +++ b/pyload/plugin/hoster/AlldebridCom.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +import re + +from random import randrange +from urllib import unquote + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHoster import MultiHoster +from pyload.utils import parseFileSize + + +class AlldebridCom(MultiHoster): + __name__ = "AlldebridCom" + __type__ = "hoster" + __version__ = "0.44" + + __pattern__ = r'https?://(?:www\.|s\d+\.)?alldebrid\.com/dl/[\w^_]+' + + __description__ = """Alldebrid.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Andy Voigt", "spamsales@online.de")] + + + def getFilename(self, url): + try: + name = unquote(url.rsplit("/", 1)[1]) + except IndexError: + name = "Unknown_Filename..." + + if name.endswith("..."): # incomplete filename, append random stuff + name += "%s.tmp" % randrange(100, 999) + + return name + + + def setup(self): + self.chunkLimit = 16 + + + def handlePremium(self, pyfile): + password = self.getPassword() + + data = json_loads(self.load("http://www.alldebrid.com/service.php", + get={'link': pyfile.url, 'json': "true", 'pw': password})) + + self.logDebug("Json data", data) + + if data['error']: + if data['error'] == "This link isn't available on the hoster website.": + self.offline() + else: + self.logWarning(data['error']) + self.tempOffline() + else: + if pyfile.name and not pyfile.name.endswith('.tmp'): + pyfile.name = data['filename'] + pyfile.size = parseFileSize(data['filesize']) + self.link = data['link'] + + if self.getConfig("ssl"): + self.link = self.link.replace("http://", "https://") + else: + self.link = self.link.replace("https://", "http://") + + if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown"): + #only use when name wasnt already set + pyfile.name = self.getFilename(self.link) + + + def checkFile(self): + if self.checkDownload({'error': "An error occured while processing your request"}) == "error": + self.retry(wait_time=60, reason=_("An error occured while generating link")) + + return super(AlldebridCom, self).checkFile() diff --git a/pyload/plugin/hoster/AndroidfilehostCom.py b/pyload/plugin/hoster/AndroidfilehostCom.py new file mode 100644 index 000000000..386df4f2c --- /dev/null +++ b/pyload/plugin/hoster/AndroidfilehostCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -* +# +# Test links: +# https://www.androidfilehost.com/?fid=95916177934518197 + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class AndroidfilehostCom(SimpleHoster): + __name__ = "AndroidfilehostCom" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?androidfilehost\.com/\?fid=\d+' + + __description__ = """Androidfilehost.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'
(?P.*?)' + SIZE_PATTERN = r'

size

\s*

(?P[\d.,]+)(?P[\w^_]+)

' + HASHSUM_PATTERN = r'

(?P.*?)

\s*

(?P.*?)

' + + OFFLINE_PATTERN = r'404 not found' + + WAIT_PATTERN = r'users must wait (\d+) secs' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + wait = re.search(self.WAIT_PATTERN, self.html) + self.logDebug("Waiting time: %s seconds" % wait.group(1)) + + fid = re.search(r'id="fid" value="(\d+)" />', self.html).group(1) + self.logDebug("fid: %s" % fid) + + html = self.load("https://www.androidfilehost.com/libs/otf/mirrors.otf.php", + post={'submit': 'submit', + 'action': 'getdownloadmirrors', + 'fid' : fid}, + decode=True) + + self.link = re.findall('"url":"(.*?)"', html)[0].replace("\\", "") + mirror_host = self.link.split("/")[2] + + self.logDebug("Mirror Host: %s" % mirror_host) + + html = self.load("https://www.androidfilehost.com/libs/otf/stats.otf.php", + get={'fid' : fid, + 'w' : 'download', + 'mirror': mirror_host}, + decode=True) diff --git a/pyload/plugin/hoster/BasketbuildCom.py b/pyload/plugin/hoster/BasketbuildCom.py new file mode 100644 index 000000000..fb34bbc40 --- /dev/null +++ b/pyload/plugin/hoster/BasketbuildCom.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -* +# +# Test links: +# https://s.basketbuild.com/filedl/devs?dev=pacman&dl=pacman/falcon/RC-3/pac_falcon-RC-3-20141103.zip +# https://s.basketbuild.com/filedl/gapps?dl=gapps-gb-20110828-signed.zip + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class BasketbuildCom(SimpleHoster): + __name__ = "BasketbuildCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'https?://(?:www\.)?(?:\w\.)?basketbuild\.com/filedl/.+' + + __description__ = """basketbuild.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'File Name: (?P.+?)
' + SIZE_PATTERN = r'File Size: (?P[\d.,]+) (?P[\w^_]+)' + OFFLINE_PATTERN = r'404 - Page Not Found' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + try: + link1 = re.search(r'href="(.+dlgate/.+)"', self.html).group(1) + self.html = self.load(link1) + + except AttributeError: + self.error(_("Hop #1 not found")) + + else: + self.logDebug("Next hop: %s" % link1) + + try: + wait = re.search(r'var sec = (\d+)', self.html).group(1) + self.logDebug("Wait %s seconds" % wait) + self.wait(wait) + + except AttributeError: + self.logDebug("No wait time found") + + try: + self.link = re.search(r'id="dlLink">\s*
\w+/\w+/[^/]+)' + + __description__ = """Bayfiles.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] diff --git a/pyload/plugin/hoster/BezvadataCz.py b/pyload/plugin/hoster/BezvadataCz.py new file mode 100644 index 000000000..cea32dd5c --- /dev/null +++ b/pyload/plugin/hoster/BezvadataCz.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class BezvadataCz(SimpleHoster): + __name__ = "BezvadataCz" + __type__ = "hoster" + __version__ = "0.26" + + __pattern__ = r'http://(?:www\.)?bezvadata\.cz/stahnout/.+' + + __description__ = """BezvaData.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'

Soubor: (?P[^<]+)

' + SIZE_PATTERN = r'
  • Velikost: (?P[^<]+)
  • ' + OFFLINE_PATTERN = r'BezvaData \| Soubor nenalezen' + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + + + def handleFree(self, pyfile): + #download button + m = re.search(r'
    ', self.html) + if m is None: + self.error(_("Page 2 URL not found")) + url = "http://bezvadata.cz%s" % m.group(1) + self.logDebug("DL URL %s" % url) + + #countdown + m = re.search(r'id="countdown">(\d\d):(\d\d)<', self.html) + wait_time = (int(m.group(1)) * 60 + int(m.group(2))) if m else 120 + self.wait(wait_time, False) + + self.download(url) + + + def checkErrors(self): + if 'images/button-download-disable.png' in self.html: + self.longWait(5 * 60, 24) #: parallel dl limit + elif '
    (?P[\d.,]+) (?P[\w^_]+)' diff --git a/pyload/plugin/hoster/BitshareCom.py b/pyload/plugin/hoster/BitshareCom.py new file mode 100644 index 000000000..81bc8dae9 --- /dev/null +++ b/pyload/plugin/hoster/BitshareCom.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re + +from pyload.plugin.internal.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class BitshareCom(SimpleHoster): + __name__ = "BitshareCom" + __type__ = "hoster" + __version__ = "0.53" + + __pattern__ = r'http://(?:www\.)?bitshare\.com/(files/)?(?(1)|\?f=)(?P\w+)(?(1)/(?P.+?)\.html)' + + __description__ = """Bitshare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Paul King", ""), + ("fragonib", "fragonib[AT]yahoo[DOT]es")] + + + COOKIES = [("bitshare.com", "language_selection", "EN")] + + INFO_PATTERN = r'Downloading (?P.+) - (?P[\d.,]+) (?P[\w^_]+)' + OFFLINE_PATTERN = r'[Ff]ile (not available|was deleted|was not found)' + + AJAXID_PATTERN = r'var ajaxdl = "(.*?)";' + TRAFFIC_USED_UP = r'Your Traffic is used up for today' + + + def setup(self): + self.multiDL = self.premium + self.chunkLimit = 1 + + + def process(self, pyfile): + if self.premium: + self.account.relogin(self.user) + + # File id + m = re.match(self.__pattern__, pyfile.url) + self.file_id = max(m.group('ID1'), m.group('ID2')) + self.logDebug("File id is [%s]" % self.file_id) + + # Load main page + self.html = self.load(pyfile.url, ref=False, decode=True) + + # Check offline + if re.search(self.OFFLINE_PATTERN, self.html): + self.offline() + + # Check Traffic used up + if re.search(self.TRAFFIC_USED_UP, self.html): + self.logInfo(_("Your Traffic is used up for today")) + self.wait(30 * 60, True) + self.retry() + + # File name + m = re.match(self.__pattern__, pyfile.url) + name1 = m.group('NAME') if m else None + + m = re.search(self.INFO_PATTERN, self.html) + name2 = m.group('N') if m else None + + pyfile.name = max(name1, name2) + + # Ajax file id + self.ajaxid = re.search(self.AJAXID_PATTERN, self.html).group(1) + self.logDebug("File ajax id is [%s]" % self.ajaxid) + + # This may either download our file or forward us to an error page + self.download(self.getDownloadUrl()) + + if self.checkDownload({"error": ">Error occured<"}): + self.retry(5, 5 * 60, "Bitshare host : Error occured") + + + def getDownloadUrl(self): + # Return location if direct download is active + if self.premium: + header = self.load(self.pyfile.url, cookies=True, just_header=True) + if 'location' in header: + return header['location'] + + # Get download info + self.logDebug("Getting download info") + res = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + post={"request": "generateID", "ajaxid": self.ajaxid}) + + self.handleErrors(res, ':') + + parts = res.split(":") + filetype = parts[0] + wait = int(parts[1]) + captcha = int(parts[2]) + + self.logDebug("Download info [type: '%s', waiting: %d, captcha: %d]" % (filetype, wait, captcha)) + + # Waiting + if wait > 0: + self.logDebug("Waiting %d seconds." % wait) + if wait < 120: + self.wait(wait, False) + else: + self.wait(wait - 55, True) + self.retry() + + # Resolve captcha + if captcha == 1: + self.logDebug("File is captcha protected") + recaptcha = ReCaptcha(self) + + # Try up to 3 times + for i in xrange(3): + response, challenge = recaptcha.challenge() + res = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + post={"request" : "validateCaptcha", + "ajaxid" : self.ajaxid, + "recaptcha_challenge_field": challenge, + "recaptcha_response_field" : response}) + if self.handleCaptchaErrors(res): + break + + # Get download URL + self.logDebug("Getting download url") + res = self.load("http://bitshare.com/files-ajax/" + self.file_id + "/request.html", + post={"request": "getDownloadURL", "ajaxid": self.ajaxid}) + + self.handleErrors(res, '#') + + url = res.split("#")[-1] + + return url + + + def handleErrors(self, res, separator): + self.logDebug("Checking response [%s]" % res) + if "ERROR:Session timed out" in res: + self.retry() + elif "ERROR" in res: + msg = res.split(separator)[-1] + self.fail(msg) + + + def handleCaptchaErrors(self, res): + self.logDebug("Result of captcha resolving [%s]" % res) + if "SUCCESS" in res: + self.correctCaptcha() + return True + elif "ERROR:SESSION ERROR" in res: + self.retry() + + self.invalidCaptcha() diff --git a/pyload/plugin/hoster/BoltsharingCom.py b/pyload/plugin/hoster/BoltsharingCom.py new file mode 100644 index 000000000..39e84cd7c --- /dev/null +++ b/pyload/plugin/hoster/BoltsharingCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class BoltsharingCom(DeadHoster): + __name__ = "BoltsharingCom" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?boltsharing\.com/\w{12}' + + __description__ = """Boltsharing.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/CatShareNet.py b/pyload/plugin/hoster/CatShareNet.py new file mode 100644 index 000000000..94a963c45 --- /dev/null +++ b/pyload/plugin/hoster/CatShareNet.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class CatShareNet(SimpleHoster): + __name__ = "CatShareNet" + __type__ = "hoster" + __version__ = "0.11" + + __pattern__ = r'http://(?:www\.)?catshare\.net/\w{16}' + + __description__ = """CatShare.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("z00nx", "z00nx0@gmail.com"), + ("prOq", ""), + ("Walter Purcaro", "vuolter@gmail.com")] + + + TEXT_ENCODING = True + + INFO_PATTERN = r'(?P<N>.+) \((?P<S>[\d.,]+) (?P<U>[\w^_]+)\)<' + OFFLINE_PATTERN = ur'Podany plik został usunięty\s*</div>' + + IP_BLOCKED_PATTERN = ur'>Nasz serwis wykrył że Twój adres IP nie pochodzi z Polski.<' + WAIT_PATTERN = r'var\scount\s=\s(\d+);' + + LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'<form action="(.+?)" method="GET">' + + + def setup(self): + self.multiDL = self.premium + self.resumeDownload = True + + + def getFileInfo(self): + m = re.search(self.IP_BLOCKED_PATTERN, self.html) + if m: + self.fail(_("Only connections from Polish IP address are allowed")) + return super(CatShareNet, self).getFileInfo() + + + def handleFree(self, pyfile): + recaptcha = ReCaptcha(self) + + response, challenge = recaptcha.challenge() + self.html = self.load(pyfile.url, + post={'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.invalidCaptcha() + self.retry(reason=_("Wrong captcha entered")) + + self.link = m.group(1) diff --git a/pyload/plugin/hoster/CloudzerNet.py b/pyload/plugin/hoster/CloudzerNet.py new file mode 100644 index 000000000..c3154e9f1 --- /dev/null +++ b/pyload/plugin/hoster/CloudzerNet.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class CloudzerNet(DeadHoster): + __name__ = "CloudzerNet" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'https?://(?:www\.)?(cloudzer\.net/file/|clz\.to/(file/)?)\w+' + + __description__ = """Cloudzer.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("gs", "I-_-I-_-I@web.de"), + ("z00nx", "z00nx0@gmail.com"), + ("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/hoster/CloudzillaTo.py b/pyload/plugin/hoster/CloudzillaTo.py new file mode 100644 index 000000000..e33e2ebe4 --- /dev/null +++ b/pyload/plugin/hoster/CloudzillaTo.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class CloudzillaTo(SimpleHoster): + __name__ = "CloudzillaTo" + __type__ = "hoster" + __version__ = "0.06" + + __pattern__ = r'http://(?:www\.)?cloudzilla\.to/share/file/(?P<ID>[\w^_]+)' + + __description__ = """Cloudzilla.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + INFO_PATTERN = r'title="(?P<N>.+?)">\1</span> <span class="size">\((?P<S>[\d.]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'>File not found...<' + + PASSWORD_PATTERN = r'<div id="pwd_protected">' + + + def checkErrors(self): + m = re.search(self.PASSWORD_PATTERN, self.html) + if m: + self.html = self.load(self.pyfile.url, get={'key': self.getPassword()}) + + if re.search(self.PASSWORD_PATTERN, self.html): + self.retry(reason="Wrong password") + + + def handleFree(self, pyfile): + self.html = self.load("http://www.cloudzilla.to/generateticket/", + post={'file_id': self.info['pattern']['ID'], 'key': self.getPassword()}) + + ticket = dict(re.findall(r'<(.+?)>([^<>]+?)</', self.html)) + + self.logDebug(ticket) + + if 'error' in ticket: + if "File is password protected" in ticket['error']: + self.retry(reason="Wrong password") + else: + self.fail(ticket['error']) + + if 'wait' in ticket: + self.wait(ticket['wait'], int(ticket['wait']) > 5) + + self.link = "http://%(server)s/download/%(file_id)s/%(ticket_id)s" % {'server' : ticket['server'], + 'file_id' : self.info['pattern']['ID'], + 'ticket_id': ticket['ticket_id']} + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) diff --git a/pyload/plugin/hoster/CramitIn.py b/pyload/plugin/hoster/CramitIn.py new file mode 100644 index 000000000..3ccb3cfc6 --- /dev/null +++ b/pyload/plugin/hoster/CramitIn.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class CramitIn(XFSHoster): + __name__ = "CramitIn" + __type__ = "hoster" + __version__ = "0.07" + + __pattern__ = r'http://(?:www\.)?cramit\.in/\w{12}' + + __description__ = """Cramit.in hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + INFO_PATTERN = r'<span class=t2>\s*(?P<N>.*?)</span>.*?<small>\s*\((?P<S>.*?)\)' + + LINK_PATTERN = r'href="(http://cramit\.in/file_download/.*?)"' diff --git a/pyload/plugin/hoster/CrockoCom.py b/pyload/plugin/hoster/CrockoCom.py new file mode 100644 index 000000000..bcbbc84a1 --- /dev/null +++ b/pyload/plugin/hoster/CrockoCom.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class CrockoCom(SimpleHoster): + __name__ = "CrockoCom" + __type__ = "hoster" + __version__ = "0.19" + + __pattern__ = r'http://(?:www\.)?(crocko|easy-share)\.com/\w+' + + __description__ = """Crocko hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'<span class="fz24">Download:\s*<strong>(?P<N>.*)' + SIZE_PATTERN = r'<span class="tip1"><span class="inner">(?P<S>[^<]+)</span></span>' + OFFLINE_PATTERN = r'<h1>Sorry,<br />the page you\'re looking for <br />isn\'t here.</h1>|File not found' + + CAPTCHA_PATTERN = re.compile(r"u='(/file_contents/captcha/\w+)';\s*w='(\d+)';") + + FORM_PATTERN = r'<form method="post" action="([^"]+)">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' + + NAME_REPLACEMENTS = [(r'<[^>]*>', '')] + + + def handleFree(self, pyfile): + if "You need Premium membership to download this file." in self.html: + self.fail(_("You need Premium membership to download this file")) + + for _i in xrange(5): + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + url, wait_time = 'http://crocko.com' + m.group(1), int(m.group(2)) + self.wait(wait_time) + self.html = self.load(url) + else: + break + + m = re.search(self.FORM_PATTERN, self.html, re.S) + if m is None: + self.error(_("FORM_PATTERN not found")) + + action, form = m.groups() + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + recaptcha = ReCaptcha(self) + + for _i in xrange(5): + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge() + self.download(action, post=inputs) + + if self.checkDownload({"captcha": recaptcha.KEY_AJAX_PATTERN}): + self.invalidCaptcha() + else: + break + else: + self.fail(_("No valid captcha solution received")) diff --git a/pyload/plugin/hoster/CyberlockerCh.py b/pyload/plugin/hoster/CyberlockerCh.py new file mode 100644 index 000000000..c4e1d41b1 --- /dev/null +++ b/pyload/plugin/hoster/CyberlockerCh.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class CyberlockerCh(DeadHoster): + __name__ = "CyberlockerCh" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?cyberlocker\.ch/\w+' + + __description__ = """Cyberlocker.ch hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/hoster/CzshareCom.py b/pyload/plugin/hoster/CzshareCom.py new file mode 100644 index 000000000..3bf371508 --- /dev/null +++ b/pyload/plugin/hoster/CzshareCom.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://czshare.com/5278880/random.bin + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster +from pyload.utils import parseFileSize + + +class CzshareCom(SimpleHoster): + __name__ = "CzshareCom" + __type__ = "hoster" + __version__ = "0.98" + + __pattern__ = r'http://(?:www\.)?(czshare|sdilej)\.(com|cz)/(\d+/|download\.php\?).+' + + __description__ = """CZshare.com hoster plugin, now Sdilej.cz""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=[^>]*>(?P<N>[^<]+)</a>' + SIZE_PATTERN = r'<div class="tab" id="category">(?:\s*<p>[^\n]*</p>)*\s*Velikost:\s*(?P<S>[\d .,]+)(?P<U>[\w^_]+)\s*</div>' + OFFLINE_PATTERN = r'<div class="header clearfix">\s*<h2 class="red">' + + SIZE_REPLACEMENTS = [(' ', '')] + URL_REPLACEMENTS = [(r'http://[^/]*/download.php\?.*?id=(\w+).*', r'http://sdilej.cz/\1/x/')] + + CHECK_TRAFFIC = True + + FREE_URL_PATTERN = r'<a href="([^"]+)" class="page-download">[^>]*alt="([^"]+)" /></a>' + FREE_FORM_PATTERN = r'<form action="download\.php" method="post">\s*<img src="captcha\.php" id="captcha" />(.*?)</form>' + PREMIUM_FORM_PATTERN = r'<form action="/profi_down\.php" method="post">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)"[^>]*/>' + MULTIDL_PATTERN = r'<p><font color=\'red\'>Z[^<]*PROFI.</font></p>' + USER_CREDIT_PATTERN = r'<div class="credit">\s*kredit: <strong>([\d .,]+)(\w+)</strong>\s*</div><!-- .credit -->' + + + def checkTrafficLeft(self): + # check if user logged in + m = re.search(self.USER_CREDIT_PATTERN, self.html) + if m is None: + self.account.relogin(self.user) + self.html = self.load(self.pyfile.url, cookies=True, decode=True) + m = re.search(self.USER_CREDIT_PATTERN, self.html) + if m is None: + return False + + # check user credit + try: + credit = parseFileSize(m.group(1).replace(' ', ''), m.group(2)) + self.logInfo(_("Premium download for %i KiB of Credit") % (self.pyfile.size / 1024)) + self.logInfo(_("User %s has %i KiB left") % (self.user, credit / 1024)) + if credit < self.pyfile.size: + self.logInfo(_("Not enough credit to download file: %s") % self.pyfile.name) + return False + except Exception, e: + # let's continue and see what happens... + self.logError(e) + + return True + + + def handlePremium(self, pyfile): + # parse download link + try: + form = re.search(self.PREMIUM_FORM_PATTERN, self.html, re.S).group(1) + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + except Exception, e: + self.logError(e) + self.resetAccount() + + # download the file, destination is determined by pyLoad + self.download("http://sdilej.cz/profi_down.php", post=inputs, disposition=True) + + + def handleFree(self, pyfile): + # get free url + m = re.search(self.FREE_URL_PATTERN, self.html) + if m is None: + self.error(_("FREE_URL_PATTERN not found")) + + parsed_url = "http://sdilej.cz" + m.group(1) + + self.logDebug("PARSED_URL:" + parsed_url) + + # get download ticket and parse html + self.html = self.load(parsed_url, cookies=True, decode=True) + if re.search(self.MULTIDL_PATTERN, self.html): + self.longWait(5 * 60, 12) + + try: + form = re.search(self.FREE_FORM_PATTERN, self.html, re.S).group(1) + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + pyfile.size = int(inputs['size']) + + except Exception, e: + self.logError(e) + self.error(_("Form")) + + # get and decrypt captcha + captcha_url = 'http://sdilej.cz/captcha.php' + for _i in xrange(5): + inputs['captchastring2'] = self.decryptCaptcha(captcha_url) + self.html = self.load(parsed_url, cookies=True, post=inputs, decode=True) + + if u"<li>Zadaný ověřovací kód nesouhlasí!</li>" in self.html: + self.invalidCaptcha() + + elif re.search(self.MULTIDL_PATTERN, self.html): + self.longWait(5 * 60, 12) + + else: + self.correctCaptcha() + break + else: + self.fail(_("No valid captcha code entered")) + + m = re.search("countdown_number = (\d+);", self.html) + self.setWait(int(m.group(1)) if m else 50) + + # download the file, destination is determined by pyLoad + self.logDebug("WAIT URL", self.req.lastEffectiveURL) + + m = re.search("free_wait.php\?server=(.*?)&(.*)", self.req.lastEffectiveURL) + if m is None: + self.error(_("Download URL not found")) + + self.link = "http://%s/download.php?%s" % (m.group(1), m.group(2)) + + self.wait() + + + def checkFile(self): + # check download + check = self.checkDownload({ + "temp offline" : re.compile(r"^Soubor je do.*asn.* nedostupn.*$"), + "credit" : re.compile(r"^Nem.*te dostate.*n.* kredit.$"), + "multi-dl" : re.compile(self.MULTIDL_PATTERN), + "captcha" : "<li>Zadaný ověřovací kód nesouhlasí!</li>" + }) + + if check == "temp offline": + self.fail(_("File not available - try later")) + + elif check == "credit": + self.resetAccount() + + elif check == "multi-dl": + self.longWait(5 * 60, 12) + + elif check == "captcha": + self.invalidCaptcha() + self.retry() + + return super(CzshareCom, self).checkFile() diff --git a/pyload/plugin/hoster/DailymotionCom.py b/pyload/plugin/hoster/DailymotionCom.py new file mode 100644 index 000000000..c212fa872 --- /dev/null +++ b/pyload/plugin/hoster/DailymotionCom.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.datatype.File import statusMap +from pyload.utils import json_loads +from pyload.network.RequestFactory import getURL +from pyload.plugin.Hoster import Hoster + + +def getInfo(urls): + result = [] + regex = re.compile(DailymotionCom.__pattern__) + apiurl = "https://api.dailymotion.com/video/%s" + request = {"fields": "access_error,status,title"} + + for url in urls: + id = regex.match(url).group('ID') + html = getURL(apiurl % id, get=request) + info = json_loads(html) + + name = info['title'] + ".mp4" if "title" in info else url + + if "error" in info or info['access_error']: + status = "offline" + else: + status = info['status'] + if status in ("ready", "published"): + status = "online" + elif status in ("waiting", "processing"): + status = "temp. offline" + else: + status = "offline" + + result.append((name, 0, statusMap[status], url)) + + return result + + +class DailymotionCom(Hoster): + __name__ = "DailymotionCom" + __type__ = "hoster" + __version__ = "0.20" + + __pattern__ = r'https?://(?:www\.)?dailymotion\.com/.*video/(?P<ID>[\w^_]+)' + __config__ = [("quality", "Lowest;LD 144p;LD 240p;SD 384p;HQ 480p;HD 720p;HD 1080p;Highest", "Quality", "Highest")] + + __description__ = """Dailymotion.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + + + def getStreams(self): + streams = [] + + for result in re.finditer(r"\"(?P<URL>http:\\/\\/www.dailymotion.com\\/cdn\\/H264-(?P<QF>.*?)\\.*?)\"", + self.html): + url = result.group('URL') + qf = result.group('QF') + + link = url.replace("\\", "") + quality = tuple(int(x) for x in qf.split("x")) + + streams.append((quality, link)) + + return sorted(streams, key=lambda x: x[0][::-1]) + + + def getQuality(self): + q = self.getConfig("quality") + + if q == "Lowest": + quality = 0 + elif q == "Highest": + quality = -1 + else: + quality = int(q.rsplit(" ")[1][:-1]) + + return quality + + + def getLink(self, streams, quality): + if quality > 0: + for x, s in [item for item in enumerate(streams)][::-1]: + qf = s[0][1] + if qf <= quality: + idx = x + break + else: + idx = 0 + else: + idx = quality + + s = streams[idx] + + self.logInfo(_("Download video quality %sx%s") % s[0]) + + return s[1] + + + def checkInfo(self, pyfile): + pyfile.name, pyfile.size, pyfile.status, pyfile.url = getInfo([pyfile.url])[0] + + if pyfile.status == 1: + self.offline() + + elif pyfile.status == 6: + self.tempOffline() + + + def process(self, pyfile): + self.checkInfo(pyfile) + + id = re.match(self.__pattern__, pyfile.url).group('ID') + self.html = self.load("http://www.dailymotion.com/embed/video/" + id, decode=True) + + streams = self.getStreams() + quality = self.getQuality() + + self.download(self.getLink(streams, quality)) diff --git a/pyload/plugin/hoster/DataHu.py b/pyload/plugin/hoster/DataHu.py new file mode 100644 index 000000000..219e73441 --- /dev/null +++ b/pyload/plugin/hoster/DataHu.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://data.hu/get/6381232/random.bin + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class DataHu(SimpleHoster): + __name__ = "DataHu" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?data\.hu/get/\w+' + + __description__ = """Data.hu hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("crash", ""), + ("stickell", "l.stickell@yahoo.it")] + + + INFO_PATTERN = ur'<title>(?P<N>.*) \((?P<S>[^)]+)\) let\xf6lt\xe9se' + OFFLINE_PATTERN = ur'Az adott f\xe1jl nem l\xe9tezik' + LINK_FREE_PATTERN = r'
    ' + + + def setup(self): + self.resumeDownload = True + self.multiDL = self.premium diff --git a/pyload/plugin/hoster/DataportCz.py b/pyload/plugin/hoster/DataportCz.py new file mode 100644 index 000000000..8a32400ec --- /dev/null +++ b/pyload/plugin/hoster/DataportCz.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class DataportCz(SimpleHoster): + __name__ = "DataportCz" + __type__ = "hoster" + __version__ = "0.41" + + __pattern__ = r'http://(?:www\.)?dataport\.cz/file/(.+)' + + __description__ = """Dataport.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'(?P[^<]+)' + SIZE_PATTERN = r'Velikost\s*(?P[^<]+)' + OFFLINE_PATTERN = r'

    Soubor nebyl nalezen

    ' + + CAPTCHA_PATTERN = r'
    \s*(\d+)
    ' + + + def handleFree(self, pyfile): + captchas = {"1": "jkeG", "2": "hMJQ", "3": "vmEK", "4": "ePQM", "5": "blBd"} + + for _i in xrange(60): + action, inputs = self.parseHtmlForm('free_download_form') + self.logDebug(action, inputs) + if not action or not inputs: + self.error(_("free_download_form")) + + if "captchaId" in inputs and inputs['captchaId'] in captchas: + inputs['captchaCode'] = captchas[inputs['captchaId']] + else: + self.error(_("captcha")) + + self.html = self.download("http://www.dataport.cz%s" % action, post=inputs) + + check = self.checkDownload({"captcha": 'alert("\u0160patn\u011b opsan\u00fd k\u00f3d z obr\u00e1zu");', + "slot" : 'alert("Je n\u00e1m l\u00edto, ale moment\u00e1ln\u011b nejsou'}) + if check == "captcha": + self.error(_("invalid captcha")) + + elif check == "slot": + self.logDebug("No free slots - wait 60s and retry") + self.wait(60, False) + self.html = self.load(pyfile.url, decode=True) + continue + + else: + break diff --git a/pyload/plugin/hoster/DateiTo.py b/pyload/plugin/hoster/DateiTo.py new file mode 100644 index 000000000..216a758c8 --- /dev/null +++ b/pyload/plugin/hoster/DateiTo.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class DateiTo(SimpleHoster): + __name__ = "DateiTo" + __type__ = "hoster" + __version__ = "0.07" + + __pattern__ = r'http://(?:www\.)?datei\.to/datei/(?P\w+)\.html' + + __description__ = """Datei.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'Dateiname:\s*(?P.*?)\s*(?P.*?)Datei wurde nicht gefunden<|>Bitte wähle deine Datei aus... <' + + WAIT_PATTERN = r'countdown\({seconds: (\d+)' + MULTIDL_PATTERN = r'>Du lädst bereits eine Datei herunter<' + + DATA_PATTERN = r'url: "(.*?)", data: "(.*?)",' + + + def handleFree(self, pyfile): + url = 'http://datei.to/ajax/download.php' + data = {'P': 'I', 'ID': self.info['pattern']['ID']} + recaptcha = ReCaptcha(self) + + for _i in xrange(10): + self.logDebug("URL", url, "POST", data) + self.html = self.load(url, post=data) + self.checkErrors() + + if url.endswith('download.php') and 'P' in data: + if data['P'] == 'I': + self.doWait() + + elif data['P'] == 'IV': + break + + m = re.search(self.DATA_PATTERN, self.html) + if m is None: + self.error(_("data")) + url = 'http://datei.to/' + m.group(1) + data = dict(x.split('=') for x in m.group(2).split('&')) + + if url.endswith('recaptcha.php'): + data['recaptcha_response_field'], data['recaptcha_challenge_field'] = recaptcha.challenge() + else: + self.fail(_("Too bad...")) + + self.download(self.html) + + + def checkErrors(self): + m = re.search(self.MULTIDL_PATTERN, self.html) + if m: + m = re.search(self.WAIT_PATTERN, self.html) + wait_time = int(m.group(1)) if m else 30 + + errmsg = self.info['error'] = _("Parallel downloads") + self.retry(wait_time=wait_time, reason=errmsg) + + self.info.pop('error', None) + + + def doWait(self): + m = re.search(self.WAIT_PATTERN, self.html) + wait_time = int(m.group(1)) if m else 30 + + self.load('http://datei.to/ajax/download.php', post={'P': 'Ads'}) + self.wait(wait_time, False) diff --git a/pyload/plugin/hoster/DdlstorageCom.py b/pyload/plugin/hoster/DdlstorageCom.py new file mode 100644 index 000000000..cfde84ad0 --- /dev/null +++ b/pyload/plugin/hoster/DdlstorageCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class DdlstorageCom(DeadHoster): + __name__ = "DdlstorageCom" + __type__ = "hoster" + __version__ = "1.02" + + __pattern__ = r'https?://(?:www\.)?ddlstorage\.com/\w+' + + __description__ = """DDLStorage.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/hoster/DebridItaliaCom.py b/pyload/plugin/hoster/DebridItaliaCom.py new file mode 100644 index 000000000..1e7e1a338 --- /dev/null +++ b/pyload/plugin/hoster/DebridItaliaCom.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHoster import MultiHoster + + +class DebridItaliaCom(MultiHoster): + __name__ = "DebridItaliaCom" + __type__ = "hoster" + __version__ = "0.17" + + __pattern__ = r'https?://(?:www\.|s\d+\.)?debriditalia\.com/dl/\d+' + + __description__ = """Debriditalia.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [("https://", "http://")] + + + def handlePremium(self, pyfile): + self.html = self.load("http://www.debriditalia.com/api.php", + get={'generate': "on", 'link': pyfile.url, 'p': self.getPassword()}) + + if "ERROR:" not in self.html: + self.link = self.html.strip() + else: + self.info['error'] = re.search(r'ERROR:(.*)', self.html).group(1).strip() + + self.html = self.load("http://debriditalia.com/linkgen2.php", + post={'xjxfun' : "convertiLink", + 'xjxargs[]': "S" % pyfile.url, + 'xjxargs[]': "S%s" % self.getPassword()}) + try: + self.link = re.search(r'
    \w+)' + + __description__ = """Depositfiles.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'' + + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + self.getFileInfo() + + # parse js variables + self.jsvars = dict((x, y.strip("'")) for x, y in re.findall(r"var (\w+) = ([\d.]+|'[^']*')", self.html)) + self.logDebug(self.jsvars) + pyfile.name = self.jsvars['ID3'] + + # determine download type - free or premium + if self.premium: + if 'UU_prihlasen' in self.jsvars: + if self.jsvars['UU_prihlasen'] == '0': + self.logWarning(_("User not logged in")) + self.relogin(self.user) + self.retry() + elif float(self.jsvars['UU_kredit']) < float(self.jsvars['kredit_odecet']): + self.logWarning(_("Not enough credit left")) + self.premium = False + + if self.premium: + self.handlePremium(pyfile) + else: + self.handleFree(pyfile) + + if self.checkDownload({"error": re.compile(r"\AChyba!")}, max_size=100): + self.fail(_("File not m or plugin defect")) + + + def handleFree(self, pyfile): + # get download url + download_url = '%s/download.php' % self.jsvars['server'] + data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ("ID1", "ID2", "ID3", "ID4")) + self.logDebug("FREE URL1:" + download_url, data) + + self.load(download_url, post=data, follow_location=False) + self.header = self.req.http.header + + m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) + if m is None: + self.fail(_("File not found")) + download_url = m.group(1) + self.logDebug("FREE URL2:" + download_url) + + # check errors + m = re.search(r'/chyba/(\d+)', download_url) + if m: + if m.group(1) == '1': + self.retry(60, 2 * 60, "This IP is already downloading") + elif m.group(1) == '2': + self.retry(60, 60, "No free slots available") + else: + self.fail(_("Error %d") % m.group(1)) + + # download file + self.download(download_url) + + + def handlePremium(self, pyfile): + download_url = '%s/download_premium.php' % self.jsvars['server'] + data = dict((x, self.jsvars[x]) for x in self.jsvars if x in ("ID1", "ID2", "ID4", "ID5")) + self.download(download_url, get=data) diff --git a/pyload/plugin/hoster/RPNetBiz.py b/pyload/plugin/hoster/RPNetBiz.py new file mode 100644 index 000000000..af56d8162 --- /dev/null +++ b/pyload/plugin/hoster/RPNetBiz.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHoster import MultiHoster +from pyload.utils import json_loads + + +class RPNetBiz(MultiHoster): + __name__ = "RPNetBiz" + __type__ = "hoster" + __version__ = "0.14" + + __description__ = """RPNet.biz multi-hoster plugin""" + __license__ = "GPLv3" + + __pattern__ = r'https?://.+rpnet\.biz' + __authors__ = [("Dman", "dmanugm@gmail.com")] + + + def setup(self): + self.chunkLimit = -1 + + + def handlePremium(self, pyfile): + user, data = self.account.selectAccount() + + # Get the download link + res = self.load("https://premium.rpnet.biz/client_api.php", + get={"username": user, + "password": data['password'], + "action" : "generate", + "links" : pyfile.url}) + + self.logDebug("JSON data: %s" % res) + link_status = json_loads(res)['links'][0] # get the first link... since we only queried one + + # Check if we only have an id as a HDD link + if 'id' in link_status: + self.logDebug("Need to wait at least 30 seconds before requery") + self.setWait(30) # wait for 30 seconds + self.wait() + # Lets query the server again asking for the status on the link, + # we need to keep doing this until we reach 100 + max_tries = 30 + my_try = 0 + while (my_try <= max_tries): + self.logDebug("Try: %d ; Max Tries: %d" % (my_try, max_tries)) + res = self.load("https://premium.rpnet.biz/client_api.php", + get={"username": user, + "password": data['password'], + "action": "downloadInformation", + "id": link_status['id']}) + self.logDebug("JSON data hdd query: %s" % res) + download_status = json_loads(res)['download'] + + if download_status['status'] == '100': + link_status['generated'] = download_status['rpnet_link'] + self.logDebug("Successfully downloaded to rpnet HDD: %s" % link_status['generated']) + break + else: + self.logDebug("At %s%% for the file download" % download_status['status']) + + self.setWait(30) + self.wait() + my_try += 1 + + if my_try > max_tries: # We went over the limit! + self.fail(_("Waited for about 15 minutes for download to finish but failed")) + + if 'generated' in link_status: + self.link = link_status['generated'] + return + elif 'error' in link_status: + self.fail(link_status['error']) + else: + self.fail(_("Something went wrong, not supposed to enter here")) diff --git a/pyload/plugin/hoster/RapideoPl.py b/pyload/plugin/hoster/RapideoPl.py new file mode 100644 index 000000000..9e9bbf58b --- /dev/null +++ b/pyload/plugin/hoster/RapideoPl.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHoster import MultiHoster + + +class RapideoPl(MultiHoster): + __name__ = "RapideoPl" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'^unmatchable$' + + __description__ = """Rapideo.pl multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("goddie", "dev@rapideo.pl")] + + + API_URL = "http://enc.rapideo.pl" + + API_QUERY = {'site' : "newrd", + 'output' : "json", + 'username': "", + 'password': "", + 'url' : ""} + + ERROR_CODES = {0 : "[%s] Incorrect login credentials", + 1 : "[%s] Not enough transfer to download - top-up your account", + 2 : "[%s] Incorrect / dead link", + 3 : "[%s] Error connecting to hosting, try again later", + 9 : "[%s] Premium account has expired", + 15: "[%s] Hosting no longer supported", + 80: "[%s] Too many incorrect login attempts, account blocked for 24h"} + + + def prepare(self): + super(RapideoPl, self).prepare() + + data = self.account.getAccountData(self.user) + + self.usr = data['usr'] + self.pwd = data['pwd'] + + + def runFileQuery(self, url, mode=None): + query = self.API_QUERY.copy() + + query["username"] = self.usr + query["password"] = self.pwd + query["url"] = url + + if mode == "fileinfo": + query['check'] = 2 + query['loc'] = 1 + + self.logDebug(query) + + return self.load(self.API_URL, post=query) + + + def handleFree(self, pyfile): + try: + data = self.runFileQuery(pyfile.url, 'fileinfo') + + except Exception: + self.logDebug("RunFileQuery error") + self.tempOffline() + + try: + parsed = json_loads(data) + + except Exception: + self.logDebug("Loads error") + self.tempOffline() + + self.logDebug(parsed) + + if "errno" in parsed.keys(): + if parsed["errno"] in self.ERROR_CODES: + # error code in known + self.fail(self.ERROR_CODES[parsed["errno"]] % self.__name__) + else: + # error code isn't yet added to plugin + self.fail( + parsed["errstring"] + or _("Unknown error (code: %s)") % parsed["errno"] + ) + + if "sdownload" in parsed: + if parsed["sdownload"] == "1": + self.fail( + _("Download from %s is possible only using Rapideo.pl website \ + directly") % parsed["hosting"]) + + pyfile.name = parsed["filename"] + pyfile.size = parsed["filesize"] + + try: + self.link = self.runFileQuery(pyfile.url, 'filedownload') + + except Exception: + self.logDebug("runFileQuery error #2") + self.tempOffline() diff --git a/pyload/plugin/hoster/RapidfileshareNet.py b/pyload/plugin/hoster/RapidfileshareNet.py new file mode 100644 index 000000000..1b1895b39 --- /dev/null +++ b/pyload/plugin/hoster/RapidfileshareNet.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class RapidfileshareNet(XFSHoster): + __name__ = "RapidfileshareNet" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?rapidfileshare\.net/\w{12}' + + __description__ = """Rapidfileshare.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("guidobelix", "guidobelix@hotmail.it")] + + + NAME_PATTERN = r'' + SIZE_PATTERN = r'>http://www.rapidfileshare.net/\w+? \((?P[\d.,]+) (?P[\w^_]+)\)' + + OFFLINE_PATTERN = r'>No such file with this filename' + TEMP_OFFLINE_PATTERN = r'The page may have been renamed, removed or be temporarily unavailable.<' diff --git a/pyload/plugin/hoster/RapidgatorNet.py b/pyload/plugin/hoster/RapidgatorNet.py new file mode 100644 index 000000000..8cd883537 --- /dev/null +++ b/pyload/plugin/hoster/RapidgatorNet.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- + +import re + +from pycurl import HTTPHEADER + +from pyload.utils import json_loads +from pyload.network.HTTPRequest import BadHeader +from pyload.plugin.internal.CaptchaService import AdsCaptcha, ReCaptcha, SolveMedia +from pyload.plugin.internal.SimpleHoster import SimpleHoster, secondsToMidnight + + +class RapidgatorNet(SimpleHoster): + __name__ = "RapidgatorNet" + __type__ = "hoster" + __version__ = "0.32" + + __pattern__ = r'http://(?:www\.)?(rapidgator\.net|rg\.to)/file/\w+' + + __description__ = """Rapidgator.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("chrox", ""), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + API_URL = "http://rapidgator.net/api/file" + + COOKIES = [("rapidgator.net", "lang", "en")] + + NAME_PATTERN = r'Download file (?P<N>.*)' + SIZE_PATTERN = r'File size:\s*(?P[\d.,]+) (?P[\w^_]+)' + OFFLINE_PATTERN = r'>(File not found|Error 404)' + + JSVARS_PATTERN = r'\s+var\s*(startTimerUrl|getDownloadUrl|captchaUrl|fid|secs)\s*=\s*\'?(.*?)\'?;' + + PREMIUM_ONLY_PATTERN = r'You can download files up to|This file can be downloaded by premium only<' + ERROR_PATTERN = r'You have reached your (daily|hourly) downloads limit' + WAIT_PATTERN = r'(Delay between downloads must be not less than|Try again in).+' + + LINK_FREE_PATTERN = r'return \'(http://\w+.rapidgator.net/.*)\';' + + RECAPTCHA_PATTERN = r'"http://api\.recaptcha\.net/challenge\?k=(.*?)"' + ADSCAPTCHA_PATTERN = r'(http://api\.adscaptcha\.com/Get\.aspx[^"\']*)' + SOLVEMEDIA_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.script\?k=(.*?)"' + + + def setup(self): + if self.account: + self.sid = self.account.getAccountInfo(self.user).get('sid', None) + else: + self.sid = None + + if self.sid: + self.premium = True + + self.resumeDownload = self.multiDL = self.premium + self.chunkLimit = 1 + + + def api_response(self, cmd): + try: + json = self.load('%s/%s' % (self.API_URL, cmd), + get={'sid': self.sid, + 'url': self.pyfile.url}, decode=True) + self.logDebug("API:%s" % cmd, json, "SID: %s" % self.sid) + json = json_loads(json) + status = json['response_status'] + msg = json['response_details'] + + except BadHeader, e: + self.logError("API: %s" % cmd, e, "SID: %s" % self.sid) + status = e.code + msg = e + + if status == 200: + return json['response'] + + elif status == 423: + self.account.empty(self.user) + self.retry() + + else: + self.account.relogin(self.user) + self.retry(wait_time=60) + + + def handlePremium(self, pyfile): + self.api_data = self.api_response('info') + self.api_data['md5'] = self.api_data['hash'] + + pyfile.name = self.api_data['filename'] + pyfile.size = self.api_data['size'] + + self.link = self.api_response('download')['url'] + + + def handleFree(self, pyfile): + jsvars = dict(re.findall(self.JSVARS_PATTERN, self.html)) + self.logDebug(jsvars) + + self.req.http.lastURL = pyfile.url + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + + url = "http://rapidgator.net%s?fid=%s" % ( + jsvars.get('startTimerUrl', '/download/AjaxStartTimer'), jsvars['fid']) + jsvars.update(self.getJsonResponse(url)) + + self.wait(jsvars.get('secs', 45), False) + + url = "http://rapidgator.net%s?sid=%s" % ( + jsvars.get('getDownloadUrl', '/download/AjaxGetDownload'), jsvars['sid']) + jsvars.update(self.getJsonResponse(url)) + + self.req.http.lastURL = pyfile.url + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + + url = "http://rapidgator.net%s" % jsvars.get('captchaUrl', '/download/captcha') + self.html = self.load(url) + + for _i in xrange(5): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + self.link = m.group(1) + break + else: + captcha, captcha_key = self.handleCaptcha() + response, challenge = captcha.challenge(captcha_key) + + self.html = self.load(url, post={'DownloadCaptchaForm[captcha]': "", + 'adcopy_challenge' : challenge, + 'adcopy_response' : response}) + + if "The verification code is incorrect" in self.html: + self.invalidCaptcha() + else: + self.correctCaptcha() + else: + self.error(_("Download link")) + + + def handleCaptcha(self): + m = re.search(self.ADSCAPTCHA_PATTERN, self.html) + if m: + captcha_key = m.group(1) + captcha = AdsCaptcha(self) + else: + m = re.search(self.RECAPTCHA_PATTERN, self.html) + if m: + captcha_key = m.group(1) + captcha = ReCaptcha(self) + else: + m = re.search(self.SOLVEMEDIA_PATTERN, self.html) + if m: + captcha_key = m.group(1) + captcha = SolveMedia(self) + else: + self.error(_("Captcha")) + + return captcha, captcha_key + + + def getJsonResponse(self, url): + res = self.load(url, decode=True) + if not res.startswith('{'): + self.retry() + self.logDebug(url, res) + return json_loads(res) diff --git a/pyload/plugin/hoster/RapiduNet.py b/pyload/plugin/hoster/RapiduNet.py new file mode 100644 index 000000000..1e12ab776 --- /dev/null +++ b/pyload/plugin/hoster/RapiduNet.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +import re + +from pycurl import HTTPHEADER +from time import time, altzone + +from pyload.utils import json_loads +from pyload.plugin.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class RapiduNet(SimpleHoster): + __name__ = "RapiduNet" + __type__ = "hoster" + __version__ = "0.07" + + __pattern__ = r'https?://(?:www\.)?rapidu\.net/(?P\d{10})' + + __description__ = """Rapidu.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("prOq", "")] + + + COOKIES = [("rapidu.net", "rapidu_lang", "en")] + + INFO_PATTERN = r'

    .*

    \s*(?P\d+(\.\d+)?)\s(?P\w+)' + OFFLINE_PATTERN = r'404 - File not found' + + ERROR_PATTERN = r'
    ' + + RECAPTCHA_KEY = r'6Ld12ewSAAAAAHoE6WVP_pSfCdJcBQScVweQh8Io' + + + def setup(self): + self.resumeDownload = True + self.multiDL = self.premium + + + def handleFree(self, pyfile): + self.req.http.lastURL = pyfile.url + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + + jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", + get={'a': "getLoadTimeToDownload"}, + post={'_go': ""}, + decode=True) + + if str(jsvars['timeToDownload']) is "stop": + t = (24 * 60 * 60) - (int(time()) % (24 * 60 * 60)) + altzone + + self.logInfo("You've reach your daily download transfer") + + self.retry(10, 10 if t < 1 else None, _("Try tomorrow again")) #@NOTE: check t in case of not synchronised clock + + else: + self.wait(int(jsvars['timeToDownload']) - int(time())) + + recaptcha = ReCaptcha(self) + + for _i in xrange(10): + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) + + jsvars = self.getJsonResponse("https://rapidu.net/ajax.php", + get={'a': "getCheckCaptcha"}, + post={'_go' : "", + 'captcha1': challenge, + 'captcha2': response, + 'fileId' : self.info['pattern']['ID']}, + decode=True) + if jsvars['message'] == 'success': + self.download(jsvars['url']) + break + + + def getJsonResponse(self, *args, **kwargs): + res = self.load(*args, **kwargs) + if not res.startswith('{'): + self.retry() + + self.logDebug(res) + + return json_loads(res) diff --git a/pyload/plugin/hoster/RarefileNet.py b/pyload/plugin/hoster/RarefileNet.py new file mode 100644 index 000000000..1010c92ca --- /dev/null +++ b/pyload/plugin/hoster/RarefileNet.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class RarefileNet(XFSHoster): + __name__ = "RarefileNet" + __type__ = "hoster" + __version__ = "0.09" + + __pattern__ = r'http://(?:www\.)?rarefile\.net/\w{12}' + + __description__ = """Rarefile.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + LINK_PATTERN = r'\1' diff --git a/pyload/plugin/hoster/RealdebridCom.py b/pyload/plugin/hoster/RealdebridCom.py new file mode 100644 index 000000000..ae6f69d7c --- /dev/null +++ b/pyload/plugin/hoster/RealdebridCom.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +import re + +from random import randrange +from urllib import unquote +from time import time + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHoster import MultiHoster +from pyload.utils import parseFileSize + + +class RealdebridCom(MultiHoster): + __name__ = "RealdebridCom" + __type__ = "hoster" + __version__ = "0.64" + + __pattern__ = r'https?://((?:www\.|s\d+\.)?real-debrid\.com/dl/|[\w^_]\.rdb\.so/d/)[\w^_]+' + + __description__ = """Real-Debrid.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Devirex Hazzard", "naibaf_11@yahoo.de")] + + + def getFilename(self, url): + try: + name = unquote(url.rsplit("/", 1)[1]) + except IndexError: + name = "Unknown_Filename..." + if not name or name.endswith(".."): #: incomplete filename, append random stuff + name += "%s.tmp" % randrange(100, 999) + return name + + + def setup(self): + self.chunkLimit = 3 + + + def handlePremium(self, pyfile): + data = json_loads(self.load("https://real-debrid.com/ajax/unrestrict.php", + get={'lang' : "en", + 'link' : pyfile.url, + 'password': self.getPassword(), + 'time' : int(time() * 1000)})) + + self.logDebug("Returned Data: %s" % data) + + if data['error'] != 0: + if data['message'] == "Your file is unavailable on the hoster.": + self.offline() + else: + self.logWarning(data['message']) + self.tempOffline() + else: + if pyfile.name is not None and pyfile.name.endswith('.tmp') and data['file_name']: + pyfile.name = data['file_name'] + pyfile.size = parseFileSize(data['file_size']) + self.link = data['generated_links'][0][-1] + + if self.getConfig("ssl"): + self.link = self.link.replace("http://", "https://") + else: + self.link = self.link.replace("https://", "http://") + + if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): + #only use when name wasnt already set + pyfile.name = self.getFilename(self.link) + + + def checkFile(self): + if self.checkDownload({"error": "An error occured while processing your request"}): + #usual this download can safely be retried + self.retry(wait_time=60, reason=_("An error occured while generating link")) + + return super(RealdebridCom, self).checkFile() diff --git a/pyload/plugin/hoster/RedtubeCom.py b/pyload/plugin/hoster/RedtubeCom.py new file mode 100644 index 000000000..1f18d09c7 --- /dev/null +++ b/pyload/plugin/hoster/RedtubeCom.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Hoster import Hoster +from pyload.utils import html_unescape + + +class RedtubeCom(Hoster): + __name__ = "RedtubeCom" + __type__ = "hoster" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?redtube\.com/\d+' + + __description__ = """Redtube.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de")] + + + def process(self, pyfile): + self.download_html() + if not self.file_exists(): + self.offline() + + pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if not self.html: + self.download_html() + + file_url = html_unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) + + return file_url + + + def get_file_name(self): + if not self.html: + self.download_html() + + return re.search('(.*?)- RedTube - Free Porn Videos', self.html).group(1).strip() + ".flv" + + + def file_exists(self): + """ returns True or False + """ + if not self.html: + self.download_html() + + if re.search(r'This video has been removed.', self.html) is not None: + return False + else: + return True diff --git a/pyload/plugin/hoster/RehostTo.py b/pyload/plugin/hoster/RehostTo.py new file mode 100644 index 000000000..3e3e88c6c --- /dev/null +++ b/pyload/plugin/hoster/RehostTo.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from urllib import unquote + +from pyload.plugin.internal.MultiHoster import MultiHoster + + +class RehostTo(MultiHoster): + __name__ = "RehostTo" + __type__ = "hoster" + __version__ = "0.21" + + __pattern__ = r'https?://.*rehost\.to\..+' + + __description__ = """Rehost.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + + def handlePremium(self, pyfile): + self.download("http://rehost.to/process_download.php", + get={'user': "cookie", + 'pass': self.account.getAccountInfo(self.user)['session'], + 'dl' : pyfile.url}, + disposition=True) diff --git a/pyload/plugin/hoster/RemixshareCom.py b/pyload/plugin/hoster/RemixshareCom.py new file mode 100644 index 000000000..a1cd2a37e --- /dev/null +++ b/pyload/plugin/hoster/RemixshareCom.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://remixshare.com/download/p946u +# +# Note: +# The remixshare.com website is very very slow, so +# if your download not starts because of pycurl timeouts: +# Adjust timeouts in /usr/share/pyload/pyload/network/HTTPRequest.py + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class RemixshareCom(SimpleHoster): + __name__ = "RemixshareCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'https?://remixshare\.com/(download|dl)/\w+' + + __description__ = """Remixshare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + INFO_PATTERN = r'title=\'.+?\'>(?P.+?) \((?P\d+) (?P[\w^_]+)\)<' + OFFLINE_PATTERN = r'

    Ooops!<' + + LINK_FREE_PATTERN = r'(http://remixshare\.com/downloadfinal/.+?)"' + TOKEN_PATTERN = r'var acc = (\d+)' + WAIT_PATTERN = r'var XYZ = r"(\d+)"' + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + b = re.search(self.LINK_FREE_PATTERN, self.html) + if not b: + self.error(_("Cannot parse download url")) + + c = re.search(self.TOKEN_PATTERN, self.html) + if not c: + self.error(_("Cannot parse file token")) + + self.link = b.group(1) + c.group(1) + + #Check if we have to wait + seconds = re.search(self.WAIT_PATTERN, self.html) + if seconds: + self.logDebug("Wait " + seconds.group(1)) + self.wait(seconds.group(1)) diff --git a/pyload/plugin/hoster/RgHostNet.py b/pyload/plugin/hoster/RgHostNet.py new file mode 100644 index 000000000..d9e548721 --- /dev/null +++ b/pyload/plugin/hoster/RgHostNet.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class RgHostNet(SimpleHoster): + __name__ = "RgHostNet" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?rghost\.net/\d+(?:r=\d+)?' + + __description__ = """RgHost.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("z00nx", "z00nx0@gmail.com")] + + + INFO_PATTERN = r'

    \s+(]+>)?(?P[^<]+)()?\s+]+>\s+\((?P[^)]+)\)\s+\s+

    ' + OFFLINE_PATTERN = r'File is deleted|this page is not found' + + LINK_FREE_PATTERN = r']+>Download' diff --git a/pyload/plugin/hoster/SafesharingEu.py b/pyload/plugin/hoster/SafesharingEu.py new file mode 100644 index 000000000..af046af62 --- /dev/null +++ b/pyload/plugin/hoster/SafesharingEu.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class SafesharingEu(XFSHoster): + __name__ = "SafesharingEu" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'https?://(?:www\.)?safesharing\.eu/\w{12}' + + __description__ = """Safesharing.eu hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + ERROR_PATTERN = r'(?:
    )(.+?)(?:
    )' diff --git a/pyload/plugin/hoster/SecureUploadEu.py b/pyload/plugin/hoster/SecureUploadEu.py new file mode 100644 index 000000000..4db413c90 --- /dev/null +++ b/pyload/plugin/hoster/SecureUploadEu.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class SecureUploadEu(XFSHoster): + __name__ = "SecureUploadEu" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'https?://(?:www\.)?secureupload\.eu/\w{12}' + + __description__ = """SecureUpload.eu hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("z00nx", "z00nx0@gmail.com")] + + + INFO_PATTERN = r'

    Downloading (?P[^<]+) \((?P[^<]+)\)

    ' diff --git a/pyload/plugin/hoster/SendspaceCom.py b/pyload/plugin/hoster/SendspaceCom.py new file mode 100644 index 000000000..4d477579d --- /dev/null +++ b/pyload/plugin/hoster/SendspaceCom.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class SendspaceCom(SimpleHoster): + __name__ = "SendspaceCom" + __type__ = "hoster" + __version__ = "0.17" + + __pattern__ = r'https?://(?:www\.)?sendspace\.com/file/\w+' + + __description__ = """Sendspace.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'

    \s*<(?:b|strong)>(?P[^<]+)\s*File Size:\s*(?P[\d.,]+)(?P[\w^_]+)\s*

    ' + OFFLINE_PATTERN = r'
    Sorry, the file you requested is not available.
    ' + + LINK_FREE_PATTERN = r'' + USER_CAPTCHA_PATTERN = r'' + + + def handleFree(self, pyfile): + params = {} + for _i in xrange(3): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + if 'captcha_hash' in params: + self.correctCaptcha() + download_url = m.group(1) + break + + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + if 'captcha_hash' in params: + self.invalidCaptcha() + captcha_url1 = "http://www.sendspace.com/" + m.group(1) + m = re.search(self.USER_CAPTCHA_PATTERN, self.html) + captcha_url2 = "http://www.sendspace.com/" + m.group(1) + params = {'captcha_hash': m.group(2), + 'captcha_submit': 'Verify', + 'captcha_answer': self.decryptCaptcha(captcha_url1) + " " + self.decryptCaptcha(captcha_url2)} + else: + params = {'download': "Regular Download"} + + self.logDebug(params) + self.html = self.load(pyfile.url, post=params) + else: + self.fail(_("Download link not found")) + + self.download(download_url) diff --git a/pyload/plugin/hoster/Share4WebCom.py b/pyload/plugin/hoster/Share4WebCom.py new file mode 100644 index 000000000..f2b4e49e9 --- /dev/null +++ b/pyload/plugin/hoster/Share4WebCom.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.hoster.UnibytesCom import UnibytesCom + + +class Share4WebCom(UnibytesCom): + __name__ = "Share4WebCom" + __type__ = "hoster" + __version__ = "0.11" + + __pattern__ = r'https?://(?:www\.)?share4web\.com/get/\w+' + + __description__ = """Share4web.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "share4web.com" diff --git a/pyload/plugin/hoster/Share76Com.py b/pyload/plugin/hoster/Share76Com.py new file mode 100644 index 000000000..d460726f2 --- /dev/null +++ b/pyload/plugin/hoster/Share76Com.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class Share76Com(DeadHoster): + __name__ = "Share76Com" + __type__ = "hoster" + __version__ = "0.04" + + __pattern__ = r'http://(?:www\.)?share76\.com/\w{12}' + + __description__ = """Share76.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [] diff --git a/pyload/plugin/hoster/ShareFilesCo.py b/pyload/plugin/hoster/ShareFilesCo.py new file mode 100644 index 000000000..edd56572e --- /dev/null +++ b/pyload/plugin/hoster/ShareFilesCo.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class ShareFilesCo(DeadHoster): + __name__ = "ShareFilesCo" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?sharefiles\.co/\w{12}' + + __description__ = """Sharefiles.co hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/hoster/SharebeesCom.py b/pyload/plugin/hoster/SharebeesCom.py new file mode 100644 index 000000000..54de21095 --- /dev/null +++ b/pyload/plugin/hoster/SharebeesCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class SharebeesCom(DeadHoster): + __name__ = "SharebeesCom" + __type__ = "hoster" + __version__ = "0.02" + + __pattern__ = r'http://(?:www\.)?sharebees\.com/\w{12}' + + __description__ = """ShareBees hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/ShareonlineBiz.py b/pyload/plugin/hoster/ShareonlineBiz.py new file mode 100644 index 000000000..f3be1aeb2 --- /dev/null +++ b/pyload/plugin/hoster/ShareonlineBiz.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- + +import re + +from time import time +from urllib import unquote +from urlparse import urlparse + +from pyload.network.RequestFactory import getURL +from pyload.plugin.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class ShareonlineBiz(SimpleHoster): + __name__ = "ShareonlineBiz" + __type__ = "hoster" + __version__ = "0.48" + + __pattern__ = r'https?://(?:www\.)?(share-online\.biz|egoshare\.com)/(download\.php\?id=|dl/)(?P\w+)' + + __description__ = """Shareonline.biz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [(__pattern__ + ".*", "http://www.share-online.biz/dl/\g")] + + CHECK_TRAFFIC = True + + RECAPTCHA_KEY = "6LdatrsSAAAAAHZrB70txiV5p-8Iv8BtVxlTtjKX" + + ERROR_PATTERN = r'

    Information:

    \s*
    \s*(.*?)' + + + @classmethod + def getInfo(cls, url="", html=""): + info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url} + + if url: + info['pattern'] = re.match(cls.__pattern__, url).groupdict() + + field = getURL("http://api.share-online.biz/linkcheck.php", + get={'md5': "1"}, + post={'links': info['pattern']['ID']}, + decode=True).split(";") + + if field[1] == "OK": + info['fileid'] = field[0] + info['status'] = 2 + info['name'] = field[2] + info['size'] = field[3] #: in bytes + info['md5'] = field[4].strip().lower().replace("\n\n", "") #: md5 + + elif field[1] in ("DELETED", "NOT FOUND"): + info['status'] = 1 + + return info + + + def setup(self): + self.resumeDownload = self.premium + self.multiDL = False + + + def handleCaptcha(self): + recaptcha = ReCaptcha(self) + + for _i in xrange(5): + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) + + m = re.search(r'var wait=(\d+);', self.html) + self.setWait(int(m.group(1)) if m else 30) + + res = self.load("%s/free/captcha/%d" % (self.pyfile.url, int(time() * 1000)), + post={'dl_free' : "1", + 'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) + if not res == '0': + self.correctCaptcha() + return res + else: + self.invalidCaptcha() + else: + self.invalidCaptcha() + self.fail(_("No valid captcha solution received")) + + + def handleFree(self, pyfile): + self.wait(3) + + self.html = self.load("%s/free/" % pyfile.url, + post={'dl_free': "1", 'choice': "free"}, + decode=True) + + self.checkErrors() + + res = self.handleCaptcha() + download_url = res.decode('base64') + + if not download_url.startswith("http://"): + self.error(_("Wrong download url")) + + self.wait() + + self.download(download_url) + + + def checkFile(self): + check = self.checkDownload({'cookie': re.compile(r'
    Share-Online")}) + + if check == "cookie": + self.invalidCaptcha() + self.retry(5, 60, _("Cookie failure")) + + elif check == "fail": + self.invalidCaptcha() + self.retry(5, 5 * 60, _("Download failed")) + + return super(ShareonlineBiz, self).checkFile() + + + def handlePremium(self, pyfile): #: should be working better loading (account) api internally + html = self.load("http://api.share-online.biz/account.php", + get={'username': self.user, + 'password': self.account.getAccountData(self.user)['password'], + 'act' : "download", + 'lid' : self.info['fileid']}) + + self.api_data = dlinfo = {} + + for line in html.splitlines(): + key, value = line.split(": ") + dlinfo[key.lower()] = value + + self.logDebug(dlinfo) + + if not dlinfo['status'] == "online": + self.offline() + else: + pyfile.name = dlinfo['name'] + pyfile.size = int(dlinfo['size']) + + dlLink = dlinfo['url'] + + if dlLink == "server_under_maintenance": + self.tempOffline() + else: + self.multiDL = True + self.download(dlLink) + + + def checkErrors(self): + m = re.search(r"/failure/(.*?)/1", self.req.lastEffectiveURL) + if m is None: + self.info.pop('error', None) + return + + errmsg = m.group(1).lower() + + try: + self.logError(errmsg, re.search(self.ERROR_PATTERN, self.html).group(1)) + except Exception: + self.logError("Unknown error occurred", errmsg) + + if errmsg is "invalid": + self.fail(_("File not available")) + + elif errmsg in ("freelimit", "size", "proxy"): + self.fail(_("Premium account needed")) + + elif errmsg in ("expired", "server"): + self.retry(wait_time=600, reason=errmsg) + + elif 'slot' in errmsg: + self.wantReconnect = True + self.retry(24, 3600, errmsg) + + else: + self.wantReconnect = True + self.retry(wait_time=60, reason=errmsg) diff --git a/pyload/plugin/hoster/ShareplaceCom.py b/pyload/plugin/hoster/ShareplaceCom.py new file mode 100644 index 000000000..61e0f8723 --- /dev/null +++ b/pyload/plugin/hoster/ShareplaceCom.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from pyload.plugin.Hoster import Hoster + + +class ShareplaceCom(Hoster): + __name__ = "ShareplaceCom" + __type__ = "hoster" + __version__ = "0.12" + + __pattern__ = r'http://(?:www\.)?shareplace\.(com|org)/\?\w+' + + __description__ = """Shareplace.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("ACCakut", "")] + + + def process(self, pyfile): + self.pyfile = pyfile + self.prepare() + self.download(self.get_file_url()) + + + def prepare(self): + if not self.file_exists(): + self.offline() + + self.pyfile.name = self.get_file_name() + + wait_time = self.get_waiting_time() + self.setWait(wait_time) + self.wait() + + + def get_waiting_time(self): + if not self.html: + self.download_html() + + #var zzipitime = 15; + m = re.search(r'var zzipitime = (\d+);', self.html) + if m: + sec = int(m.group(1)) + else: + sec = 0 + + return sec + + + def download_html(self): + url = re.sub("shareplace.com\/\?", "shareplace.com//index1.php/?a=", self.pyfile.url) + self.html = self.load(url, decode=True) + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + url = re.search(r"var beer = '(.*?)';", self.html) + if url: + url = url.group(1) + url = unquote( + url.replace("http://http:/", "").replace("vvvvvvvvv", "").replace("lllllllll", "").replace( + "teletubbies", "")) + self.logDebug("URL: %s" % url) + return url + else: + self.error(_("Absolute filepath not found")) + + + def get_file_name(self): + if not self.html: + self.download_html() + + return re.search("\s*(.*?)\s*", self.html).group(1) + + + def file_exists(self): + """ returns True or False + """ + if not self.html: + self.download_html() + + if re.search(r"HTTP Status 404", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugin/hoster/SharingmatrixCom.py b/pyload/plugin/hoster/SharingmatrixCom.py new file mode 100644 index 000000000..d1892be14 --- /dev/null +++ b/pyload/plugin/hoster/SharingmatrixCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class SharingmatrixCom(DeadHoster): + __name__ = "SharingmatrixCom" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?sharingmatrix\.com/file/\w+' + + __description__ = """Sharingmatrix.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de"), + ("paulking", "")] diff --git a/pyload/plugin/hoster/ShragleCom.py b/pyload/plugin/hoster/ShragleCom.py new file mode 100644 index 000000000..628537108 --- /dev/null +++ b/pyload/plugin/hoster/ShragleCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class ShragleCom(DeadHoster): + __name__ = "ShragleCom" + __type__ = "hoster" + __version__ = "0.22" + + __pattern__ = r'http://(?:www\.)?(cloudnator|shragle)\.com/files/(?P.+?)/' + + __description__ = """Cloudnator.com (Shragle.com) hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/SimplyPremiumCom.py b/pyload/plugin/hoster/SimplyPremiumCom.py new file mode 100644 index 000000000..50b278985 --- /dev/null +++ b/pyload/plugin/hoster/SimplyPremiumCom.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +import re + +from datetime import datetime, timedelta + +from pyload.plugin.internal.MultiHoster import MultiHoster +from pyload.plugin.internal.SimpleHoster import secondsToMidnight + + +class SimplyPremiumCom(MultiHoster): + __name__ = "SimplyPremiumCom" + __type__ = "hoster" + __version__ = "0.08" + + __pattern__ = r'https?://.+simply-premium\.com' + + __description__ = """Simply-Premium.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("EvolutionClip", "evolutionclip@live.de")] + + + def setup(self): + self.chunkLimit = 16 + + + def checkErrors(self): + if '0' in self.html or ( + "You are not allowed to download from this host" in self.html and self.premium): + self.account.relogin(self.user) + self.retry() + + elif "NOTFOUND" in self.html: + self.offline() + + elif "downloadlimit" in self.html: + self.logWarning(_("Reached maximum connctions")) + self.retry(5, 60, _("Reached maximum connctions")) + + elif "trafficlimit" in self.html: + self.logWarning(_("Reached daily limit for this host")) + self.retry(wait_time=secondsToMidnight(gmt=2), reason="Daily limit for this host reached") + + elif "hostererror" in self.html: + self.logWarning(_("Hoster temporarily unavailable, waiting 1 minute and retry")) + self.retry(5, 60, _("Hoster is temporarily unavailable")) + + + def handlePremium(self, pyfile): + for i in xrange(5): + self.html = self.load("http://www.simply-premium.com/premium.php", get={'info': "", 'link': self.pyfile.url}) + + if self.html: + self.logDebug("JSON data: " + self.html) + break + else: + self.logInfo(_("Unable to get API data, waiting 1 minute and retry")) + self.retry(5, 60, _("Unable to get API data")) + + self.checkErrors() + + try: + self.pyfile.name = re.search(r'([^<]+)', self.html).group(1) + + except AttributeError: + self.pyfile.name = "" + + try: + self.pyfile.size = re.search(r'(\d+)', self.html).group(1) + + except AttributeError: + self.pyfile.size = 0 + + try: + self.link = re.search(r'([^<]+)', self.html).group(1) + + except AttributeError: + self.link = 'http://www.simply-premium.com/premium.php?link=' + self.pyfile.url diff --git a/pyload/plugin/hoster/SimplydebridCom.py b/pyload/plugin/hoster/SimplydebridCom.py new file mode 100644 index 000000000..4b740ff20 --- /dev/null +++ b/pyload/plugin/hoster/SimplydebridCom.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.MultiHoster import MultiHoster, replace_patterns + + +class SimplydebridCom(MultiHoster): + __name__ = "SimplydebridCom" + __type__ = "hoster" + __version__ = "0.15" + + __pattern__ = r'http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/sd\.php' + + __description__ = """Simply-debrid.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] + + + def handlePremium(self, pyfile): + #fix the links for simply-debrid.com! + self.link = replace_patterns(pyfile.url, [("clz.to", "cloudzer.net/file") + ("http://share-online", "http://www.share-online") + ("ul.to", "uploaded.net/file") + ("uploaded.com", "uploaded.net") + ("filerio.com", "filerio.in") + ("lumfile.com", "lumfile.se")] + + if 'fileparadox' in self.link: + self.link = self.link.replace("http://", "https://") + + self.html = self.load("http://simply-debrid.com/api.php", get={'dl': self.link}) + if 'tiger Link' in self.html or 'Invalid Link' in self.html or ('API' in self.html and 'ERROR' in self.html): + self.error(_("Unable to unrestrict link")) + + self.link = self.html + + self.wait(5) + + + def checkFile(self): + if self.checkDownload({"error": "No address associated with hostname"}): + self.retry(24, 3 * 60, _("Bad file downloaded")) + + return super(SimplydebridCom, self).checkFile() diff --git a/pyload/plugin/hoster/SmoozedCom.py b/pyload/plugin/hoster/SmoozedCom.py new file mode 100644 index 000000000..715d99b40 --- /dev/null +++ b/pyload/plugin/hoster/SmoozedCom.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHoster import MultiHoster + + +class SmoozedCom(MultiHoster): + __name__ = "SmoozedCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' #: Since we want to allow the user to specify the list of hoster to use we let MultiHoster.activate + + __description__ = """Smoozed.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("", "")] + + + def handlePremium(self, pyfile): + # In some cases hostsers do not supply us with a filename at download, so we + # are going to set a fall back filename (e.g. for freakshare or xfileshare) + pyfile.name = pyfile.name.split('/').pop() # Remove everthing before last slash + + # Correction for automatic assigned filename: Removing html at end if needed + suffix_to_remove = ["html", "htm", "php", "php3", "asp", "shtm", "shtml", "cfml", "cfm"] + temp = pyfile.name.split('.') + + if temp.pop() in suffix_to_remove: + pyfile.name = ".".join(temp) + + # Check the link + get_data = {'session_key': self.account.getAccountInfo(self.user)['session'], + 'url' : pyfile.url} + + data = json_loads(self.load("http://www2.smoozed.com/api/check", get=get_data)) + + if data["state"] != "ok": + self.fail(data["message"]) + + if data["data"].get("state", "ok") != "ok": + if data["data"] == "Offline": + self.offline() + else: + self.fail(data["data"]["message"]) + + pyfile.name = data["data"]["name"] + pyfile.size = int(data["data"]["size"]) + + # Start the download + header = self.load("http://www2.smoozed.com/api/download", get=get_data, just_header=True) + + if not "location" in header: + self.fail(_("Unable to initialize download")) + else: + self.link = header["location"][-1] if isinstance(header["location"], list) else header["location"] + + + def checkFile(self): + if self.checkDownload({'error': '{"state":"error"}', + 'retry': '{"state":"retry"}'}): + self.fail(_("Error response received")) + + return super(SmoozedCom, self).checkFile() diff --git a/pyload/plugin/hoster/SockshareCom.py b/pyload/plugin/hoster/SockshareCom.py new file mode 100644 index 000000000..e903e3daf --- /dev/null +++ b/pyload/plugin/hoster/SockshareCom.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class SockshareCom(DeadHoster): + __name__ = "SockshareCom" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'http://(?:www\.)?sockshare\.com/(mobile/)?(file|embed)/(?P\w+)' + + __description__ = """Sockshare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] diff --git a/pyload/plugin/hoster/SoundcloudCom.py b/pyload/plugin/hoster/SoundcloudCom.py new file mode 100644 index 000000000..fd5d1ea6c --- /dev/null +++ b/pyload/plugin/hoster/SoundcloudCom.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +import pycurl +import re + +from pyload.plugin.Hoster import Hoster + + +class SoundcloudCom(Hoster): + __name__ = "SoundcloudCom" + __type__ = "hoster" + __version__ = "0.10" + + __pattern__ = r'https?://(?:www\.)?soundcloud\.com/(?P.+?)/(?P.+)' + + __description__ = """SoundCloud.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Peekayy", "peekayy.dev@gmail.com")] + + + def process(self, pyfile): + # default UserAgent of HTTPRequest fails for this hoster so we use this one + self.req.http.c.setopt(pycurl.USERAGENT, 'Mozilla/5.0') + self.html = self.load(pyfile.url) + m = re.search(r'
    .*?)"', self.html) + if m: + clientId = m.group('CID') + + if len(clientId) <= 0: + clientId = "b45b1aa10f1ac2941910a7f0d10f8e28" + + m = re.search(r'\s(?P.*?)\s</em>', self.html) + if m: + pyfile.name = m.group('TITLE') + ".mp3" + else: + pyfile.name = re.match(self.__pattern__, pyfile.url).group('SID') + ".mp3" + + # url to retrieve the actual song url + self.html = self.load("https://api.sndcdn.com/i1/tracks/%s/streams" % songId, get={"client_id": clientId}) + # getting streams + # for now we choose the first stream found in all cases + # it could be improved if relevant for this hoster + streams = [ + (result.group('QUALITY'), result.group('URL')) + for result in re.finditer(r'"(?P<QUALITY>.*?)":"(?P<URL>.*?)"', self.html) + ] + self.logDebug("Found Streams", streams) + self.logDebug("Downloading", streams[0][0], streams[0][1]) + self.download(streams[0][1]) diff --git a/pyload/plugin/hoster/SpeedLoadOrg.py b/pyload/plugin/hoster/SpeedLoadOrg.py new file mode 100644 index 000000000..a20f87902 --- /dev/null +++ b/pyload/plugin/hoster/SpeedLoadOrg.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class SpeedLoadOrg(DeadHoster): + __name__ = "SpeedLoadOrg" + __type__ = "hoster" + __version__ = "1.02" + + __pattern__ = r'http://(?:www\.)?speedload\.org/(?P<ID>\w+)' + + __description__ = """Speedload.org hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/pyload/plugin/hoster/SpeedfileCz.py b/pyload/plugin/hoster/SpeedfileCz.py new file mode 100644 index 000000000..981ba861d --- /dev/null +++ b/pyload/plugin/hoster/SpeedfileCz.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class SpeedfileCz(DeadHoster): + __name__ = "SpeedFileCz" + __type__ = "hoster" + __version__ = "0.32" + + __pattern__ = r'http://(?:www\.)?speedfile\.cz/.+' + + __description__ = """Speedfile.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/SpeedyshareCom.py b/pyload/plugin/hoster/SpeedyshareCom.py new file mode 100644 index 000000000..c92522d70 --- /dev/null +++ b/pyload/plugin/hoster/SpeedyshareCom.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://speedy.sh/ep2qY/Zapp-Brannigan.jpg + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class SpeedyshareCom(SimpleHoster): + __name__ = "SpeedyshareCom" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'https?://(?:www\.)?(speedyshare\.com|speedy\.sh)/\w+' + + __description__ = """Speedyshare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de")] + + + NAME_PATTERN = r'class=downloadfilename>(?P<N>.*)</span></td>' + SIZE_PATTERN = r'class=sizetagtext>(?P<S>.*) (?P<U>[kKmM]?[iI]?[bB]?)</div>' + + OFFLINE_PATTERN = r'class=downloadfilenamenotfound>.*</span>' + + LINK_FREE_PATTERN = r'<a href=\'(.*)\'><img src=/gf/slowdownload\.png alt=\'Slow Download\' border=0' + + + def setup(self): + self.multiDL = False + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.link = m.group(1) diff --git a/pyload/plugin/hoster/StorageTo.py b/pyload/plugin/hoster/StorageTo.py new file mode 100644 index 000000000..78b9a60bb --- /dev/null +++ b/pyload/plugin/hoster/StorageTo.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class StorageTo(DeadHoster): + __name__ = "StorageTo" + __type__ = "hoster" + __version__ = "0.01" + + __pattern__ = r'http://(?:www\.)?storage\.to/get/.+' + + __description__ = """Storage.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] diff --git a/pyload/plugin/hoster/StreamCz.py b/pyload/plugin/hoster/StreamCz.py new file mode 100644 index 000000000..95e69abf4 --- /dev/null +++ b/pyload/plugin/hoster/StreamCz.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.network.RequestFactory import getURL +from pyload.plugin.Hoster import Hoster + + +def getInfo(urls): + result = [] + + for url in urls: + + html = getURL(url) + if re.search(StreamCz.OFFLINE_PATTERN, html): + # File offline + result.append((url, 0, 1, url)) + else: + result.append((url, 0, 2, url)) + yield result + + +class StreamCz(Hoster): + __name__ = "StreamCz" + __type__ = "hoster" + __version__ = "0.20" + + __pattern__ = r'https?://(?:www\.)?stream\.cz/[^/]+/\d+' + + __description__ = """Stream.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'<link rel="video_src" href="http://www\.stream\.cz/\w+/(\d+)-([^"]+)" />' + OFFLINE_PATTERN = r'<h1 class="commonTitle">Str.nku nebylo mo.n. nal.zt \(404\)</h1>' + + CDN_PATTERN = r'<param name="flashvars" value="[^"]*&id=(?P<ID>\d+)(?:&cdnLQ=(?P<cdnLQ>\d*))?(?:&cdnHQ=(?P<cdnHQ>\d*))?(?:&cdnHD=(?P<cdnHD>\d*))?&' + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + + if re.search(self.OFFLINE_PATTERN, self.html): + self.offline() + + m = re.search(self.CDN_PATTERN, self.html) + if m is None: + self.error(_("CDN_PATTERN not found")) + cdn = m.groupdict() + self.logDebug(cdn) + for cdnkey in ("cdnHD", "cdnHQ", "cdnLQ"): + if cdnkey in cdn and cdn[cdnkey] > '': + cdnid = cdn[cdnkey] + break + else: + self.fail(_("Stream URL not found")) + + m = re.search(self.NAME_PATTERN, self.html) + if m is None: + self.error(_("NAME_PATTERN not found")) + pyfile.name = "%s-%s.%s.mp4" % (m.group(2), m.group(1), cdnkey[-2:]) + + download_url = "http://cdn-dispatcher.stream.cz/?id=" + cdnid + self.logInfo(_("STREAM: %s") % cdnkey[-2:], download_url) + self.download(download_url) diff --git a/pyload/plugin/hoster/StreamcloudEu.py b/pyload/plugin/hoster/StreamcloudEu.py new file mode 100644 index 000000000..01e334491 --- /dev/null +++ b/pyload/plugin/hoster/StreamcloudEu.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class StreamcloudEu(XFSHoster): + __name__ = "StreamcloudEu" + __type__ = "hoster" + __version__ = "0.10" + + __pattern__ = r'http://(?:www\.)?streamcloud\.eu/\w{12}' + + __description__ = """Streamcloud.eu hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("seoester", "seoester@googlemail.com")] + + + WAIT_PATTERN = r'var count = (\d+)' + + LINK_PATTERN = r'file: "(http://(stor|cdn)\d+\.streamcloud\.eu:?\d*/.*/video\.(mp4|flv))",' + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + self.resumeDownload = self.premium diff --git a/pyload/plugin/hoster/TurbobitNet.py b/pyload/plugin/hoster/TurbobitNet.py new file mode 100644 index 000000000..c69158c39 --- /dev/null +++ b/pyload/plugin/hoster/TurbobitNet.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- + +import random +import re +import time + +from Crypto.Cipher import ARC4 +from binascii import hexlify, unhexlify +from pycurl import HTTPHEADER +from urllib import quote + +from pyload.network.RequestFactory import getURL +from pyload.plugin.internal.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster, timestamp + + +class TurbobitNet(SimpleHoster): + __name__ = "TurbobitNet" + __type__ = "hoster" + __version__ = "0.19" + + __pattern__ = r'http://(?:www\.)?turbobit\.net/(?:download/free/)?(?P<ID>\w+)' + + __description__ = """Turbobit.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("prOq", "")] + + + URL_REPLACEMENTS = [(__pattern__ + ".*", "http://turbobit.net/\g<ID>.html")] + + COOKIES = [("turbobit.net", "user_lang", "en")] + + NAME_PATTERN = r'id="file-title">(?P<N>.+?)<' + SIZE_PATTERN = r'class="file-size">(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File (?:was )?not found' + + LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'(/download/redirect/[^"\']+)' + + LIMIT_WAIT_PATTERN = r'<div id=\'timeout\'>(\d+)<' + CAPTCHA_PATTERN = r'<img alt="Captcha" src="(.+?)"' + + + def handleFree(self, pyfile): + self.html = self.load("http://turbobit.net/download/free/%s" % self.info['pattern']['ID'], + decode=True) + + rtUpdate = self.getRtUpdate() + + self.solveCaptcha() + + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + + self.html = self.load(self.getDownloadUrl(rtUpdate)) + + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + self.link = m.group(1) + + + def solveCaptcha(self): + for _i in xrange(5): + m = re.search(self.LIMIT_WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.wait(wait_time, wait_time > 60) + self.retry() + + action, inputs = self.parseHtmlForm("action='#'") + if not inputs: + self.error(_("Captcha form not found")) + self.logDebug(inputs) + + if inputs['captcha_type'] == 'recaptcha': + recaptcha = ReCaptcha(self) + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge() + else: + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m is None: + self.error(_("captcha")) + captcha_url = m.group(1) + inputs['captcha_response'] = self.decryptCaptcha(captcha_url) + + self.logDebug(inputs) + self.html = self.load(self.url, post=inputs) + + if '<div class="captcha-error">Incorrect, try again!<' in self.html: + self.invalidCaptcha() + else: + self.correctCaptcha() + break + else: + self.fail(_("Invalid captcha")) + + + def getRtUpdate(self): + rtUpdate = self.getStorage("rtUpdate") + if not rtUpdate: + if self.getStorage("version") != self.__version__ \ + or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): + # that's right, we are even using jdownloader updates + rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") + rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) + # but we still need to fix the syntax to work with other engines than rhino + rtUpdate = re.sub(r'for each\(var (\w+) in(\[[^\]]+\])\)\{', + r'zza=\2;for(var zzi=0;zzi<zza.length;zzi++){\1=zza[zzi];', rtUpdate) + rtUpdate = re.sub(r"for\((\w+)=", r"for(var \1=", rtUpdate) + + self.setStorage("rtUpdate", rtUpdate) + self.setStorage("timestamp", timestamp()) + self.setStorage("version", self.__version__) + else: + self.logError(_("Unable to download, wait for update...")) + self.tempOffline() + + return rtUpdate + + + def getDownloadUrl(self, rtUpdate): + self.req.http.lastURL = self.url + + m = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) + if m: + url = "http://turbobit.net%s%s" % m.groups() + else: + url = "http://turbobit.net/files/timeout.js?ver=%s" % "".join(random.choice('0123456789ABCDEF') for _i in xrange(32)) + + fun = self.load(url) + + self.setWait(65, False) + + for b in [1, 3]: + self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % ( + self.info['pattern']['ID'], b, quote(fun), rtUpdate) + + try: + out = self.js.eval(self.jscode) + self.logDebug("URL", self.js.engine, out) + if out.startswith('/download/'): + return "http://turbobit.net%s" % out.strip() + + except Exception, e: + self.logError(e) + else: + if self.retries >= 2: + # retry with updated js + self.delStorage("rtUpdate") + else: + self.retry() + + self.wait() + + + def decrypt(self, data): + cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) + return unhexlify(cipher.encrypt(unhexlify(data))) + + + def getLocalTimeString(self): + lt = time.localtime() + tz = time.altzone if lt.tm_isdst else time.timezone + return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) diff --git a/pyload/plugin/hoster/TurbouploadCom.py b/pyload/plugin/hoster/TurbouploadCom.py new file mode 100644 index 000000000..e964d1365 --- /dev/null +++ b/pyload/plugin/hoster/TurbouploadCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class TurbouploadCom(DeadHoster): + __name__ = "TurbouploadCom" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?turboupload\.com/(\w+)' + + __description__ = """Turboupload.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/TusfilesNet.py b/pyload/plugin/hoster/TusfilesNet.py new file mode 100644 index 000000000..20a948925 --- /dev/null +++ b/pyload/plugin/hoster/TusfilesNet.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from pyload.network.HTTPRequest import BadHeader +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class TusfilesNet(XFSHoster): + __name__ = "TusfilesNet" + __type__ = "hoster" + __version__ = "0.09" + + __pattern__ = r'https?://(?:www\.)?tusfiles\.net/\w{12}' + + __description__ = """Tusfiles.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("guidobelix", "guidobelix@hotmail.it")] + + + INFO_PATTERN = r'\](?P<N>.+) - (?P<S>[\d.,]+) (?P<U>[\w^_]+)\[' + OFFLINE_PATTERN = r'>File Not Found|<Title>TusFiles - Fast Sharing Files!|The file you are trying to download is no longer available' + + + def setup(self): + self.chunkLimit = -1 + self.multiDL = True + self.resumeDownload = True + + + def downloadLink(self, link): + try: + return super(TusfilesNet, self).downloadLink(link) + + except BadHeader, e: + if e.code is 503: + self.multiDL = False + raise Retry("503") diff --git a/pyload/plugin/hoster/TwoSharedCom.py b/pyload/plugin/hoster/TwoSharedCom.py new file mode 100644 index 000000000..8c6864414 --- /dev/null +++ b/pyload/plugin/hoster/TwoSharedCom.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class TwoSharedCom(SimpleHoster): + __name__ = "TwoSharedCom" + __type__ = "hoster" + __version__ = "0.13" + + __pattern__ = r'http://(?:www\.)?2shared\.com/(account/)?(download|get|file|document|photo|video|audio)/.+' + + __description__ = """2Shared.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'<h1>(?P<N>.*)</h1>' + SIZE_PATTERN = r'<span class="dtitle">File size:</span>\s*(?P<S>[\d.,]+) (?P<U>[\w^_]+)' + OFFLINE_PATTERN = r'The file link that you requested is not valid\.|This file was deleted\.' + + LINK_FREE_PATTERN = r'window.location =\'(.+?)\';' + + + def setup(self): + self.resumeDownload = True + self.multiDL = True diff --git a/pyload/plugin/hoster/UlozTo.py b/pyload/plugin/hoster/UlozTo.py new file mode 100644 index 000000000..0e4156ad0 --- /dev/null +++ b/pyload/plugin/hoster/UlozTo.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from pyload.utils import json_loads +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +def convertDecimalPrefix(m): + # decimal prefixes used in filesize and traffic + return ("%%.%df" % {'k': 3, 'M': 6, 'G': 9}[m.group(2)] % float(m.group(1))).replace('.', '') + + +class UlozTo(SimpleHoster): + __name__ = "UlozTo" + __type__ = "hoster" + __version__ = "1.04" + + __pattern__ = r'http://(?:www\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj\.cz|zachowajto\.pl)/(?:live/)?(?P<ID>\w+/[^/?]*)' + + __description__ = """Uloz.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + INFO_PATTERN = r'<p>File <strong>(?P<N>[^<]+)</strong> is password protected</p>' + NAME_PATTERN = r'<title>(?P<N>[^<]+) \| Uloz\.to' + SIZE_PATTERN = r'.*?(?P[\d.,]+\s[kMG]?B)' + OFFLINE_PATTERN = r'404 - Page not found|

    File (has been deleted|was banned)

    ' + + URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "www.ulozto.net")] + SIZE_REPLACEMENTS = [('([\d.]+)\s([kMG])B', convertDecimalPrefix)] + + ADULT_PATTERN = r'' + PASSWD_PATTERN = r'
    ' + VIPLINK_PATTERN = r'' + TOKEN_PATTERN = r'\s*
  • Error rewriting the text.
  • '), + "offline" : re.compile(self.OFFLINE_PATTERN), + "passwd" : self.PASSWD_PATTERN, + "server_error" : 'src="http://img.ulozto.cz/error403/vykricnik.jpg"', # paralell dl, server overload etc. + "not_found" : "Ulož.to" + }) + + if check == "wrong_captcha": + #self.delStorage("captcha_id") + #self.delStorage("captcha_text") + self.invalidCaptcha() + self.retry(reason=_("Wrong captcha code")) + + elif check == "offline": + self.offline() + + elif check == "passwd": + self.fail(_("Wrong password")) + + elif check == "server_error": + self.logError(_("Server error, try downloading later")) + self.multiDL = False + self.wait(1 * 60 * 60, True) + self.retry() + + elif check == "not_found": + self.fail(_("Server error - file not downloadable")) diff --git a/pyload/plugin/hoster/UloziskoSk.py b/pyload/plugin/hoster/UloziskoSk.py new file mode 100644 index 000000000..ce40c9f7a --- /dev/null +++ b/pyload/plugin/hoster/UloziskoSk.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class UloziskoSk(SimpleHoster): + __name__ = "UloziskoSk" + __type__ = "hoster" + __version__ = "0.25" + + __pattern__ = r'http://(?:www\.)?ulozisko\.sk/.+' + + __description__ = """Ulozisko.sk hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'
    (?P[^<]+)
    ' + SIZE_PATTERN = ur'Veľkosť súboru: (?P[\d.,]+) (?P[\w^_]+)
    ' + OFFLINE_PATTERN = ur'Zadaný súbor neexistuje z jedného z nasledujúcich dôvodov:' + + LINK_FREE_PATTERN = r'' + ID_PATTERN = r'' + CAPTCHA_PATTERN = r'' + IMG_PATTERN = ur'PRE ZVÄČŠENIE KLIKNITE NA OBRÁZOK
    ' + + + def process(self, pyfile): + self.html = self.load(pyfile.url, decode=True) + self.getFileInfo() + + m = re.search(self.IMG_PATTERN, self.html) + if m: + url = "http://ulozisko.sk" + m.group(1) + self.download(url) + else: + self.handleFree(pyfile) + + + def handleFree(self, pyfile): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("LINK_FREE_PATTERN not found")) + parsed_url = 'http://www.ulozisko.sk' + m.group(1) + + m = re.search(self.ID_PATTERN, self.html) + if m is None: + self.error(_("ID_PATTERN not found")) + id = m.group(1) + + self.logDebug("URL:" + parsed_url + ' ID:' + id) + + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m is None: + self.error(_("CAPTCHA_PATTERN not found")) + captcha_url = 'http://www.ulozisko.sk' + m.group(1) + + captcha = self.decryptCaptcha(captcha_url, cookies=True) + + self.logDebug("CAPTCHA_URL:" + captcha_url + ' CAPTCHA:' + captcha) + + self.download(parsed_url, + post={"antispam": captcha, + "id" : id, + "name" : pyfile.name, + "but" : "++++STIAHNI+S%DABOR++++"}) diff --git a/pyload/plugin/hoster/UnibytesCom.py b/pyload/plugin/hoster/UnibytesCom.py new file mode 100644 index 000000000..2c9b9ca5f --- /dev/null +++ b/pyload/plugin/hoster/UnibytesCom.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class UnibytesCom(SimpleHoster): + __name__ = "UnibytesCom" + __type__ = "hoster" + __version__ = "0.12" + + __pattern__ = r'https?://(?:www\.)?unibytes\.com/[\w .-]{11}B' + + __description__ = """UniBytes.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + HOSTER_DOMAIN = "unibytes.com" + + INFO_PATTERN = r']*?id="fileName"[^>]*>(?P[^>]+)\s*\((?P\d.*?)\)' + + WAIT_PATTERN = r'Wait for (\d+) sec' + LINK_FREE_PATTERN = r'Download' + + + def handleFree(self, pyfile): + domain = "http://www.%s/" % self.HOSTER_DOMAIN + action, post_data = self.parseHtmlForm('id="startForm"') + + + for _i in xrange(8): + self.logDebug(action, post_data) + self.html = self.load(urljoin(domain, action), post=post_data, follow_location=False) + + m = re.search(r'location:\s*(\S+)', self.req.http.header, re.I) + if m: + url = m.group(1) + break + + if '>Somebody else is already downloading using your IP-address<' in self.html: + self.wait(10 * 60, True) + self.retry() + + if post_data['step'] == 'last': + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m: + url = m.group(1) + self.correctCaptcha() + break + else: + self.invalidCaptcha() + + last_step = post_data['step'] + action, post_data = self.parseHtmlForm('id="stepForm"') + + if last_step == 'timer': + m = re.search(self.WAIT_PATTERN, self.html) + self.wait(m.group(1) if m else 60, False) + + elif last_step in ("captcha", "last"): + post_data['captcha'] = self.decryptCaptcha(urljoin(domain, "/captcha.jpg")) + + else: + self.fail(_("No valid captcha code entered")) + + self.download(url) diff --git a/pyload/plugin/hoster/UnrestrictLi.py b/pyload/plugin/hoster/UnrestrictLi.py new file mode 100644 index 000000000..c81bb0554 --- /dev/null +++ b/pyload/plugin/hoster/UnrestrictLi.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.utils import json_loads +from pyload.plugin.internal.MultiHoster import MultiHoster +from pyload.plugin.internal.SimpleHoster import secondsToMidnight + + +class UnrestrictLi(MultiHoster): + __name__ = "UnrestrictLi" + __type__ = "hoster" + __version__ = "0.21" + + __pattern__ = r'https?://(?:www\.)?(unrestrict|unr)\.li/dl/[\w^_]+' + + __description__ = """Unrestrict.li multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + LOGIN_ACCOUNT = False + + + def setup(self): + self.chunkLimit = 16 + self.resumeDownload = True + + + def handleFree(self, pyfile): + for _i in xrange(5): + self.html = self.load('https://unrestrict.li/unrestrict.php', + post={'link': pyfile.url, 'domain': 'long'}) + + self.logDebug("JSON data: " + self.html) + + if self.html: + break + else: + self.logInfo(_("Unable to get API data, waiting 1 minute and retry")) + self.retry(5, 60, "Unable to get API data") + + if 'Expired session' in self.html \ + or ("You are not allowed to download from this host" in self.html and self.premium): + self.account.relogin(self.user) + self.retry() + + elif "File offline" in self.html: + self.offline() + + elif "You are not allowed to download from this host" in self.html: + self.fail(_("You are not allowed to download from this host")) + + elif "You have reached your daily limit for this host" in self.html: + self.logWarning(_("Reached daily limit for this host")) + self.retry(5, secondsToMidnight(gmt=2), "Daily limit for this host reached") + + elif "ERROR_HOSTER_TEMPORARILY_UNAVAILABLE" in self.html: + self.logInfo(_("Hoster temporarily unavailable, waiting 1 minute and retry")) + self.retry(5, 60, "Hoster is temporarily unavailable") + + self.html = json_loads(self.html) + self.link = self.html.keys()[0] + self.api_data = self.html[self.link] + + if hasattr(self, 'api_data'): + self.setNameSize() + + + def checkFile(self): + super(UnrestrictLi, self).checkFile() + + if self.getConfig("history"): + self.load("https://unrestrict.li/history/", get={'delete': "all"}) + self.logInfo(_("Download history deleted")) + + + def setNameSize(self): + if 'name' in self.api_data: + self.pyfile.name = self.api_data['name'] + if 'size' in self.api_data: + self.pyfile.size = self.api_data['size'] diff --git a/pyload/plugin/hoster/UpleaCom.py b/pyload/plugin/hoster/UpleaCom.py new file mode 100644 index 000000000..855d9ed65 --- /dev/null +++ b/pyload/plugin/hoster/UpleaCom.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class UpleaCom(XFSHoster): + __name__ = "UpleaCom" + __type__ = "hoster" + __version__ = "0.06" + + __pattern__ = r'https?://(?:www\.)?uplea\.com/dl/\w{15}' + + __description__ = """Uplea.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Redleon", "")] + + + NAME_PATTERN = r'class="agmd size18">(?P.+?)<' + SIZE_PATTERN = r'size14">(?P[\d.,]+) (?P[\w^_])' + + OFFLINE_PATTERN = r'>You followed an invalid or expired link' + + LINK_PATTERN = r'"(http?://\w+\.uplea\.com/anonym/.*?)"' + + WAIT_PATTERN = r'timeText:([\d.]+),' + STEP_PATTERN = r'' + + + def setup(self): + self.multiDL = False + self.chunkLimit = 1 + self.resumeDownload = True + + + def handleFree(self, pyfile): + m = re.search(self.STEP_PATTERN, self.html) + if m is None: + self.error(_("STEP_PATTERN not found")) + + self.html = self.load(urljoin("http://uplea.com/", m.group(1))) + + m = re.search(self.WAIT_PATTERN, self.html) + if m: + self.wait(m.group(1), True) + self.retry() + + m = re.search(self.LINK_PATTERN, self.html) + if m is None: + self.error(_("LINK_PATTERN not found")) + + self.link = m.group(1) + self.wait(15) diff --git a/pyload/plugin/hoster/UploadStationCom.py b/pyload/plugin/hoster/UploadStationCom.py new file mode 100644 index 000000000..d77bdb760 --- /dev/null +++ b/pyload/plugin/hoster/UploadStationCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class UploadStationCom(DeadHoster): + __name__ = "UploadStationCom" + __type__ = "hoster" + __version__ = "0.52" + + __pattern__ = r'http://(?:www\.)?uploadstation\.com/file/(?P\w+)' + + __description__ = """UploadStation.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("fragonib", "fragonib[AT]yahoo[DOT]es"), + ("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/UploadableCh.py b/pyload/plugin/hoster/UploadableCh.py new file mode 100644 index 000000000..c54ecb495 --- /dev/null +++ b/pyload/plugin/hoster/UploadableCh.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from pyload.plugin.captcha import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class UploadableCh(SimpleHoster): + __name__ = "UploadableCh" + __type__ = "hoster" + __version__ = "0.08" + + __pattern__ = r'http://(?:www\.)?uploadable\.ch/file/(?P\w+)' + + __description__ = """Uploadable.ch hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zapp-brannigan", "fuerst.reinje@web.de"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://www.uploadable.ch/file/\g')] + + INFO_PATTERN = r'div id=\"file_name\" title=.*>(?P.+)\((?P[\d.]+) (?P\w+)\)<' + + OFFLINE_PATTERN = r'>(File not available|This file is no longer available)' + TEMP_OFFLINE_PATTERN = r'
    ' + + WAIT_PATTERN = r'>Please wait.+?<' + + RECAPTCHA_KEY = "6LdlJuwSAAAAAPJbPIoUhyqOJd7-yrah5Nhim5S3" + + + def handleFree(self, pyfile): + # Click the "free user" button and wait + a = self.load(pyfile.url, cookies=True, post={'downloadLink': "wait"}, decode=True) + self.logDebug(a) + + self.wait(30) + + # Make the recaptcha appear and show it the pyload interface + b = self.load(pyfile.url, cookies=True, post={'checkDownload': "check"}, decode=True) + self.logDebug(b) #: Expected output: {"success":"showCaptcha"} + + recaptcha = ReCaptcha(self) + + response, challenge = recaptcha.challenge(self.RECAPTCHA_KEY) + + # Submit the captcha solution + self.load("http://www.uploadable.ch/checkReCaptcha.php", + cookies=True, + post={'recaptcha_challenge_field' : challenge, + 'recaptcha_response_field' : response, + 'recaptcha_shortencode_field': self.info['pattern']['ID']}, + decode=True) + + self.wait(3) + + # Get ready for downloading + self.load(pyfile.url, cookies=True, post={'downloadLink': "show"}, decode=True) + + self.wait(3) + + # Download the file + self.download(pyfile.url, cookies=True, post={'download': "normal"}, disposition=True) + + + def checkFile(self): + if self.checkDownload({'wait': re.compile("Please wait for")}): + self.logInfo("Downloadlimit reached, please wait or reconnect") + self.wait(60 * 60, True) + self.retry() + + return super(UploadableCh, self).checkFile() diff --git a/pyload/plugin/hoster/UploadboxCom.py b/pyload/plugin/hoster/UploadboxCom.py new file mode 100644 index 000000000..01e4c6854 --- /dev/null +++ b/pyload/plugin/hoster/UploadboxCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class UploadboxCom(DeadHoster): + __name__ = "Uploadbox" + __type__ = "hoster" + __version__ = "0.05" + + __pattern__ = r'http://(?:www\.)?uploadbox\.com/files/.+' + + __description__ = """UploadBox.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/UploadedTo.py b/pyload/plugin/hoster/UploadedTo.py new file mode 100644 index 000000000..f9b121bfe --- /dev/null +++ b/pyload/plugin/hoster/UploadedTo.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from pyload.network.RequestFactory import getURL +from pyload.plugin.internal.CaptchaService import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class UploadedTo(SimpleHoster): + __name__ = "UploadedTo" + __type__ = "hoster" + __version__ = "0.84" + + __pattern__ = r'https?://(?:www\.)?(uploaded\.(to|net)|ul\.to)(/file/|/?\?id=|.*?&id=|/)(?P\w+)' + + __description__ = """Uploaded.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + API_KEY = "lhF2IeeprweDfu9ccWlxXVVypA5nA3EL" + + URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://uploaded.net/file/\g')] + + LINK_PREMIUM_PATTERN = r'
    (\d+)' + DL_LIMIT_ERROR = r'You have reached the max. number of possible free downloads for this hour' + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + info = super(UploadedTo, cls).apiInfo(url) + + for _i in xrange(5): + html = getURL("http://uploaded.net/api/filemultiple", + get={"apikey": cls.API_KEY, 'id_0': re.match(cls.__pattern__, url).group('ID')}, + decode=True) + + if html != "can't find request": + api = html.split(",", 4) + if api[0] == "online": + info.update({'name': api[4].strip(), 'size': api[2], 'status': 2}) + else: + info['status'] = 1 + break + else: + sleep(3) + + return info + + + def setup(self): + self.multiDL = self.resumeDownload = self.premium + self.chunkLimit = 1 # critical problems with more chunks + + + def checkErrors(self): + if 'var free_enabled = false;' in self.html: + self.logError(_("Free-download capacities exhausted")) + self.retry(24, 5 * 60) + + elif "limit-size" in self.html: + self.fail(_("File too big for free download")) + + elif "limit-slot" in self.html: # Temporary restriction so just wait a bit + self.wait(30 * 60, True) + self.retry() + + elif "limit-parallel" in self.html: + self.fail(_("Cannot download in parallel")) + + elif "limit-dl" in self.html or self.DL_LIMIT_ERROR in self.html: # limit-dl + self.wait(3 * 60 * 60, True) + self.retry() + + elif '"err":"captcha"' in self.html: + self.invalidCaptcha() + + else: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + self.wait(m.group(1)) + + + def handleFree(self, pyfile): + self.load("http://uploaded.net/language/en", just_header=True) + + self.html = self.load("http://uploaded.net/js/download.js", decode=True) + + recaptcha = ReCaptcha(self) + response, challenge = recaptcha.challenge() + + self.html = self.load("http://uploaded.net/io/ticket/captcha/%s" % self.info['pattern']['ID'], + post={'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) + + if "type:'download'" in self.html: + self.correctCaptcha() + try: + self.link = re.search("url:'([^']+)", self.html).group(1) + + except Exception: + pass + + self.checkErrors() + + + def checkFile(self): + if self.checkDownload({'limit-dl': self.DL_LIMIT_ERROR}): + self.wait(3 * 60 * 60, True) + self.retry() + + return super(UploadedTo, self).checkFile() diff --git a/pyload/plugin/hoster/UploadhereCom.py b/pyload/plugin/hoster/UploadhereCom.py new file mode 100644 index 000000000..77970a97b --- /dev/null +++ b/pyload/plugin/hoster/UploadhereCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class UploadhereCom(DeadHoster): + __name__ = "UploadhereCom" + __type__ = "hoster" + __version__ = "0.12" + + __pattern__ = r'http://(?:www\.)?uploadhere\.com/\w{10}' + + __description__ = """Uploadhere.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/pyload/plugin/hoster/UploadheroCom.py b/pyload/plugin/hoster/UploadheroCom.py new file mode 100644 index 000000000..917053f18 --- /dev/null +++ b/pyload/plugin/hoster/UploadheroCom.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# http://uploadhero.co/dl/wQBRAVSM + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class UploadheroCom(SimpleHoster): + __name__ = "UploadheroCom" + __type__ = "hoster" + __version__ = "0.17" + + __pattern__ = r'http://(?:www\.)?uploadhero\.com?/dl/\w+' + + __description__ = """UploadHero.co plugin""" + __license__ = "GPLv3" + __authors__ = [("mcmyst", "mcmyst@hotmail.fr"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'
    (?P.*?)
    ' + SIZE_PATTERN = r'Taille du fichier : (?P.*?)' + OFFLINE_PATTERN = r'

    |

    Le lien du fichier ci-dessus n\'existe plus.' + + COOKIES = [("uploadhero.co", "lang", "en")] + + IP_BLOCKED_PATTERN = r'href="(/lightbox_block_download\.php\?min=.*?)"' + IP_WAIT_PATTERN = r'(\d+).*\s*(\d+)' + + CAPTCHA_PATTERN = r'"(/captchadl\.php\?\w+)"' + + LINK_FREE_PATTERN = r'var magicomfg = \'"/]+)"' + LINK_PREMIUM_PATTERN = r'\w+)' + + __description__ = """Uploading.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de"), + ("mkaay", "mkaay@mkaay.de"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'id="file_title">(?P.+)(?P[\d.,]+) (?P[\w^_]+)<' + OFFLINE_PATTERN = r'(Page|file) not found' + + COOKIES = [("uploading.com", "lang", "1"), + (".uploading.com", "language", "1"), + (".uploading.com", "setlang", "en"), + (".uploading.com", "_lang", "en")] + + + def process(self, pyfile): + if not "/get/" in pyfile.url: + pyfile.url = pyfile.url.replace("/files", "/files/get") + + self.html = self.load(pyfile.url, decode=True) + self.getFileInfo() + + if self.premium: + self.handlePremium(pyfile) + else: + self.handleFree(pyfile) + + + def handlePremium(self, pyfile): + postData = {'action': 'get_link', + 'code' : self.info['pattern']['ID'], + 'pass' : 'undefined'} + + self.html = self.load('http://uploading.com/files/get/?JsHttpRequest=%d-xml' % timestamp(), post=postData) + url = re.search(r'"link"\s*:\s*"(.*?)"', self.html) + if url: + url = url.group(1).replace("\\/", "/") + self.download(url) + + raise Exception("Plugin defect") + + + def handleFree(self, pyfile): + m = re.search('

    ((Daily )?Download Limit)

    ', self.html) + if m: + pyfile.error = m.group(1) + self.logWarning(pyfile.error) + self.retry(6, (6 * 60 if m.group(2) else 15) * 60, pyfile.error) + + ajax_url = "http://uploading.com/files/get/?ajax" + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.lastURL = pyfile.url + + res = json_loads(self.load(ajax_url, post={'action': 'second_page', 'code': self.info['pattern']['ID']})) + + if 'answer' in res and 'wait_time' in res['answer']: + wait_time = int(res['answer']['wait_time']) + self.logInfo(_("Waiting %d seconds") % wait_time) + self.wait(wait_time) + else: + self.error(_("No AJAX/WAIT")) + + res = json_loads(self.load(ajax_url, post={'action': 'get_link', 'code': self.info['pattern']['ID'], 'pass': 'false'})) + + if 'answer' in res and 'link' in res['answer']: + url = res['answer']['link'] + else: + self.error(_("No AJAX/URL")) + + self.html = self.load(url) + m = re.search(r'.*?
    \s*\n

    (?P.*?)

    \s*\n
    \s*\n\s*(?P[\d.,]+) (?P[\w^_]+)' + OFFLINE_PATTERN = r'File not found' + + WAIT_PATTERN = r'var sec = (\d+)' + CHASH_PATTERN = r'' + LINK_FREE_PATTERN = r'' + + + def handleFree(self, pyfile): + # STAGE 1: get link to continue + m = re.search(self.CHASH_PATTERN, self.html) + if m is None: + self.error(_("CHASH_PATTERN not found")) + chash = m.group(1) + self.logDebug("Read hash " + chash) + # continue to stage2 + post_data = {'hash': chash, 'free': 'Slow download'} + self.html = self.load(pyfile.url, post=post_data, decode=True) + + # STAGE 2: solv captcha and wait + # first get the infos we need: recaptcha key and wait time + recaptcha = ReCaptcha(self) + + # try the captcha 5 times + for i in xrange(5): + m = re.search(self.WAIT_PATTERN, self.html) + if m is None: + self.error(_("Wait pattern not found")) + wait_time = int(m.group(1)) + + # then, do the waiting + self.wait(wait_time) + + # then, handle the captcha + response, challenge = recaptcha.challenge() + post_data.update({'recaptcha_challenge_field': challenge, + 'recaptcha_response_field' : response}) + + self.html = self.load(pyfile.url, post=post_data, decode=True) + + # STAGE 3: get direct link + m = re.search(self.LINK_FREE_PATTERN, self.html, re.S) + if m: + break + + if m is None: + self.error(_("Download link not found")) + + self.link = m.group(1) diff --git a/pyload/plugin/hoster/UptoboxCom.py b/pyload/plugin/hoster/UptoboxCom.py new file mode 100644 index 000000000..da93b3c6b --- /dev/null +++ b/pyload/plugin/hoster/UptoboxCom.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class UptoboxCom(XFSHoster): + __name__ = "UptoboxCom" + __type__ = "hoster" + __version__ = "0.17" + + __pattern__ = r'https?://(?:www\.)?(uptobox|uptostream)\.com/\w{12}' + + __description__ = """Uptobox.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + INFO_PATTERN = r'"para_title">(?P.+) \((?P[\d.,]+) (?P[\w^_]+)\)' + OFFLINE_PATTERN = r'>(File not found|Access Denied|404 Not Found)' + + LINK_PATTERN = r'"(https?://\w+\.uptobox\.com/d/.*?)"' + + ERROR_PATTERN = r'>(You have to wait.+till next download.)<' #@TODO: Check XFSHoster ERROR_PATTERN + + + def setup(self): + self.multiDL = True + self.chunkLimit = 1 + self.resumeDownload = True diff --git a/pyload/plugin/hoster/VeehdCom.py b/pyload/plugin/hoster/VeehdCom.py new file mode 100644 index 000000000..91d7cc443 --- /dev/null +++ b/pyload/plugin/hoster/VeehdCom.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Hoster import Hoster + + +class VeehdCom(Hoster): + __name__ = "VeehdCom" + __type__ = "hoster" + __version__ = "0.23" + + __pattern__ = r'http://veehd\.com/video/\d+_\S+' + __config__ = [("filename_spaces", "bool", "Allow spaces in filename", False), + ("replacement_char", "str", "Filename replacement character", "_")] + + __description__ = """Veehd.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("cat", "cat@pyload")] + + + def setup(self): + self.multiDL = True + self.req.canContinue = True + + + def process(self, pyfile): + self.download_html() + if not self.file_exists(): + self.offline() + + pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + + def download_html(self): + url = self.pyfile.url + self.logDebug("Requesting page: %s" % url) + self.html = self.load(url) + + + def file_exists(self): + if not self.html: + self.download_html() + + if 'Veehd' in self.html: + return False + return True + + + def get_file_name(self): + if not self.html: + self.download_html() + + m = re.search(r']*>([^<]+) on Veehd', self.html) + if m is None: + self.error(_("Video title not found")) + + name = m.group(1) + + # replace unwanted characters in filename + if self.getConfig('filename_spaces'): + pattern = '[^\w ]+' + else: + pattern = '[^\w.]+' + + return re.sub(pattern, self.getConfig('replacement_char'), name) + '.avi' + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if not self.html: + self.download_html() + + m = re.search(r'v\w+)' + __config__ = [("quality", "Low;High;Auto", "Quality", "Auto")] + + __description__ = """Veoh.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'Sorry, we couldn\'t find the video you were looking for' + + URL_REPLACEMENTS = [(__pattern__ + ".*", r'http://www.veoh.com/watch/\g')] + + COOKIES = [("veoh.com", "lassieLocale", "en")] + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + self.chunkLimit = -1 + + + def handleFree(self, pyfile): + quality = self.getConfig("quality") + if quality == "Auto": + quality = ("High", "Low") + + for q in quality: + pattern = r'"fullPreviewHash%sPath":"(.+?)"' % q + m = re.search(pattern, self.html) + if m: + pyfile.name += ".mp4" + link = m.group(1).replace("\\", "") + self.download(link) + return + else: + self.logInfo(_("No %s quality video found") % q.upper()) + else: + self.fail(_("No video found!")) diff --git a/pyload/plugin/hoster/VidPlayNet.py b/pyload/plugin/hoster/VidPlayNet.py new file mode 100644 index 000000000..094da09bd --- /dev/null +++ b/pyload/plugin/hoster/VidPlayNet.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Test links: +# BigBuckBunny_320x180.mp4 - 61.7 Mb - http://vidplay.net/38lkev0h3jv0 + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class VidPlayNet(XFSHoster): + __name__ = "VidPlayNet" + __type__ = "hoster" + __version__ = "0.04" + + __pattern__ = r'https?://(?:www\.)?vidplay\.net/\w{12}' + + __description__ = """VidPlay.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("t4skforce", "t4skforce1337[AT]gmail[DOT]com")] + + + NAME_PATTERN = r'Password:
    \s*(?P[^<]+)' diff --git a/pyload/plugin/hoster/VimeoCom.py b/pyload/plugin/hoster/VimeoCom.py new file mode 100644 index 000000000..c48ad3414 --- /dev/null +++ b/pyload/plugin/hoster/VimeoCom.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class VimeoCom(SimpleHoster): + __name__ = "VimeoCom" + __type__ = "hoster" + __version__ = "0.04" + + __pattern__ = r'https?://(?:www\.)?(player\.)?vimeo\.com/(video/)?(?P\d+)' + __config__ = [("quality", "Lowest;Mobile;SD;HD;Highest", "Quality", "Highest"), + ("original", "bool", "Try to download the original file first", True)] + + __description__ = """Vimeo.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + NAME_PATTERN = r'(?P<N>.+) on Vimeo<' + OFFLINE_PATTERN = r'class="exception_header"' + TEMP_OFFLINE_PATTERN = r'Please try again in a few minutes.<' + + URL_REPLACEMENTS = [(__pattern__ + ".*", r'https://www.vimeo.com/\g<ID>')] + + COOKIES = [("vimeo.com", "language", "en")] + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + self.chunkLimit = -1 + + + def handleFree(self, pyfile): + password = self.getPassword() + + if self.js and 'class="btn iconify_down_b"' in self.html: + html = self.js.eval(self.load(pyfile.url, get={'action': "download", 'password': password}, decode=True)) + pattern = r'href="(?P<URL>http://vimeo\.com.+?)".*?\>(?P<QL>.+?) ' + else: + html = self.load("https://player.vimeo.com/video/" + self.info['pattern']['ID'], get={'password': password}) + pattern = r'"(?P<QL>\w+)":{"profile".*?"(?P<URL>http://pdl\.vimeocdn\.com.+?)"' + + link = dict((l.group('QL').lower(), l.group('URL')) for l in re.finditer(pattern, html)) + + if self.getConfig("original"): + if "original" in link: + self.download(link[q]) + return + else: + self.logInfo(_("Original file not downloadable")) + + quality = self.getConfig("quality") + if quality == "Highest": + qlevel = ("hd", "sd", "mobile") + elif quality == "Lowest": + qlevel = ("mobile", "sd", "hd") + else: + qlevel = quality.lower() + + for q in qlevel: + if q in link: + self.download(link[q]) + return + else: + self.logInfo(_("No %s quality video found") % q.upper()) + else: + self.fail(_("No video found!")) diff --git a/pyload/plugin/hoster/Vipleech4UCom.py b/pyload/plugin/hoster/Vipleech4UCom.py new file mode 100644 index 000000000..100def197 --- /dev/null +++ b/pyload/plugin/hoster/Vipleech4UCom.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class Vipleech4UCom(DeadHoster): + __name__ = "Vipleech4UCom" + __type__ = "hoster" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?vipleech4u\.com/manager\.php' + + __description__ = """Vipleech4u.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Kagenoshin", "kagenoshin@gmx.ch")] diff --git a/pyload/plugin/hoster/WarserverCz.py b/pyload/plugin/hoster/WarserverCz.py new file mode 100644 index 000000000..d44b751f9 --- /dev/null +++ b/pyload/plugin/hoster/WarserverCz.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class WarserverCz(DeadHoster): + __name__ = "WarserverCz" + __type__ = "hoster" + __version__ = "0.13" + + __pattern__ = r'http://(?:www\.)?warserver\.cz/stahnout/\d+' + + __description__ = """Warserver.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] diff --git a/pyload/plugin/hoster/WebshareCz.py b/pyload/plugin/hoster/WebshareCz.py new file mode 100644 index 000000000..58c9c6a44 --- /dev/null +++ b/pyload/plugin/hoster/WebshareCz.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.network.RequestFactory import getURL +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class WebshareCz(SimpleHoster): + __name__ = "WebshareCz" + __type__ = "hoster" + __version__ = "0.16" + + __pattern__ = r'https?://(?:www\.)?webshare\.cz/(?:#/)?file/(?P<ID>\w+)' + + __description__ = """WebShare.cz hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("rush", "radek.senfeld@gmail.com")] + + + @classmethod + def getInfo(cls, url="", html=""): + info = super(WebshareCz, cls).getInfo(url, html) + + if url: + info['pattern'] = re.match(cls.__pattern__, url).groupdict() + + api_data = getURL("https://webshare.cz/api/file_info/", + post={'ident': info['pattern']['ID']}, + decode=True) + + if 'File not found' in api_data: + info['status'] = 1 + else: + info["status"] = 2 + info['name'] = re.search('<name>(.+)</name>', api_data).group(1) or info['name'] + info['size'] = re.search('<size>(.+)</size>', api_data).group(1) or info['size'] + + return info + + + def handleFree(self, pyfile): + wst = self.account.infos['wst'] if self.account and 'wst' in self.account.infos else "" + + api_data = getURL('https://webshare.cz/api/file_link/', + post={'ident': self.info['pattern']['ID'], 'wst': wst}, + decode=True) + + self.logDebug("API data: " + api_data) + + m = re.search('<link>(.+)</link>', api_data) + if m is None: + self.error(_("Unable to detect direct link")) + + self.link = m.group(1) + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) diff --git a/pyload/plugin/hoster/WrzucTo.py b/pyload/plugin/hoster/WrzucTo.py new file mode 100644 index 000000000..de78d7af2 --- /dev/null +++ b/pyload/plugin/hoster/WrzucTo.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import re + +from pycurl import HTTPHEADER + +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class WrzucTo(SimpleHoster): + __name__ = "WrzucTo" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'http://(?:www\.)?wrzuc\.to/(\w+(\.wt|\.html)|(\w+/?linki/\w+))' + + __description__ = """Wrzuc.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'id="file_info">\s*<strong>(?P<N>.*?)</strong>' + SIZE_PATTERN = r'class="info">\s*<tr>\s*<td>(?P<S>.*?)</td>' + + COOKIES = [("wrzuc.to", "language", "en")] + + + def setup(self): + self.multiDL = True + + + def handleFree(self, pyfile): + data = dict(re.findall(r'(md5|file): "(.*?)"', self.html)) + if len(data) != 2: + self.error(_("No file ID")) + + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.req.http.lastURL = pyfile.url + self.load("http://www.wrzuc.to/ajax/server/prepair", post={"md5": data['md5']}) + + self.req.http.lastURL = pyfile.url + self.html = self.load("http://www.wrzuc.to/ajax/server/download_link", post={"file": data['file']}) + + data.update(re.findall(r'"(download_link|server_id)":"(.*?)"', self.html)) + if len(data) != 4: + self.error(_("No download URL")) + + download_url = "http://%s.wrzuc.to/pobierz/%s" % (data['server_id'], data['download_link']) + self.download(download_url) diff --git a/pyload/plugin/hoster/WuploadCom.py b/pyload/plugin/hoster/WuploadCom.py new file mode 100644 index 000000000..729db1d4d --- /dev/null +++ b/pyload/plugin/hoster/WuploadCom.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class WuploadCom(DeadHoster): + __name__ = "WuploadCom" + __type__ = "hoster" + __version__ = "0.23" + + __pattern__ = r'http://(?:www\.)?wupload\..+?/file/((\w+/)?\d+)(/.*)?' + + __description__ = """Wupload.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de"), + ("Paul King", "")] diff --git a/pyload/plugin/hoster/X7To.py b/pyload/plugin/hoster/X7To.py new file mode 100644 index 000000000..ac01bc5ff --- /dev/null +++ b/pyload/plugin/hoster/X7To.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class X7To(DeadHoster): + __name__ = "X7To" + __type__ = "hoster" + __version__ = "0.41" + + __pattern__ = r'http://(?:www\.)?x7\.to/' + + __description__ = """X7.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("ernieb", "ernieb")] diff --git a/pyload/plugin/hoster/XFileSharingPro.py b/pyload/plugin/hoster/XFileSharingPro.py new file mode 100644 index 000000000..8b9f7fcfa --- /dev/null +++ b/pyload/plugin/hoster/XFileSharingPro.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.XFSHoster import XFSHoster + + +class XFileSharingPro(XFSHoster): + __name__ = "XFileSharingPro" + __type__ = "hoster" + __version__ = "0.44" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharingPro dummy hoster plugin for hook""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + URL_REPLACEMENTS = [("/embed-", "/")] + + + def _log(self, type, args): + msg = " | ".join(str(a).strip() for a in args if a) + logger = getattr(self.log, type) + logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + + + def init(self): + super(XFileSharingPro, self).init() + + self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + + self.HOSTER_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() + self.HOSTER_NAME = "".join(part.capitalize() for part in re.split(r'(\.|\d+)', self.HOSTER_DOMAIN) if part != '.') + + if self.HOSTER_NAME[0].isdigit(): + self.HOSTER_NAME = 'X' + self.HOSTER_NAME + + account = self.core.accountManager.getAccountPlugin(self.HOSTER_NAME) + + if account and account.canUse(): + self.account = account + + elif self.account: + self.account.HOSTER_DOMAIN = self.HOSTER_DOMAIN + + else: + return + + self.user, data = self.account.selectAccount() + self.req = self.account.getAccountRequest(self.user) + self.premium = self.account.isPremium(self.user) + + + def setup(self): + self.chunkLimit = 1 + self.resumeDownload = self.premium + self.multiDL = True diff --git a/pyload/plugin/hoster/XHamsterCom.py b/pyload/plugin/hoster/XHamsterCom.py new file mode 100644 index 000000000..92340152f --- /dev/null +++ b/pyload/plugin/hoster/XHamsterCom.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from pyload.utils import json_loads +from pyload.plugin.Hoster import Hoster + + +def clean_json(json_expr): + json_expr = re.sub('[\n\r]', '', json_expr) + json_expr = re.sub(' +', '', json_expr) + json_expr = re.sub('\'', '"', json_expr) + + return json_expr + + +class XHamsterCom(Hoster): + __name__ = "XHamsterCom" + __type__ = "hoster" + __version__ = "0.12" + + __pattern__ = r'http://(?:www\.)?xhamster\.com/movies/.+' + __config__ = [("type", ".mp4;.flv", "Preferred type", ".mp4")] + + __description__ = """XHamster.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [] + + + def process(self, pyfile): + self.pyfile = pyfile + + if not self.file_exists(): + self.offline() + + if self.getConfig("type"): + self.desired_fmt = self.getConfig("type") + + pyfile.name = self.get_file_name() + self.desired_fmt + self.download(self.get_file_url()) + + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if not self.html: + self.download_html() + + flashvar_pattern = re.compile('flashvars = ({.*?});', re.S) + json_flashvar = flashvar_pattern.search(self.html) + + if not json_flashvar: + self.error(_("flashvar not found")) + + j = clean_json(json_flashvar.group(1)) + flashvars = json_loads(j) + + if flashvars['srv']: + srv_url = flashvars['srv'] + '/' + else: + self.error(_("srv_url not found")) + + if flashvars['url_mode']: + url_mode = flashvars['url_mode'] + + + else: + self.error(_("url_mode not found")) + + if self.desired_fmt == ".mp4": + file_url = re.search(r"<a href=\"" + srv_url + "(.+?)\"", self.html) + if file_url is None: + self.error(_("file_url not found")) + file_url = file_url.group(1) + long_url = srv_url + file_url + self.logDebug("long_url = " + long_url) + else: + if flashvars['file']: + file_url = unquote(flashvars['file']) + else: + self.error(_("file_url not found")) + + if url_mode == '3': + long_url = file_url + self.logDebug("long_url = " + long_url) + else: + long_url = srv_url + "key=" + file_url + self.logDebug("long_url = " + long_url) + + return long_url + + + def get_file_name(self): + if not self.html: + self.download_html() + + pattern = r'<title>(.*?) - xHamster\.com' + name = re.search(pattern, self.html) + if name is None: + pattern = r'

    (.*)

    ' + name = re.search(pattern, self.html) + if name is None: + pattern = r'http://[www.]+xhamster\.com/movies/.*/(.*?)\.html?' + name = re.match(file_name_pattern, self.pyfile.url) + if name is None: + pattern = r'' + name = re.search(pattern, self.html) + if name is None: + return "Unknown" + + return name.group(1) + + + def file_exists(self): + """ returns True or False + """ + if not self.html: + self.download_html() + if re.search(r"(.*Video not found.*)", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugin/hoster/XVideosCom.py b/pyload/plugin/hoster/XVideosCom.py new file mode 100644 index 000000000..c5e2921cb --- /dev/null +++ b/pyload/plugin/hoster/XVideosCom.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from pyload.plugin.Hoster import Hoster + + +class XVideosCom(Hoster): + __name__ = "XVideos.com" + __type__ = "hoster" + __version__ = "0.10" + + __pattern__ = r'http://(?:www\.)?xvideos\.com/video(\d+)' + + __description__ = """XVideos.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [] + + + def process(self, pyfile): + site = self.load(pyfile.url) + pyfile.name = "%s (%s).flv" % ( + re.search(r"

    ([^<]+)Filename:\s*
    \s*(?P.*?)\n' + SIZE_PATTERN = r'\s*
    \s*(?P[\d.,]+)(?P[\w^_]+)' + OFFLINE_PATTERN = r' Device Filter

    ' + + + def setup(self): + self.multiDL = True + self.resumeDownload = True + self.chunkLimit = 1 + + + def handleFree(self, pyfile): + self.link = pyfile.url + "&task=get" #@TODO: Revert to `get={'task': "get"}` in 0.4.10 diff --git a/pyload/plugin/hoster/Xdcc.py b/pyload/plugin/hoster/Xdcc.py new file mode 100644 index 000000000..277b7ad0c --- /dev/null +++ b/pyload/plugin/hoster/Xdcc.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- + +import re +import socket +import struct +import sys + +from os import makedirs +from os.path import exists, join +from select import select +from time import time + +from pyload.plugin.Hoster import Hoster +from pyload.utils import safe_join + + +class Xdcc(Hoster): + __name__ = "Xdcc" + __type__ = "hoster" + __version__ = "0.32" + + __config__ = [("nick", "str", "Nickname", "pyload"), + ("ident", "str", "Ident", "pyloadident"), + ("realname", "str", "Realname", "pyloadreal")] + + __description__ = """Download from IRC XDCC bot""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.com")] + + + def setup(self): + self.debug = 0 # 0,1,2 + self.timeout = 30 + self.multiDL = False + + + def process(self, pyfile): + # change request type + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__, type="XDCC") + + self.pyfile = pyfile + for _i in xrange(0, 3): + try: + nmn = self.doDownload(pyfile.url) + self.logDebug("Download of %s finished." % nmn) + return + except socket.error, e: + if hasattr(e, "errno"): + errno = e.errno + else: + errno = e.args[0] + + if errno == 10054: + self.logDebug("Server blocked our ip, retry in 5 min") + self.setWait(300) + self.wait() + continue + + self.fail(_("Failed due to socket errors. Code: %d") % errno) + + self.fail(_("Server blocked our ip, retry again later manually")) + + + def doDownload(self, url): + self.pyfile.setStatus("waiting") # real link + + m = re.match(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url) + server = m.group(1) + chan = m.group(2) + bot = m.group(3) + pack = m.group(4) + nick = self.getConfig('nick') + ident = self.getConfig('ident') + real = self.getConfig('realname') + + temp = server.split(':') + ln = len(temp) + if ln == 2: + host, port = temp + elif ln == 1: + host, port = temp[0], 6667 + else: + self.fail(_("Invalid hostname for IRC Server: %s") % server) + + ####################### + # CONNECT TO IRC AND IDLE FOR REAL LINK + dl_time = time() + + sock = socket.socket() + sock.connect((host, int(port))) + if nick == "pyload": + nick = "pyload-%d" % (time() % 1000) # last 3 digits + sock.send("NICK %s\r\n" % nick) + sock.send("USER %s %s bla :%s\r\n" % (ident, host, real)) + + self.setWait(3) + self.wait() + + sock.send("JOIN #%s\r\n" % chan) + sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) + + # IRC recv loop + readbuffer = "" + done = False + retry = None + m = None + while True: + + # done is set if we got our real link + if done: + break + + if retry: + if time() > retry: + retry = None + dl_time = time() + sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) + + else: + if (dl_time + self.timeout) < time(): # todo: add in config + sock.send("QUIT :byebye\r\n") + sock.close() + self.fail(_("XDCC Bot did not answer")) + + fdset = select([sock], [], [], 0) + if sock not in fdset[0]: + continue + + readbuffer += sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + if self.debug is 2: + print "*> " + unicode(line, errors='ignore') + line = line.rstrip() + first = line.split() + + if first[0] == "PING": + sock.send("PONG %s\r\n" % first[1]) + + if first[0] == "ERROR": + self.fail(_("IRC-Error: %s") % line) + + msg = line.split(None, 3) + if len(msg) != 4: + continue + + msg = { + "origin": msg[0][1:], + "action": msg[1], + "target": msg[2], + "text": msg[3][1:] + } + + if nick == msg['target'][0:len(nick)] and "PRIVMSG" == msg['action']: + if msg['text'] == "\x01VERSION\x01": + self.logDebug("Sending CTCP VERSION") + sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) + elif msg['text'] == "\x01TIME\x01": + self.logDebug("Sending CTCP TIME") + sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time())) + elif msg['text'] == "\x01LAG\x01": + pass # don't know how to answer + + if not (bot == msg['origin'][0:len(bot)] + and nick == msg['target'][0:len(nick)] + and msg['action'] in ("PRIVMSG", "NOTICE")): + continue + + if self.debug is 1: + print "%s: %s" % (msg['origin'], msg['text']) + + if "You already requested that pack" in msg['text']: + retry = time() + 300 + + if "you must be on a known channel to request a pack" in msg['text']: + self.fail(_("Wrong channel")) + + m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg['text']) + if m: + done = True + + # get connection data + ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2))))) + port = int(m.group(3)) + packname = m.group(1) + + if len(m.groups()) > 3: + self.req.filesize = int(m.group(4)) + + self.pyfile.name = packname + + download_folder = self.config['general']['download_folder'] + filename = safe_join(download_folder, packname) + + self.logInfo(_("Downloading %s from %s:%d") % (packname, ip, port)) + + self.pyfile.setStatus("downloading") + newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress) + if newname and newname != filename: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": self.pyfile.name, "newname": newname}) + filename = newname + + # kill IRC socket + # sock.send("QUIT :byebye\r\n") + sock.close() + + self.lastDownload = filename + return self.lastDownload diff --git a/pyload/plugin/hoster/YibaishiwuCom.py b/pyload/plugin/hoster/YibaishiwuCom.py new file mode 100644 index 000000000..a53acdaf5 --- /dev/null +++ b/pyload/plugin/hoster/YibaishiwuCom.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.utils import json_loads +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class YibaishiwuCom(SimpleHoster): + __name__ = "YibaishiwuCom" + __type__ = "hoster" + __version__ = "0.14" + + __pattern__ = r'http://(?:www\.)?(?:u\.)?115\.com/file/(?P\w+)' + + __description__ = """115.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + NAME_PATTERN = r'file_name: \'(?P.+?)\'' + SIZE_PATTERN = r'file_size: \'(?P.+?)\'' + OFFLINE_PATTERN = ur'

    哎呀!提取码不存在!不妨搜搜看吧!

    ' + + LINK_FREE_PATTERN = r'(/\?ct=(pickcode|download)[^"\']+)' + + + def handleFree(self, pyfile): + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("LINK_FREE_PATTERN not found")) + + url = m.group(1) + + self.logDebug(('FREEUSER' if m.group(2) == 'download' else 'GUEST') + ' URL', url) + + res = json_loads(self.load("http://115.com" + url, decode=False)) + if "urls" in res: + mirrors = res['urls'] + + elif "data" in res: + mirrors = res['data'] + + else: + mirrors = None + + for mr in mirrors: + try: + url = mr['url'].replace("\\", "") + self.logDebug("Trying URL: " + url) + self.download(url) + break + except Exception: + continue + else: + self.fail(_("No working link found")) diff --git a/pyload/plugin/hoster/YoupornCom.py b/pyload/plugin/hoster/YoupornCom.py new file mode 100644 index 000000000..75606935d --- /dev/null +++ b/pyload/plugin/hoster/YoupornCom.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.Hoster import Hoster + + +class YoupornCom(Hoster): + __name__ = "YoupornCom" + __type__ = "hoster" + __version__ = "0.20" + + __pattern__ = r'http://(?:www\.)?youporn\.com/watch/.+' + + __description__ = """Youporn.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("willnix", "willnix@pyload.org")] + + + def process(self, pyfile): + self.pyfile = pyfile + + if not self.file_exists(): + self.offline() + + pyfile.name = self.get_file_name() + self.download(self.get_file_url()) + + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url, post={"user_choice": "Enter"}, cookies=False) + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if not self.html: + self.download_html() + + return re.search(r'(http://download\.youporn\.com/download/\d+\?save=1)">', self.html).group(1) + + + def get_file_name(self): + if not self.html: + self.download_html() + + file_name_pattern = r'(.+) - ' + return re.search(file_name_pattern, self.html).group(1).replace("&", "&").replace("/", "") + '.flv' + + + def file_exists(self): + """ returns True or False + """ + if not self.html: + self.download_html() + if re.search(r"(.*invalid video_id.*)", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugin/hoster/YourfilesTo.py b/pyload/plugin/hoster/YourfilesTo.py new file mode 100644 index 000000000..a600d822f --- /dev/null +++ b/pyload/plugin/hoster/YourfilesTo.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from pyload.plugin.Hoster import Hoster + + +class YourfilesTo(Hoster): + __name__ = "YourfilesTo" + __type__ = "hoster" + __version__ = "0.22" + + __pattern__ = r'http://(?:www\.)?yourfiles\.(to|biz)/\?d=\w+' + + __description__ = """Youfiles.to hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("jeix", "jeix@hasnomail.de"), + ("skydancer", "skydancer@hasnomail.de")] + + + def process(self, pyfile): + self.pyfile = pyfile + self.prepare() + self.download(self.get_file_url()) + + + def prepare(self): + if not self.file_exists(): + self.offline() + + self.pyfile.name = self.get_file_name() + + wait_time = self.get_waiting_time() + self.setWait(wait_time) + self.wait() + + + def get_waiting_time(self): + if not self.html: + self.download_html() + + #var zzipitime = 15; + m = re.search(r'var zzipitime = (\d+);', self.html) + if m: + sec = int(m.group(1)) + else: + sec = 0 + + return sec + + + def download_html(self): + url = self.pyfile.url + self.html = self.load(url) + + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + url = re.search(r"var bla = '(.*?)';", self.html) + if url: + url = url.group(1) + url = unquote(url.replace("http://http:/http://", "http://").replace("dumdidum", "")) + return url + else: + self.error(_("Absolute filepath not found")) + + + def get_file_name(self): + if not self.html: + self.download_html() + + return re.search("<title>(.*)", self.html).group(1) + + + def file_exists(self): + """ returns True or False + """ + if not self.html: + self.download_html() + + if re.search(r"HTTP Status 404", self.html) is not None: + return False + else: + return True diff --git a/pyload/plugin/hoster/YoutubeCom.py b/pyload/plugin/hoster/YoutubeCom.py new file mode 100644 index 000000000..bf8785022 --- /dev/null +++ b/pyload/plugin/hoster/YoutubeCom.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +import os +import re +import subprocess + +from urllib import unquote + +from pyload.plugin.Hoster import Hoster +from pyload.plugin.internal.SimpleHoster import replace_patterns +from pyload.utils import html_unescape + + +def which(program): + """Works exactly like the unix command which + Courtesy of http://stackoverflow.com/a/377028/675646""" + + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) + + fpath, fname = os.path.split(program) + + if fpath: + if isExe(program): + return program + else: + for path in os.environ['PATH'].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if isExe(exe_file): + return exe_file + + +class YoutubeCom(Hoster): + __name__ = "YoutubeCom" + __type__ = "hoster" + __version__ = "0.41" + + __pattern__ = r'https?://(?:[^/]*\.)?(youtube\.com|youtu\.be)/watch\?(?:.*&)?v=.+' + __config__ = [("quality", "sd;hd;fullhd;240p;360p;480p;720p;1080p;3072p", "Quality Setting" , "hd" ), + ("fmt" , "int" , "FMT/ITAG Number (0 for auto)", 0 ), + (".mp4" , "bool" , "Allow .mp4" , True ), + (".flv" , "bool" , "Allow .flv" , True ), + (".webm" , "bool" , "Allow .webm" , False), + (".3gp" , "bool" , "Allow .3gp" , False), + ("3d" , "bool" , "Prefer 3D" , False)] + + __description__ = """Youtube.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("spoob", "spoob@pyload.org"), + ("zoidberg", "zoidberg@mujmail.cz")] + + + URL_REPLACEMENTS = [(r'youtu\.be/', 'youtube.com/')] + + # Invalid characters that must be removed from the file name + invalidChars = u'\u2605:?><"|\\' + + # name, width, height, quality ranking, 3D + formats = {5 : (".flv" , 400 , 240 , 1 , False), + 6 : (".flv" , 640 , 400 , 4 , False), + 17 : (".3gp" , 176 , 144 , 0 , False), + 18 : (".mp4" , 480 , 360 , 2 , False), + 22 : (".mp4" , 1280, 720 , 8 , False), + 43 : (".webm", 640 , 360 , 3 , False), + 34 : (".flv" , 640 , 360 , 4 , False), + 35 : (".flv" , 854 , 480 , 6 , False), + 36 : (".3gp" , 400 , 240 , 1 , False), + 37 : (".mp4" , 1920, 1080, 9 , False), + 38 : (".mp4" , 4096, 3072, 10, False), + 44 : (".webm", 854 , 480 , 5 , False), + 45 : (".webm", 1280, 720 , 7 , False), + 46 : (".webm", 1920, 1080, 9 , False), + 82 : (".mp4" , 640 , 360 , 3 , True ), + 83 : (".mp4" , 400 , 240 , 1 , True ), + 84 : (".mp4" , 1280, 720 , 8 , True ), + 85 : (".mp4" , 1920, 1080, 9 , True ), + 100: (".webm", 640 , 360 , 3 , True ), + 101: (".webm", 640 , 360 , 4 , True ), + 102: (".webm", 1280, 720 , 8 , True )} + + + def setup(self): + self.resumeDownload = True + self.multiDL = True + + + def process(self, pyfile): + pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) + html = self.load(pyfile.url, decode=True) + + if re.search(r'
    ', html): + self.offline() + + if "We have been receiving a large volume of requests from your network." in html: + self.tempOffline() + + #get config + use3d = self.getConfig("3d") + + if use3d: + quality = {"sd": 82, "hd": 84, "fullhd": 85, "240p": 83, "360p": 82, + "480p": 82, "720p": 84, "1080p": 85, "3072p": 85} + else: + quality = {"sd": 18, "hd": 22, "fullhd": 37, "240p": 5, "360p": 18, + "480p": 35, "720p": 22, "1080p": 37, "3072p": 38} + + desired_fmt = self.getConfig("fmt") + + if not desired_fmt: + desired_fmt = quality.get(self.getConfig("quality"), 18) + + elif desired_fmt not in self.formats: + self.logWarning(_("FMT %d unknown, using default") % desired_fmt) + desired_fmt = 0 + + #parse available streams + streams = re.search(r'"url_encoded_fmt_stream_map":"(.+?)",', html).group(1) + streams = [x.split('\u0026') for x in streams.split(',')] + streams = [dict((y.split('=', 1)) for y in x) for x in streams] + streams = [(int(x['itag']), unquote(x['url'])) for x in streams] + + # self.logDebug("Found links: %s" % streams) + + self.logDebug("AVAILABLE STREAMS: %s" % [x[0] for x in streams]) + + #build dictionary of supported itags (3D/2D) + allowed = lambda x: self.getConfig(self.formats[x][0]) + streams = [x for x in streams if x[0] in self.formats and allowed(x[0])] + + if not streams: + self.fail(_("No available stream meets your preferences")) + + fmt_dict = dict([x for x in streams if self.formats[x[0]][4] == use3d] or streams) + + self.logDebug("DESIRED STREAM: ITAG:%d (%s) %sfound, %sallowed" % + (desired_fmt, "%s %dx%d Q:%d 3D:%s" % self.formats[desired_fmt], + "" if desired_fmt in fmt_dict else "NOT ", "" if allowed(desired_fmt) else "NOT ")) + + #return fmt nearest to quality index + if desired_fmt in fmt_dict and allowed(desired_fmt): + fmt = desired_fmt + else: + sel = lambda x: self.formats[x][3] # select quality index + comp = lambda x, y: abs(sel(x) - sel(y)) + + self.logDebug("Choosing nearest fmt: %s" % [(x, allowed(x), comp(x, desired_fmt)) for x in fmt_dict.keys()]) + + fmt = reduce(lambda x, y: x if comp(x, desired_fmt) <= comp(y, desired_fmt) and + sel(x) > sel(y) else y, fmt_dict.keys()) + + self.logDebug("Chosen fmt: %s" % fmt) + + url = fmt_dict[fmt] + + self.logDebug("URL: %s" % url) + + #set file name + file_suffix = self.formats[fmt][0] if fmt in self.formats else ".flv" + file_name_pattern = '' + name = re.search(file_name_pattern, html).group(1).replace("/", "") + + # Cleaning invalid characters from the file name + name = name.encode('ascii', 'replace') + for c in self.invalidChars: + name = name.replace(c, '_') + + pyfile.name = html_unescape(name) + + time = re.search(r"t=((\d+)m)?(\d+)s", pyfile.url) + ffmpeg = which("ffmpeg") + if ffmpeg and time: + m, s = time.groups()[1:] + if m is None: + m = "0" + + pyfile.name += " (starting at %s:%s)" % (m, s) + + pyfile.name += file_suffix + filename = self.download(url) + + if ffmpeg and time: + inputfile = filename + "_" + os.rename(filename, inputfile) + + subprocess.call([ + ffmpeg, + "-ss", "00:%s:%s" % (m, s), + "-i", inputfile, + "-vcodec", "copy", + "-acodec", "copy", + filename]) + + os.remove(inputfile) diff --git a/pyload/plugin/hoster/ZDF.py b/pyload/plugin/hoster/ZDF.py new file mode 100644 index 000000000..da6d6448e --- /dev/null +++ b/pyload/plugin/hoster/ZDF.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import re + +from xml.etree.ElementTree import fromstring + +from pyload.plugin.Hoster import Hoster + + +# Based on zdfm by Roland Beermann (http://github.com/enkore/zdfm/) +class ZDF(Hoster): + __name__ = "ZDF Mediathek" + __type__ = "hoster" + __version__ = "0.80" + + __pattern__ = r'http://(?:www\.)?zdf\.de/ZDFmediathek/\D*(\d+)\D*' + + __description__ = """ZDF.de hoster plugin""" + __license__ = "GPLv3" + __authors__ = [] + + XML_API = "http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?id=%i" + + + @staticmethod + def video_key(video): + return ( + int(video.findtext("videoBitrate", "0")), + any(f.text == "progressive" for f in video.iter("facet")), + ) + + + @staticmethod + def video_valid(video): + return video.findtext("url").startswith("http") and video.findtext("url").endswith(".mp4") and \ + video.findtext("facets/facet").startswith("progressive") + + + @staticmethod + def get_id(url): + return int(re.search(r"\D*(\d{4,})\D*", url).group(1)) + + + def process(self, pyfile): + xml = fromstring(self.load(self.XML_API % self.get_id(pyfile.url))) + + status = xml.findtext("./status/statuscode") + if status != "ok": + self.fail(_("Error retrieving manifest")) + + video = xml.find("video") + title = video.findtext("information/title") + + pyfile.name = title + + target_url = sorted((v for v in video.iter("formitaet") if self.video_valid(v)), + key=self.video_key)[-1].findtext("url") + + self.download(target_url) diff --git a/pyload/plugin/hoster/ZShareNet.py b/pyload/plugin/hoster/ZShareNet.py new file mode 100644 index 000000000..12c65b206 --- /dev/null +++ b/pyload/plugin/hoster/ZShareNet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.DeadHoster import DeadHoster + + +class ZShareNet(DeadHoster): + __name__ = "ZShareNet" + __type__ = "hoster" + __version__ = "0.21" + + __pattern__ = r'https?://(?:ww[2w]\.)?zshares?\.net/.+' + + __description__ = """ZShare.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("espes", ""), + ("Cptn Sandwich", "")] diff --git a/pyload/plugin/hoster/ZeveraCom.py b/pyload/plugin/hoster/ZeveraCom.py new file mode 100644 index 000000000..264a62a7d --- /dev/null +++ b/pyload/plugin/hoster/ZeveraCom.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.plugin.internal.MultiHoster import MultiHoster + + +class ZeveraCom(MultiHoster): + __name__ = "ZeveraCom" + __type__ = "hoster" + __version__ = "0.28" + + __pattern__ = r'https?://(?:www\.)zevera\.com/(getFiles\.ashx|Members/download\.ashx)\?.*ourl=.+' + + __description__ = """Zevera.com multi-hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def handlePremium(self, pyfile): + self.link = "https://%s/getFiles.ashx?ourl=%s" % (self.account.HOSTER_DOMAIN, pyfile.url) + + + def checkFile(self): + if self.checkDownload({"error": 'action="ErrorDownload.aspx'}): + self.fail(_("Error response received")) + + return super(ZeveraCom, self).checkFile() diff --git a/pyload/plugin/hoster/ZippyshareCom.py b/pyload/plugin/hoster/ZippyshareCom.py new file mode 100644 index 000000000..0aa59fdaa --- /dev/null +++ b/pyload/plugin/hoster/ZippyshareCom.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.CaptchaService import ReCaptcha +from pyload.plugin.internal.SimpleHoster import SimpleHoster + + +class ZippyshareCom(SimpleHoster): + __name__ = "ZippyshareCom" + __type__ = "hoster" + __version__ = "0.72" + + __pattern__ = r'http://www\d{0,2}\.zippyshare\.com/v(/|iew\.jsp.*key=)(?P[\w^_]+)' + + __description__ = """Zippyshare.com hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + COOKIES = [("zippyshare.com", "ziplocale", "en")] + + NAME_PATTERN = r'("\d{6,}/"[ ]*\+.+?"/|Zippyshare.com - )(?P<N>.+?)("|)' + SIZE_PATTERN = r'>Size:.+?">(?P[\d.,]+) (?P[\w^_]+)' + OFFLINE_PATTERN = r'>File does not exist on this server' + + LINK_PREMIUM_PATTERN = r'document.location = \'(.+?)\'' + + + def setup(self): + self.chunkLimit = -1 + self.multiDL = True + self.resumeDownload = True + + + def handleFree(self, pyfile): + recaptcha = ReCaptcha(self) + captcha_key = recaptcha.detect_key() + + if captcha_key: + try: + self.link = re.search(self.LINK_PREMIUM_PATTERN, self.html) + recaptcha.challenge() + + except Exception, e: + self.error(e) + + else: + self.link = '/'.join(("d", self.info['pattern']['KEY'], str(self.get_checksum()), self.pyfile.name)) + + + def get_checksum(self): + try: + n = 2 + b = int(re.search(r'var b = (\d+)', self.html).group(1)) + checksum = int("%d3" % (n + n * 2 + b)) + + except Exception: + self.error(_("Unable to calculate checksum")) + + else: + return checksum diff --git a/pyload/plugin/hoster/__init__.py b/pyload/plugin/hoster/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/hoster/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/pyload/plugin/internal/BasePlugin.py b/pyload/plugin/internal/BasePlugin.py new file mode 100644 index 000000000..103e0d5cb --- /dev/null +++ b/pyload/plugin/internal/BasePlugin.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote +from urlparse import urljoin, urlparse + +from pyload.network.HTTPRequest import BadHeader +from pyload.plugin.internal.SimpleHoster import fileUrl +from pyload.plugin.Hoster import Hoster + + +class BasePlugin(Hoster): + __name__ = "BasePlugin" + __type__ = "hoster" + __version__ = "0.34" + + __pattern__ = r'^unmatchable$' + + __description__ = """Base plugin when any other didnt fit""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + url = unquote(url) + url_p = urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + + def setup(self): + self.chunkLimit = -1 + self.multiDL = True + self.resumeDownload = True + + + def process(self, pyfile): + """main function""" + + pyfile.name = self.getInfo(pyfile.url)['name'] + + if not pyfile.url.startswith("http"): + self.fail(_("No plugin matched")) + + for _i in xrange(5): + try: + link = fileUrl(self, unquote(pyfile.url)) + + if link: + self.download(link, ref=False, disposition=True) + else: + self.fail(_("File not found")) + + except BadHeader, e: + if e.code is 404: + self.offline() + + elif e.code in (401, 403): + self.logDebug("Auth required", "Received HTTP status code: %d" % e.code) + + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc + + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.getAccountData(server)['password']) + else: + pwd = self.getPassword() + if ':' in pwd: + self.req.addAuth(pwd) + else: + self.fail(_("Authorization required")) + else: + self.fail(e) + else: + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 + + check = self.checkDownload({'empty file': re.compile(r'\A\Z'), + 'html file' : re.compile(r'\A\s*)?\d{3}(\Z|\s+)')}) + if check: + self.fail(check.capitalize()) diff --git a/pyload/plugin/internal/DeadCrypter.py b/pyload/plugin/internal/DeadCrypter.py new file mode 100644 index 000000000..ddeb0431d --- /dev/null +++ b/pyload/plugin/internal/DeadCrypter.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Crypter import Crypter as _Crypter + + +class DeadCrypter(_Crypter): + __name__ = "DeadCrypter" + __type__ = "crypter" + __version__ = "0.04" + + __pattern__ = r'^unmatchable$' + + __description__ = """Crypter is no longer available""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadCrypter, self).apiInfo(url, get, post) + api['status'] = 1 + return api + + + def setup(self): + self.pyfile.error = "Crypter is no longer available" + self.offline() #@TODO: self.offline("Crypter is no longer available") diff --git a/pyload/plugin/internal/DeadHoster.py b/pyload/plugin/internal/DeadHoster.py new file mode 100644 index 000000000..1596943ae --- /dev/null +++ b/pyload/plugin/internal/DeadHoster.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Hoster import Hoster as _Hoster + + +class DeadHoster(_Hoster): + __name__ = "DeadHoster" + __type__ = "hoster" + __version__ = "0.14" + + __pattern__ = r'^unmatchable$' + + __description__ = """Hoster is no longer available""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadHoster, self).apiInfo(url, get, post) + api['status'] = 1 + return api + + + def setup(self): + self.pyfile.error = "Hoster is no longer available" + self.offline() #@TODO: self.offline("Hoster is no longer available") diff --git a/pyload/plugin/internal/MultiHook.py b/pyload/plugin/internal/MultiHook.py new file mode 100644 index 000000000..2beccfcc5 --- /dev/null +++ b/pyload/plugin/internal/MultiHook.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from pyload.plugin.Hook import Hook +from pyload.utils import decode, remove_chars + + +class MultiHook(Hook): + __name__ = "MultiHook" + __type__ = "hook" + __version__ = "0.37" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Hook plugin for multi hoster/crypter""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + MIN_INTERVAL = 1 * 60 * 60 + + DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), + (r'1fichier\.com' , "onefichier.com" ), + (r'2shared\.com' , "twoshared.com" ), + (r'4shared\.com' , "fourshared.com" ), + (r'bayfiles\.net' , "bayfiles.com" ), + (r'cloudnator\.com' , "shragle.com" ), + (r'dfiles\.eu' , "depositfiles.com" ), + (r'easy-share\.com' , "crocko.com" ), + (r'freakshare\.net' , "freakshare.com" ), + (r'hellshare\.com' , "hellshare.cz" ), + (r'ifile\.it' , "filecloud.io" ), + (r'nowdownload\.\w+', "nowdownload.sx" ), + (r'nowvideo\.\w+' , "nowvideo.sx" ), + (r'putlocker\.com' , "firedrive.com" ), + (r'share-?rapid\.cz', "multishare.cz" ), + (r'ul\.to' , "uploaded.to" ), + (r'uploaded\.net' , "uploaded.to" ), + (r'uploadhero\.co' , "uploadhero.com" ), + (r'zshares\.net' , "zshare.net" ), + (r'(\d+.+)' , "X\1" )] + + + def setup(self): + self.plugins = [] + self.supported = [] + self.new_supported = [] + + self.account = None + self.pluginclass = None + self.pluginmodule = None + self.pluginname = None + self.plugintype = None + + self._initPlugin() + + + def _initPlugin(self): + plugin, type = self.core.pluginManager.findPlugin(self.__name__) + + if not plugin: + self.logWarning("Hook plugin will be deactivated due missing plugin reference") + self.setConfig('activated', False) + else: + self.pluginname = self.__name__ + self.plugintype = type + self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__) + self.pluginclass = getattr(self.pluginmodule, self.__name__) + + + def _loadAccount(self): + self.account = self.core.accountManager.getAccountPlugin(self.pluginname) + + if self.account and not self.account.canUse(): + self.account = None + + if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: + self.logWarning("Hook plugin will be deactivated due missing account reference") + self.setConfig('activated', False) + + + def activate(self): + self._loadAccount() + + + def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 + """ see HTTPRequest for argument list """ + h = pyreq.getHTTPRequest(timeout=120) + try: + if not 'decode' in kwargs: + kwargs['decode'] = True + rep = h.load(*args, **kwargs) + finally: + h.close() + + return rep + + + def getConfig(self, option, default=''): + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + + except KeyError: + return default + + + def pluginsCached(self): + if self.plugins: + return self.plugins + + for _i in xrange(3): + try: + pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) + + except Exception, e: + self.logError(e, "Waiting 1 minute and retry") + sleep(60) + + else: + break + else: + return list() + + try: + configmode = self.getConfig("pluginmode", 'all') + if configmode in ("listed", "unlisted"): + pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + configset = self._pluginSet(pluginlist) + + if configmode == "listed": + pluginset &= configset + else: + pluginset -= configset + + except Exception, e: + self.logError(e) + + self.plugins = list(pluginset) + + return self.plugins + + + def _pluginSet(self, plugins): + plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) + + for rf, rt in self.DOMAIN_REPLACEMENTS: + regex = re.compile(rf) + for p in filter(lambda x: regex.match(x), plugins): + plugins.remove(p) + plugins.add(re.sub(rf, rt, p)) + + plugins.discard('') + + return plugins + + + def getHosters(self): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def getCrypters(self): + """Load list of supported crypters + + :return: List of domain names + """ + raise NotImplementedError + + + def periodical(self): + """reload plugin list periodically""" + self.logInfo(_("Reloading supported %s list") % self.plugintype) + + old_supported = self.supported + + self.supported = [] + self.new_supported = [] + self.plugins = [] + + self.overridePlugins() + + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + + if old_supported: + self.logDebug("Unload: %s" % ", ".join(old_supported)) + for plugin in old_supported: + self.unloadPlugin(plugin) + + if self.getConfig("reload", True): + self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL) + else: + self.core.scheduler.removeJob(self.cb) + self.cb = None + + + def overridePlugins(self): + excludedList = [] + + if self.plugintype == "hoster": + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) + accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] + + for plugin in self.pluginsCached(): + name = remove_chars(plugin, "-.") + + if name in accountList: + excludedList.append(plugin) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(plugin) + + if not self.supported and not self.new_supported: + self.logError(_("No %s loaded") % self.plugintype) + return + + # inject plugin plugin + self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + hdict['new_module'] = self.pluginmodule + hdict['new_name'] = self.pluginname + + if excludedList: + self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + + if self.new_supported: + plugins = sorted(self.new_supported) + + self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + + # create new regexp + regexp = r'.*(?P%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: + regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) + + self.logDebug("Regexp: %s" % regexp) + + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unloadPlugin(self, plugin): + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + if "module" in hdict: + del hdict['module'] + + if "new_module" in hdict: + del hdict['new_module'] + del hdict['new_name'] + + + def deactivate(self): + """Remove override for all plugins. Scheduler job is removed by hookmanager""" + for plugin in self.supported: + self.unloadPlugin(plugin) + + # reset pattern + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + + hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) + + + def downloadFailed(self, pyfile): + """remove plugin override if download fails but not if file is offline/temp.offline""" + if pyfile.status != 8 or not self.getConfig("revertfailed", True): + return + + hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] + if "new_name" in hdict and hdict['new_name'] == self.pluginname: + if pyfile.error == "MultiHook": + self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) + self.unloadPlugin(pyfile.pluginname) + pyfile.setStatus("queued") + pyfile.sync() + else: + retries = max(self.getConfig("retry", 10), 0) + wait_time = max(self.getConfig("retryinterval", 1), 0) + + if 0 < retries > pyfile.plugin.retries: + self.logInfo(_("Retrying: %s") % pyfile.name) + pyfile.setCustomStatus("MultiHook", "queued") + pyfile.sync() + + pyfile.plugin.retries += 1 + pyfile.plugin.setWait(wait_time) + pyfile.plugin.wait() diff --git a/pyload/plugin/internal/MultiHoster.py b/pyload/plugin/internal/MultiHoster.py new file mode 100644 index 000000000..ed425ffaa --- /dev/null +++ b/pyload/plugin/internal/MultiHoster.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies + + +class MultiHoster(SimpleHoster): + __name__ = "MultiHoster" + __type__ = "hoster" + __version__ = "0.37" + + __pattern__ = r'^unmatchable$' + + __description__ = """Multi hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + LOGIN_ACCOUNT = True + + + def setup(self): + self.chunkLimit = 1 + self.multiDL = bool(self.account) + self.resumeDownload = self.premium + + + def prepare(self): + self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + if self.DIRECT_LINK is None: + self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + else: + self.directDL = self.DIRECT_LINK + + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + + + def process(self, pyfile): + self.prepare() + + if self.directDL: + self.checkInfo() + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) + + if not self.link and not self.lastDownload: + self.preload() + + self.checkErrors() + self.checkStatus(getinfo=False) + + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) + + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) + + self.downloadLink(self.link, True) + self.checkFile() + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) + + + def handleFree(self, pyfile): + if self.premium: + raise NotImplementedError + else: + self.fail(_("Required premium account not found")) diff --git a/pyload/plugin/internal/SimpleCrypter.py b/pyload/plugin/internal/SimpleCrypter.py new file mode 100644 index 000000000..e4b8874f3 --- /dev/null +++ b/pyload/plugin/internal/SimpleCrypter.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin, urlparse + +from pyload.plugin.Crypter import Crypter +from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from pyload.utils import fixup + + +class SimpleCrypter(Crypter, SimpleHoster): + __name__ = "SimpleCrypter" + __type__ = "crypter" + __version__ = "0.43" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Simple decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each crypter: + + LINK_PATTERN: Download link or regex to catch links in group(1) + example: LINK_PATTERN = r'