From 181ab47c5f775b8b48569361da6b88a801784a30 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 26 Mar 2015 18:15:41 +0100 Subject: Cleanup --- pyload/plugin/crypter/DailymotionComFolder.py | 106 ++++++++++++++++++ pyload/plugin/crypter/FilecryptCc.py | 2 +- pyload/plugin/crypter/YoutubeComFolder.py | 148 ++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 pyload/plugin/crypter/DailymotionComFolder.py create mode 100644 pyload/plugin/crypter/YoutubeComFolder.py (limited to 'pyload/plugin/crypter') diff --git a/pyload/plugin/crypter/DailymotionComFolder.py b/pyload/plugin/crypter/DailymotionComFolder.py new file mode 100644 index 000000000..3c5000515 --- /dev/null +++ b/pyload/plugin/crypter/DailymotionComFolder.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.utils import json_loads +from pyload.plugin.Crypter import Crypter +from pyload.utils import fs_join + + +class DailymotionComFolder(Crypter): + __name__ = "DailymotionComFolder" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'https?://(?:www\.)?dailymotion\.com/((playlists/)?(?Pplaylist|user)/)?(?P[\w^_]+)(?(TYPE)|#)' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] + + __description__ = """Dailymotion.com channel & playlist decrypter""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def api_response(self, ref, req=None): + url = urljoin("https://api.dailymotion.com/", ref) + html = self.load(url, get=req) + return json_loads(html) + + + def getPlaylistInfo(self, id): + ref = "playlist/" + id + req = {"fields": "name,owner.screenname"} + playlist = self.api_response(ref, req) + + if "error" in playlist: + return + + name = playlist['name'] + owner = playlist['owner.screenname'] + return name, owner + + + def _getPlaylists(self, user_id, page=1): + ref = "user/%s/playlists" % user_id + req = {"fields": "id", "page": page, "limit": 100} + user = self.api_response(ref, req) + + if "error" in user: + return + + for playlist in user['list']: + yield playlist['id'] + + if user['has_more']: + for item in self._getPlaylists(user_id, page + 1): + yield item + + + def getPlaylists(self, user_id): + return [(id,) + self.getPlaylistInfo(id) for id in self._getPlaylists(user_id)] + + + def _getVideos(self, id, page=1): + ref = "playlist/%s/videos" % id + req = {"fields": "url", "page": page, "limit": 100} + playlist = self.api_response(ref, req) + + if "error" in playlist: + return + + for video in playlist['list']: + yield video['url'] + + if playlist['has_more']: + for item in self._getVideos(id, page + 1): + yield item + + + def getVideos(self, playlist_id): + return list(self._getVideos(playlist_id))[::-1] + + + def decrypt(self, pyfile): + m = re.match(self.__pattern__, pyfile.url) + m_id = m.group('ID') + m_type = m.group('TYPE') + + if m_type == "playlist": + self.logDebug("Url recognized as Playlist") + p_info = self.getPlaylistInfo(m_id) + playlists = [(m_id,) + p_info] if p_info else None + else: + self.logDebug("Url recognized as Channel") + playlists = self.getPlaylists(m_id) + self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), m_id)) + + if not playlists: + self.fail(_("No playlist available")) + + for p_id, p_name, p_owner in playlists: + p_videos = self.getVideos(p_id) + p_folder = fs_join(self.config['general']['download_folder'], p_owner, p_name) + self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name)) + self.packages.append((p_name, p_videos, p_folder)) #: folder is NOT recognized by pyload 0.4.9! diff --git a/pyload/plugin/crypter/FilecryptCc.py b/pyload/plugin/crypter/FilecryptCc.py index c052573e1..1d8388dcd 100644 --- a/pyload/plugin/crypter/FilecryptCc.py +++ b/pyload/plugin/crypter/FilecryptCc.py @@ -10,7 +10,7 @@ from Crypto.Cipher import AES from urlparse import urljoin from pyload.plugin.Crypter import Crypter -from pyload.plugin.internal.CaptchaService import ReCaptcha +from pyload.plugin.captcha.ReCaptcha import ReCaptcha class FilecryptCc(Crypter): diff --git a/pyload/plugin/crypter/YoutubeComFolder.py b/pyload/plugin/crypter/YoutubeComFolder.py new file mode 100644 index 000000000..e42615af0 --- /dev/null +++ b/pyload/plugin/crypter/YoutubeComFolder.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin + +from pyload.utils import json_loads +from pyload.plugin.Crypter import Crypter +from pyload.utils import fs_join + + +class YoutubeComFolder(Crypter): + __name__ = "YoutubeComFolder" + __type__ = "crypter" + __version__ = "1.01" + + __pattern__ = r'https?://(?:www\.|m\.)?youtube\.com/(?Puser|playlist|view_play_list)(/|.*?[?&](?:list|p)=)(?P[\w-]+)' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True ), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True ), + ("likes" , "bool", "Grab user (channel) liked videos" , False), + ("favorites" , "bool", "Grab user (channel) favorite videos", False), + ("uploads" , "bool", "Grab channel unplaylisted videos" , True )] + + __description__ = """Youtube.com channel & playlist decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + API_KEY = "AIzaSyCKnWLNlkX-L4oD1aEzqqhRw1zczeD6_k0" + + + def api_response(self, ref, req): + req.update({"key": self.API_KEY}) + url = urljoin("https://www.googleapis.com/youtube/v3/", ref) + html = self.load(url, get=req) + return json_loads(html) + + + def getChannel(self, user): + channels = self.api_response("channels", {"part": "id,snippet,contentDetails", "forUsername": user, "maxResults": "50"}) + if channels['items']: + channel = channels['items'][0] + return {"id": channel['id'], + "title": channel['snippet']['title'], + "relatedPlaylists": channel['contentDetails']['relatedPlaylists'], + "user": user} # One lone channel for user? + + + def getPlaylist(self, p_id): + playlists = self.api_response("playlists", {"part": "snippet", "id": p_id}) + if playlists['items']: + playlist = playlists['items'][0] + return {"id": p_id, + "title": playlist['snippet']['title'], + "channelId": playlist['snippet']['channelId'], + "channelTitle": playlist['snippet']['channelTitle']} + + + def _getPlaylists(self, id, token=None): + req = {"part": "id", "maxResults": "50", "channelId": id} + if token: + req.update({"pageToken": token}) + + playlists = self.api_response("playlists", req) + + for playlist in playlists['items']: + yield playlist['id'] + + if "nextPageToken" in playlists: + for item in self._getPlaylists(id, playlists['nextPageToken']): + yield item + + + def getPlaylists(self, ch_id): + return map(self.getPlaylist, self._getPlaylists(ch_id)) + + + def _getVideosId(self, id, token=None): + req = {"part": "contentDetails", "maxResults": "50", "playlistId": id} + if token: + req.update({"pageToken": token}) + + playlist = self.api_response("playlistItems", req) + + for item in playlist['items']: + yield item['contentDetails']['videoId'] + + if "nextPageToken" in playlist: + for item in self._getVideosId(id, playlist['nextPageToken']): + yield item + + + def getVideosId(self, p_id): + return list(self._getVideosId(p_id)) + + + def decrypt(self, pyfile): + m = re.match(self.__pattern__, pyfile.url) + m_id = m.group('ID') + m_type = m.group('TYPE') + + if m_type == "user": + self.logDebug("Url recognized as Channel") + user = m_id + channel = self.getChannel(user) + + if channel: + playlists = self.getPlaylists(channel['id']) + self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), channel['title'])) + + relatedplaylist = {p_name: self.getPlaylist(p_id) for p_name, p_id in channel['relatedPlaylists'].iteritems()} + self.logDebug("Channel's related playlists found = %s" % relatedplaylist.keys()) + + relatedplaylist['uploads']['title'] = "Unplaylisted videos" + relatedplaylist['uploads']['checkDups'] = True #: checkDups flag + + for p_name, p_data in relatedplaylist.iteritems(): + if self.getConfig(p_name): + p_data['title'] += " of " + user + playlists.append(p_data) + else: + playlists = [] + else: + self.logDebug("Url recognized as Playlist") + playlists = [self.getPlaylist(m_id)] + + if not playlists: + self.fail(_("No playlist available")) + + addedvideos = [] + urlize = lambda x: "https://www.youtube.com/watch?v=" + x + for p in playlists: + p_name = p['title'] + p_videos = self.getVideosId(p['id']) + p_folder = fs_join(self.config['general']['download_folder'], p['channelTitle'], p_name) + self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name)) + + if not p_videos: + continue + elif "checkDups" in p: + p_urls = [urlize(v_id) for v_id in p_videos if v_id not in addedvideos] + self.logDebug("%s video\s available on playlist \"%s\" after duplicates cleanup" % (len(p_urls), p_name)) + else: + p_urls = map(urlize, p_videos) + + self.packages.append((p_name, p_urls, p_folder)) #: folder is NOT recognized by pyload 0.4.9! + + addedvideos.extend(p_videos) -- cgit v1.2.3