diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 51caefd4d7..ea5fa118c2 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -208,7 +208,11 @@ from .bandcamp import ( BandcampUserIE, BandcampWeeklyIE, ) -from .bannedvideo import BannedVideoIE +from .bannedvideo import ( + BannedVideoChannelIE, + BannedVideoIE, + BannedVideoPlaylistIE, +) from .bbc import ( BBCIE, BBCCoUkArticleIE, diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index 46f2978f7f..a8e5a737f1 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -1,31 +1,19 @@ +import itertools import json +import re from .common import InfoExtractor from ..utils import ( float_or_none, int_or_none, + traverse_obj, try_get, unified_timestamp, url_or_none, ) -class BannedVideoIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P[0-f]{24})' - _TESTS = [{ - 'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11', - 'md5': '14b6e81d41beaaee2215cd75c6ed56e4', - 'info_dict': { - 'id': '5e7a859644e02200c6ef5f11', - 'ext': 'mp4', - 'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement', - 'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/', - 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', - 'upload_date': '20200324', - 'timestamp': 1585087895, - }, - }] - +class BannedVideoBaseIE(InfoExtractor): _GRAPHQL_GETMETADATA_QUERY = ''' query GetVideoAndComments($id: String!) { getVideo(id: $id) { @@ -62,6 +50,29 @@ query GetVideoAndComments($id: String!) { } }''' + _GRAPHQL_GETMETADATA_FALLBACK_QUERY = ''' +query GetVideo($id: String!) { + getVideo(id: $id) { + streamUrl + directUrl + unlisted + live + tags { + name + } + title + summary + playCount + largeImage + videoDuration + channel { + _id + title + } + createdAt + } +}''' + _GRAPHQL_GETCOMMENTSREPLIES_QUERY = ''' query GetCommentReplies($id: String!) { getCommentReplies(id: $id, limit: 999999, offset: 0) { @@ -79,30 +90,140 @@ query GetCommentReplies($id: String!) { } }''' + _GRAPHQL_GETCHANNEL_QUERY = ''' +query GetChannel($id: String!) { + getChannelByIdOrTitle(id: $id) { + _id + title + summary + avatar + coverImage + } +}''' + + _GRAPHQL_GETCHANNELVIDEOS_QUERY = ''' +query GetChannelVideos($id: String!, $limit: Float, $offset: Float) { + getChannel(id: $id) { + videos(limit: $limit, offset: $offset) { + ...DisplayVideoFields + } + } +} + +fragment DisplayVideoFields on Video { + _id +}''' + + _GRAPHQL_GETPLAYLIST_QUERY = ''' +query GetPlaylist($id: String!) { + getPlaylist(id: $id) { + title + summary + } + }''' + + _GRAPHQL_GETPLAYLISTVIDEOS_QUERY = ''' +query GetPlaylistVideos($id: String!, $limit: Float, $offset: Float) { + getPlaylist(id: $id) { + videos(limit: $limit, offset: $offset) { + ...DisplayVideoFields + } + } +} + +fragment DisplayVideoFields on Video { + _id +}''' + _GRAPHQL_QUERIES = { 'GetVideoAndComments': _GRAPHQL_GETMETADATA_QUERY, + 'GetVideo': _GRAPHQL_GETMETADATA_FALLBACK_QUERY, 'GetCommentReplies': _GRAPHQL_GETCOMMENTSREPLIES_QUERY, + 'GetChannel': _GRAPHQL_GETCHANNEL_QUERY, + 'GetChannelVideos': _GRAPHQL_GETCHANNELVIDEOS_QUERY, + 'GetPlaylist': _GRAPHQL_GETPLAYLIST_QUERY, + 'GetPlaylistVideos': _GRAPHQL_GETPLAYLISTVIDEOS_QUERY, } - def _call_api(self, video_id, id_var, operation, note): + _API_HEADERS = { + 'apollographql-client-name': 'banned-web', + 'apollographql-client-version': '1.3', + 'Content-Type': 'application/json; charset=utf-8', + 'Origin': 'https://banned.video', + 'User-Agent': 'bannedVideoFrontEnd', + } + + def _call_api(self, video_id, operation, note, variables=None): return self._download_json( - 'https://api.infowarsmedia.com/graphql', video_id, note=note, - headers={ - 'Content-Type': 'application/json; charset=utf-8', - }, data=json.dumps({ - 'variables': {'id': id_var}, + 'https://api.banned.video/graphql', video_id, note=note, + headers=self._API_HEADERS, data=json.dumps({ + 'variables': variables or {'id': video_id}, 'operationName': operation, 'query': self._GRAPHQL_QUERIES[operation], }).encode('utf8')).get('data') + def _paginate(self, playlist_id, query): + for i in itertools.count(0): + page_json = self._call_api( + playlist_id, query, f'Downloading playlist page {i + 1}', + {'id': playlist_id, 'limit': 1000, 'offset': 1000 * i}) + + videos = traverse_obj(page_json, (..., 'videos', ...), expected_type=dict) + + for v in videos: + yield v['_id'] + + if not videos: + return + + +class BannedVideoIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P[0-f]{24})' + _TESTS = [{ + 'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11', + 'md5': '14b6e81d41beaaee2215cd75c6ed56e4', + 'info_dict': { + 'id': '5e7a859644e02200c6ef5f11', + 'ext': 'mp4', + 'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement', + 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', + 'thumbnail': 'https://download.assets.video/images/78bacefe-2bd7-434f-b584-ac2a20d1fac4-large.jpg', + 'timestamp': 1585087895, + 'upload_date': '20200324', + 'channel_id': '5b885d33e6646a0015a6fa2d', + 'duration': 322.6223, + 'channel': 'The Alex Jones Show', + 'view_count': int, + 'tags': ['fentynol', 'alex jones', 'china', 'dragon', 'communist', 'chicom'], + }, + }, + # requires fallback video metadata call (graphql api returns error about comments) + { + 'url': 'https://banned.video/watch?id=60461f187d2e5c334e31177c', + 'info_dict': { + 'id': '60461f187d2e5c334e31177c', + 'ext': 'mp4', + 'title': 'Roger Stone: Who Controls Joe Biden', + 'view_count': int, + 'duration': 1946.561283, + 'channel': 'The Alex Jones Show', + 'thumbnail': 'https://download.assets.video/images/77981308-99e8-401a-bd96-12c7ceda5c75-large.jpg', + 'upload_date': '20210308', + 'timestamp': 1615208216, + 'description': 'Alex Jones interviews Roger Stone in South Florida', + 'channel_id': '5b885d33e6646a0015a6fa2d', + 'tags': ['alex jones', 'roger stone'], + }, + }] + def _get_comments(self, video_id, comments, comment_data): yield from comments for comment in comment_data.copy(): comment_id = comment.get('_id') if comment.get('replyCount') > 0: reply_json = self._call_api( - video_id, comment_id, 'GetCommentReplies', - f'Downloading replies for comment {comment_id}') + video_id, 'GetCommentReplies', f'Downloading replies for comment {comment_id}', + {'id': comment_id}) for reply in reply_json.get('getCommentReplies'): yield self._parse_comment(reply, comment_id) @@ -120,10 +241,12 @@ query GetCommentReplies($id: String!) { def _real_extract(self, url): video_id = self._match_id(url) - video_json = self._call_api(video_id, video_id, 'GetVideoAndComments', 'Downloading video metadata') + video_json = self._call_api(video_id, 'GetVideoAndComments', 'Downloading video metadata') + if not video_json: + video_json = self._call_api(video_id, 'GetVideo', 'Downloading video metadata (fallback)') video_info = video_json['getVideo'] is_live = video_info.get('live') - comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments')] + comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments', [])] formats = [{ 'format_id': 'direct', @@ -131,10 +254,11 @@ query GetCommentReplies($id: String!) { 'url': video_info.get('directUrl'), 'ext': 'mp4', }] if url_or_none(video_info.get('directUrl')) else [] - if video_info.get('streamUrl'): + if video_info.get('streamUrl') and not re.search(r'\.mp4$', video_info.get('streamUrl') or ''): formats.extend(self._extract_m3u8_formats( video_info.get('streamUrl'), video_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id='hls', live=True)) + entry_protocol='m3u8_native', m3u8_id='hls', + live=True, fatal=False)) return { 'id': video_id, @@ -153,3 +277,67 @@ query GetCommentReplies($id: String!) { 'comments': comments, '__post_extractor': self.extract_comments(video_id, comments, video_json.get('getVideoComments')), } + + +class BannedVideoChannelIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/channel/(?P[\w-]+)' + _TESTS = [{ + 'url': 'https://banned.video/channel/war-room-with-owen-shroyer', + 'playlist_mincount': 9810, + 'info_dict': { + 'id': '5b9301172abf762e22bc22fd', + 'title': 'War Room With Owen Shroyer', + 'description': 'md5:821d45563613ec1f5a2b68046d13e2ff', + 'thumbnail': 'https://download.assets.video/images/acb63e7e-d23f-4901-9bac-cf3edda8eb89-large.png', + }, + }, { + 'url': 'https://banned.video/channel/the-best-of-tucker-carlson-', + 'playlist_mincount': 21, + 'info_dict': { + 'thumbnail': 'https://download.assets.video/images/b1416a59-4902-480c-979e-246e62c34861-large.jpg', + 'id': '64598c46b1e3f80b32930313', + 'title': 'The Best of Tucker Carlson ', + 'description': 'Channel dedicated to Tucker', + }, + }] + + def _real_extract(self, url): + channel_info = self._call_api( + self._match_id(url), 'GetChannel', 'Downloading channel metadata')['getChannelByIdOrTitle'] + channel_id = channel_info['_id'] + + return self.playlist_result( + [self.url_result(f'https://banned.video/watch?id={vid}', url_transparent=True) + for vid in self._paginate(channel_id, 'GetChannelVideos')], + channel_id, channel_info['title'], channel_info.get('summary'), + thumbnail=channel_info.get('coverImage')) + + +class BannedVideoPlaylistIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/playlist/(?P[\w-]+)' + _TESTS = [{ + 'url': 'https://banned.video/playlist/5d81058ce2ea200013c01580', + 'playlist_mincount': 1507, + 'info_dict': { + 'title': 'Full Alex Jones Shows', + 'id': '5d81058ce2ea200013c01580', + 'description': '', + }, + }, { + 'url': 'https://banned.video/playlist/5db8bac40d7a4400199b73ca', + 'playlist_mincount': 92, + 'info_dict': { + 'title': 'Owen Shroyer Man On The Street Interviews', + 'id': '5db8bac40d7a4400199b73ca', + 'description': '', + }, + }] + + def _real_extract(self, url): + playlist_id = self._match_id(url) + playlist_info = self._call_api(playlist_id, 'GetPlaylist', 'Downloading playlist metadata')['getPlaylist'] + + return self.playlist_result( + [self.url_result(f'https://banned.video/watch?id={vid}', url_transparent=True) + for vid in self._paginate(playlist_id, 'GetPlaylistVideos')], playlist_id, + playlist_info['title'], playlist_info.get('summary'))