From 0372246647092344761b7b203a61d238f41553bf Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Sun, 9 Jun 2024 20:00:44 -0400 Subject: [PATCH 1/7] upgrade bannedvideo extractor * add channel extractor * add playlist extractor --- yt_dlp/extractor/_extractors.py | 6 +- yt_dlp/extractor/bannedvideo.py | 187 +++++++++++++++++++++++++++----- 2 files changed, 167 insertions(+), 26 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index e9cd38a651..2339b264a8 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -207,7 +207,11 @@ from .bandcamp import ( BandcampUserIE, BandcampWeeklyIE, ) -from .bannedvideo import BannedVideoIE +from .bannedvideo import ( + BannedVideoIE, + BannedVideoChannelIE, + BannedVideoPlaylistIE, +) from .bbc import ( BBCIE, BBCCoUkArticleIE, diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index 82dc9ab025..030ad267bb 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -1,31 +1,18 @@ +import itertools import json from .common import InfoExtractor from ..utils import ( float_or_none, int_or_none, + traverse_obj, try_get, unified_timestamp, url_or_none, ) -class BannedVideoIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P[0-f]{24})' - _TESTS = [{ - 'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11', - 'md5': '14b6e81d41beaaee2215cd75c6ed56e4', - 'info_dict': { - 'id': '5e7a859644e02200c6ef5f11', - 'ext': 'mp4', - 'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement', - 'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/', - 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', - 'upload_date': '20200324', - 'timestamp': 1585087895, - } - }] - +class BannedVideoBaseIE(InfoExtractor): _GRAPHQL_GETMETADATA_QUERY = ''' query GetVideoAndComments($id: String!) { getVideo(id: $id) { @@ -79,30 +66,116 @@ query GetCommentReplies($id: String!) { } }''' + _GRAPHQL_GETCHANNEL_QUERY = ''' +query GetChannel($id: String!) { + getChannelByIdOrTitle(id: $id) { + _id + title + summary + avatar + coverImage + } +}''' + + _GRAPHQL_GETCHANNELVIDEOS_QUERY = ''' +query GetChannelVideos($id: String!, $limit: Float, $offset: Float) { + getChannel(id: $id) { + videos(limit: $limit, offset: $offset) { + ...DisplayVideoFields + } + } +} + +fragment DisplayVideoFields on Video { + _id +}''' + + _GRAPHQL_GETPLAYLIST_QUERY = ''' +query GetPlaylist($id: String!) { + getPlaylist(id: $id) { + title + summary + } + }''' + + _GRAPHQL_GETPLAYLISTVIDEOS_QUERY = ''' +query GetPlaylistVideos($id: String!, $limit: Float, $offset: Float) { + getPlaylist(id: $id) { + videos(limit: $limit, offset: $offset) { + ...DisplayVideoFields + } + } +} + +fragment DisplayVideoFields on Video { + _id +}''' + _GRAPHQL_QUERIES = { 'GetVideoAndComments': _GRAPHQL_GETMETADATA_QUERY, 'GetCommentReplies': _GRAPHQL_GETCOMMENTSREPLIES_QUERY, + 'GetChannel': _GRAPHQL_GETCHANNEL_QUERY, + 'GetChannelVideos': _GRAPHQL_GETCHANNELVIDEOS_QUERY, + 'GetPlaylist': _GRAPHQL_GETPLAYLIST_QUERY, + 'GetPlaylistVideos': _GRAPHQL_GETPLAYLISTVIDEOS_QUERY, } - def _call_api(self, video_id, id, operation, note): + _API_HEADERS = { + 'apollographql-client-name': 'banned-web', + 'apollographql-client-version': '1.3', + 'Content-Type': 'application/json; charset=utf-8', + 'Origin': 'https://banned.video', + 'User-Agent': 'bannedVideoFrontEnd', + } + + def _call_api(self, video_id, operation, note, variables=None): return self._download_json( - 'https://api.infowarsmedia.com/graphql', video_id, note=note, - headers={ - 'Content-Type': 'application/json; charset=utf-8' - }, data=json.dumps({ - 'variables': {'id': id}, + 'https://api.banned.video/graphql', video_id, note=note, + headers=self._API_HEADERS, data=json.dumps({ + 'variables': variables or {'id': video_id}, 'operationName': operation, 'query': self._GRAPHQL_QUERIES[operation] }).encode('utf8')).get('data') + def _paginate(self, playlist_id, query): + for i in itertools.count(0): + page_json = self._call_api( + playlist_id, query, f'Downloading playlist page {i + 1}', + {'id': playlist_id, 'limit': 1000, 'offset': 1000 * i}) + + videos = traverse_obj(page_json, (..., 'videos', ...), expected_type=dict) + + for v in videos: + yield v['_id'] + + if not videos: + return + + +class BannedVideoIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P[0-f]{24})' + _TESTS = [{ + 'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11', + 'md5': '14b6e81d41beaaee2215cd75c6ed56e4', + 'info_dict': { + 'id': '5e7a859644e02200c6ef5f11', + 'ext': 'mp4', + 'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement', + 'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/', + 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', + 'upload_date': '20200324', + 'timestamp': 1585087895, + } + }] + def _get_comments(self, video_id, comments, comment_data): yield from comments for comment in comment_data.copy(): comment_id = comment.get('_id') if comment.get('replyCount') > 0: reply_json = self._call_api( - video_id, comment_id, 'GetCommentReplies', - f'Downloading replies for comment {comment_id}') + video_id, 'GetCommentReplies', f'Downloading replies for comment {comment_id}', + {'id': comment_id}) for reply in reply_json.get('getCommentReplies'): yield self._parse_comment(reply, comment_id) @@ -120,7 +193,7 @@ query GetCommentReplies($id: String!) { def _real_extract(self, url): video_id = self._match_id(url) - video_json = self._call_api(video_id, video_id, 'GetVideoAndComments', 'Downloading video metadata') + video_json = self._call_api(video_id, 'GetVideoAndComments', 'Downloading video metadata') video_info = video_json['getVideo'] is_live = video_info.get('live') comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments')] @@ -153,3 +226,67 @@ query GetCommentReplies($id: String!) { 'comments': comments, '__post_extractor': self.extract_comments(video_id, comments, video_json.get('getVideoComments')) } + + +class BannedVideoChannelIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/channel/(?P[\w-]+)' + _TESTS = [{ + 'url': 'https://banned.video/channel/war-room-with-owen-shroyer', + 'playlist_mincount': 9810, + 'info_dict': { + 'id': '5b9301172abf762e22bc22fd', + 'title': 'War Room With Owen Shroyer', + 'description': 'md5:821d45563613ec1f5a2b68046d13e2ff', + 'thumbnail': 'https://download.assets.video/images/acb63e7e-d23f-4901-9bac-cf3edda8eb89-large.png', + }, + }, { + 'url': 'https://banned.video/channel/the-best-of-tucker-carlson-', + 'playlist_mincount': 21, + 'info_dict': { + 'thumbnail': 'https://download.assets.video/images/b1416a59-4902-480c-979e-246e62c34861-large.jpg', + 'id': '64598c46b1e3f80b32930313', + 'title': 'The Best of Tucker Carlson ', + 'description': 'Channel dedicated to Tucker', + }, + }] + + def _real_extract(self, url): + channel_info = self._call_api( + self._match_id(url), 'GetChannel', 'Downloading channel metadata')['getChannelByIdOrTitle'] + channel_id = channel_info['_id'] + + return self.playlist_result( + [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True) + for id in self._paginate(channel_id, 'GetChannelVideos')], + channel_id, channel_info['title'], channel_info.get('summary'), + thumbnail=channel_info.get('coverImage')) + + +class BannedVideoPlaylistIE(BannedVideoBaseIE): + _VALID_URL = r'https?://(?:www\.)?banned\.video/playlist/(?P[\w-]+)' + _TESTS = [{ + 'url': 'https://banned.video/playlist/5d81058ce2ea200013c01580', + 'playlist_mincount': 1507, + 'info_dict': { + 'title': 'Full Alex Jones Shows', + 'id': '5d81058ce2ea200013c01580', + 'description': '', + }, + }, { + 'url': 'https://banned.video/playlist/5db8bac40d7a4400199b73ca', + 'playlist_mincount': 92, + 'info_dict': { + 'title': 'Owen Shroyer Man On The Street Interviews', + 'id': '5db8bac40d7a4400199b73ca', + 'description': '', + }, + }] + + def _real_extract(self, url): + playlist_id = self._match_id(url) + playlist_info = self._call_api(playlist_id, 'GetPlaylist', 'Downloading playlist metadata')['getPlaylist'] + + return self.playlist_result( + [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True) + for id in self._paginate(playlist_id, 'GetPlaylistVideos')], playlist_id, + playlist_info['title'], playlist_info.get('summary')) From fd545b5d8d37b669328a5287881cd6fade86bf22 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Sun, 9 Jun 2024 20:15:47 -0400 Subject: [PATCH 2/7] fix _extractors.py linter error --- yt_dlp/extractor/_extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 2339b264a8..8d261808c0 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -208,8 +208,8 @@ from .bandcamp import ( BandcampWeeklyIE, ) from .bannedvideo import ( - BannedVideoIE, BannedVideoChannelIE, + BannedVideoIE, BannedVideoPlaylistIE, ) from .bbc import ( From 722ac25c98e3d361bc02abccf9e3b92c3d1c6be8 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:32:29 -0400 Subject: [PATCH 3/7] don't treat direct mp4 urls as manifests --- yt_dlp/extractor/bannedvideo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index f77431a775..0726f42b91 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -1,5 +1,6 @@ import itertools import json +import re from .common import InfoExtractor from ..utils import ( @@ -165,7 +166,7 @@ class BannedVideoIE(BannedVideoBaseIE): 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', 'upload_date': '20200324', 'timestamp': 1585087895, - } + }, }] def _get_comments(self, video_id, comments, comment_data): @@ -204,7 +205,7 @@ class BannedVideoIE(BannedVideoBaseIE): 'url': video_info.get('directUrl'), 'ext': 'mp4', }] if url_or_none(video_info.get('directUrl')) else [] - if video_info.get('streamUrl'): + if video_info.get('streamUrl') and not re.search(r'\.mp4$', video_info.get('streamUrl') or ''): formats.extend(self._extract_m3u8_formats( video_info.get('streamUrl'), video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', live=True)) From 0217522df6b39cd488faa6e4710d0ad1b8ff4b00 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Sat, 2 Nov 2024 08:36:45 -0400 Subject: [PATCH 4/7] add video metadata fallback api call, fix linter errors --- yt_dlp/extractor/bannedvideo.py | 35 ++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index 0726f42b91..929cabbaf9 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -50,6 +50,28 @@ query GetVideoAndComments($id: String!) { } }''' + _GRAPHQL_GETMETADATA_FALLBACK_QUERY = ''' +query GetVideo($id: String!) { + getVideo(id: $id) { + ...DisplayVideoFields streamUrl directUrl liked disliked audio unlisted live tags { + _id name __typename + } + sale { + _id text textSale description url videoUpload { + cloudflareVideoUID __typename + } + __typename + } + __typename + } +} +fragment DisplayVideoFields on Video { + _id title summary playCount likeCount angerCount largeImage embedUrl published videoDuration channel { + _id title avatar __typename + } + createdAt __typename +}''' + _GRAPHQL_GETCOMMENTSREPLIES_QUERY = ''' query GetCommentReplies($id: String!) { getCommentReplies(id: $id, limit: 999999, offset: 0) { @@ -114,6 +136,7 @@ fragment DisplayVideoFields on Video { _GRAPHQL_QUERIES = { 'GetVideoAndComments': _GRAPHQL_GETMETADATA_QUERY, + 'GetVideo': _GRAPHQL_GETMETADATA_FALLBACK_QUERY, 'GetCommentReplies': _GRAPHQL_GETCOMMENTSREPLIES_QUERY, 'GetChannel': _GRAPHQL_GETCHANNEL_QUERY, 'GetChannelVideos': _GRAPHQL_GETCHANNELVIDEOS_QUERY, @@ -195,9 +218,11 @@ class BannedVideoIE(BannedVideoBaseIE): def _real_extract(self, url): video_id = self._match_id(url) video_json = self._call_api(video_id, 'GetVideoAndComments', 'Downloading video metadata') + if not video_json: + video_json = self._call_api(video_id, 'GetVideo', 'Downloading video metadata (fallback)') video_info = video_json['getVideo'] is_live = video_info.get('live') - comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments')] + comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments', [])] formats = [{ 'format_id': 'direct', @@ -257,8 +282,8 @@ class BannedVideoChannelIE(BannedVideoBaseIE): channel_id = channel_info['_id'] return self.playlist_result( - [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True) - for id in self._paginate(channel_id, 'GetChannelVideos')], + [self.url_result(f'https://banned.video/watch?id={vid}', url_transparent=True) + for vid in self._paginate(channel_id, 'GetChannelVideos')], channel_id, channel_info['title'], channel_info.get('summary'), thumbnail=channel_info.get('coverImage')) @@ -288,6 +313,6 @@ class BannedVideoPlaylistIE(BannedVideoBaseIE): playlist_info = self._call_api(playlist_id, 'GetPlaylist', 'Downloading playlist metadata')['getPlaylist'] return self.playlist_result( - [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True) - for id in self._paginate(playlist_id, 'GetPlaylistVideos')], playlist_id, + [self.url_result(f'https://banned.video/watch?id={vid}', url_transparent=True) + for vid in self._paginate(playlist_id, 'GetPlaylistVideos')], playlist_id, playlist_info['title'], playlist_info.get('summary')) From 4041fd0523fcdb7d31c176ea7a7203465a484403 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Sat, 2 Nov 2024 21:16:16 -0400 Subject: [PATCH 5/7] cleanup gql query, update tests --- yt_dlp/extractor/bannedvideo.py | 56 +++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index 929cabbaf9..beb3a22c1a 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -53,23 +53,24 @@ query GetVideoAndComments($id: String!) { _GRAPHQL_GETMETADATA_FALLBACK_QUERY = ''' query GetVideo($id: String!) { getVideo(id: $id) { - ...DisplayVideoFields streamUrl directUrl liked disliked audio unlisted live tags { - _id name __typename + streamUrl + directUrl + unlisted + live + tags { + name } - sale { - _id text textSale description url videoUpload { - cloudflareVideoUID __typename - } - __typename + title + summary + playCount + largeImage + videoDuration + channel { + _id + title } - __typename + createdAt } -} -fragment DisplayVideoFields on Video { - _id title summary playCount likeCount angerCount largeImage embedUrl published videoDuration channel { - _id title avatar __typename - } - createdAt __typename }''' _GRAPHQL_GETCOMMENTSREPLIES_QUERY = ''' @@ -185,10 +186,33 @@ class BannedVideoIE(BannedVideoBaseIE): 'id': '5e7a859644e02200c6ef5f11', 'ext': 'mp4', 'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement', - 'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/', 'description': 'md5:560d96f02abbebe6c6b78b47465f6b28', - 'upload_date': '20200324', + 'thumbnail': 'https://download.assets.video/images/78bacefe-2bd7-434f-b584-ac2a20d1fac4-large.jpg', 'timestamp': 1585087895, + 'upload_date': '20200324', + 'channel_id': '5b885d33e6646a0015a6fa2d', + 'duration': 322.6223, + 'channel': 'The Alex Jones Show', + 'view_count': int, + 'tags': ['fentynol', 'alex jones', 'china', 'dragon', 'communist', 'chicom'], + }, + }, + # requires fallback video metadata call (graphql api returns error about comments) + { + 'url': 'https://banned.video/watch?id=60461f187d2e5c334e31177c', + 'info_dict': { + 'id': '60461f187d2e5c334e31177c', + 'ext': 'mp4', + 'title': 'Roger Stone: Who Controls Joe Biden', + 'view_count': int, + 'duration': 1946.561283, + 'channel': 'The Alex Jones Show', + 'thumbnail': 'https://download.assets.video/images/77981308-99e8-401a-bd96-12c7ceda5c75-large.jpg', + 'upload_date': '20210308', + 'timestamp': 1615208216, + 'description': 'Alex Jones interviews Roger Stone in South Florida', + 'channel_id': '5b885d33e6646a0015a6fa2d', + 'tags': ['alex jones', 'roger stone'], }, }] From f4e3a4e46d012772cea91a783dd1a0b3877b1d84 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:48:55 -0500 Subject: [PATCH 6/7] don't fail on m3u8 error make m3u8 HTTP errors non-fatal (424 noticed) --- yt_dlp/extractor/bannedvideo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index beb3a22c1a..3c662e64e5 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -254,10 +254,14 @@ class BannedVideoIE(BannedVideoBaseIE): 'url': video_info.get('directUrl'), 'ext': 'mp4', }] if url_or_none(video_info.get('directUrl')) else [] + + print(formats) + if video_info.get('streamUrl') and not re.search(r'\.mp4$', video_info.get('streamUrl') or ''): formats.extend(self._extract_m3u8_formats( video_info.get('streamUrl'), video_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id='hls', live=True)) + entry_protocol='m3u8_native', m3u8_id='hls', + live=True, fatal=False)) return { 'id': video_id, From 9106506dfe45595cb47848c4f7b2a7f4fa401cc3 Mon Sep 17 00:00:00 2001 From: Audrey <45548254+tntmod54321@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:51:42 -0500 Subject: [PATCH 7/7] remove print statement --- yt_dlp/extractor/bannedvideo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/yt_dlp/extractor/bannedvideo.py b/yt_dlp/extractor/bannedvideo.py index 3c662e64e5..a8e5a737f1 100644 --- a/yt_dlp/extractor/bannedvideo.py +++ b/yt_dlp/extractor/bannedvideo.py @@ -254,9 +254,6 @@ class BannedVideoIE(BannedVideoBaseIE): 'url': video_info.get('directUrl'), 'ext': 'mp4', }] if url_or_none(video_info.get('directUrl')) else [] - - print(formats) - if video_info.get('streamUrl') and not re.search(r'\.mp4$', video_info.get('streamUrl') or ''): formats.extend(self._extract_m3u8_formats( video_info.get('streamUrl'), video_id, 'mp4',