upgrade bannedvideo extractor

* add channel extractor
* add playlist extractor

yt_dlp/extractor/_extractors.py

@@ -207,7 +207,11 @@ from .bandcamp import (
     BandcampUserIE,
     BandcampWeeklyIE,
 )
-from .bannedvideo import BannedVideoIE
+from .bannedvideo import (
+    BannedVideoIE,
+    BannedVideoChannelIE,
+    BannedVideoPlaylistIE,
+)
 from .bbc import (
     BBCIE,
     BBCCoUkArticleIE,

yt_dlp/extractor/bannedvideo.py

@@ -1,31 +1,18 @@
+import itertools
 import json
 
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
+    traverse_obj,
     try_get,
     unified_timestamp,
     url_or_none,
 )
 
 
-class BannedVideoIE(InfoExtractor):
+class BannedVideoBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P<id>[0-f]{24})'
-    _TESTS = [{
-        'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11',
-        'md5': '14b6e81d41beaaee2215cd75c6ed56e4',
-        'info_dict': {
-            'id': '5e7a859644e02200c6ef5f11',
-            'ext': 'mp4',
-            'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement',
-            'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/',
-            'description': 'md5:560d96f02abbebe6c6b78b47465f6b28',
-            'upload_date': '20200324',
-            'timestamp': 1585087895,
-        }
-    }]
-
     _GRAPHQL_GETMETADATA_QUERY = '''
 query GetVideoAndComments($id: String!) {
     getVideo(id: $id) {
@@ -79,30 +66,116 @@ query GetCommentReplies($id: String!) {
     }
 }'''
 
+    _GRAPHQL_GETCHANNEL_QUERY = '''
+query GetChannel($id: String!) {
+    getChannelByIdOrTitle(id: $id) {
+      _id
+      title
+      summary
+      avatar
+      coverImage
+    }
+}'''
+
+    _GRAPHQL_GETCHANNELVIDEOS_QUERY = '''
+query GetChannelVideos($id: String!, $limit: Float, $offset: Float) {
+  getChannel(id: $id) {
+    videos(limit: $limit, offset: $offset) {
+      ...DisplayVideoFields
+    }
+  }
+}
+
+fragment DisplayVideoFields on Video {
+  _id
+}'''
+
+    _GRAPHQL_GETPLAYLIST_QUERY = '''
+query GetPlaylist($id: String!) {
+    getPlaylist(id: $id) {
+      title
+      summary
+    }
+  }'''
+
+    _GRAPHQL_GETPLAYLISTVIDEOS_QUERY = '''
+query GetPlaylistVideos($id: String!, $limit: Float, $offset: Float) {
+  getPlaylist(id: $id) {
+    videos(limit: $limit, offset: $offset) {
+      ...DisplayVideoFields
+    }
+  }
+}
+
+fragment DisplayVideoFields on Video {
+  _id
+}'''
+
     _GRAPHQL_QUERIES = {
         'GetVideoAndComments': _GRAPHQL_GETMETADATA_QUERY,
         'GetCommentReplies': _GRAPHQL_GETCOMMENTSREPLIES_QUERY,
+        'GetChannel': _GRAPHQL_GETCHANNEL_QUERY,
+        'GetChannelVideos': _GRAPHQL_GETCHANNELVIDEOS_QUERY,
+        'GetPlaylist': _GRAPHQL_GETPLAYLIST_QUERY,
+        'GetPlaylistVideos': _GRAPHQL_GETPLAYLISTVIDEOS_QUERY,
     }
 
-    def _call_api(self, video_id, id, operation, note):
+    _API_HEADERS = {
+        'apollographql-client-name': 'banned-web',
+        'apollographql-client-version': '1.3',
+        'Content-Type': 'application/json; charset=utf-8',
+        'Origin': 'https://banned.video',
+        'User-Agent': 'bannedVideoFrontEnd',
+    }
+
+    def _call_api(self, video_id, operation, note, variables=None):
         return self._download_json(
-            'https://api.infowarsmedia.com/graphql', video_id, note=note,
+            'https://api.banned.video/graphql', video_id, note=note,
-            headers={
+            headers=self._API_HEADERS, data=json.dumps({
-                'Content-Type': 'application/json; charset=utf-8'
+                'variables': variables or {'id': video_id},
-            }, data=json.dumps({
-                'variables': {'id': id},
                 'operationName': operation,
                 'query': self._GRAPHQL_QUERIES[operation]
             }).encode('utf8')).get('data')
 
+    def _paginate(self, playlist_id, query):
+        for i in itertools.count(0):
+            page_json = self._call_api(
+                playlist_id, query, f'Downloading playlist page {i + 1}',
+                {'id': playlist_id, 'limit': 1000, 'offset': 1000 * i})
+
+            videos = traverse_obj(page_json, (..., 'videos', ...), expected_type=dict)
+
+            for v in videos:
+                yield v['_id']
+
+            if not videos:
+                return
+
+
+class BannedVideoIE(BannedVideoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?banned\.video/watch\?id=(?P<id>[0-f]{24})'
+    _TESTS = [{
+        'url': 'https://banned.video/watch?id=5e7a859644e02200c6ef5f11',
+        'md5': '14b6e81d41beaaee2215cd75c6ed56e4',
+        'info_dict': {
+            'id': '5e7a859644e02200c6ef5f11',
+            'ext': 'mp4',
+            'title': 'China Discovers Origin of Corona Virus: Issues Emergency Statement',
+            'thumbnail': r're:^https?://(?:www\.)?assets\.infowarsmedia.com/images/',
+            'description': 'md5:560d96f02abbebe6c6b78b47465f6b28',
+            'upload_date': '20200324',
+            'timestamp': 1585087895,
+        }
+    }]
+
     def _get_comments(self, video_id, comments, comment_data):
         yield from comments
         for comment in comment_data.copy():
             comment_id = comment.get('_id')
             if comment.get('replyCount') > 0:
                 reply_json = self._call_api(
-                    video_id, comment_id, 'GetCommentReplies',
+                    video_id, 'GetCommentReplies', f'Downloading replies for comment {comment_id}',
-                    f'Downloading replies for comment {comment_id}')
+                    {'id': comment_id})
                 for reply in reply_json.get('getCommentReplies'):
                     yield self._parse_comment(reply, comment_id)
 
@@ -120,7 +193,7 @@ query GetCommentReplies($id: String!) {
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_json = self._call_api(video_id, video_id, 'GetVideoAndComments', 'Downloading video metadata')
+        video_json = self._call_api(video_id, 'GetVideoAndComments', 'Downloading video metadata')
         video_info = video_json['getVideo']
         is_live = video_info.get('live')
         comments = [self._parse_comment(comment, 'root') for comment in video_json.get('getVideoComments')]
@@ -153,3 +226,67 @@ query GetCommentReplies($id: String!) {
             'comments': comments,
             '__post_extractor': self.extract_comments(video_id, comments, video_json.get('getVideoComments'))
         }
+
+
+class BannedVideoChannelIE(BannedVideoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?banned\.video/channel/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://banned.video/channel/war-room-with-owen-shroyer',
+        'playlist_mincount': 9810,
+        'info_dict': {
+            'id': '5b9301172abf762e22bc22fd',
+            'title': 'War Room With Owen Shroyer',
+            'description': 'md5:821d45563613ec1f5a2b68046d13e2ff',
+            'thumbnail': 'https://download.assets.video/images/acb63e7e-d23f-4901-9bac-cf3edda8eb89-large.png',
+        },
+    }, {
+        'url': 'https://banned.video/channel/the-best-of-tucker-carlson-',
+        'playlist_mincount': 21,
+        'info_dict': {
+            'thumbnail': 'https://download.assets.video/images/b1416a59-4902-480c-979e-246e62c34861-large.jpg',
+            'id': '64598c46b1e3f80b32930313',
+            'title': 'The Best of Tucker Carlson ',
+            'description': 'Channel dedicated to Tucker',
+        },
+    }]
+
+    def _real_extract(self, url):
+        channel_info = self._call_api(
+            self._match_id(url), 'GetChannel', 'Downloading channel metadata')['getChannelByIdOrTitle']
+        channel_id = channel_info['_id']
+
+        return self.playlist_result(
+            [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True)
+             for id in self._paginate(channel_id, 'GetChannelVideos')],
+            channel_id, channel_info['title'], channel_info.get('summary'),
+            thumbnail=channel_info.get('coverImage'))
+
+
+class BannedVideoPlaylistIE(BannedVideoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?banned\.video/playlist/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://banned.video/playlist/5d81058ce2ea200013c01580',
+        'playlist_mincount': 1507,
+        'info_dict': {
+            'title': 'Full Alex Jones Shows',
+            'id': '5d81058ce2ea200013c01580',
+            'description': '',
+        },
+    }, {
+        'url': 'https://banned.video/playlist/5db8bac40d7a4400199b73ca',
+        'playlist_mincount': 92,
+        'info_dict': {
+            'title': 'Owen Shroyer Man On The Street Interviews',
+            'id': '5db8bac40d7a4400199b73ca',
+            'description': '',
+        },
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        playlist_info = self._call_api(playlist_id, 'GetPlaylist', 'Downloading playlist metadata')['getPlaylist']
+
+        return self.playlist_result(
+            [self.url_result(f'https://banned.video/watch?id={id}', url_transparent=True)
+             for id in self._paginate(playlist_id, 'GetPlaylistVideos')], playlist_id,
+            playlist_info['title'], playlist_info.get('summary'))