yt_dlp/extractor/substack.py

@@ -86,51 +86,6 @@ class SubstackIE(InfoExtractor):
             'duration': 25.939592,
             'uploader_id': '535611',
         },
-    }, {
-        # A "podcast" page which contains embedded video
-        'url': 'https://substack.com/home/post/p-161501224',
-        'info_dict': {
-            'id': '161501224',
-            'title': 'Videos for Global Medical Tyranny Just Got Real | Daily Pulse',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '161501224',
-                'ext': 'mp3',
-                'title': 'Global Medical Tyranny Just Got Real | Daily Pulse',
-                'uploader_id': '975571',
-                'uploader': 'The Vigilant Fox',
-                'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
-                'description': 'The news you weren’t supposed to see.',
-            },
-        }, {
-            'info_dict': {
-                'id': '95b4b5a7-3873-465d-a6f4-8d3c499f29f2',
-                'ext': 'mp4',
-                'title': 'WHO.mp4',
-                'uploader_id': '82027648',
-                'duration': 593.3333,
-                'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
-            },
-        }, {
-            'info_dict': {
-                'id': '235db8bf-c6e8-439a-b51a-eb7566fc7ac1',
-                'ext': 'mp4',
-                'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED.mp4',
-                'uploader_id': '82027648',
-                'duration': 498.86667,
-                'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
-            },
-        }, {
-            'info_dict': {
-                'id': 'ef3fe4c1-d748-4665-ba9c-33396008e75b',
-                'ext': 'mp4',
-                'title': 'The Vigilant Fox\'s Video - Apr 16, 2025-VEED (1).mp4',
-                'uploader_id': '82027648',
-                'duration': 410.53333,
-                'thumbnail': r're:https://substack.*video.*/video_upload/post/.+\.png',
-            },
-        }],
     }]
 
     @classmethod
@@ -178,9 +133,7 @@ class SubstackIE(InfoExtractor):
             canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl()
 
         post_type = webpage_info['post']['type']
-        post_title = traverse_obj(webpage_info, ('post', 'title'))
         formats, subtitles = [], {}
-        all_results = []
         result_info = {
             'id': str(webpage_info['post']['id']),
             'subtitles': subtitles,
@@ -191,9 +144,7 @@ class SubstackIE(InfoExtractor):
             'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))),
             'webpage_url': canonical_url,
         }
-        # handle specific post types which are based on media
         if post_type == 'podcast':
-            podcast_result = dict(result_info)
             fmt = {'url': webpage_info['post']['podcast_url']}
             if not determine_ext(fmt['url'], default_ext=None):
                 # The redirected format URL expires but the original URL doesn't,
@@ -202,87 +153,89 @@ class SubstackIE(InfoExtractor):
                     HEADRequest(fmt['url']), display_id,
                     'Resolving podcast file extension',
                     'Podcast URL is invalid').url)
-            podcast_result['formats'] = [fmt]
+            result_info['formats'] = [fmt]
-            all_results.append(podcast_result)
+            return result_info
-        if post_type == 'video':
+        elif post_type == 'video':
-            video_result = dict(result_info)
             formats, subtitles = self._extract_video_formats(
                 webpage_info['post']['videoUpload']['id'], canonical_url)
-            video_result.update({
+            result_info.update({
                 'formats': formats,
                 'subtitles': subtitles,
             })
-            all_results.append(video_result)
+            return result_info
+        elif post_type == 'newsletter':
+            results = []
+            found_items = []
+            post_id = str(webpage_info['post']['id'])
+            post_title = traverse_obj(webpage_info, ('post', 'title'))
+            assert result_info['uploader_id'] is not None, 'newsletter posted without user_id'
 
-        # search for embedded players on the page
+            video_players = re.finditer(
-        found_items = []
+                r'<div[^>]*data-component-name="VideoEmbedPlayer"[^>]*>',
-        post_id = str(webpage_info['post']['id'])
+                webpage)
-        assert result_info['uploader_id'] is not None, 'newsletter posted without user_id'
+            for vp in video_players:
+                video_id = self._search_regex(r'id="([^"]+)"',
+                                              vp.group(0),
+                                              'video id', group=1).replace('media-', '')
+                video_metadata_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}')
+                json_vid_data = self._download_json(video_metadata_url, video_id)
+                assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
+                json_vid_data['_type'] = 'video'
+                found_items.append(json_vid_data)
 
-        video_players = re.finditer(
+            audio_players = re.finditer(
-            r'<div[^>]*data-component-name="VideoEmbedPlayer"[^>]*>',
+                r'<div[^>]*data-component-name="AudioEmbedPlayer"[^>]*>',
-            webpage)
+                webpage)
-        for vp in video_players:
+            for ap in audio_players:
-            video_id = self._search_regex(r'id="([^"]+)"',
+                video_uri = self._search_regex(r'src="(/api/v1/audio/upload/[^"]+)"',
-                                          vp.group(0),
+                                               webpage[ap.start():],
-                                          'video id', group=1).replace('media-', '')
+                                               'video uri', group=1)
-            video_metadata_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}')
+                video_metadata_url = urllib.parse.urljoin(url, video_uri.replace('/src', ''))
-            json_vid_data = self._download_json(video_metadata_url, video_id)
+                video_id = self._search_regex(r'upload/([^/]+)/src',
-            assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
+                                              video_uri,
-            json_vid_data['_type'] = 'video'
+                                              'video id', group=1)
-            found_items.append(json_vid_data)
+                json_vid_data = self._download_json(video_metadata_url, video_id)
+                assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
+                json_vid_data['_type'] = 'audio'
+                json_vid_data['_uri'] = video_uri
+                found_items.append(json_vid_data)
 
-        audio_players = re.finditer(
+            for json_vid_data in found_items:
-            r'<div[^>]*data-component-name="AudioEmbedPlayer"[^>]*>',
+                video_id = json_vid_data['id']
-            webpage)
+                if 'video' in json_vid_data['_type']:
-        for ap in audio_players:
+                    formats, subtitles = self._extract_video_formats(
-            video_uri = self._search_regex(r'src="(/api/v1/audio/upload/[^"]+)"',
+                        video_id, canonical_url,
-                                           webpage[ap.start():],
+                        query_params={'override_publication_id': result_info['uploader_id']})
-                                           'video uri', group=1)
+                else:
-            video_metadata_url = urllib.parse.urljoin(url, video_uri.replace('/src', ''))
+                    fmt = {'url': urllib.parse.urljoin(url, json_vid_data['_uri'])}
-            video_id = self._search_regex(r'upload/([^/]+)/src',
+                    if not determine_ext(fmt['url'], default_ext=None):
-                                          video_uri,
+                        # The redirected format URL expires but the original URL doesn't,
-                                          'video id', group=1)
+                        # so we only want to extract the extension from this request
-            json_vid_data = self._download_json(video_metadata_url, video_id)
+                        fmt['ext'] = determine_ext(self._request_webpage(
-            assert video_id == json_vid_data['id'], 'unexpected json metadata retrieved'
+                            HEADRequest(fmt['url']), video_id,
-            json_vid_data['_type'] = 'audio'
+                            'Resolving audio file extension',
-            json_vid_data['_uri'] = video_uri
+                            'Embedded audio URL is invalid').url)
-            found_items.append(json_vid_data)
+                    formats.append(fmt)
 
-        for json_vid_data in found_items:
+                new_result = dict(result_info)
-            video_id = json_vid_data['id']
+                new_result.update({
-            if 'video' in json_vid_data['_type']:
+                    'formats': formats,
-                formats, subtitles = self._extract_video_formats(
+                    'subtitles': subtitles,
-                    video_id, canonical_url,
+                    'id': video_id,
-                    query_params={'override_publication_id': result_info['uploader_id']})
+                    'title': json_vid_data.get('name', None),
+                    'description': None,
+                    'thumbnail': json_vid_data.get('thumbnail_url', None),
+                    'duration': float_or_none(json_vid_data.get('duration', None)),
+                    # videos can be cross-embedded, so the publication id should be of the uploader
+                    'uploader_id': str_or_none(json_vid_data.get('user_id', None)),
+                    'uploader': None,  # video uploader username is not included
+                })
+                results.append(new_result)
+
+            if len(results) > 0:
+                playlist_title = f'Videos for {post_title}'
+                return self.playlist_result(results, post_id, playlist_title)
             else:
-                fmt = {'url': urllib.parse.urljoin(url, json_vid_data['_uri'])}
+                self.raise_no_formats(f'Page type "{post_type}" contains no supported embeds')
-                if not determine_ext(fmt['url'], default_ext=None):
-                    # The redirected format URL expires but the original URL doesn't,
-                    # so we only want to extract the extension from this request
-                    fmt['ext'] = determine_ext(self._request_webpage(
-                        HEADRequest(fmt['url']), video_id,
-                        'Resolving audio file extension',
-                        'Embedded audio URL is invalid').url)
-                formats.append(fmt)
 
-            new_result = dict(result_info)
+        self.raise_no_formats(f'Page type "{post_type}" is not supported')
-            new_result.update({
-                'formats': formats,
-                'subtitles': subtitles,
-                'id': video_id,
-                'title': json_vid_data.get('name', None),
-                'description': None,
-                'thumbnail': json_vid_data.get('thumbnail_url', None),
-                'duration': float_or_none(json_vid_data.get('duration', None)),
-                # videos can be cross-embedded, so the publication id should be of the uploader
-                'uploader_id': str_or_none(json_vid_data.get('user_id', None)),
-                'uploader': None,  # video uploader username is not included
-            })
-            all_results.append(new_result)
-
-        if len(all_results) > 0:
-            playlist_title = f'Videos for {post_title}'
-            return self.playlist_result(all_results, post_id, playlist_title)
-        else:
-            self.raise_no_formats(f'Page type "{post_type}" contains no supported embeds')