Rework extractor

yt_dlp/extractor/_extractors.py

@@ -1983,8 +1983,8 @@ from .stretchinternet import StretchInternetIE
 from .stripchat import StripchatIE
 from .stv import STVPlayerIE
 from .subsplash import (
+    SubsplashIE,
     SubsplashPlaylistIE,
-    SubsplashVideoIE,
 )
 from .substack import SubstackIE
 from .sunporno import SunPornoIE

yt_dlp/extractor/subsplash.py

@@ -1,19 +1,56 @@
 import functools
+import json
 import math
 
 from .common import InfoExtractor
 from ..utils import (
     InAdvancePagedList,
+    get_element_by_id,
     int_or_none,
-    str_or_none,
+    parse_iso8601,
-    traverse_obj,
+    try_call,
-    unified_strdate,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
-class SubsplashVideoIE(InfoExtractor):
+class SubsplashBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?subsplash\.com/u/[^/]+/media/d/(?P<id>\w+)'
+    def _get_headers(self, url, display_id):
+        token = try_call(lambda: self._get_cookies(url)['ss-token-guest'].value)
+        if not token:
+            webpage, urlh = self._download_webpage_handle(url, display_id)
+            token = try_call(lambda: self._get_cookies(url)['ss-token-guest'].value)
+            if not token:
+                token = urlh.get_header('x-api-token')
+            if not token:
+                token = traverse_obj(get_element_by_id('shoebox-tokens', webpage), ({json.loads}, 'apiToken', {str}))
+            if not token:
+                token = self._search_regex(r'\\"tokens\\":{\\"guest\\":\\"([A-Za-z0-9._-]+)\\"', webpage, 'token', default=None)
+
+        if not token:
+            return None
+        return {'Authorization': f'Bearer {token}'}
+
+    def _extract_video(self, data, display_id):
+        return traverse_obj(data, {
+            'id': ('id', {str}),
+            'title': ('title', {str}),
+            'description': ('summary_text', {str}),
+            'thumbnail': ('_embedded', 'images', 0, '_links', 'related', 'href', {url_or_none}),
+            'duration': ('_embedded', 'video', 'duration', {int_or_none(scale=1000)}),
+            'timestamp': ('date', {parse_iso8601}),
+            'release_timestamp': ('published_at', {parse_iso8601}),
+            'modified_timestamp': ('updated_at', {parse_iso8601}),
+            'formats': ('_embedded', 'video', '_embedded', 'playlists', 0, '_links', 'related', 'href',
+                        {lambda url: self._extract_m3u8_formats(url, display_id)}),
+        })
+
+
+class SubsplashIE(SubsplashBaseIE):
+    _VALID_URL = [
+        r'https?://(?:www\.)?subsplash\.com/u/[^/]+/media/d/(?P<id>\w+)',
+        r'https?://(?:\w+\.)?subspla.sh/(?P<id>\w+)',
+    ]
     _TESTS = [{
         'url': 'https://subsplash.com/u/skywatchtv/media/d/5whnx5s-the-grand-delusion-taking-place-right-now',
         'md5': '2d67c50deac3c6c49c6e25c4a5b25afe',
@@ -25,6 +62,11 @@ class SubsplashVideoIE(InfoExtractor):
             'upload_date': '20240901',
             'duration': 1710,
             'thumbnail': r're:https?://.*\.(?:jpg|png)$',
+            'modified_date': '20240901',
+            'release_date': '20240901',
+            'release_timestamp': 1725195600,
+            'timestamp': 1725148800,
+            'modified_timestamp': 1725195657,
         },
     }, {
         'url': 'https://subsplash.com/u/prophecywatchers/media/d/n4dr8b2-the-transhumanist-plan-for-humanity-billy-crone',
@@ -37,34 +79,31 @@ class SubsplashVideoIE(InfoExtractor):
             'upload_date': '20240903',
             'duration': 1709,
             'thumbnail': r're:https?://.*\.(?:jpg|png)$',
+            'timestamp': 1725321600,
+            'modified_date': '20241010',
+            'release_date': '20240903',
+            'release_timestamp': 1725379200,
+            'modified_timestamp': 1728577804,
         },
+    }, {
+        'url': 'https://prophecywatchers.subspla.sh/8gps8cx',
+        'only_matching': True,
     }]
 
-    def _fetch_json(self, url, display_id, token):
-        return self._download_json(url, display_id, headers={'Authorization': f'Bearer {token}'})
-
-    def _extract_metadata(self, data, display_id):
-        return traverse_obj(data, {
-            'id': ('id', {str_or_none}),
-            'title': ('title', {str_or_none}),
-            'description': ('summary_text', {str_or_none}),
-            'thumbnail': ('_embedded', 'images', 0, '_links', 'related', 'href', {url_or_none}),
-            'duration': ('_embedded', 'video', 'duration', {lambda x: int_or_none(x, 1000)}),
-            'upload_date': ('published_at', {unified_strdate}),
-            'formats': ('_embedded', 'video', '_embedded', 'playlists', 0, '_links', 'related', 'href',
-                        {lambda url: self._extract_m3u8_formats(url, display_id)}),
-        })
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage, urlh = self._download_webpage_handle(url, display_id)
+        headers = self._get_headers(url, display_id)
-        token = urlh.get_header('set-cookie').split(';')[0].split('=')[1].strip()
+
-        metadata_url = f'https://core.subsplash.com/media/v1/media-items?filter[short_code]={display_id}&include=images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document,broadcast'
+        data = self._download_json(
-        metadata = self._fetch_json(metadata_url, display_id, token)
+            'https://core.subsplash.com/media/v1/media-items', display_id, headers=headers,
-        return self._extract_metadata(traverse_obj(metadata, ('_embedded', 'media-items', 0, {dict})), display_id)
+            query={
+                'filter[short_code]': display_id,
+                'include': 'images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document,broadcast',
+            })
+        return self._extract_video(traverse_obj(data, ('_embedded', 'media-items', 0)), display_id)
 
 
-class SubsplashPlaylistIE(SubsplashVideoIE):
+class SubsplashPlaylistIE(SubsplashBaseIE):
     IE_NAME = 'subsplash:playlist'
     _VALID_URL = r'https?://(?:www\.)?subsplash\.com/[^/]+/(?:our-videos|media)/ms/\+(?P<id>\w+)'
     _PAGE_SIZE = 15
@@ -87,26 +126,43 @@ class SubsplashPlaylistIE(SubsplashVideoIE):
         'only_matching': True,
     }]
 
-    def _get_entries(self, token, series_id, page):
+    def _entries(self, series_id, headers, page):
-        url = f'https://core.subsplash.com/media/v1/media-items?filter[broadcast.status|broadcast.status]=null|on-demand&filter[media_series]={series_id}&filter[status]=published&include=images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document&page[number]={page + 1}&page[size]={self._PAGE_SIZE}&sort=-position'
+        data = self._download_json(
-        data = self._fetch_json(url, f'{series_id}_{page}', token)
+            'https://core.subsplash.com/media/v1/media-items', series_id, headers=headers,
-        entries = traverse_obj(data, ('_embedded', 'media-items', {list}))
+            query={
+                'filter[broadcast.status|broadcast.status]': 'null|on-demand',
+                'filter[media_series]': series_id,
+                'filter[status]': 'published',
+                'include': 'images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document',
+                'page[number]': page + 1,
+                'page[size]': self._PAGE_SIZE,
+                'sort': '-position',
+            }, note=f'Downloading page {page + 1}')
+
+        entries = traverse_obj(data, ('_embedded', 'media-items', lambda _, v: v['short_code']))
         for entry in entries:
-            yield self._extract_metadata(entry, series_id)
+            info = self._extract_video(entry, entry.get('id'))
+            yield {
+                **info,
+                'webpage_url': f'https://subspla.sh/{entry["short_code"]}',
+                'extractor_key': SubsplashIE.ie_key(),
+                'extractor': SubsplashIE.IE_NAME,
+            }
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage, urlh = self._download_webpage_handle(url, display_id)
+        headers = self._get_headers(url, display_id)
-        token = urlh.get_header('x-api-token')
 
-        series_url = f'https://core.subsplash.com/media/v1/media-series?filter[short_code]={display_id}'
+        data = self._download_json(
-        json_data = self._fetch_json(series_url, display_id, token)
+            'https://core.subsplash.com/media/v1/media-series', display_id, headers=headers,
-        series_data = traverse_obj(json_data, ('_embedded', 'media-series', 0, {
+            query={'filter[short_code]': display_id})
+        series_data = traverse_obj(data, ('_embedded', 'media-series', 0, {
             'id': ('id', {str}),
             'title': ('title', {str}),
             'count': ('media_items_count', {int}),
         }))
         total_pages = math.ceil(series_data['count'] / self._PAGE_SIZE)
 
-        entries = InAdvancePagedList(functools.partial(self._get_entries, token, series_data['id']), total_pages, self._PAGE_SIZE)
+        return self.playlist_result(
-        return self.playlist_result(entries, display_id, series_data['title'])
+            InAdvancePagedList(functools.partial(self._entries, series_data['id'], headers), total_pages, self._PAGE_SIZE),
+            display_id, series_data['title'])