diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 384937633c..5b09d82a78 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1983,8 +1983,8 @@ from .stretchinternet import StretchInternetIE from .stripchat import StripchatIE from .stv import STVPlayerIE from .subsplash import ( + SubsplashIE, SubsplashPlaylistIE, - SubsplashVideoIE, ) from .substack import SubstackIE from .sunporno import SunPornoIE diff --git a/yt_dlp/extractor/subsplash.py b/yt_dlp/extractor/subsplash.py index 564464aa16..c7cd80a36f 100644 --- a/yt_dlp/extractor/subsplash.py +++ b/yt_dlp/extractor/subsplash.py @@ -1,19 +1,56 @@ import functools +import json import math from .common import InfoExtractor from ..utils import ( InAdvancePagedList, + get_element_by_id, int_or_none, - str_or_none, - traverse_obj, - unified_strdate, + parse_iso8601, + try_call, url_or_none, ) +from ..utils.traversal import traverse_obj -class SubsplashVideoIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?subsplash\.com/u/[^/]+/media/d/(?P\w+)' +class SubsplashBaseIE(InfoExtractor): + def _get_headers(self, url, display_id): + token = try_call(lambda: self._get_cookies(url)['ss-token-guest'].value) + if not token: + webpage, urlh = self._download_webpage_handle(url, display_id) + token = try_call(lambda: self._get_cookies(url)['ss-token-guest'].value) + if not token: + token = urlh.get_header('x-api-token') + if not token: + token = traverse_obj(get_element_by_id('shoebox-tokens', webpage), ({json.loads}, 'apiToken', {str})) + if not token: + token = self._search_regex(r'\\"tokens\\":{\\"guest\\":\\"([A-Za-z0-9._-]+)\\"', webpage, 'token', default=None) + + if not token: + return None + return {'Authorization': f'Bearer {token}'} + + def _extract_video(self, data, display_id): + return traverse_obj(data, { + 'id': ('id', {str}), + 'title': ('title', {str}), + 'description': ('summary_text', {str}), + 'thumbnail': ('_embedded', 'images', 0, '_links', 'related', 'href', {url_or_none}), + 'duration': ('_embedded', 'video', 'duration', {int_or_none(scale=1000)}), + 'timestamp': ('date', {parse_iso8601}), + 'release_timestamp': ('published_at', {parse_iso8601}), + 'modified_timestamp': ('updated_at', {parse_iso8601}), + 'formats': ('_embedded', 'video', '_embedded', 'playlists', 0, '_links', 'related', 'href', + {lambda url: self._extract_m3u8_formats(url, display_id)}), + }) + + +class SubsplashIE(SubsplashBaseIE): + _VALID_URL = [ + r'https?://(?:www\.)?subsplash\.com/u/[^/]+/media/d/(?P\w+)', + r'https?://(?:\w+\.)?subspla.sh/(?P\w+)', + ] _TESTS = [{ 'url': 'https://subsplash.com/u/skywatchtv/media/d/5whnx5s-the-grand-delusion-taking-place-right-now', 'md5': '2d67c50deac3c6c49c6e25c4a5b25afe', @@ -25,6 +62,11 @@ class SubsplashVideoIE(InfoExtractor): 'upload_date': '20240901', 'duration': 1710, 'thumbnail': r're:https?://.*\.(?:jpg|png)$', + 'modified_date': '20240901', + 'release_date': '20240901', + 'release_timestamp': 1725195600, + 'timestamp': 1725148800, + 'modified_timestamp': 1725195657, }, }, { 'url': 'https://subsplash.com/u/prophecywatchers/media/d/n4dr8b2-the-transhumanist-plan-for-humanity-billy-crone', @@ -37,34 +79,31 @@ class SubsplashVideoIE(InfoExtractor): 'upload_date': '20240903', 'duration': 1709, 'thumbnail': r're:https?://.*\.(?:jpg|png)$', + 'timestamp': 1725321600, + 'modified_date': '20241010', + 'release_date': '20240903', + 'release_timestamp': 1725379200, + 'modified_timestamp': 1728577804, }, + }, { + 'url': 'https://prophecywatchers.subspla.sh/8gps8cx', + 'only_matching': True, }] - def _fetch_json(self, url, display_id, token): - return self._download_json(url, display_id, headers={'Authorization': f'Bearer {token}'}) - - def _extract_metadata(self, data, display_id): - return traverse_obj(data, { - 'id': ('id', {str_or_none}), - 'title': ('title', {str_or_none}), - 'description': ('summary_text', {str_or_none}), - 'thumbnail': ('_embedded', 'images', 0, '_links', 'related', 'href', {url_or_none}), - 'duration': ('_embedded', 'video', 'duration', {lambda x: int_or_none(x, 1000)}), - 'upload_date': ('published_at', {unified_strdate}), - 'formats': ('_embedded', 'video', '_embedded', 'playlists', 0, '_links', 'related', 'href', - {lambda url: self._extract_m3u8_formats(url, display_id)}), - }) - def _real_extract(self, url): display_id = self._match_id(url) - webpage, urlh = self._download_webpage_handle(url, display_id) - token = urlh.get_header('set-cookie').split(';')[0].split('=')[1].strip() - metadata_url = f'https://core.subsplash.com/media/v1/media-items?filter[short_code]={display_id}&include=images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document,broadcast' - metadata = self._fetch_json(metadata_url, display_id, token) - return self._extract_metadata(traverse_obj(metadata, ('_embedded', 'media-items', 0, {dict})), display_id) + headers = self._get_headers(url, display_id) + + data = self._download_json( + 'https://core.subsplash.com/media/v1/media-items', display_id, headers=headers, + query={ + 'filter[short_code]': display_id, + 'include': 'images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document,broadcast', + }) + return self._extract_video(traverse_obj(data, ('_embedded', 'media-items', 0)), display_id) -class SubsplashPlaylistIE(SubsplashVideoIE): +class SubsplashPlaylistIE(SubsplashBaseIE): IE_NAME = 'subsplash:playlist' _VALID_URL = r'https?://(?:www\.)?subsplash\.com/[^/]+/(?:our-videos|media)/ms/\+(?P\w+)' _PAGE_SIZE = 15 @@ -87,26 +126,43 @@ class SubsplashPlaylistIE(SubsplashVideoIE): 'only_matching': True, }] - def _get_entries(self, token, series_id, page): - url = f'https://core.subsplash.com/media/v1/media-items?filter[broadcast.status|broadcast.status]=null|on-demand&filter[media_series]={series_id}&filter[status]=published&include=images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document&page[number]={page + 1}&page[size]={self._PAGE_SIZE}&sort=-position' - data = self._fetch_json(url, f'{series_id}_{page}', token) - entries = traverse_obj(data, ('_embedded', 'media-items', {list})) + def _entries(self, series_id, headers, page): + data = self._download_json( + 'https://core.subsplash.com/media/v1/media-items', series_id, headers=headers, + query={ + 'filter[broadcast.status|broadcast.status]': 'null|on-demand', + 'filter[media_series]': series_id, + 'filter[status]': 'published', + 'include': 'images,audio.audio-outputs,audio.video,video.video-outputs,video.playlists,document', + 'page[number]': page + 1, + 'page[size]': self._PAGE_SIZE, + 'sort': '-position', + }, note=f'Downloading page {page + 1}') + + entries = traverse_obj(data, ('_embedded', 'media-items', lambda _, v: v['short_code'])) for entry in entries: - yield self._extract_metadata(entry, series_id) + info = self._extract_video(entry, entry.get('id')) + yield { + **info, + 'webpage_url': f'https://subspla.sh/{entry["short_code"]}', + 'extractor_key': SubsplashIE.ie_key(), + 'extractor': SubsplashIE.IE_NAME, + } def _real_extract(self, url): display_id = self._match_id(url) - webpage, urlh = self._download_webpage_handle(url, display_id) - token = urlh.get_header('x-api-token') + headers = self._get_headers(url, display_id) - series_url = f'https://core.subsplash.com/media/v1/media-series?filter[short_code]={display_id}' - json_data = self._fetch_json(series_url, display_id, token) - series_data = traverse_obj(json_data, ('_embedded', 'media-series', 0, { + data = self._download_json( + 'https://core.subsplash.com/media/v1/media-series', display_id, headers=headers, + query={'filter[short_code]': display_id}) + series_data = traverse_obj(data, ('_embedded', 'media-series', 0, { 'id': ('id', {str}), 'title': ('title', {str}), 'count': ('media_items_count', {int}), })) total_pages = math.ceil(series_data['count'] / self._PAGE_SIZE) - entries = InAdvancePagedList(functools.partial(self._get_entries, token, series_data['id']), total_pages, self._PAGE_SIZE) - return self.playlist_result(entries, display_id, series_data['title']) + return self.playlist_result( + InAdvancePagedList(functools.partial(self._entries, series_data['id'], headers), total_pages, self._PAGE_SIZE), + display_id, series_data['title'])