diff --git a/README.md b/README.md index a2017443e8..356738ad28 100644 --- a/README.md +++ b/README.md @@ -1969,6 +1969,9 @@ The following extractors use this feature: #### sonylivseries * `sort_order`: Episode sort order for series extraction - one of `asc` (ascending, oldest first) or `desc` (descending, newest first). Default is `asc` +#### streaks +* `api_key`: API key for the `X-Streaks-Api-Key` header + #### tver * `backend`: Backend API to use for extraction - one of `streaks` (default) or `brightcove` (deprecated) diff --git a/yt_dlp/extractor/streaks.py b/yt_dlp/extractor/streaks.py index 642e0527e3..ce2309b6a3 100644 --- a/yt_dlp/extractor/streaks.py +++ b/yt_dlp/extractor/streaks.py @@ -5,12 +5,13 @@ from .common import InfoExtractor from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, + clean_html, filter_dict, float_or_none, + int_or_none, join_nonempty, mimetype2ext, parse_iso8601, - unsmuggle_url, update_url_query, url_or_none, ) @@ -22,22 +23,28 @@ class StreaksBaseIE(InfoExtractor): _GEO_BYPASS = False _GEO_COUNTRIES = ['JP'] - def _extract_from_streaks_api(self, project_id, media_id, headers=None, query=None, ssai=False, live_from_start=False): + def _streaks_playback_api_url(self, project_id, media_id): + return self._API_URL_TEMPLATE.format('playback', project_id, media_id, '') + + def _download_streaks_playback_json(self, project_id, media_id, headers=None): + return self._download_json( + self._streaks_playback_api_url(project_id, media_id), + media_id, 'Downloading STREAKS playback API JSON', headers={ + 'Accept': 'application/json', + 'Origin': 'https://players.streaks.jp', + **self.geo_verification_headers(), + **(headers or {}), + }) + + def _extract_from_streaks_api(self, project_id, media_id, headers=None, query=None, live_from_start=False): try: - response = self._download_json( - self._API_URL_TEMPLATE.format('playback', project_id, media_id, ''), - media_id, 'Downloading STREAKS playback API JSON', headers={ - 'Accept': 'application/json', - 'Origin': 'https://players.streaks.jp', - **self.geo_verification_headers(), - **(headers or {}), - }) + response = self._download_streaks_playback_json(project_id, media_id, headers=headers) except ExtractorError as e: if isinstance(e.cause, HTTPError) and e.cause.status in (403, 404): error = self._parse_json(e.cause.response.read().decode(), media_id, fatal=False) - message = traverse_obj(error, ('message', {str})) - code = traverse_obj(error, ('code', {str})) - error_id = traverse_obj(error, ('id', {int})) + message = traverse_obj(error, ('message', {clean_html}, filter)) + code = traverse_obj(error, ('code', {clean_html}, filter)) + error_id = traverse_obj(error, ('id', {int_or_none})) if code == 'REQUEST_FAILED': if error_id == 124: self.raise_geo_restricted(countries=self._GEO_COUNTRIES) @@ -59,8 +66,12 @@ class StreaksBaseIE(InfoExtractor): formats, subtitles = [], {} drm_formats = False + sources = response['sources'] + ssai = traverse_obj(sources, (..., 'ssai', {dict}, any)) - for source in traverse_obj(response, ('sources', lambda _, v: v['src'])): + for source in traverse_obj(sources, ( + lambda _, v: url_or_none(v['src']), + )): if source.get('key_systems'): drm_formats = True continue @@ -108,12 +119,12 @@ class StreaksBaseIE(InfoExtractor): 'subtitles': subtitles, 'uploader_id': project_id, **traverse_obj(response, { - 'title': ('name', {str}), - 'description': ('description', {str}, filter), + 'title': ('name', {clean_html}, filter), + 'description': ('description', {clean_html}, filter), 'duration': ('duration', {float_or_none}), 'modified_timestamp': ('updated_at', {parse_iso8601}), - 'tags': ('tags', ..., {str}), - 'thumbnails': (('poster', 'thumbnail'), 'src', {'url': {url_or_none}}), + 'tags': ('tags', ..., {clean_html}, filter, all, filter), + 'thumbnail': (('thumbnail', 'poster'), 'src', {url_or_none}, any), 'timestamp': ('created_at', {parse_iso8601}), }), } @@ -121,26 +132,27 @@ class StreaksBaseIE(InfoExtractor): class StreaksIE(StreaksBaseIE): _VALID_URL = [ - r'https?://players\.streaks\.jp/(?P[\w-]+)/[\da-f]+/index\.html\?(?:[^#]+&)?m=(?P(?:ref:)?[\w-]+)', + r'https?://players\.streaks\.jp/(?P[\w-]+)/(?P[\da-f]+)/index\.html\?(?:[^#]+&)?m=(?P(?:ref:)?[\w-]+)', r'https?://playback\.api\.streaks\.jp/v1/projects/(?P[\w-]+)/medias/(?P(?:ref:)?[\w-]+)', ] _EMBED_REGEX = [rf']*\bsrc\s*=\s*["\'](?P{_VALID_URL[0]})'] _TESTS = [{ - 'url': 'https://players.streaks.jp/tipness/08155cd19dc14c12bebefb69b92eafcc/index.html?m=dbdf2df35b4d483ebaeeaeb38c594647', + # https://online.tipness.co.jp/contents/9a064492-a62b-5d03-4668-6b40d6325b1e + 'url': 'https://players.streaks.jp/tipness/08155cd19dc14c12bebefb69b92eafcc/index.html?m=ba2c253508914d9ea061a5f26bc58b20', 'info_dict': { - 'id': 'dbdf2df35b4d483ebaeeaeb38c594647', + 'id': 'ba2c253508914d9ea061a5f26bc58b20', 'ext': 'mp4', - 'title': '3shunenCM_edit.mp4', - 'display_id': 'dbdf2df35b4d483ebaeeaeb38c594647', - 'duration': 47.533, + 'title': 'tarun_suimin.mp4', + 'duration': 265.344, 'live_status': 'not_live', - 'modified_date': '20230726', - 'modified_timestamp': 1690356180, - 'timestamp': 1690355996, - 'upload_date': '20230726', + 'modified_date': '20230908', + 'modified_timestamp': 1694146842, + 'timestamp': 1694145352, + 'upload_date': '20230908', 'uploader_id': 'tipness', }, }, { + # https://www.ktv.jp/mycoffeetime/ 'url': 'https://players.streaks.jp/ktv-web/0298e8964c164ab384c07ef6e08c444b/index.html?m=ref:mycoffeetime_250317', 'info_dict': { 'id': 'dccdc079e3fd41f88b0c8435e2d453ab', @@ -157,22 +169,24 @@ class StreaksIE(StreaksBaseIE): 'uploader_id': 'ktv-web', }, }, { - 'url': 'https://playback.api.streaks.jp/v1/projects/ktv-web/medias/b5411938e1e5435dac71edf829dd4813', + # https://www.ktv.jp/news/articles/?id=28105 + 'url': 'https://playback.api.streaks.jp/v1/projects/ktv-news/medias/714171c4c53c409bb41e1572997ebfcf', 'info_dict': { - 'id': 'b5411938e1e5435dac71edf829dd4813', + 'id': '714171c4c53c409bb41e1572997ebfcf', 'ext': 'mp4', - 'title': 'KANTELE_SYUSEi_0630', - 'display_id': 'b5411938e1e5435dac71edf829dd4813', + 'title': '28105.mp4', + 'duration': 49.984, 'live_status': 'not_live', - 'modified_date': '20250122', - 'modified_timestamp': 1737522999, + 'modified_date': '20260702', + 'modified_timestamp': 1783028559, 'thumbnail': r're:https?://.+\.jpg', - 'timestamp': 1735205137, - 'upload_date': '20241226', - 'uploader_id': 'ktv-web', + 'timestamp': 1783028407, + 'upload_date': '20260702', + 'uploader_id': 'ktv-news', }, + 'params': {'extractor_args': {'streaks': {'api_key': ['0ff2ccfb6381401582d6ee60e3cb66a1']}}}, }, { - # TVer Olympics: website already down, but api remains accessible + # https://tver.jp/olympic/paris2024/live/FBLMTEAM11------------SFNL000100--/ 'url': 'https://playback.api.streaks.jp/v1/projects/tver-olympic/medias/ref:sp_240806_1748_dvr', 'info_dict': { 'id': 'c10f7345adb648cf804d7578ab93b2e3', @@ -187,8 +201,10 @@ class StreaksIE(StreaksBaseIE): 'upload_date': '20240804', 'uploader_id': 'tver-olympic', }, + 'params': {'extractor_args': {'streaks': {'api_key': ['e09168c4383d4b18949067022558f071']}}}, + 'skip': 'Invalid URL', }, { - # TBS FREE: 24-hour stream + # https://cu.tbs.co.jp/simul/simul-02 'url': 'https://playback.api.streaks.jp/v1/projects/tbs/medias/ref:simul-02', 'info_dict': { 'id': 'c4e83a7b48f4409a96adacec674b4e22', @@ -202,12 +218,14 @@ class StreaksIE(StreaksBaseIE): 'upload_date': '20240117', 'uploader_id': 'tbs', }, + 'skip': 'Invalid URL', }, { # DRM protected 'url': 'https://players.streaks.jp/sp-jbc/a12d7ee0f40c49d6a0a2bff520639677/index.html?m=5f89c62f37ee4a68be8e6e3b1396c7d8', 'only_matching': True, }] _WEBPAGE_TESTS = [{ + # https://players.streaks.jp/play/719af2a1d2d544e89bcad3456eeae5d9/index.html?m=2d975178293140dc8074a7fc536a7604 'url': 'https://event.play.jp/playnext2023/', 'info_dict': { 'id': '2d975178293140dc8074a7fc536a7604', @@ -222,6 +240,7 @@ class StreaksIE(StreaksBaseIE): 'modified_date': '20250213', 'live_status': 'not_live', }, + 'params': {'nocheckcertificate': True}, }, { 'url': 'https://wowshop.jp/Page/special/cooking_goods/?bid=wowshop&srsltid=AfmBOor_phUNoPEE_UCPiGGSCMrJE5T2US397smvsbrSdLqUxwON0el4', 'playlist_mincount': 2, @@ -232,13 +251,15 @@ class StreaksIE(StreaksBaseIE): 'age_limit': 0, 'thumbnail': 'https://wowshop.jp/Page/special/cooking_goods/images/ogp.jpg', }, + 'skip': 'Invalid URL', }] def _real_extract(self, url): - url, smuggled_data = unsmuggle_url(url, {}) - project_id, media_id = self._match_valid_url(url).group('project_id', 'id') + mobj = self._match_valid_url(url).groupdict() + project_id, media_id = mobj['project_id'], mobj['id'] + api_key = mobj.get('api_key') or self._configuration_arg('api_key', [None])[0] return self._extract_from_streaks_api( project_id, media_id, headers=filter_dict({ - 'X-Streaks-Api-Key': smuggled_data.get('api_key'), + 'X-Streaks-Api-Key': api_key, }))