diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index ac6c10e71d..94c4e5fabc 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1314,6 +1314,11 @@ from .oftv import ( ) from .oktoberfesttv import OktoberfestTVIE from .olympics import OlympicsReplayIE +from .omnyfm import ( + OmnyfmIE, + OmnyfmPlaylistIE, + OmnyfmShowIE, +) from .on24 import On24IE from .ondemandkorea import ( OnDemandKoreaIE, diff --git a/yt_dlp/extractor/omnyfm.py b/yt_dlp/extractor/omnyfm.py new file mode 100644 index 0000000000..caf4b7f798 --- /dev/null +++ b/yt_dlp/extractor/omnyfm.py @@ -0,0 +1,272 @@ +import functools +import itertools + +from .common import InfoExtractor +from ..utils import ( + OnDemandPagedList, + clean_html, + filter_dict, + float_or_none, + int_or_none, + parse_duration, + parse_iso8601, + parse_qs, + update_url, + url_or_none, +) +from ..utils.traversal import ( + require, + traverse_obj, + trim_str, +) + + +class OmnyfmIE(InfoExtractor): + _VALID_URL = r'https?://omny\.fm/shows/(?P[\w-]+)/(?P(?!playlists(?:[/?#"\']|$))[\w-]+)(?:/embed)?(?=[?#"\']|$)' + _EMBED_REGEX = [rf']+\bsrc\s*=\s*(["\'])(?P{_VALID_URL}[^"\']*)\1'] + _TESTS = [{ + 'url': 'https://omny.fm/shows/sleep-hub/cannabinoids-and-sleep', + 'md5': 'e45ec0ce43da757a0be6ca117ec01bdc', + 'info_dict': { + 'id': 'cannabinoids-and-sleep', + 'ext': 'mp3', + 'title': 'Cannabinoids and Sleep', + 'categories': 'count:1', + 'chapters': [ + {'start_time': 0, 'title': 'Introduction'}, + {'start_time': 138, 'title': 'Theme: Cannabinoids and Sleep'}, + {'start_time': 1487, 'title': 'Clinical Tip'}, + {'start_time': 1635, 'title': 'Pick of the Month'}, + {'start_time': 1795, 'title': 'What\'s Coming Up?'}, + ], + 'description': 'md5:c0fd2d29f3148382d344cfbd012fb00d', + 'duration': 1840.274, + 'episode': 'Episode 48', + 'episode_number': 48, + 'modified_date': r're:\d{8}', + 'modified_timestamp': int, + 'tags': 'count:6', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + 'timestamp': 1574013600, + 'upload_date': '20191117', + 'uploader': 'Sleep Talk', + 'uploader_id': 'sleep-hub', + }, + }, { + 'url': 'https://omny.fm/shows/the-origin-of-things/a-song-of-hope/embed', + 'md5': 'd7600ef33e3f139ff1bb8946f3651b15', + 'info_dict': { + 'id': 'a-song-of-hope', + 'ext': 'mp3', + 'title': 'A song of hope', + 'categories': 'count:3', + 'description': 'md5:f8e710765c341a48cfda8dacd428f56d', + 'duration': 478.955, + 'episode': 'Episode 17', + 'episode_number': 17, + 'modified_date': r're:\d{8}', + 'modified_timestamp': int, + 'season': 'Season 3', + 'season_number': 3, + 'tags': 'count:27', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + 'timestamp': 1679445000, + 'upload_date': '20230322', + 'uploader': 'The Origin Of Things', + 'uploader_id': 'the-origin-of-things', + }, + }] + _WEBPAGE_TESTS = [{ + 'url': 'https://www.asahi.com/special/podcasts/item/?itemid=311a5f48-ad71-4548-b1f2-af5e00747fbc', + 'md5': '4c788bf03323734524a7c0f98d9956ed', + 'info_dict': { + 'id': 'sdgs-271', + 'ext': 'mp3', + 'title': '「どこかのだれかの人生のにおいがする」 SDGsを音声番組で身近に #271', + 'categories': 'count:5', + 'description': 'md5:fa086ecce764d81c51648a82d0fe4850', + 'duration': 1870.524, + 'episode': 'Episode 271', + 'episode_number': 271, + 'modified_date': r're:\d{8}', + 'modified_timestamp': int, + 'season': 'Season 1', + 'season_number': 1, + 'tags': 'count:4', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + 'timestamp': 1670266800, + 'upload_date': '20221205', + 'uploader': '朝日新聞ポッドキャスト', + 'uploader_id': 'asahi', + 'webpage_url': 'https://omny.fm/shows/asahi/sdgs-271', + }, + }] + + def _real_extract(self, url): + uploader_id, audio_id = self._match_valid_url(url).group('uploader_id', 'id') + webpage = self._download_webpage(url, audio_id) + nextjs_data = self._search_nextjs_data(webpage, audio_id) + clip = traverse_obj(nextjs_data, ('props', 'pageProps', 'clip', {dict})) + + return { + 'id': audio_id, + 'section_start': traverse_obj(url, ({parse_qs}, 't', -1, {parse_duration})), + 'uploader_id': uploader_id, + 'vcodec': 'none', + **traverse_obj(clip, { + 'title': ('Title', {clean_html}, filter), + 'chapters': ('Chapters', lambda _, v: parse_duration(v['Position']) is not None, { + 'title': ('Name', {clean_html}, filter), + 'start_time': ('Position', {parse_duration}), + }), + 'description': ('Description', {clean_html}, filter), + 'duration': ('DurationSeconds', {float_or_none}), + 'episode_number': ('Episode', {int_or_none}), + 'filesize': ('PublishedAudioSizeInBytes', {int_or_none}), + 'modified_timestamp': ('ModifiedAtUtc', {parse_iso8601}), + 'season_number': ('Season', {int_or_none}), + 'tags': ('Tags', ..., {clean_html}, filter, all, filter), + 'thumbnail': ('ImageUrl', {update_url(query=None)}), + 'timestamp': ('PublishedUtc', {parse_iso8601}), + 'url': ('AudioUrl', {url_or_none}, {require('audio URL')}), + 'webpage_url': ('PublishedUrl', {url_or_none}), + }), + **traverse_obj(clip, ('Program', { + 'categories': ('Categories', ..., {clean_html}, filter, all, filter), + 'uploader': ('Name', {clean_html}, filter), + })), + } + + +class OmnyfmPlaylistBaseIE(InfoExtractor): + _API_BASE = 'https://api.omny.fm' + _BASE_URL = 'https://omny.fm/shows' + _PAGE_SIZE = 100 + + def _yield_clips(self, clips, uploader_id): + for audio_id in traverse_obj(clips, ( + 'Clips', ..., 'Slug', {str}, + )): + yield self.url_result( + f'{self._BASE_URL}/{uploader_id}/{audio_id}', OmnyfmIE) + + +class OmnyfmPlaylistIE(OmnyfmPlaylistBaseIE): + _VALID_URL = r'https?://omny\.fm/shows/(?P[\w-]+)/playlists(?:/(?P[\w-]+))?(?:/embed)?/?(?=[?#"\']|$)' + _EMBED_REGEX = [fr']+\bsrc=(["\'])(?P{_VALID_URL}[^"\']*)\1'] + _TESTS = [{ + 'url': 'https://omny.fm/shows/sleep-hub/playlists/sleep-talk', + 'info_dict': { + 'id': 'sleep-talk', + 'title': 'Sleep Talk - Talking all things sleep', + 'description': 'md5:c1d7e5bf32100a432307d2d32c4ab74a', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + }, + 'playlist_mincount': 79, + }, { + 'url': 'https://omny.fm/shows/bayfm-program03/playlists', + 'info_dict': { + 'id': 'bayfm-program03', + }, + 'playlist_count': 4, + }] + _WEBPAGE_TESTS = [{ + 'url': 'https://www.asahi.com/articles/ASP763WDKP4JDIFI002.html', + 'info_dict': { + 'id': 'podcast', + 'title': 'ニュースの現場から', + 'description': 'md5:ed1f78462ebed09258ca31b1da5ff640', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + 'webpage_url': 'https://omny.fm/shows/asahi/playlists/podcast', + }, + 'playlist_mincount': 2517, + }] + + def _entries(self, uploader_id, playlist_id): + clip_id = None + + for page in itertools.count(1): + clips = self._download_json( + f'{self._API_BASE}/programs/{uploader_id}/playlists/{playlist_id}/clips', + playlist_id, f'Downloading page {page}', query=filter_dict({ + 'clipId': clip_id, + 'direction': 'AfterExclusive', + 'pageSize': self._PAGE_SIZE, + })) + + yield from self._yield_clips(clips, uploader_id) + + if not clips.get('NextClipsAvailable'): + break + + clip_id = traverse_obj(clips, ('Clips', -1, 'Id', {str})) + if not clip_id: + break + + def _real_extract(self, url): + uploader_id, playlist_id = self._match_valid_url(url).group('uploader_id', 'id') + webpage = self._download_webpage(url, playlist_id or uploader_id) + nextjs_data = self._search_nextjs_data(webpage, playlist_id or uploader_id) + page_props = traverse_obj(nextjs_data, ('props', 'pageProps', {dict})) + + if not playlist_id: + entries = [self.url_result( + f'{self._BASE_URL}/{uploader_id}/playlists/{playlist_id}', OmnyfmPlaylistIE, + ) for playlist_id in traverse_obj(page_props, ( + 'playlistsWithClips', ..., 'playlist', 'Slug', {str}, + ))] + + return self.playlist_result(entries, uploader_id) + + return self.playlist_result( + self._entries(uploader_id, playlist_id), playlist_id, + **traverse_obj(page_props, ('playlist', { + 'title': ('Title', {clean_html}, filter), + 'description': ('Description', {clean_html}, filter), + 'thumbnail': ('ArtworkUrl', {update_url(query=None)}), + 'webpage_url': ('EmbedUrl', {url_or_none}, {trim_str(end='/embed')}), + }))) + + +class OmnyfmShowIE(OmnyfmPlaylistBaseIE): + _VALID_URL = r'https?://omny\.fm/shows/(?P[\w-]+)/?(?:[?#]|$)' + _TESTS = [{ + 'url': 'https://omny.fm/shows/the-origin-of-things', + 'info_dict': { + 'id': 'the-origin-of-things', + 'title': 'The Origin Of Things', + 'description': 'md5:52b7fba08201d050639c78ea88cc782e', + 'thumbnail': r're:https?://www\.omnycontent\.com/.+', + }, + 'playlist_mincount': 75, + }] + + def _fetch_page(self, uploader_id, organization_id, program_id, page): + clips = self._download_json( + f'{self._API_BASE}/orgs/{organization_id}/programs/{program_id}/clips', + uploader_id, f'Downloading page {page + 1}', query={ + 'cursor': page, + 'pageSize': self._PAGE_SIZE, + }) + + yield from self._yield_clips(clips, uploader_id) + + def _real_extract(self, url): + uploader_id = self._match_id(url) + webpage = self._download_webpage(url, uploader_id) + nextjs_data = self._search_nextjs_data(webpage, uploader_id) + program = traverse_obj(nextjs_data, ('props', 'pageProps', 'program', {dict})) + + organization_id = traverse_obj(program, ( + 'OrganizationId', {str}, {require('organization ID')})) + program_id = traverse_obj(program, ('Id', {str}, {require('program ID')})) + entries = OnDemandPagedList( + functools.partial(self._fetch_page, uploader_id, organization_id, program_id), self._PAGE_SIZE) + + return self.playlist_result( + entries, uploader_id, + **traverse_obj(program, { + 'title': ('Name', {clean_html}, filter), + 'description': ('Description', {clean_html}, filter), + 'thumbnail': ('ArtworkUrl', {update_url(query=None)}), + }))