mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2026-07-06 18:29:14 +00:00
[ie/omnyfm] Add extractors (#15942)
Closes #10201, Closes #11035 Authored by: doe1080
This commit is contained in:
doe1080
GitHub
parent
5aa335ecd9
commit
fa383a9efa
@ -1314,6 +1314,11 @@ from .oftv import (
|
|||||||
)
|
)
|
||||||
from .oktoberfesttv import OktoberfestTVIE
|
from .oktoberfesttv import OktoberfestTVIE
|
||||||
from .olympics import OlympicsReplayIE
|
from .olympics import OlympicsReplayIE
|
||||||
|
from .omnyfm import (
|
||||||
|
OmnyfmIE,
|
||||||
|
OmnyfmPlaylistIE,
|
||||||
|
OmnyfmShowIE,
|
||||||
|
)
|
||||||
from .on24 import On24IE
|
from .on24 import On24IE
|
||||||
from .ondemandkorea import (
|
from .ondemandkorea import (
|
||||||
OnDemandKoreaIE,
|
OnDemandKoreaIE,
|
||||||
|
|||||||
272
yt_dlp/extractor/omnyfm.py
Normal file
272
yt_dlp/extractor/omnyfm.py
Normal file
@ -0,0 +1,272 @@
|
|||||||
|
import functools
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
OnDemandPagedList,
|
||||||
|
clean_html,
|
||||||
|
filter_dict,
|
||||||
|
float_or_none,
|
||||||
|
int_or_none,
|
||||||
|
parse_duration,
|
||||||
|
parse_iso8601,
|
||||||
|
parse_qs,
|
||||||
|
update_url,
|
||||||
|
url_or_none,
|
||||||
|
)
|
||||||
|
from ..utils.traversal import (
|
||||||
|
require,
|
||||||
|
traverse_obj,
|
||||||
|
trim_str,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class OmnyfmIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://omny\.fm/shows/(?P<uploader_id>[\w-]+)/(?P<id>(?!playlists(?:[/?#"\']|$))[\w-]+)(?:/embed)?(?=[?#"\']|$)'
|
||||||
|
_EMBED_REGEX = [rf'<iframe[^>]+\bsrc\s*=\s*(["\'])(?P<url>{_VALID_URL}[^"\']*)\1']
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://omny.fm/shows/sleep-hub/cannabinoids-and-sleep',
|
||||||
|
'md5': 'e45ec0ce43da757a0be6ca117ec01bdc',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cannabinoids-and-sleep',
|
||||||
|
'ext': 'mp3',
|
||||||
|
'title': 'Cannabinoids and Sleep',
|
||||||
|
'categories': 'count:1',
|
||||||
|
'chapters': [
|
||||||
|
{'start_time': 0, 'title': 'Introduction'},
|
||||||
|
{'start_time': 138, 'title': 'Theme: Cannabinoids and Sleep'},
|
||||||
|
{'start_time': 1487, 'title': 'Clinical Tip'},
|
||||||
|
{'start_time': 1635, 'title': 'Pick of the Month'},
|
||||||
|
{'start_time': 1795, 'title': 'What\'s Coming Up?'},
|
||||||
|
],
|
||||||
|
'description': 'md5:c0fd2d29f3148382d344cfbd012fb00d',
|
||||||
|
'duration': 1840.274,
|
||||||
|
'episode': 'Episode 48',
|
||||||
|
'episode_number': 48,
|
||||||
|
'modified_date': r're:\d{8}',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'tags': 'count:6',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
'timestamp': 1574013600,
|
||||||
|
'upload_date': '20191117',
|
||||||
|
'uploader': 'Sleep Talk',
|
||||||
|
'uploader_id': 'sleep-hub',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://omny.fm/shows/the-origin-of-things/a-song-of-hope/embed',
|
||||||
|
'md5': 'd7600ef33e3f139ff1bb8946f3651b15',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'a-song-of-hope',
|
||||||
|
'ext': 'mp3',
|
||||||
|
'title': 'A song of hope',
|
||||||
|
'categories': 'count:3',
|
||||||
|
'description': 'md5:f8e710765c341a48cfda8dacd428f56d',
|
||||||
|
'duration': 478.955,
|
||||||
|
'episode': 'Episode 17',
|
||||||
|
'episode_number': 17,
|
||||||
|
'modified_date': r're:\d{8}',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'season': 'Season 3',
|
||||||
|
'season_number': 3,
|
||||||
|
'tags': 'count:27',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
'timestamp': 1679445000,
|
||||||
|
'upload_date': '20230322',
|
||||||
|
'uploader': 'The Origin Of Things',
|
||||||
|
'uploader_id': 'the-origin-of-things',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
_WEBPAGE_TESTS = [{
|
||||||
|
'url': 'https://www.asahi.com/special/podcasts/item/?itemid=311a5f48-ad71-4548-b1f2-af5e00747fbc',
|
||||||
|
'md5': '4c788bf03323734524a7c0f98d9956ed',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'sdgs-271',
|
||||||
|
'ext': 'mp3',
|
||||||
|
'title': '「どこかのだれかの人生のにおいがする」 SDGsを音声番組で身近に #271',
|
||||||
|
'categories': 'count:5',
|
||||||
|
'description': 'md5:fa086ecce764d81c51648a82d0fe4850',
|
||||||
|
'duration': 1870.524,
|
||||||
|
'episode': 'Episode 271',
|
||||||
|
'episode_number': 271,
|
||||||
|
'modified_date': r're:\d{8}',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'season': 'Season 1',
|
||||||
|
'season_number': 1,
|
||||||
|
'tags': 'count:4',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
'timestamp': 1670266800,
|
||||||
|
'upload_date': '20221205',
|
||||||
|
'uploader': '朝日新聞ポッドキャスト',
|
||||||
|
'uploader_id': 'asahi',
|
||||||
|
'webpage_url': 'https://omny.fm/shows/asahi/sdgs-271',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
uploader_id, audio_id = self._match_valid_url(url).group('uploader_id', 'id')
|
||||||
|
webpage = self._download_webpage(url, audio_id)
|
||||||
|
nextjs_data = self._search_nextjs_data(webpage, audio_id)
|
||||||
|
clip = traverse_obj(nextjs_data, ('props', 'pageProps', 'clip', {dict}))
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': audio_id,
|
||||||
|
'section_start': traverse_obj(url, ({parse_qs}, 't', -1, {parse_duration})),
|
||||||
|
'uploader_id': uploader_id,
|
||||||
|
'vcodec': 'none',
|
||||||
|
**traverse_obj(clip, {
|
||||||
|
'title': ('Title', {clean_html}, filter),
|
||||||
|
'chapters': ('Chapters', lambda _, v: parse_duration(v['Position']) is not None, {
|
||||||
|
'title': ('Name', {clean_html}, filter),
|
||||||
|
'start_time': ('Position', {parse_duration}),
|
||||||
|
}),
|
||||||
|
'description': ('Description', {clean_html}, filter),
|
||||||
|
'duration': ('DurationSeconds', {float_or_none}),
|
||||||
|
'episode_number': ('Episode', {int_or_none}),
|
||||||
|
'filesize': ('PublishedAudioSizeInBytes', {int_or_none}),
|
||||||
|
'modified_timestamp': ('ModifiedAtUtc', {parse_iso8601}),
|
||||||
|
'season_number': ('Season', {int_or_none}),
|
||||||
|
'tags': ('Tags', ..., {clean_html}, filter, all, filter),
|
||||||
|
'thumbnail': ('ImageUrl', {update_url(query=None)}),
|
||||||
|
'timestamp': ('PublishedUtc', {parse_iso8601}),
|
||||||
|
'url': ('AudioUrl', {url_or_none}, {require('audio URL')}),
|
||||||
|
'webpage_url': ('PublishedUrl', {url_or_none}),
|
||||||
|
}),
|
||||||
|
**traverse_obj(clip, ('Program', {
|
||||||
|
'categories': ('Categories', ..., {clean_html}, filter, all, filter),
|
||||||
|
'uploader': ('Name', {clean_html}, filter),
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class OmnyfmPlaylistBaseIE(InfoExtractor):
|
||||||
|
_API_BASE = 'https://api.omny.fm'
|
||||||
|
_BASE_URL = 'https://omny.fm/shows'
|
||||||
|
_PAGE_SIZE = 100
|
||||||
|
|
||||||
|
def _yield_clips(self, clips, uploader_id):
|
||||||
|
for audio_id in traverse_obj(clips, (
|
||||||
|
'Clips', ..., 'Slug', {str},
|
||||||
|
)):
|
||||||
|
yield self.url_result(
|
||||||
|
f'{self._BASE_URL}/{uploader_id}/{audio_id}', OmnyfmIE)
|
||||||
|
|
||||||
|
|
||||||
|
class OmnyfmPlaylistIE(OmnyfmPlaylistBaseIE):
|
||||||
|
_VALID_URL = r'https?://omny\.fm/shows/(?P<uploader_id>[\w-]+)/playlists(?:/(?P<id>[\w-]+))?(?:/embed)?/?(?=[?#"\']|$)'
|
||||||
|
_EMBED_REGEX = [fr'<iframe[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL}[^"\']*)\1']
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://omny.fm/shows/sleep-hub/playlists/sleep-talk',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'sleep-talk',
|
||||||
|
'title': 'Sleep Talk - Talking all things sleep',
|
||||||
|
'description': 'md5:c1d7e5bf32100a432307d2d32c4ab74a',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 79,
|
||||||
|
}, {
|
||||||
|
'url': 'https://omny.fm/shows/bayfm-program03/playlists',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'bayfm-program03',
|
||||||
|
},
|
||||||
|
'playlist_count': 4,
|
||||||
|
}]
|
||||||
|
_WEBPAGE_TESTS = [{
|
||||||
|
'url': 'https://www.asahi.com/articles/ASP763WDKP4JDIFI002.html',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'podcast',
|
||||||
|
'title': 'ニュースの現場から',
|
||||||
|
'description': 'md5:ed1f78462ebed09258ca31b1da5ff640',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
'webpage_url': 'https://omny.fm/shows/asahi/playlists/podcast',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 2517,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _entries(self, uploader_id, playlist_id):
|
||||||
|
clip_id = None
|
||||||
|
|
||||||
|
for page in itertools.count(1):
|
||||||
|
clips = self._download_json(
|
||||||
|
f'{self._API_BASE}/programs/{uploader_id}/playlists/{playlist_id}/clips',
|
||||||
|
playlist_id, f'Downloading page {page}', query=filter_dict({
|
||||||
|
'clipId': clip_id,
|
||||||
|
'direction': 'AfterExclusive',
|
||||||
|
'pageSize': self._PAGE_SIZE,
|
||||||
|
}))
|
||||||
|
|
||||||
|
yield from self._yield_clips(clips, uploader_id)
|
||||||
|
|
||||||
|
if not clips.get('NextClipsAvailable'):
|
||||||
|
break
|
||||||
|
|
||||||
|
clip_id = traverse_obj(clips, ('Clips', -1, 'Id', {str}))
|
||||||
|
if not clip_id:
|
||||||
|
break
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
uploader_id, playlist_id = self._match_valid_url(url).group('uploader_id', 'id')
|
||||||
|
webpage = self._download_webpage(url, playlist_id or uploader_id)
|
||||||
|
nextjs_data = self._search_nextjs_data(webpage, playlist_id or uploader_id)
|
||||||
|
page_props = traverse_obj(nextjs_data, ('props', 'pageProps', {dict}))
|
||||||
|
|
||||||
|
if not playlist_id:
|
||||||
|
entries = [self.url_result(
|
||||||
|
f'{self._BASE_URL}/{uploader_id}/playlists/{playlist_id}', OmnyfmPlaylistIE,
|
||||||
|
) for playlist_id in traverse_obj(page_props, (
|
||||||
|
'playlistsWithClips', ..., 'playlist', 'Slug', {str},
|
||||||
|
))]
|
||||||
|
|
||||||
|
return self.playlist_result(entries, uploader_id)
|
||||||
|
|
||||||
|
return self.playlist_result(
|
||||||
|
self._entries(uploader_id, playlist_id), playlist_id,
|
||||||
|
**traverse_obj(page_props, ('playlist', {
|
||||||
|
'title': ('Title', {clean_html}, filter),
|
||||||
|
'description': ('Description', {clean_html}, filter),
|
||||||
|
'thumbnail': ('ArtworkUrl', {update_url(query=None)}),
|
||||||
|
'webpage_url': ('EmbedUrl', {url_or_none}, {trim_str(end='/embed')}),
|
||||||
|
})))
|
||||||
|
|
||||||
|
|
||||||
|
class OmnyfmShowIE(OmnyfmPlaylistBaseIE):
|
||||||
|
_VALID_URL = r'https?://omny\.fm/shows/(?P<id>[\w-]+)/?(?:[?#]|$)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://omny.fm/shows/the-origin-of-things',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'the-origin-of-things',
|
||||||
|
'title': 'The Origin Of Things',
|
||||||
|
'description': 'md5:52b7fba08201d050639c78ea88cc782e',
|
||||||
|
'thumbnail': r're:https?://www\.omnycontent\.com/.+',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 75,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _fetch_page(self, uploader_id, organization_id, program_id, page):
|
||||||
|
clips = self._download_json(
|
||||||
|
f'{self._API_BASE}/orgs/{organization_id}/programs/{program_id}/clips',
|
||||||
|
uploader_id, f'Downloading page {page + 1}', query={
|
||||||
|
'cursor': page,
|
||||||
|
'pageSize': self._PAGE_SIZE,
|
||||||
|
})
|
||||||
|
|
||||||
|
yield from self._yield_clips(clips, uploader_id)
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
uploader_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, uploader_id)
|
||||||
|
nextjs_data = self._search_nextjs_data(webpage, uploader_id)
|
||||||
|
program = traverse_obj(nextjs_data, ('props', 'pageProps', 'program', {dict}))
|
||||||
|
|
||||||
|
organization_id = traverse_obj(program, (
|
||||||
|
'OrganizationId', {str}, {require('organization ID')}))
|
||||||
|
program_id = traverse_obj(program, ('Id', {str}, {require('program ID')}))
|
||||||
|
entries = OnDemandPagedList(
|
||||||
|
functools.partial(self._fetch_page, uploader_id, organization_id, program_id), self._PAGE_SIZE)
|
||||||
|
|
||||||
|
return self.playlist_result(
|
||||||
|
entries, uploader_id,
|
||||||
|
**traverse_obj(program, {
|
||||||
|
'title': ('Name', {clean_html}, filter),
|
||||||
|
'description': ('Description', {clean_html}, filter),
|
||||||
|
'thumbnail': ('ArtworkUrl', {update_url(query=None)}),
|
||||||
|
}))
|
||||||
Loading…
x
Reference in New Issue
Block a user