Compare commits

..
7 Commits
Author SHA1 Message Date
seproandGitHub d925e92b71 [ie/vevo] Restore extractors (#14203)
Partially reverts 6f4c1bb593

Authored by: seproDev
2025-08-31 00:41:52 +02:00
seproandGitHub ed24640943 [ie/lrt] Fix extractors (#14193)
Closes #13501
Authored by: seproDev
2025-08-30 00:28:44 +02:00
seproandGitHub 76bb46002c Fix --id deprecation warning (#14190)
Authored by: seproDev
2025-08-29 22:06:53 +02:00
InvalidUsernameExceptionandGitHub 1e28f6bf74 [ie/kick:vod] Support ongoing livestream VODs (#14154)
Authored by: InvalidUsernameException
2025-08-28 01:26:49 +00:00
garret1317andGitHub 0b51005b48 [ie/ITVBTCC] Fix extractor (#14161)
Closes #14156
Authored by: garret1317
2025-08-28 01:19:25 +00:00
AbdulmohsenandGitHub 223baa81f6 [ie/tver] Extract more metadata (#14165)
Authored by: arabcoders
2025-08-28 01:18:10 +00:00
Gegham ZakaryanandGitHub 18fe696df9 [ie/googledrive] Fix subtitles extraction (#14139)
Authored by: zakaryan2004
2025-08-28 01:12:08 +00:00
10 changed files with 488 additions and 73 deletions
+1
View File
@@ -979,6 +979,7 @@ def parse_options(argv=None):
'geo_bypass': opts.geo_bypass, 'geo_bypass': opts.geo_bypass,
'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_country': opts.geo_bypass_country,
'geo_bypass_ip_block': opts.geo_bypass_ip_block, 'geo_bypass_ip_block': opts.geo_bypass_ip_block,
'useid': opts.useid or None,
'warn_when_outdated': opts.update_self is None, 'warn_when_outdated': opts.update_self is None,
'_warnings': warnings, '_warnings': warnings,
'_deprecation_warnings': deprecation_warnings, '_deprecation_warnings': deprecation_warnings,
+4
View File
@@ -2288,6 +2288,10 @@ from .varzesh3 import Varzesh3IE
from .vbox7 import Vbox7IE from .vbox7 import Vbox7IE
from .veo import VeoIE from .veo import VeoIE
from .vesti import VestiIE from .vesti import VestiIE
from .vevo import (
VevoIE,
VevoPlaylistIE,
)
from .vgtv import ( from .vgtv import (
VGTVIE, VGTVIE,
BTArticleIE, BTArticleIE,
+4
View File
@@ -90,6 +90,10 @@ class DisneyIE(InfoExtractor):
webpage, 'embed data'), video_id) webpage, 'embed data'), video_id)
video_data = page_data['video'] video_data = page_data['video']
for external in video_data.get('externals', []):
if external.get('source') == 'vevo':
return self.url_result('vevo:' + external['data_id'], 'Vevo')
video_id = video_data['id'] video_id = video_data['id']
title = video_data['title'] title = video_data['title']
+14 -10
View File
@@ -12,6 +12,7 @@ from ..utils import (
get_element_html_by_id, get_element_html_by_id,
int_or_none, int_or_none,
lowercase_escape, lowercase_escape,
parse_qs,
try_get, try_get,
update_url_query, update_url_query,
) )
@@ -111,14 +112,18 @@ class GoogleDriveIE(InfoExtractor):
self._caption_formats_ext.append(f.attrib['fmt_code']) self._caption_formats_ext.append(f.attrib['fmt_code'])
def _get_captions_by_type(self, video_id, subtitles_id, caption_type, def _get_captions_by_type(self, video_id, subtitles_id, caption_type,
origin_lang_code=None): origin_lang_code=None, origin_lang_name=None):
if not subtitles_id or not caption_type: if not subtitles_id or not caption_type:
return return
captions = {} captions = {}
for caption_entry in self._captions_xml.findall( for caption_entry in self._captions_xml.findall(
self._CAPTIONS_ENTRY_TAG[caption_type]): self._CAPTIONS_ENTRY_TAG[caption_type]):
caption_lang_code = caption_entry.attrib.get('lang_code') caption_lang_code = caption_entry.attrib.get('lang_code')
if not caption_lang_code: caption_name = caption_entry.attrib.get('name') or origin_lang_name
if not caption_lang_code or not caption_name:
self.report_warning(f'Missing necessary caption metadata. '
f'Need lang_code and name attributes. '
f'Found: {caption_entry.attrib}')
continue continue
caption_format_data = [] caption_format_data = []
for caption_format in self._caption_formats_ext: for caption_format in self._caption_formats_ext:
@@ -129,7 +134,7 @@ class GoogleDriveIE(InfoExtractor):
'lang': (caption_lang_code if origin_lang_code is None 'lang': (caption_lang_code if origin_lang_code is None
else origin_lang_code), else origin_lang_code),
'type': 'track', 'type': 'track',
'name': '', 'name': caption_name,
'kind': '', 'kind': '',
} }
if origin_lang_code is not None: if origin_lang_code is not None:
@@ -155,14 +160,15 @@ class GoogleDriveIE(InfoExtractor):
self._download_subtitles_xml(video_id, subtitles_id, hl) self._download_subtitles_xml(video_id, subtitles_id, hl)
if not self._captions_xml: if not self._captions_xml:
return return
track = self._captions_xml.find('track') track = next((t for t in self._captions_xml.findall('track') if t.attrib.get('cantran') == 'true'), None)
if track is None: if track is None:
return return
origin_lang_code = track.attrib.get('lang_code') origin_lang_code = track.attrib.get('lang_code')
if not origin_lang_code: origin_lang_name = track.attrib.get('name')
if not origin_lang_code or not origin_lang_name:
return return
return self._get_captions_by_type( return self._get_captions_by_type(
video_id, subtitles_id, 'automatic_captions', origin_lang_code) video_id, subtitles_id, 'automatic_captions', origin_lang_code, origin_lang_name)
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
@@ -268,10 +274,8 @@ class GoogleDriveIE(InfoExtractor):
subtitles_id = None subtitles_id = None
ttsurl = get_value('ttsurl') ttsurl = get_value('ttsurl')
if ttsurl: if ttsurl:
# the video Id for subtitles will be the last value in the ttsurl # the subtitles ID is the vid param of the ttsurl query
# query string subtitles_id = parse_qs(ttsurl).get('vid', [None])[-1]
subtitles_id = ttsurl.encode().decode(
'unicode_escape').split('=')[-1]
self.cookiejar.clear(domain='.google.com', path='/', name='NID') self.cookiejar.clear(domain='.google.com', path='/', name='NID')
+3 -1
View File
@@ -18,6 +18,7 @@ from ..utils import (
url_or_none, url_or_none,
urljoin, urljoin,
) )
from ..utils.traversal import traverse_obj
class ITVIE(InfoExtractor): class ITVIE(InfoExtractor):
@@ -223,6 +224,7 @@ class ITVBTCCIE(InfoExtractor):
}, },
'playlist_count': 12, 'playlist_count': 12,
}, { }, {
# news page, can have absent `data` field
'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike', 'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
'info_dict': { 'info_dict': {
'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike', 'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
@@ -243,7 +245,7 @@ class ITVBTCCIE(InfoExtractor):
entries = [] entries = []
for video in json_map: for video in json_map:
if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')): if not any(traverse_obj(video, ('data', attr)) == 'Brightcove' for attr in ('name', 'type')):
continue continue
video_id = video['data']['id'] video_id = video['data']['id']
account_id = video['data']['accountId'] account_id = video['data']['accountId']
+34 -12
View File
@@ -95,26 +95,47 @@ class KickVODIE(KickBaseIE):
IE_NAME = 'kick:vod' IE_NAME = 'kick:vod'
_VALID_URL = r'https?://(?:www\.)?kick\.com/[\w-]+/videos/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' _VALID_URL = r'https?://(?:www\.)?kick\.com/[\w-]+/videos/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
_TESTS = [{ _TESTS = [{
'url': 'https://kick.com/xqc/videos/8dd97a8d-e17f-48fb-8bc3-565f88dbc9ea', # Regular VOD
'md5': '3870f94153e40e7121a6e46c068b70cb', 'url': 'https://kick.com/xqc/videos/5c697a87-afce-4256-b01f-3c8fe71ef5cb',
'info_dict': { 'info_dict': {
'id': '8dd97a8d-e17f-48fb-8bc3-565f88dbc9ea', 'id': '5c697a87-afce-4256-b01f-3c8fe71ef5cb',
'ext': 'mp4', 'ext': 'mp4',
'title': '18+ #ad 🛑LIVE🛑CLICK🛑DRAMA🛑NEWS🛑STUFF🛑REACT🛑GET IN HHERE🛑BOP BOP🛑WEEEE WOOOO🛑', 'title': '🐗LIVE🐗CLICK🐗HERE🐗DRAMA🐗ALL DAY🐗NEWS🐗VIDEOS🐗CLIPS🐗GAMES🐗STUFF🐗WOW🐗IM HERE🐗LETS GO🐗COOL🐗VERY NICE🐗',
'description': 'THE BEST AT ABSOLUTELY EVERYTHING. THE JUICER. LEADER OF THE JUICERS.', 'description': 'THE BEST AT ABSOLUTELY EVERYTHING. THE JUICER. LEADER OF THE JUICERS.',
'channel': 'xqc',
'channel_id': '668',
'uploader': 'xQc', 'uploader': 'xQc',
'uploader_id': '676', 'uploader_id': '676',
'upload_date': '20240909', 'channel': 'xqc',
'timestamp': 1725919141, 'channel_id': '668',
'duration': 10155.0,
'thumbnail': r're:^https?://.*\.jpg',
'view_count': int, 'view_count': int,
'categories': ['Just Chatting'], 'age_limit': 18,
'age_limit': 0, 'duration': 22278.0,
'thumbnail': r're:^https?://.*\.jpg',
'categories': ['Deadlock'],
'timestamp': 1756082443,
'upload_date': '20250825',
}, },
'params': {'skip_download': 'm3u8'}, 'params': {'skip_download': 'm3u8'},
}, {
# VOD of ongoing livestream (at the time of writing the test, ID rotates every two days)
'url': 'https://kick.com/a-log-burner/videos/5230df84-ea38-46e1-be4f-f5949ae55641',
'info_dict': {
'id': '5230df84-ea38-46e1-be4f-f5949ae55641',
'ext': 'mp4',
'title': r're:😴 Cozy Fireplace ASMR 🔥 | Relax, Focus, Sleep 💤',
'description': 'md5:080bc713eac0321a7b376a1b53816d1b',
'uploader': 'A_Log_Burner',
'uploader_id': '65114691',
'channel': 'a-log-burner',
'channel_id': '63967687',
'view_count': int,
'age_limit': 18,
'thumbnail': r're:^https?://.*\.jpg',
'categories': ['Other, Watch Party'],
'timestamp': int,
'upload_date': str,
'live_status': 'is_live',
},
'skip': 'live',
}] }]
def _real_extract(self, url): def _real_extract(self, url):
@@ -137,6 +158,7 @@ class KickVODIE(KickBaseIE):
'categories': ('livestream', 'categories', ..., 'name', {str}), 'categories': ('livestream', 'categories', ..., 'name', {str}),
'view_count': ('views', {int_or_none}), 'view_count': ('views', {int_or_none}),
'age_limit': ('livestream', 'is_mature', {bool}, {lambda x: 18 if x else 0}), 'age_limit': ('livestream', 'is_mature', {bool}, {lambda x: 18 if x else 0}),
'is_live': ('livestream', 'is_live', {bool}),
}), }),
} }
+65 -49
View File
@@ -1,22 +1,14 @@
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ( from ..utils import (
clean_html, clean_html,
merge_dicts,
traverse_obj,
unified_timestamp, unified_timestamp,
url_or_none, url_or_none,
urljoin, urljoin,
) )
from ..utils.traversal import traverse_obj
class LRTBaseIE(InfoExtractor): class LRTStreamIE(InfoExtractor):
def _extract_js_var(self, webpage, var_name, default=None):
return self._search_regex(
fr'{var_name}\s*=\s*(["\'])((?:(?!\1).)+)\1',
webpage, var_name.replace('_', ' '), default, group=2)
class LRTStreamIE(LRTBaseIE):
_VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/tiesiogiai/(?P<id>[\w-]+)' _VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/tiesiogiai/(?P<id>[\w-]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.lrt.lt/mediateka/tiesiogiai/lrt-opus', 'url': 'https://www.lrt.lt/mediateka/tiesiogiai/lrt-opus',
@@ -31,86 +23,110 @@ class LRTStreamIE(LRTBaseIE):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
streams_data = self._download_json(self._extract_js_var(webpage, 'tokenURL'), video_id)
# TODO: Use _search_nextjs_v13_data once fixed
get_stream_url = self._search_regex(
r'\\"get_streams_url\\":\\"([^"]+)\\"', webpage, 'stream URL')
streams_data = self._download_json(get_stream_url, video_id)
formats, subtitles = [], {} formats, subtitles = [], {}
for stream_url in traverse_obj(streams_data, ( for stream_url in traverse_obj(streams_data, (
'response', 'data', lambda k, _: k.startswith('content')), expected_type=url_or_none): 'response', 'data', lambda k, _: k.startswith('content'), {url_or_none})):
fmts, subs = self._extract_m3u8_formats_and_subtitles(stream_url, video_id, 'mp4', m3u8_id='hls', live=True) fmts, subs = self._extract_m3u8_formats_and_subtitles(
stream_url, video_id, 'mp4', m3u8_id='hls', live=True)
formats.extend(fmts) formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs) subtitles = self._merge_subtitles(subtitles, subs)
stream_title = self._extract_js_var(webpage, 'video_title', 'LRT')
return { return {
'id': video_id, 'id': video_id,
'formats': formats, 'formats': formats,
'subtitles': subtitles, 'subtitles': subtitles,
'is_live': True, 'is_live': True,
'title': f'{self._og_search_title(webpage)} - {stream_title}', 'title': self._og_search_title(webpage),
} }
class LRTVODIE(LRTBaseIE): class LRTVODIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?lrt\.lt(?P<path>/mediateka/irasas/(?P<id>[0-9]+))' _VALID_URL = [
r'https?://(?:(?:www|archyvai)\.)?lrt\.lt/mediateka/irasas/(?P<id>[0-9]+)',
r'https?://(?:(?:www|archyvai)\.)?lrt\.lt/mediateka/video/[^?#]+\?(?:[^#]*&)?episode=(?P<id>[0-9]+)',
]
_TESTS = [{ _TESTS = [{
# m3u8 download # m3u8 download
'url': 'https://www.lrt.lt/mediateka/irasas/2000127261/greita-ir-gardu-sicilijos-ikvepta-klasikiniu-makaronu-su-baklazanais-vakariene', 'url': 'https://www.lrt.lt/mediateka/irasas/2000127261/greita-ir-gardu-sicilijos-ikvepta-klasikiniu-makaronu-su-baklazanais-vakariene',
'info_dict': { 'info_dict': {
'id': '2000127261', 'id': '2000127261',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Greita ir gardu: Sicilijos įkvėpta klasikinių makaronų su baklažanais vakarienė', 'title': 'Nustebinkite svečius klasikiniu makaronų su baklažanais receptu',
'description': 'md5:ad7d985f51b0dc1489ba2d76d7ed47fa', 'description': 'md5:ad7d985f51b0dc1489ba2d76d7ed47fa',
'duration': 3035, 'timestamp': 1604086200,
'timestamp': 1604079000,
'upload_date': '20201030', 'upload_date': '20201030',
'tags': ['LRT TELEVIZIJA', 'Beatos virtuvė', 'Beata Nicholson', 'Makaronai', 'Baklažanai', 'Vakarienė', 'Receptas'], 'tags': ['LRT TELEVIZIJA', 'Beatos virtuvė', 'Beata Nicholson', 'Makaronai', 'Baklažanai', 'Vakarienė', 'Receptas'],
'thumbnail': 'https://www.lrt.lt/img/2020/10/30/764041-126478-1287x836.jpg', 'thumbnail': 'https://www.lrt.lt/img/2020/10/30/764041-126478-1287x836.jpg',
'channel': 'Beatos virtuvė',
}, },
}, { }, {
# direct mp3 download # audio download
'url': 'http://www.lrt.lt/mediateka/irasas/1013074524/', 'url': 'https://www.lrt.lt/mediateka/irasas/1013074524/kita-tema',
'md5': '389da8ca3cad0f51d12bed0c844f6a0a', 'md5': 'fc982f10274929c66fdff65f75615cb0',
'info_dict': { 'info_dict': {
'id': '1013074524', 'id': '1013074524',
'ext': 'mp3', 'ext': 'mp4',
'title': 'Kita tema 2016-09-05 15:05', 'title': 'Kita tema',
'description': 'md5:1b295a8fc7219ed0d543fc228c931fb5', 'description': 'md5:1b295a8fc7219ed0d543fc228c931fb5',
'duration': 3008, 'channel': 'Kita tema',
'view_count': int, 'timestamp': 1473087900,
'like_count': int, 'upload_date': '20160905',
}, },
}, {
'url': 'https://www.lrt.lt/mediateka/video/auksinis-protas-vasara?episode=2000420320&season=%2Fmediateka%2Fvideo%2Fauksinis-protas-vasara%2F2025',
'info_dict': {
'id': '2000420320',
'ext': 'mp4',
'title': 'Kuris senovės romėnų poetas aprašė Narcizo mitą?',
'description': 'Intelektinė viktorina. Ved. Arūnas Valinskas ir Andrius Tapinas.',
'channel': 'Auksinis protas. Vasara',
'thumbnail': 'https://www.lrt.lt/img/2025/06/09/2094343-987905-1287x836.jpg',
'tags': ['LRT TELEVIZIJA', 'Auksinis protas'],
'timestamp': 1749851040,
'upload_date': '20250613',
},
}, {
'url': 'https://archyvai.lrt.lt/mediateka/video/ziniu-riteriai-ir-damos?episode=49685&season=%2Fmediateka%2Fvideo%2Fziniu-riteriai-ir-damos%2F2013',
'only_matching': True,
}, {
'url': 'https://archyvai.lrt.lt/mediateka/irasas/2000077058/panorama-1989-baltijos-kelias',
'only_matching': True,
}] }]
def _real_extract(self, url): def _real_extract(self, url):
path, video_id = self._match_valid_url(url).group('path', 'id') video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
media_url = self._extract_js_var(webpage, 'main_url', path) # TODO: Use _search_nextjs_v13_data once fixed
media = self._download_json(self._extract_js_var( canonical_url = (
webpage, 'media_info_url', self._search_regex(r'\\"(?:article|data)\\":{[^}]*\\"url\\":\\"(/[^"]+)\\"', webpage, 'content URL', fatal=False)
'https://www.lrt.lt/servisai/stream_url/vod/media_info/'), or self._search_regex(r'<link\s+rel="canonical"\s*href="(/[^"]+)"', webpage, 'canonical URL'))
video_id, query={'url': media_url})
media = self._download_json(
'https://www.lrt.lt/servisai/stream_url/vod/media_info/',
video_id, query={'url': canonical_url})
jw_data = self._parse_jwplayer_data( jw_data = self._parse_jwplayer_data(
media['playlist_item'], video_id, base_url=url) media['playlist_item'], video_id, base_url=url)
json_ld_data = self._search_json_ld(webpage, video_id) return {
**jw_data,
tags = [] **traverse_obj(media, {
for tag in (media.get('tags') or []): 'id': ('id', {str}),
tag_name = tag.get('name') 'title': ('title', {str}),
if not tag_name: 'description': ('content', {clean_html}),
continue 'timestamp': ('date', {lambda x: x.replace('.', '/')}, {unified_timestamp}),
tags.append(tag_name) 'tags': ('tags', ..., 'name', {str}),
}),
clean_info = {
'description': clean_html(media.get('content')),
'tags': tags,
} }
return merge_dicts(clean_info, jw_data, json_ld_data)
class LRTRadioIE(InfoExtractor):
class LRTRadioIE(LRTBaseIE):
_VALID_URL = r'https?://(?:www\.)?lrt\.lt/radioteka/irasas/(?P<id>\d+)/(?P<path>[^?#/]+)' _VALID_URL = r'https?://(?:www\.)?lrt\.lt/radioteka/irasas/(?P<id>\d+)/(?P<path>[^?#/]+)'
_TESTS = [{ _TESTS = [{
# m3u8 download # m3u8 download
+5 -1
View File
@@ -111,8 +111,12 @@ class MySpaceIE(InfoExtractor):
search_data('stream-url'), search_data('hls-stream-url'), search_data('stream-url'), search_data('hls-stream-url'),
search_data('http-stream-url')) search_data('http-stream-url'))
if not formats: if not formats:
vevo_id = search_data('vevo-id')
youtube_id = search_data('youtube-id') youtube_id = search_data('youtube-id')
if youtube_id: if vevo_id:
self.to_screen(f'Vevo video detected: {vevo_id}')
return self.url_result(f'vevo:{vevo_id}', ie='Vevo')
elif youtube_id:
self.to_screen(f'Youtube video detected: {youtube_id}') self.to_screen(f'Youtube video detected: {youtube_id}')
return self.url_result(youtube_id, ie='Youtube') return self.url_result(youtube_id, ie='Youtube')
else: else:
+6
View File
@@ -45,6 +45,8 @@ class TVerIE(StreaksBaseIE):
'release_timestamp': 1651453200, 'release_timestamp': 1651453200,
'release_date': '20220502', 'release_date': '20220502',
'_old_archive_ids': ['brightcovenew ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068'], '_old_archive_ids': ['brightcovenew ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068'],
'series_id': 'sru35hwdd2',
'season_id': 'ss2lcn4af6',
}, },
}, { }, {
# via Brightcove backend (deprecated) # via Brightcove backend (deprecated)
@@ -67,6 +69,8 @@ class TVerIE(StreaksBaseIE):
'upload_date': '20220501', 'upload_date': '20220501',
'release_timestamp': 1651453200, 'release_timestamp': 1651453200,
'release_date': '20220502', 'release_date': '20220502',
'series_id': 'sru35hwdd2',
'season_id': 'ss2lcn4af6',
}, },
'params': {'extractor_args': {'tver': {'backend': ['brightcove']}}}, 'params': {'extractor_args': {'tver': {'backend': ['brightcove']}}},
}, { }, {
@@ -202,6 +206,8 @@ class TVerIE(StreaksBaseIE):
'description': ('description', {str}), 'description': ('description', {str}),
'release_timestamp': ('viewStatus', 'startAt', {int_or_none}), 'release_timestamp': ('viewStatus', 'startAt', {int_or_none}),
'episode_number': ('no', {int_or_none}), 'episode_number': ('no', {int_or_none}),
'series_id': ('seriesID', {str}),
'season_id': ('seasonID', {str}),
}), }),
} }
+352
View File
@@ -0,0 +1,352 @@
import json
import re
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
parse_iso8601,
parse_qs,
)
class VevoBaseIE(InfoExtractor):
def _extract_json(self, webpage, video_id):
return self._parse_json(
self._search_regex(
r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
webpage, 'initial store'),
video_id)
class VevoIE(VevoBaseIE):
"""
Accepts urls from vevo.com or in the format 'vevo:{id}'
(currently used by MTVIE and MySpaceIE)
"""
_VALID_URL = r'''(?x)
(?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
https?://cache\.vevo\.com/m/html/embed\.html\?video=|
https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
https?://embed\.vevo\.com/.*?[?&]isrc=|
https?://tv\.vevo\.com/watch/artist/(?:[^/]+)/|
vevo:)
(?P<id>[^&?#]+)'''
_EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1']
_TESTS = [{
'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
'md5': '95ee28ee45e70130e3ab02b0f579ae23',
'info_dict': {
'id': 'GB1101300280',
'ext': 'mp4',
'title': 'Hurts - Somebody to Die For',
'timestamp': 1372057200,
'upload_date': '20130624',
'uploader': 'Hurts',
'track': 'Somebody to Die For',
'artist': 'Hurts',
'genre': 'Pop',
},
'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
}, {
'note': 'v3 SMIL format',
'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
'info_dict': {
'id': 'USUV71302923',
'ext': 'mp4',
'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
'timestamp': 1392796919,
'upload_date': '20140219',
'uploader': 'Cassadee Pope',
'track': 'I Wish I Could Break Your Heart',
'artist': 'Cassadee Pope',
'genre': 'Country',
},
'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
}, {
'note': 'Age-limited video',
'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
'info_dict': {
'id': 'USRV81300282',
'ext': 'mp4',
'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
'age_limit': 18,
'timestamp': 1372888800,
'upload_date': '20130703',
'uploader': 'Justin Timberlake',
'track': 'Tunnel Vision (Explicit)',
'artist': 'Justin Timberlake',
'genre': 'Pop',
},
'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
}, {
'note': 'No video_info',
'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
'info_dict': {
'id': 'USUV71503000',
'ext': 'mp4',
'title': 'K Camp ft. T.I. - Till I Die',
'age_limit': 18,
'timestamp': 1449468000,
'upload_date': '20151207',
'uploader': 'K Camp',
'track': 'Till I Die',
'artist': 'K Camp',
'genre': 'Hip-Hop',
},
'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
}, {
'note': 'Featured test',
'url': 'https://www.vevo.com/watch/lemaitre/Wait/USUV71402190',
'md5': 'd28675e5e8805035d949dc5cf161071d',
'info_dict': {
'id': 'USUV71402190',
'ext': 'mp4',
'title': 'Lemaitre ft. LoLo - Wait',
'age_limit': 0,
'timestamp': 1413432000,
'upload_date': '20141016',
'uploader': 'Lemaitre',
'track': 'Wait',
'artist': 'Lemaitre',
'genre': 'Electronic',
},
'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
}, {
'note': 'Only available via webpage',
'url': 'http://www.vevo.com/watch/GBUV71600656',
'md5': '67e79210613865b66a47c33baa5e37fe',
'info_dict': {
'id': 'GBUV71600656',
'ext': 'mp4',
'title': 'ABC - Viva Love',
'age_limit': 0,
'timestamp': 1461830400,
'upload_date': '20160428',
'uploader': 'ABC',
'track': 'Viva Love',
'artist': 'ABC',
'genre': 'Pop',
},
'expected_warnings': ['Failed to download video versions info'],
}, {
# no genres available
'url': 'http://www.vevo.com/watch/INS171400764',
'only_matching': True,
}, {
# Another case available only via the webpage; using streams/streamsV3 formats
# Geo-restricted to Netherlands/Germany
'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909',
'only_matching': True,
}, {
'url': 'https://embed.vevo.com/?isrc=USH5V1923499&partnerId=4d61b777-8023-4191-9ede-497ed6c24647&partnerAdCode=',
'only_matching': True,
}, {
'url': 'https://tv.vevo.com/watch/artist/janet-jackson/US0450100550',
'only_matching': True,
}]
_VERSIONS = {
0: 'youtube', # only in AuthenticateVideo videoVersions
1: 'level3',
2: 'akamai',
3: 'level3',
4: 'amazon',
}
def _initialize_api(self, video_id):
webpage = self._download_webpage(
'https://accounts.vevo.com/token', None,
note='Retrieving oauth token',
errnote='Unable to retrieve oauth token',
data=json.dumps({
'client_id': 'SPupX1tvqFEopQ1YS6SS',
'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
}).encode(),
headers={
'Content-Type': 'application/json',
})
if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
self.raise_geo_restricted(
f'{self.IE_NAME} said: This page is currently unavailable in your region')
auth_info = self._parse_json(webpage, video_id)
self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']
def _call_api(self, path, *args, **kwargs):
try:
data = self._download_json(self._api_url_template % path, *args, **kwargs)
except ExtractorError as e:
if isinstance(e.cause, HTTPError):
errors = self._parse_json(e.cause.response.read().decode(), None)['errors']
error_message = ', '.join([error['message'] for error in errors])
raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True)
raise
return data
def _real_extract(self, url):
video_id = self._match_id(url)
self._initialize_api(video_id)
video_info = self._call_api(
f'video/{video_id}', video_id, 'Downloading api video info',
'Failed to download video info')
video_versions = self._call_api(
f'video/{video_id}/streams', video_id,
'Downloading video versions info',
'Failed to download video versions info',
fatal=False)
# Some videos are only available via webpage (e.g.
# https://github.com/ytdl-org/youtube-dl/issues/9366)
if not video_versions:
webpage = self._download_webpage(url, video_id)
json_data = self._extract_json(webpage, video_id)
if 'streams' in json_data.get('default', {}):
video_versions = json_data['default']['streams'][video_id][0]
else:
video_versions = [
value
for key, value in json_data['apollo']['data'].items()
if key.startswith(f'{video_id}.streams')]
uploader = None
artist = None
featured_artist = None
artists = video_info.get('artists')
for curr_artist in artists:
if curr_artist.get('role') == 'Featured':
featured_artist = curr_artist['name']
else:
artist = uploader = curr_artist['name']
formats = []
for video_version in video_versions:
version = self._VERSIONS.get(video_version.get('version'), 'generic')
version_url = video_version.get('url')
if not version_url:
continue
if '.ism' in version_url:
continue
elif '.mpd' in version_url:
formats.extend(self._extract_mpd_formats(
version_url, video_id, mpd_id=f'dash-{version}',
note=f'Downloading {version} MPD information',
errnote=f'Failed to download {version} MPD information',
fatal=False))
elif '.m3u8' in version_url:
formats.extend(self._extract_m3u8_formats(
version_url, video_id, 'mp4', 'm3u8_native',
m3u8_id=f'hls-{version}',
note=f'Downloading {version} m3u8 information',
errnote=f'Failed to download {version} m3u8 information',
fatal=False))
else:
m = re.search(r'''(?xi)
_(?P<quality>[a-z0-9]+)
_(?P<width>[0-9]+)x(?P<height>[0-9]+)
_(?P<vcodec>[a-z0-9]+)
_(?P<vbr>[0-9]+)
_(?P<acodec>[a-z0-9]+)
_(?P<abr>[0-9]+)
\.(?P<ext>[a-z0-9]+)''', version_url)
if not m:
continue
formats.append({
'url': version_url,
'format_id': f'http-{version}-{video_version.get("quality") or m.group("quality")}',
'vcodec': m.group('vcodec'),
'acodec': m.group('acodec'),
'vbr': int(m.group('vbr')),
'abr': int(m.group('abr')),
'ext': m.group('ext'),
'width': int(m.group('width')),
'height': int(m.group('height')),
})
track = video_info['title']
if featured_artist:
artist = f'{artist} ft. {featured_artist}'
title = f'{artist} - {track}' if artist else track
genres = video_info.get('genres')
genre = (
genres[0] if genres and isinstance(genres, list)
and isinstance(genres[0], str) else None)
is_explicit = video_info.get('isExplicit')
if is_explicit is True:
age_limit = 18
elif is_explicit is False:
age_limit = 0
else:
age_limit = None
return {
'id': video_id,
'title': title,
'formats': formats,
'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
'timestamp': parse_iso8601(video_info.get('releaseDate')),
'uploader': uploader,
'duration': int_or_none(video_info.get('duration')),
'view_count': int_or_none(video_info.get('views', {}).get('total')),
'age_limit': age_limit,
'track': track,
'artist': uploader,
'genre': genre,
}
class VevoPlaylistIE(VevoBaseIE):
_VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'http://www.vevo.com/watch/genre/rock',
'info_dict': {
'id': 'rock',
'title': 'Rock',
},
'playlist_count': 20,
}, {
'url': 'http://www.vevo.com/watch/genre/rock?index=0',
'only_matching': True,
}]
def _real_extract(self, url):
mobj = self._match_valid_url(url)
playlist_id = mobj.group('id')
playlist_kind = mobj.group('kind')
webpage = self._download_webpage(url, playlist_id)
qs = parse_qs(url)
index = qs.get('index', [None])[0]
if index:
video_id = self._search_regex(
r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',
webpage, 'video id', default=None, group='id')
if video_id:
return self.url_result(f'vevo:{video_id}', VevoIE.ie_key())
playlists = self._extract_json(webpage, playlist_id)['default'][f'{playlist_kind}s']
playlist = (next(iter(playlists.values()))
if playlist_kind == 'playlist' else playlists[playlist_id])
entries = [
self.url_result(f'vevo:{src}', VevoIE.ie_key())
for src in playlist['isrcs']]
return self.playlist_result(
entries, playlist.get('playlistId') or playlist_id,
playlist.get('name'), playlist.get('description'))