artem/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2026-08-28 06:43:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into yt-live-from-start-range

Browse Source
This commit is contained in:
Elyse
2023-06-03 14:39:32 -06:00
parent 8ee942a9c8 2fb35f6004
commit 1f7974690e
98 changed files with 7110 additions and 3283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yt_dlp/extractor/_extractors.py
+52 -17
View File
@@ -204,7 +204,11 @@ from .bfmtv import (
BFMTVLiveIE,
BFMTVArticleIE,
)
from .bibeltv import BibelTVIE
from .bibeltv import (
BibelTVLiveIE,
BibelTVSeriesIE,
BibelTVVideoIE,
)
from .bigflix import BigflixIE
from .bigo import BigoIE
from .bild import BildIE
@@ -247,7 +251,6 @@ from .bongacams import BongaCamsIE
from .bostonglobe import BostonGlobeIE
from .box import BoxIE
from .boxcast import BoxCastVideoIE
from .booyah import BooyahClipsIE
from .bpb import BpbIE
from .br import (
BRIE,
@@ -281,6 +284,10 @@ from .camdemy import (
CamdemyIE,
CamdemyFolderIE
)
from .camfm import (
CamFMEpisodeIE,
CamFMShowIE
)
from .cammodels import CamModelsIE
from .camsoda import CamsodaIE
from .camtasia import CamtasiaEmbedIE
@@ -288,12 +295,6 @@ from .camwithher import CamWithHerIE
from .canalalpha import CanalAlphaIE
from .canalplus import CanalplusIE
from .canalc2 import Canalc2IE
from .canvas import (
CanvasIE,
CanvasEenIE,
VrtNUIE,
DagelijkseKostIE,
)
from .carambatv import (
CarambaTVIE,
CarambaTVPageIE,
@@ -310,14 +311,14 @@ from .cbs import (
CBSIE,
ParamountPressExpressIE,
)
from .cbslocal import (
CBSLocalIE,
CBSLocalArticleIE,
)
from .cbsinteractive import CBSInteractiveIE
from .cbsnews import (
CBSNewsEmbedIE,
CBSNewsIE,
CBSLocalIE,
CBSLocalArticleIE,
CBSLocalLiveIE,
CBSNewsLiveIE,
CBSNewsLiveVideoIE,
)
from .cbssports import (
@@ -404,9 +405,12 @@ from .crowdbunker import (
CrowdBunkerIE,
CrowdBunkerChannelIE,
)
from .crtvg import CrtvgIE
from .crunchyroll import (
CrunchyrollBetaIE,
CrunchyrollBetaShowIE,
CrunchyrollMusicIE,
CrunchyrollArtistIE,
)
from .cspan import CSpanIE, CSpanCongressIE
from .ctsnews import CtsNewsIE
@@ -423,6 +427,10 @@ from .cybrary import (
CybraryIE,
CybraryCourseIE
)
from .dacast import (
DacastVODIE,
DacastPlaylistIE,
)
from .daftsex import DaftsexIE
from .dailymail import DailyMailIE
from .dailymotion import (
@@ -536,6 +544,7 @@ from .ehow import EHowIE
from .eighttracks import EightTracksIE
from .einthusan import EinthusanIE
from .eitb import EitbIE
from .elevensports import ElevenSportsIE
from .ellentube import (
EllenTubeIE,
EllenTubeVideoIE,
@@ -784,6 +793,7 @@ from .ichinanalive import (
IchinanaLiveIE,
IchinanaLiveClipIE,
)
from .idolplus import IdolPlusIE
from .ign import (
IGNIE,
IGNVideoIE,
@@ -868,6 +878,7 @@ from .japandiet import (
from .jeuxvideo import JeuxVideoIE
from .jove import JoveIE
from .joj import JojIE
from .jstream import JStreamIE
from .jwplatform import JWPlatformIE
from .kakao import KakaoIE
from .kaltura import KalturaIE
@@ -877,7 +888,6 @@ from .karaoketv import KaraoketvIE
from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
from .kelbyone import KelbyOneIE
from .ketnet import KetnetIE
from .khanacademy import (
KhanAcademyIE,
KhanAcademyUnitIE,
@@ -1147,6 +1157,7 @@ from .myvi import (
)
from .myvideoge import MyVideoGeIE
from .myvidster import MyVidsterIE
from .mzaalo import MzaaloIE
from .n1 import (
N1InfoAssetIE,
N1InfoIIE,
@@ -1195,6 +1206,7 @@ from .nebula import (
NebulaSubscriptionsIE,
NebulaChannelIE,
)
from .nekohacker import NekoHackerIE
from .nerdcubed import NerdCubedFeedIE
from .netzkino import NetzkinoIE
from .neteasemusic import (
@@ -1264,6 +1276,7 @@ from .niconico import (
NicovideoSearchIE,
NicovideoSearchURLIE,
NicovideoTagURLIE,
NiconicoLiveIE,
)
from .ninecninemedia import (
NineCNineMediaIE,
@@ -1373,6 +1386,7 @@ from .orf import (
ORFIPTVIE,
)
from .outsidetv import OutsideTVIE
from .owncloud import OwnCloudIE
from .packtpub import (
PacktPubIE,
PacktPubCourseIE,
@@ -1474,7 +1488,6 @@ from .polskieradio import (
PolskieRadioPlayerIE,
PolskieRadioPodcastIE,
PolskieRadioPodcastListIE,
PolskieRadioRadioKierowcowIE,
)
from .popcorntimes import PopcorntimesIE
from .popcorntv import PopcornTVIE
@@ -1544,6 +1557,8 @@ from .radlive import (
RadLiveSeasonIE,
)
from .rai import (
RaiIE,
RaiCulturaIE,
RaiPlayIE,
RaiPlayLiveIE,
RaiPlayPlaylistIE,
@@ -1552,7 +1567,6 @@ from .rai import (
RaiPlaySoundPlaylistIE,
RaiNewsIE,
RaiSudtirolIE,
RaiIE,
)
from .raywenderlich import (
RayWenderlichIE,
@@ -1574,6 +1588,7 @@ from .rcti import (
RCTIPlusTVIE,
)
from .rds import RDSIE
from .recurbate import RecurbateIE
from .redbee import ParliamentLiveUKIE, RTBFIE
from .redbulltv import (
RedBullTVIE,
@@ -2080,7 +2095,6 @@ from .tvp import (
)
from .tvplay import (
TVPlayIE,
ViafreeIE,
TVPlayHomeIE,
)
from .tvplayer import TVPlayerIE
@@ -2264,7 +2278,12 @@ from .voxmedia import (
VoxMediaVolumeIE,
VoxMediaIE,
)
from .vrt import VRTIE
from .vrt import (
VRTIE,
VrtNUIE,
KetnetIE,
DagelijkseKostIE,
)
from .vrak import VrakIE
from .vrv import (
VRVIE,
@@ -2315,7 +2334,16 @@ from .weibo import (
WeiboMobileIE
)
from .weiqitv import WeiqiTVIE
from .weverse import (
WeverseIE,
WeverseMediaIE,
WeverseMomentIE,
WeverseLiveTabIE,
WeverseMediaTabIE,
WeverseLiveIE,
)
from .wevidi import WeVidiIE
from .weyyak import WeyyakIE
from .whyp import WhypIE
from .wikimedia import WikimediaIE
from .willow import WillowIE
@@ -2344,6 +2372,12 @@ from .wsj import (
WSJArticleIE,
)
from .wwe import WWEIE
from .wykop import (
WykopDigIE,
WykopDigCommentIE,
WykopPostIE,
WykopPostCommentIE,
)
from .xanimu import XanimuIE
from .xbef import XBefIE
from .xboxclips import XboxClipsIE
@@ -2463,6 +2497,7 @@ from .zingmp3 import (
ZingMp3WeekChartIE,
ZingMp3ChartMusicVideoIE,
ZingMp3UserIE,
ZingMp3HubIE,
)
from .zoom import ZoomIE
from .zype import ZypeIE
yt_dlp/extractor/aenetworks.py
+13 -2
View File
@@ -3,6 +3,8 @@ from ..utils import (
ExtractorError,
GeoRestrictedError,
int_or_none,
remove_start,
traverse_obj,
update_url_query,
urlencode_postdata,
)
@@ -72,7 +74,14 @@ class AENetworksBaseIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
requestor_id, brand = self._DOMAIN_MAP[domain]
result = self._download_json(
'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
filter_value, query={'filter[%s]' % filter_key: filter_value})
result = traverse_obj(
result, ('results',
lambda k, v: k == 0 and v[filter_key] == filter_value),
get_all=False)
if not result:
raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
video_id=remove_start(filter_value, '/'))
title = result['title']
video_id = result['id']
media_url = result['publicUrl']
@@ -123,7 +132,7 @@ class AENetworksIE(AENetworksBaseIE):
'skip_download': True,
},
'add_ie': ['ThePlatform'],
'skip': 'This video is only available for users of participating TV providers.',
'skip': 'Geo-restricted - This content is not available in your location.'
}, {
'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
'info_dict': {
@@ -140,6 +149,7 @@ class AENetworksIE(AENetworksBaseIE):
'skip_download': True,
},
'add_ie': ['ThePlatform'],
'skip': 'This video is only available for users of participating TV providers.',
}, {
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
'only_matching': True
@@ -303,6 +313,7 @@ class HistoryTopicIE(AENetworksBaseIE):
class HistoryPlayerIE(AENetworksBaseIE):
IE_NAME = 'history:player'
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
_TESTS = []
def _real_extract(self, url):
domain, video_id = self._match_valid_url(url).groups()
yt_dlp/extractor/anvato.py
+1 -9
View File
@@ -336,7 +336,7 @@ class AnvatoIE(InfoExtractor):
elif media_format == 'm3u8-variant' or ext == 'm3u8':
# For some videos the initial m3u8 URL returns JSON instead
manifest_json = self._download_json(
video_url, video_id, note='Downloading manifest JSON', errnote=False)
video_url, video_id, note='Downloading manifest JSON', fatal=False)
if manifest_json:
video_url = manifest_json.get('master_m3u8')
if not video_url:
@@ -392,14 +392,6 @@ class AnvatoIE(InfoExtractor):
url = smuggle_url(url, {'token': anvplayer_data['token']})
yield cls.url_result(url, AnvatoIE, video_id)
def _extract_anvato_videos(self, webpage, video_id):
anvplayer_data = self._parse_json(
self._html_search_regex(
self._ANVP_RE, webpage, 'Anvato player data', group='anvp'),
video_id)
return self._get_anvato_videos(
anvplayer_data['accessKey'], anvplayer_data['video'], 'default') # cbslocal token = 'default'
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
self._initialize_geo_bypass({
yt_dlp/extractor/ard.py
+36 -8
View File
@@ -13,6 +13,7 @@ from ..utils import (
try_get,
unified_strdate,
unified_timestamp,
update_url,
update_url_query,
url_or_none,
xpath_text,
@@ -408,6 +409,23 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
(?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
_TESTS = [{
'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
'md5': '3fd5fead7a370a819341129c8d713136',
'info_dict': {
'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
'id': '12172961',
'title': 'Wolfsland - Die traurigen Schwestern',
'description': r're:^Als der Polizeiobermeister Raaben',
'duration': 5241,
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
'timestamp': 1670710500,
'upload_date': '20221210',
'ext': 'mp4',
'age_limit': 12,
'episode': 'Wolfsland - Die traurigen Schwestern',
'series': 'Filme im MDR'
},
}, {
'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
'info_dict': {
@@ -424,7 +442,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
'skip': 'Error',
}, {
'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
'md5': 'f1837e563323b8a642a8ddeff0131f51',
'md5': '1e73ded21cb79bac065117e80c81dc88',
'info_dict': {
'id': '10049223',
'ext': 'mp4',
@@ -432,13 +450,11 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
'timestamp': 1636398000,
'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
'upload_date': '20211108',
},
}, {
'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
'playlist_count': 6,
'info_dict': {
'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
'title': 'beforeigners/beforeigners/staffel-1',
'display_id': 'tagesschau-oder-tagesschau-20-00-uhr/das-erste',
'duration': 915,
'episode': 'tagesschau, 20:00 Uhr',
'series': 'tagesschau',
'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
},
}, {
'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -602,6 +618,9 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
show {
title
}
image {
src
}
synopsis
title
tracking {
@@ -640,6 +659,15 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
'description': description,
'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
'series': try_get(player_page, lambda x: x['show']['title']),
'thumbnail': (media_collection.get('_previewImage')
or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
or self.get_thumbnail_from_html(display_id, url)),
})
info.update(self._ARD_extract_episode_info(info['title']))
return info
def get_thumbnail_from_html(self, display_id, url):
webpage = self._download_webpage(url, display_id, fatal=False) or ''
return (
self._og_search_thumbnail(webpage, default=None)
or self._html_search_meta('thumbnailUrl', webpage, default=None))
yt_dlp/extractor/bibeltv.py
+189 -19
View File
@@ -1,27 +1,197 @@
from functools import partial
from .common import InfoExtractor
from ..utils import (
ExtractorError,
clean_html,
determine_ext,
format_field,
int_or_none,
js_to_json,
orderedSet,
parse_iso8601,
traverse_obj,
url_or_none,
)
class BibelTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?:crn/)?(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.bibeltv.de/mediathek/videos/329703-sprachkurs-in-malaiisch',
'md5': '252f908192d611de038b8504b08bf97f',
'info_dict': {
'id': 'ref:329703',
'ext': 'mp4',
'title': 'Sprachkurs in Malaiisch',
'description': 'md5:3e9f197d29ee164714e67351cf737dfe',
'timestamp': 1608316701,
'uploader_id': '5840105145001',
'upload_date': '20201218',
class BibelTVBaseIE(InfoExtractor):
_GEO_COUNTRIES = ['AT', 'CH', 'DE']
_GEO_BYPASS = False
API_URL = 'https://www.bibeltv.de/mediathek/api'
AUTH_TOKEN = 'j88bRXY8DsEqJ9xmTdWhrByVi5Hm'
def _extract_formats_and_subtitles(self, data, crn_id, *, is_live=False):
formats = []
subtitles = {}
for media_url in traverse_obj(data, (..., 'src', {url_or_none})):
media_ext = determine_ext(media_url)
if media_ext == 'm3u8':
m3u8_formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
media_url, crn_id, live=is_live)
formats.extend(m3u8_formats)
subtitles.update(m3u8_subs)
elif media_ext == 'mpd':
mpd_formats, mpd_subs = self._extract_mpd_formats_and_subtitles(media_url, crn_id)
formats.extend(mpd_formats)
subtitles.update(mpd_subs)
elif media_ext == 'mp4':
formats.append({'url': media_url})
else:
self.report_warning(f'Unknown format {media_ext!r}')
return formats, subtitles
@staticmethod
def _extract_base_info(data):
return {
'id': data['crn'],
**traverse_obj(data, {
'title': 'title',
'description': 'description',
'duration': ('duration', {partial(int_or_none, scale=1000)}),
'timestamp': ('schedulingStart', {parse_iso8601}),
'season_number': 'seasonNumber',
'episode_number': 'episodeNumber',
'view_count': 'viewCount',
'like_count': 'likeCount',
}),
'thumbnails': orderedSet(traverse_obj(data, ('images', ..., {
'url': ('url', {url_or_none}),
}))),
}
}, {
'url': 'https://www.bibeltv.de/mediathek/videos/crn/326374',
'only_matching': True,
def _extract_url_info(self, data):
return {
'_type': 'url',
'url': format_field(data, 'slug', 'https://www.bibeltv.de/mediathek/videos/%s'),
**self._extract_base_info(data),
}
def _extract_video_info(self, data):
crn_id = data['crn']
if data.get('drm'):
self.report_drm(crn_id)
json_data = self._download_json(
format_field(data, 'id', f'{self.API_URL}/video/%s'), crn_id,
headers={'Authorization': self.AUTH_TOKEN}, fatal=False,
errnote='No formats available') or {}
formats, subtitles = self._extract_formats_and_subtitles(
traverse_obj(json_data, ('video', 'videoUrls', ...)), crn_id)
return {
'_type': 'video',
**self._extract_base_info(data),
'formats': formats,
'subtitles': subtitles,
}
class BibelTVVideoIE(BibelTVBaseIE):
IE_DESC = 'BibelTV single video'
_VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?P<id>\d+)[\w-]+'
IE_NAME = 'bibeltv:video'
_TESTS = [{
'url': 'https://www.bibeltv.de/mediathek/videos/344436-alte-wege',
'md5': 'ec1c07efe54353780512e8a4103b612e',
'info_dict': {
'id': '344436',
'ext': 'mp4',
'title': 'Alte Wege',
'description': 'md5:2f4eb7294c9797a47b8fd13cccca22e9',
'timestamp': 1677877071,
'duration': 150.0,
'upload_date': '20230303',
'thumbnail': r're:https://bibeltv\.imgix\.net/[\w-]+\.jpg',
'episode': 'Episode 1',
'episode_number': 1,
'view_count': int,
'like_count': int,
},
'params': {
'format': '6',
},
}]
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5840105145001/default_default/index.html?videoId=ref:%s'
def _real_extract(self, url):
crn_id = self._match_id(url)
return self.url_result(
self.BRIGHTCOVE_URL_TEMPLATE % crn_id, 'BrightcoveNew')
video_data = traverse_obj(
self._search_nextjs_data(self._download_webpage(url, crn_id), crn_id),
('props', 'pageProps', 'videoPageData', 'videos', 0, {dict}))
if not video_data:
raise ExtractorError('Missing video data.')
return self._extract_video_info(video_data)
class BibelTVSeriesIE(BibelTVBaseIE):
IE_DESC = 'BibelTV series playlist'
_VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/serien/(?P<id>\d+)[\w-]+'
IE_NAME = 'bibeltv:series'
_TESTS = [{
'url': 'https://www.bibeltv.de/mediathek/serien/333485-ein-wunder-fuer-jeden-tag',
'playlist_mincount': 400,
'info_dict': {
'id': '333485',
'title': 'Ein Wunder für jeden Tag',
'description': 'Tägliche Kurzandacht mit Déborah Rosenkranz.',
},
}]
def _real_extract(self, url):
crn_id = self._match_id(url)
webpage = self._download_webpage(url, crn_id)
nextjs_data = self._search_nextjs_data(webpage, crn_id)
series_data = traverse_obj(nextjs_data, ('props', 'pageProps', 'seriePageData', {dict}))
if not series_data:
raise ExtractorError('Missing series data.')
return self.playlist_result(
traverse_obj(series_data, ('videos', ..., {dict}, {self._extract_url_info})),
crn_id, series_data.get('title'), clean_html(series_data.get('description')))
class BibelTVLiveIE(BibelTVBaseIE):
IE_DESC = 'BibelTV live program'
_VALID_URL = r'https?://(?:www\.)?bibeltv\.de/livestreams/(?P<id>[\w-]+)'
IE_NAME = 'bibeltv:live'
_TESTS = [{
'url': 'https://www.bibeltv.de/livestreams/bibeltv/',
'info_dict': {
'id': 'bibeltv',
'ext': 'mp4',
'title': 're:Bibel TV',
'live_status': 'is_live',
'thumbnail': 'https://streampreview.bibeltv.de/bibeltv.webp',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.bibeltv.de/livestreams/impuls/',
'only_matching': True,
}]
def _real_extract(self, url):
stream_id = self._match_id(url)
webpage = self._download_webpage(url, stream_id)
stream_data = self._search_json(
r'\\"video\\":', webpage, 'bibeltvData', stream_id,
transform_source=lambda jstring: js_to_json(jstring.replace('\\"', '"')))
formats, subtitles = self._extract_formats_and_subtitles(
traverse_obj(stream_data, ('src', ...)), stream_id, is_live=True)
return {
'id': stream_id,
'title': stream_data.get('title'),
'thumbnail': stream_data.get('poster'),
'is_live': True,
'formats': formats,
'subtitles': subtitles,
}
yt_dlp/extractor/bilibili.py
+52 -5
View File
@@ -1,7 +1,9 @@
import base64
import functools
import hashlib
import itertools
import math
import time
import urllib.error
import urllib.parse
@@ -26,6 +28,7 @@ from ..utils import (
srt_subtitles_timecode,
str_or_none,
traverse_obj,
try_call,
unified_timestamp,
unsmuggle_url,
url_or_none,
@@ -514,19 +517,63 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
'id': '3985676',
},
'playlist_mincount': 178,
}, {
'url': 'https://space.bilibili.com/313580179/video',
'info_dict': {
'id': '313580179',
},
'playlist_mincount': 92,
}]
def _extract_signature(self, playlist_id):
session_data = self._download_json('https://api.bilibili.com/x/web-interface/nav', playlist_id, fatal=False)
key_from_url = lambda x: x[x.rfind('/') + 1:].split('.')[0]
img_key = traverse_obj(
session_data, ('data', 'wbi_img', 'img_url', {key_from_url})) or '34478ba821254d9d93542680e3b86100'
sub_key = traverse_obj(
session_data, ('data', 'wbi_img', 'sub_url', {key_from_url})) or '7e16a90d190a4355a78fd00b32a38de6'
session_key = img_key + sub_key
signature_values = []
for position in (
46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39,
12, 38, 41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63,
57, 62, 11, 36, 20, 34, 44, 52
):
char_at_position = try_call(lambda: session_key[position])
if char_at_position:
signature_values.append(char_at_position)
return ''.join(signature_values)[:32]
def _real_extract(self, url):
playlist_id, is_video_url = self._match_valid_url(url).group('id', 'video')
if not is_video_url:
self.to_screen('A channel URL was given. Only the channel\'s videos will be downloaded. '
'To download audios, add a "/audio" to the URL')
signature = self._extract_signature(playlist_id)
def fetch_page(page_idx):
query = {
'keyword': '',
'mid': playlist_id,
'order': 'pubdate',
'order_avoided': 'true',
'platform': 'web',
'pn': page_idx + 1,
'ps': 30,
'tid': 0,
'web_location': 1550101,
'wts': int(time.time()),
}
query['w_rid'] = hashlib.md5(f'{urllib.parse.urlencode(query)}{signature}'.encode()).hexdigest()
try:
response = self._download_json('https://api.bilibili.com/x/space/arc/search',
playlist_id, note=f'Downloading page {page_idx}',
query={'mid': playlist_id, 'pn': page_idx + 1, 'jsonp': 'jsonp'})
response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
playlist_id, note=f'Downloading page {page_idx}', query=query)
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
raise ExtractorError(
@@ -556,9 +603,9 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
class BilibiliSpaceAudioIE(BilibiliSpaceBaseIE):
_VALID_URL = r'https?://space\.bilibili\.com/(?P<id>\d+)/audio'
_TESTS = [{
'url': 'https://space.bilibili.com/3985676/audio',
'url': 'https://space.bilibili.com/313580179/audio',
'info_dict': {
'id': '3985676',
'id': '313580179',
},
'playlist_mincount': 1,
}]
yt_dlp/extractor/booyah.py
-86
View File
@@ -1,86 +0,0 @@
from .common import InfoExtractor
from ..utils import int_or_none, str_or_none, traverse_obj
class BooyahBaseIE(InfoExtractor):
_BOOYAH_SESSION_KEY = None
def _real_initialize(self):
BooyahBaseIE._BOOYAH_SESSION_KEY = self._request_webpage(
'https://booyah.live/api/v3/auths/sessions', None, data=b'').getheader('booyah-session-key')
def _get_comments(self, video_id):
comment_json = self._download_json(
f'https://booyah.live/api/v3/playbacks/{video_id}/comments/tops', video_id,
headers={'Booyah-Session-Key': self._BOOYAH_SESSION_KEY}, fatal=False) or {}
return [{
'id': comment.get('comment_id'),
'author': comment.get('from_nickname'),
'author_id': comment.get('from_uid'),
'author_thumbnail': comment.get('from_thumbnail'),
'text': comment.get('content'),
'timestamp': comment.get('create_time'),
'like_count': comment.get('like_cnt'),
} for comment in comment_json.get('comment_list') or ()]
class BooyahClipsIE(BooyahBaseIE):
_VALID_URL = r'https?://booyah.live/clips/(?P<id>\d+)'
_TESTS = [{
'url': 'https://booyah.live/clips/13887261322952306617',
'info_dict': {
'id': '13887261322952306617',
'ext': 'mp4',
'view_count': int,
'duration': 30,
'channel_id': 90565760,
'like_count': int,
'title': 'Cayendo con estilo 😎',
'uploader': '♡LɪꜱGΛ​MER​',
'comment_count': int,
'uploader_id': '90565760',
'thumbnail': 'https://resmambet-a.akamaihd.net/mambet-storage/Clip/90565760/90565760-27204374-fba0-409d-9d7b-63a48b5c0e75.jpg',
'upload_date': '20220617',
'timestamp': 1655490556,
'modified_timestamp': 1655490556,
'modified_date': '20220617',
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
json_data = self._download_json(
f'https://booyah.live/api/v3/playbacks/{video_id}', video_id,
headers={'Booyah-Session-key': self._BOOYAH_SESSION_KEY})
formats = []
for video_data in json_data['playback']['endpoint_list']:
formats.extend(({
'url': video_data.get('stream_url'),
'ext': 'mp4',
'height': video_data.get('resolution'),
}, {
'url': video_data.get('download_url'),
'ext': 'mp4',
'format_note': 'Watermarked',
'height': video_data.get('resolution'),
'preference': -10,
}))
return {
'id': video_id,
'title': traverse_obj(json_data, ('playback', 'name')),
'thumbnail': traverse_obj(json_data, ('playback', 'thumbnail_url')),
'formats': formats,
'view_count': traverse_obj(json_data, ('playback', 'views')),
'like_count': traverse_obj(json_data, ('playback', 'likes')),
'duration': traverse_obj(json_data, ('playback', 'duration')),
'comment_count': traverse_obj(json_data, ('playback', 'comment_cnt')),
'channel_id': traverse_obj(json_data, ('playback', 'channel_id')),
'uploader': traverse_obj(json_data, ('user', 'nickname')),
'uploader_id': str_or_none(traverse_obj(json_data, ('user', 'uid'))),
'modified_timestamp': int_or_none(traverse_obj(json_data, ('playback', 'update_time_ms')), 1000),
'timestamp': int_or_none(traverse_obj(json_data, ('playback', 'create_time_ms')), 1000),
'__post_extractor': self.extract_comments(video_id, self._get_comments(video_id)),
}
yt_dlp/extractor/bravotv.py
+6 -2
View File
@@ -1,5 +1,6 @@
from .adobepass import AdobePassIE
from ..utils import (
HEADRequest,
extract_attributes,
float_or_none,
get_element_html_by_class,
@@ -153,8 +154,11 @@ class BravoTVIE(AdobePassIE):
if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
chapters = None
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
update_url_query(f'{tp_url}/stream.m3u8', query), video_id, 'mp4', m3u8_id='hls')
m3u8_url = self._request_webpage(HEADRequest(
update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
if 'mpeg_cenc' in m3u8_url:
self.report_drm(video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
return {
'id': video_id,
yt_dlp/extractor/camfm.py
+85
View File
@@ -0,0 +1,85 @@
import re
from .common import InfoExtractor
from ..utils import (
clean_html,
get_element_by_class,
get_elements_by_class,
join_nonempty,
traverse_obj,
unified_timestamp,
urljoin,
)
class CamFMShowIE(InfoExtractor):
_VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
_TESTS = [{
'playlist_mincount': 5,
'url': 'https://camfm.co.uk/shows/soul-mining/',
'info_dict': {
'id': 'soul-mining',
'thumbnail': 'md5:6a873091f92c936f23bdcce80f75e66a',
'title': 'Soul Mining',
'description': 'Telling the stories of jazz, funk and soul from all corners of the world.',
},
}]
def _real_extract(self, url):
show_id = self._match_id(url)
page = self._download_webpage(url, show_id)
return {
'_type': 'playlist',
'id': show_id,
'entries': [self.url_result(urljoin('https://camfm.co.uk', i), CamFMEpisodeIE)
for i in re.findall(r"javascript:popup\('(/player/[^']+)', 'listen'", page)],
'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
r'<img[^>]+class="thumb-expand"[^>]+src="([^"]+)"', page, 'thumbnail', fatal=False)),
'title': self._html_search_regex('<h1>([^<]+)</h1>', page, 'title', fatal=False),
'description': clean_html(get_element_by_class('small-12 medium-8 cell', page))
}
class CamFMEpisodeIE(InfoExtractor):
_VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/player/(?P<id>[^/]+)'
_TESTS = [{
'url': 'https://camfm.co.uk/player/43336',
'skip': 'Episode will expire - don\'t actually know when, but it will go eventually',
'info_dict': {
'id': '43336',
'title': 'AITAA: Am I the Agony Aunt? - 19:00 Tue 16/05/2023',
'ext': 'mp3',
'upload_date': '20230516',
'description': 'md5:f165144f94927c0f1bfa2ee6e6ab7bbf',
'timestamp': 1684263600,
'series': 'AITAA: Am I the Agony Aunt?',
'thumbnail': 'md5:5980a831360d0744c3764551be3d09c1',
'categories': ['Entertainment'],
}
}]
def _real_extract(self, url):
episode_id = self._match_id(url)
page = self._download_webpage(url, episode_id)
audios = self._parse_html5_media_entries('https://audio.camfm.co.uk', page, episode_id)
caption = get_element_by_class('caption', page)
series = clean_html(re.sub(r'<span[^<]+<[^<]+>', '', caption))
card_section = get_element_by_class('card-section', page)
date = self._html_search_regex('>Aired at ([^<]+)<', card_section, 'air date', fatal=False)
return {
'id': episode_id,
'title': join_nonempty(series, date, delim=' - '),
'formats': traverse_obj(audios, (..., 'formats', ...)),
'timestamp': unified_timestamp(date), # XXX: Does not account for UK's daylight savings
'series': series,
'description': clean_html(re.sub(r'<b>[^<]+</b><br[^>]+/>', '', card_section)),
'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)',
page, 'thumbnail', fatal=False)),
'categories': get_elements_by_class('label', caption),
'was_live': True,
}
yt_dlp/extractor/canvas.py
-383
View File
@@ -1,383 +0,0 @@
import json
from .common import InfoExtractor
from .gigya import GigyaBaseIE
from ..compat import compat_HTTPError
from ..utils import (
ExtractorError,
clean_html,
extract_attributes,
float_or_none,
get_element_by_class,
int_or_none,
merge_dicts,
str_or_none,
strip_or_none,
url_or_none,
urlencode_postdata
)
class CanvasIE(InfoExtractor):
_VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
'info_dict': {
'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
'ext': 'mp4',
'title': 'Nachtwacht: De Greystook',
'description': 'Nachtwacht: De Greystook',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1468.02,
},
'expected_warnings': ['is not a supported codec'],
}, {
'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
'only_matching': True,
}]
_GEO_BYPASS = False
_HLS_ENTRY_PROTOCOLS_MAP = {
'HLS': 'm3u8_native',
'HLS_AES': 'm3u8_native',
}
_REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
def _real_extract(self, url):
mobj = self._match_valid_url(url)
site_id, video_id = mobj.group('site_id'), mobj.group('id')
data = None
if site_id != 'vrtvideo':
# Old API endpoint, serves more formats but may fail for some videos
data = self._download_json(
'https://mediazone.vrt.be/api/v1/%s/assets/%s'
% (site_id, video_id), video_id, 'Downloading asset JSON',
'Unable to download asset JSON', fatal=False)
# New API endpoint
if not data:
vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
video_id, note='refreshtoken: Retrieve vrtnutoken',
errnote='refreshtoken failed')['vrtnutoken']
headers = self.geo_verification_headers()
headers.update({'Content-Type': 'application/json; charset=utf-8'})
vrtPlayerToken = self._download_json(
'%s/tokens' % self._REST_API_BASE, video_id,
'Downloading token', headers=headers, data=json.dumps({
'identityToken': vrtnutoken
}).encode('utf-8'))['vrtPlayerToken']
data = self._download_json(
'%s/videos/%s' % (self._REST_API_BASE, video_id),
video_id, 'Downloading video JSON', query={
'vrtPlayerToken': vrtPlayerToken,
'client': 'null',
}, expected_status=400)
if 'title' not in data:
code = data.get('code')
if code == 'AUTHENTICATION_REQUIRED':
self.raise_login_required()
elif code == 'INVALID_LOCATION':
self.raise_geo_restricted(countries=['BE'])
raise ExtractorError(data.get('message') or code, expected=True)
# Note: The title may be an empty string
title = data['title'] or f'{site_id} {video_id}'
description = data.get('description')
formats = []
subtitles = {}
for target in data['targetUrls']:
format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
if not format_url or not format_type:
continue
format_type = format_type.upper()
if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
fmts, subs = self._extract_m3u8_formats_and_subtitles(
format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
m3u8_id=format_type, fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
elif format_type == 'HDS':
formats.extend(self._extract_f4m_formats(
format_url, video_id, f4m_id=format_type, fatal=False))
elif format_type == 'MPEG_DASH':
fmts, subs = self._extract_mpd_formats_and_subtitles(
format_url, video_id, mpd_id=format_type, fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
elif format_type == 'HSS':
fmts, subs = self._extract_ism_formats_and_subtitles(
format_url, video_id, ism_id='mss', fatal=False)
formats.extend(fmts)
subtitles = self._merge_subtitles(subtitles, subs)
else:
formats.append({
'format_id': format_type,
'url': format_url,
})
subtitle_urls = data.get('subtitleUrls')
if isinstance(subtitle_urls, list):
for subtitle in subtitle_urls:
subtitle_url = subtitle.get('url')
if subtitle_url and subtitle.get('type') == 'CLOSED':
subtitles.setdefault('nl', []).append({'url': subtitle_url})
return {
'id': video_id,
'display_id': video_id,
'title': title,
'description': description,
'formats': formats,
'duration': float_or_none(data.get('duration'), 1000),
'thumbnail': data.get('posterImageUrl'),
'subtitles': subtitles,
}
class CanvasEenIE(InfoExtractor):
IE_DESC = 'canvas.be and een.be'
_VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
'md5': 'ed66976748d12350b118455979cca293',
'info_dict': {
'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
'ext': 'flv',
'title': 'De afspraak veilt voor de Warmste Week',
'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 49.02,
},
'expected_warnings': ['is not a supported codec'],
}, {
# with subtitles
'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
'info_dict': {
'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
'display_id': 'pieter-0167',
'ext': 'mp4',
'title': 'Pieter 0167',
'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 2553.08,
'subtitles': {
'nl': [{
'ext': 'vtt',
}],
},
},
'params': {
'skip_download': True,
},
'skip': 'Pagina niet gevonden',
}, {
'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
'info_dict': {
'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
'display_id': 'emma-pakt-thilly-aan',
'ext': 'mp4',
'title': 'Emma pakt Thilly aan',
'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 118.24,
},
'params': {
'skip_download': True,
},
'expected_warnings': ['is not a supported codec'],
}, {
'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
'only_matching': True,
}]
def _real_extract(self, url):
mobj = self._match_valid_url(url)
site_id, display_id = mobj.group('site_id'), mobj.group('id')
webpage = self._download_webpage(url, display_id)
title = strip_or_none(self._search_regex(
r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
webpage, 'title', default=None) or self._og_search_title(
webpage, default=None))
video_id = self._html_search_regex(
r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
group='id')
return {
'_type': 'url_transparent',
'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
'ie_key': CanvasIE.ie_key(),
'id': video_id,
'display_id': display_id,
'title': title,
'description': self._og_search_description(webpage),
}
class VrtNUIE(GigyaBaseIE):
IE_DESC = 'VrtNU.be'
_VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
_TESTS = [{
# Available via old API endpoint
'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
'info_dict': {
'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
'ext': 'mp4',
'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
'duration': 1457.04,
'thumbnail': r're:^https?://.*\.jpg$',
'series': 'Postbus X',
'season': 'Seizoen 1989',
'season_number': 1989,
'episode': 'De zwarte weduwe',
'episode_number': 1,
'timestamp': 1595822400,
'upload_date': '20200727',
},
'skip': 'This video is only available for registered users',
'expected_warnings': ['is not a supported codec'],
}, {
# Only available via new API endpoint
'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
'info_dict': {
'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
'ext': 'mp4',
'title': 'Aflevering 5',
'description': 'Wie valt door de mand tijdens een missie?',
'duration': 2967.06,
'season': 'Season 1',
'season_number': 1,
'episode_number': 5,
},
'skip': 'This video is only available for registered users',
'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
}]
_NETRC_MACHINE = 'vrtnu'
_APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
_CONTEXT_ID = 'R3595707040'
def _perform_login(self, username, password):
auth_info = self._gigya_login({
'APIKey': self._APIKEY,
'targetEnv': 'jssdk',
'loginID': username,
'password': password,
'authMode': 'cookie',
})
if auth_info.get('errorDetails'):
raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
# Sometimes authentication fails for no good reason, retry
login_attempt = 1
while login_attempt <= 3:
try:
self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
post_data = {
'UID': auth_info['UID'],
'UIDSignature': auth_info['UIDSignature'],
'signatureTimestamp': auth_info['signatureTimestamp'],
'_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
}
self._request_webpage(
'https://login.vrt.be/perform_login',
None, note='Performing login', errnote='perform login failed',
headers={}, query={
'client_id': 'vrtnu-site'
}, data=urlencode_postdata(post_data))
except ExtractorError as e:
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
login_attempt += 1
self.report_warning('Authentication failed')
self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
else:
raise e
else:
break
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
attrs = extract_attributes(self._search_regex(
r'(<nui-media[^>]+>)', webpage, 'media element'))
video_id = attrs['videoid']
publication_id = attrs.get('publicationid')
if publication_id:
video_id = publication_id + '$' + video_id
page = (self._parse_json(self._search_regex(
r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
default='{}'), video_id, fatal=False) or {}).get('page') or {}
info = self._search_json_ld(webpage, display_id, default={})
return merge_dicts(info, {
'_type': 'url_transparent',
'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
'ie_key': CanvasIE.ie_key(),
'id': video_id,
'display_id': display_id,
'season_number': int_or_none(page.get('episode_season')),
})
class DagelijkseKostIE(InfoExtractor):
IE_DESC = 'dagelijksekost.een.be'
_VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
_TEST = {
'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
'md5': '30bfffc323009a3e5f689bef6efa2365',
'info_dict': {
'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
'display_id': 'hachis-parmentier-met-witloof',
'ext': 'mp4',
'title': 'Hachis parmentier met witloof',
'description': 'md5:9960478392d87f63567b5b117688cdc5',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 283.02,
},
'expected_warnings': ['is not a supported codec'],
}
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
title = strip_or_none(get_element_by_class(
'dish-metadata__title', webpage
) or self._html_search_meta(
'twitter:title', webpage))
description = clean_html(get_element_by_class(
'dish-description', webpage)
) or self._html_search_meta(
('description', 'twitter:description', 'og:description'),
webpage)
video_id = self._html_search_regex(
r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
group='id')
return {
'_type': 'url_transparent',
'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
'ie_key': CanvasIE.ie_key(),
'id': video_id,
'display_id': display_id,
'title': title,
'description': description,
}
yt_dlp/extractor/cbc.py
+4 -2
View File
@@ -351,7 +351,9 @@ class CBCGemIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
video_info = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/assets/' + video_id, video_id)
video_info = self._download_json(
f'https://services.radio-canada.ca/ott/cbc-api/v2/assets/{video_id}',
video_id, expected_status=426)
email, password = self._get_login_info()
if email and password:
@@ -426,7 +428,7 @@ class CBCGemPlaylistIE(InfoExtractor):
match = self._match_valid_url(url)
season_id = match.group('id')
show = match.group('show')
show_info = self._download_json(self._API_BASE + show, season_id)
show_info = self._download_json(self._API_BASE + show, season_id, expected_status=426)
season = int(match.group('season'))
season_info = next((s for s in show_info['seasons'] if s.get('season') == season), None)
yt_dlp/extractor/cbslocal.py
-116
View File
@@ -1,116 +0,0 @@
from .anvato import AnvatoIE
from .sendtonews import SendtoNewsIE
from ..compat import compat_urlparse
from ..utils import (
parse_iso8601,
unified_timestamp,
)
class CBSLocalIE(AnvatoIE): # XXX: Do not subclass from concrete IE
_VALID_URL_BASE = r'https?://[a-z]+\.cbslocal\.com/'
_VALID_URL = _VALID_URL_BASE + r'video/(?P<id>\d+)'
_TESTS = [{
'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
'info_dict': {
'id': '3580809',
'ext': 'mp4',
'title': 'A Very Blue Anniversary',
'description': 'CBS2’s Cindy Hsu has more.',
'thumbnail': 're:^https?://.*',
'timestamp': int,
'upload_date': r're:^\d{8}$',
'uploader': 'CBS',
'subtitles': {
'en': 'mincount:5',
},
'categories': [
'Stations\\Spoken Word\\WCBSTV',
'Syndication\\AOL',
'Syndication\\MSN',
'Syndication\\NDN',
'Syndication\\Yahoo',
'Content\\News',
'Content\\News\\Local News',
],
'tags': ['CBS 2 News Weekends', 'Cindy Hsu', 'Blue Man Group'],
},
'params': {
'skip_download': True,
},
}]
def _real_extract(self, url):
mcp_id = self._match_id(url)
return self.url_result(
'anvato:anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67:' + mcp_id, 'Anvato', mcp_id)
class CBSLocalArticleIE(AnvatoIE): # XXX: Do not subclass from concrete IE
_VALID_URL = CBSLocalIE._VALID_URL_BASE + r'\d+/\d+/\d+/(?P<id>[0-9a-z-]+)'
_TESTS = [{
# Anvato backend
'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
'md5': 'f0ee3081e3843f575fccef901199b212',
'info_dict': {
'id': '3401037',
'ext': 'mp4',
'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
'thumbnail': 're:^https?://.*',
'timestamp': 1463440500,
'upload_date': '20160516',
'uploader': 'CBS',
'subtitles': {
'en': 'mincount:5',
},
'categories': [
'Stations\\Spoken Word\\KCBSTV',
'Syndication\\MSN',
'Syndication\\NDN',
'Syndication\\AOL',
'Syndication\\Yahoo',
'Syndication\\Tribune',
'Syndication\\Curb.tv',
'Content\\News'
],
'tags': ['CBS 2 News Evening'],
},
}, {
# SendtoNews embed
'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
'info_dict': {
'id': 'GxfCe0Zo7D-175909-5588',
},
'playlist_count': 9,
'params': {
# m3u8 download
'skip_download': True,
},
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
sendtonews_url = SendtoNewsIE._extract_url(webpage)
if sendtonews_url:
return self.url_result(
compat_urlparse.urljoin(url, sendtonews_url),
ie=SendtoNewsIE.ie_key())
info_dict = self._extract_anvato_videos(webpage, display_id)
timestamp = unified_timestamp(self._html_search_regex(
r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
'released date', default=None)) or parse_iso8601(
self._html_search_meta('uploadDate', webpage))
info_dict.update({
'display_id': display_id,
'timestamp': timestamp,
})
return info_dict
yt_dlp/extractor/cbsnews.py
+341 -41
View File
@@ -1,36 +1,153 @@
import base64
import re
import urllib.error
import urllib.parse
import zlib
from .anvato import AnvatoIE
from .common import InfoExtractor
from .cbs import CBSIE
from ..compat import (
compat_b64decode,
compat_urllib_parse_unquote,
)
from .paramountplus import ParamountPlusIE
from ..utils import (
ExtractorError,
HEADRequest,
UserNotLive,
determine_ext,
float_or_none,
format_field,
int_or_none,
make_archive_id,
mimetype2ext,
parse_duration,
smuggle_url,
traverse_obj,
url_or_none,
)
class CBSNewsEmbedIE(CBSIE): # XXX: Do not subclass from concrete IE
class CBSNewsBaseIE(InfoExtractor):
_LOCALES = {
'atlanta': None,
'baltimore': 'BAL',
'boston': 'BOS',
'chicago': 'CHI',
'colorado': 'DEN',
'detroit': 'DET',
'losangeles': 'LA',
'miami': 'MIA',
'minnesota': 'MIN',
'newyork': 'NY',
'philadelphia': 'PHI',
'pittsburgh': 'PIT',
'sacramento': 'SAC',
'sanfrancisco': 'SF',
'texas': 'DAL',
}
_LOCALE_RE = '|'.join(map(re.escape, _LOCALES))
_ANVACK = '5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl'
def _get_item(self, webpage, display_id):
return traverse_obj(self._search_json(
r'CBSNEWS\.defaultPayload\s*=', webpage, 'payload', display_id,
default={}), ('items', 0, {dict})) or {}
def _get_video_url(self, item):
return traverse_obj(item, 'video', 'video2', expected_type=url_or_none)
def _extract_playlist(self, webpage, playlist_id):
entries = [self.url_result(embed_url, CBSNewsEmbedIE) for embed_url in re.findall(
r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage)]
if entries:
return self.playlist_result(
entries, playlist_id, self._html_search_meta(['og:title', 'twitter:title'], webpage),
self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
def _extract_video(self, item, video_url, video_id):
if mimetype2ext(item.get('format'), default=determine_ext(video_url)) == 'mp4':
formats = [{'url': video_url, 'ext': 'mp4'}]
else:
manifest = self._download_webpage(video_url, video_id, note='Downloading m3u8 information')
anvato_id = self._search_regex(r'anvato-(\d+)', manifest, 'Anvato ID', default=None)
# Prefer Anvato if available; cbsnews.com m3u8 formats are re-encoded from Anvato source
if anvato_id:
return self.url_result(
smuggle_url(f'anvato:{self._ANVACK}:{anvato_id}', {'token': 'default'}),
AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
formats, _ = self._parse_m3u8_formats_and_subtitles(
manifest, video_url, 'mp4', m3u8_id='hls', video_id=video_id)
def get_subtitles(subs_url):
return {
'en': [{
'url': subs_url,
'ext': 'dfxp', # TTAF1
}],
} if url_or_none(subs_url) else None
episode_meta = traverse_obj(item, {
'season_number': ('season', {int_or_none}),
'episode_number': ('episode', {int_or_none}),
}) if item.get('isFullEpisode') else {}
return {
'id': video_id,
'formats': formats,
**traverse_obj(item, {
'title': (None, ('fulltitle', 'title')),
'description': 'dek',
'timestamp': ('timestamp', {lambda x: float_or_none(x, 1000)}),
'duration': ('duration', {float_or_none}),
'subtitles': ('captions', {get_subtitles}),
'thumbnail': ('images', ('hd', 'sd'), {url_or_none}),
'is_live': ('type', {lambda x: x == 'live'}),
}, get_all=False),
**episode_meta,
}
class CBSNewsEmbedIE(CBSNewsBaseIE):
IE_NAME = 'cbsnews:embed'
_VALID_URL = r'https?://(?:www\.)?cbsnews\.com/embed/video[^#]*#(?P<id>.+)'
_TESTS = [{
'url': 'https://www.cbsnews.com/embed/video/?v=1.c9b5b61492913d6660db0b2f03579ef25e86307a#1Vb7b9s2EP5XBAHbT6Gt98PAMKTJ0se6LVjWYWtdGBR1stlIpEBSTtwi%2F%2FvuJNkNhmHdGxgM2NL57vjd6zt%2B8PngdN%2Fyg79qeGvhzN%2FLGrS%2F%2BuBLB531V28%2B%2BO7Qg7%2Fy97r2z3xZ42NW8yLhDbA0S0KWlHnIijwKWJBHZZnHBa8Cgbpdf%2F89NM9Hi9fXifhpr8sr%2FlP848tn%2BTdXycX25zh4cdX%2FvHl6PmmPqnWQv9w8Ed%2B9GjYRim07bFEqdG%2BZVHuwTm65A7bVRrYtR5lAyMox7pigF6W4k%2By91mjspGsJ%2BwVae4%2BsvdnaO1p73HkXs%2FVisUDTGm7R8IcdnOROeq%2B19qT1amhA1VJtPenoTUgrtfKc9m7Rq8dP7nnjwOB7wg7ADdNt7VX64DWAWlKhPtmDEq22g4GF99x6Dk9E8OSsankHXqPNKDxC%2FdK7MLKTircTDgsI3mmj4OBdSq64dy7fd1x577RU1rt4cvMtOaulFYOd%2FLewRWvDO9lIgXFpZSnkZmjbv5SxKTPoQXClFbpsf%2Fhbbpzs0IB3vb8KkyzJQ%2BywOAgCrMpgRrz%2BKk4fvb7kFbR4XJCu0gAdtNO7woCwZTu%2BBUs9bam%2Fds71drVerpeisgrubLjAB4nnOSkWQnfr5W6o1ku5Xpr1MgrCbL0M0vUyDtfLLK15WiYp47xKWSLyjFVpwVmVJSLIoCjSOFkv3W7oKsVliwZJcB9nwXpZ5GEQQwY8jNKqKCBrgjTLeFxgdCIpazojDgnRtn43J6kG7nZ6cAbxh0EeFFk4%2B1u867cY5u4344n%2FxXjCqAjucdTHgLKojNKmSfO8KRsOFY%2FzKEYCKEJBzv90QA9nfm9gL%2BHulaFqUkz9ULUYxl62B3U%2FRVNLA8IhggaPycOoBuwOCESciDQVSSUgiOMsROB%2FhKfwCKOzEk%2B4k6rWd4uuT%2FwTDz7K7t3d3WLO8ISD95jSPQbayBacthbz86XVgxHwhex5zawzgDOmtp%2F3GPcXn0VXHdSS029%2Fj99UC%2FwJUvyKQ%2FzKyixIEVlYJOn4RxxuaH43Ty9fbJ5OObykHH435XAzJTHeOF4hhEUXD8URe%2FQ%2FBT%2BMpf8d5GN02Ox%2FfiGsl7TA7POu1xZ5%2BbTzcAVKMe48mqcC21hkacVEVScM26liVVBnrKkC4CLKyzAvHu0lhEaTKMFwI3a4SN9MsrfYzdBLq2vkwRD1gVviLT8kY9h2CHH6Y%2Bix6609weFtey4ESp60WtyeWMy%2BsmBuhsoKIyuoT%2Bq2R%2FrW5qi3g%2FvzS2j40DoixDP8%2BKP0yUdpXJ4l6Vla%2Bg9vce%2BC4yM5YlUcbA%2F0jLKdpmTwvsdN5z88nAIe08%2F0HgxeG1iv%2B6Hlhjh7uiW0SDzYNI92L401uha3JKYk268UVRzdOzNQvAaJqoXzAc80dAV440NZ1WVVAAMRYQ2KrGJFmDUsq8saWSnjvIj8t78y%2FRa3JRnbHVfyFpfwoDiGpPgjzekyUiKNlU3OMlwuLMmzgvEojllYVE2Z1HhImvsnk%2BuhusTEoB21PAtSFodeFK3iYhXEH9WOG2%2FkOE833sfeG%2Ff5cfHtEFNXgYes0%2FXj7aGivUgJ9XpusCtoNcNYVVnJVrrDo0OmJAutHCpuZul4W9lLcfy7BnuLPT02%2ByXsCTk%2B9zhzswIN04YueNSK%2BPtM0jS88QdLqSLJDTLsuGZJNolm2yO0PXh3UPnz9Ix5bfIAqxPjvETQsDCEiPG4QbqNyhBZISxybLnZYCrW5H3Axp690%2F0BJdXtDZ5ITuM4xj3f4oUHGzc5JeJmZKpp%2FjwKh4wMV%2FV1yx3emLoR0MwbG4K%2F%2BZgVep3PnzXGDHZ6a3i%2Fk%2BJrONDN13%2Bnq6tBTYk4o7cLGhBtqCC4KwacGHpEVuoH5JNro%2FE6JfE6d5RydbiR76k%2BW5wioDHBIjw1euhHjUGRB0y5A97KoaPx6MlL%2BwgboUVtUFRI%2FLemgTpdtF59ii7pab08kuPcfWzs0l%2FRI5takWnFpka0zOgWRtYcuf9aIxZMxlwr6IiGpsb6j2DQUXPl%2FimXI599Ev7fWjoPD78A',
'only_matching': True,
'info_dict': {
'id': '6ZP4cXvo9FaX3VLH7MF4CgY30JFpY_GA',
'ext': 'mp4',
'title': 'Cops investigate gorilla incident at Cincinnati Zoo',
'description': 'md5:fee7441ab8aaeb3c693482394738102b',
'duration': 350,
'timestamp': 1464719713,
'upload_date': '20160531',
'thumbnail': r're:^https?://.*\.jpg$',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
item = self._parse_json(zlib.decompress(compat_b64decode(
compat_urllib_parse_unquote(self._match_id(url))),
-zlib.MAX_WBITS).decode('utf-8'), None)['video']['items'][0]
return self._extract_video_info(item['mpxRefId'], 'cbsnews')
item = traverse_obj(self._parse_json(zlib.decompress(base64.b64decode(
urllib.parse.unquote(self._match_id(url))),
-zlib.MAX_WBITS).decode(), None), ('video', 'items', 0, {dict})) or {}
video_id = item['mpxRefId']
video_url = self._get_video_url(item)
if not video_url:
# Old embeds redirect user to ParamountPlus but most links are 404
pplus_url = f'https://www.paramountplus.com/shows/video/{video_id}'
try:
self._request_webpage(HEADRequest(pplus_url), video_id)
return self.url_result(pplus_url, ParamountPlusIE)
except ExtractorError:
self.raise_no_formats('This video is no longer available', True, video_id)
return self._extract_video(item, video_url, video_id)
class CBSNewsIE(CBSIE): # XXX: Do not subclass from concrete IE
class CBSNewsIE(CBSNewsBaseIE):
IE_NAME = 'cbsnews'
IE_DESC = 'CBS News'
_VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\da-z_-]+)'
_VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\w-]+)'
_TESTS = [
{
@@ -47,10 +164,7 @@ class CBSNewsIE(CBSIE): # XXX: Do not subclass from concrete IE
'timestamp': 1476046464,
'upload_date': '20161009',
},
'params': {
# rtmp download
'skip_download': True,
},
'skip': 'This video is no longer available',
},
{
'url': 'https://www.cbsnews.com/video/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -61,48 +175,234 @@ class CBSNewsIE(CBSIE): # XXX: Do not subclass from concrete IE
'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
'upload_date': '20140404',
'timestamp': 1396650660,
'uploader': 'CBSI-NEW',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 205,
'subtitles': {
'en': [{
'ext': 'ttml',
'ext': 'dfxp',
}],
},
},
'params': {
# m3u8 download
'skip_download': True,
'skip_download': 'm3u8',
},
},
{
# 48 hours
'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
'info_dict': {
'id': 'maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved',
'title': 'Cold as Ice',
'description': 'Can a childhood memory solve the 1957 murder of 7-year-old Maria Ridulph?',
},
'playlist_mincount': 7,
},
{
'url': 'https://www.cbsnews.com/video/032823-cbs-evening-news/',
'info_dict': {
'id': '_2wuO7hD9LwtyM_TwSnVwnKp6kxlcXgE',
'ext': 'mp4',
'title': 'CBS Evening News, March 28, 2023',
'description': 'md5:db20615aae54adc1d55a1fd69dc75d13',
'duration': 1189,
'timestamp': 1680042600,
'upload_date': '20230328',
'season': 'Season 2023',
'season_number': 2023,
'episode': 'Episode 83',
'episode_number': 83,
'thumbnail': r're:^https?://.*\.jpg$',
},
'params': {
'skip_download': 'm3u8',
},
},
]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
entries = []
for embed_url in re.findall(r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage):
entries.append(self.url_result(embed_url, CBSNewsEmbedIE.ie_key()))
if entries:
return self.playlist_result(
entries, playlist_title=self._html_search_meta(['og:title', 'twitter:title'], webpage),
playlist_description=self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
playlist = self._extract_playlist(webpage, display_id)
if playlist:
return playlist
item = self._parse_json(self._html_search_regex(
r'CBSNEWS\.defaultPayload\s*=\s*({.+})',
webpage, 'video JSON info'), display_id)['items'][0]
return self._extract_video_info(item['mpxRefId'], 'cbsnews')
item = self._get_item(webpage, display_id)
video_id = item.get('mpxRefId') or display_id
video_url = self._get_video_url(item)
if not video_url:
self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
return self._extract_video(item, video_url, video_id)
class CBSLocalBaseIE(CBSNewsBaseIE):
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
item = self._get_item(webpage, display_id)
video_id = item.get('mpxRefId') or display_id
anvato_id = None
video_url = self._get_video_url(item)
if not video_url:
anv_params = self._search_regex(
r'<iframe[^>]+\bdata-src="https?://w3\.mp\.lura\.live/player/prod/v3/anvload\.html\?key=([^"]+)"',
webpage, 'Anvato URL', default=None)
if not anv_params:
playlist = self._extract_playlist(webpage, display_id)
if playlist:
return playlist
self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
anv_data = self._parse_json(base64.urlsafe_b64decode(f'{anv_params}===').decode(), video_id)
anvato_id = anv_data['v']
return self.url_result(
smuggle_url(f'anvato:{anv_data.get("anvack") or self._ANVACK}:{anvato_id}', {
'token': anv_data.get('token') or 'default',
}), AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
return self._extract_video(item, video_url, video_id)
class CBSLocalIE(CBSLocalBaseIE):
_VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/(?:live/)?video/(?P<id>[\w-]+)'
_TESTS = [{
# Anvato video via defaultPayload JSON
'url': 'https://www.cbsnews.com/newyork/video/1st-cannabis-dispensary-opens-in-queens/',
'info_dict': {
'id': '6376747',
'ext': 'mp4',
'title': '1st cannabis dispensary opens in Queens',
'description': 'The dispensary is women-owned and located in Jamaica.',
'uploader': 'CBS',
'duration': 20,
'timestamp': 1680193657,
'upload_date': '20230330',
'categories': ['Stations\\Spoken Word\\WCBSTV', 'Content\\Google', 'Content\\News', 'Content\\News\\Local News'],
'tags': 'count:11',
'thumbnail': 're:^https?://.*',
'_old_archive_ids': ['cbslocal 6376747'],
},
'params': {'skip_download': 'm3u8'},
}, {
# cbsnews.com video via defaultPayload JSON
'url': 'https://www.cbsnews.com/newyork/live/video/20230330171655-the-city-is-sounding-the-alarm-on-dangerous-social-media-challenges/',
'info_dict': {
'id': 'sJqfw7YvgSC6ant2zVmzt3y1jYKoL5J3',
'ext': 'mp4',
'title': 'the city is sounding the alarm on dangerous social media challenges',
'description': 'md5:8eccc9b1b73be5138a52e9c4350d2cd6',
'thumbnail': 'https://images-cbsn.cbsnews.com/prod/2023/03/30/story_22509622_1680196925.jpg',
'duration': 41.0,
'timestamp': 1680196615,
'upload_date': '20230330',
},
'params': {'skip_download': 'm3u8'},
}]
class CBSLocalArticleIE(CBSLocalBaseIE):
_VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/news/(?P<id>[\w-]+)'
_TESTS = [{
# Anvato video via iframe embed
'url': 'https://www.cbsnews.com/newyork/news/mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service/',
'playlist_count': 2,
'info_dict': {
'id': 'mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service',
'title': 'MTA station agents begin leaving their booths to provide more direct customer service',
'description': 'The more than 2,200 agents will provide face-to-face customer service to passengers.',
},
}, {
'url': 'https://www.cbsnews.com/losangeles/news/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis/',
'md5': 'f0ee3081e3843f575fccef901199b212',
'info_dict': {
'id': '3401037',
'ext': 'mp4',
'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
'thumbnail': 're:^https?://.*',
'timestamp': 1463440500,
'upload_date': '20160516',
},
'skip': 'Video has been removed',
}]
class CBSNewsLiveBaseIE(CBSNewsBaseIE):
def _get_id(self, url):
raise NotImplementedError('This method must be implemented by subclasses')
def _real_extract(self, url):
video_id = self._get_id(url)
if not video_id:
raise ExtractorError('Livestream is not available', expected=True)
data = traverse_obj(self._download_json(
'https://feeds-cbsn.cbsnews.com/2.0/rundown/', video_id, query={
'partner': 'cbsnsite',
'edition': video_id,
'type': 'live',
}), ('navigation', 'data', 0, {dict}))
video_url = traverse_obj(data, (('videoUrlDAI', ('videoUrl', 'base')), {url_or_none}), get_all=False)
if not video_url:
raise UserNotLive(video_id=video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', m3u8_id='hls')
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'is_live': True,
**traverse_obj(data, {
'title': 'headline',
'description': 'rundown_slug',
'thumbnail': ('images', 'thumbnail_url_hd', {url_or_none}),
}),
}
class CBSLocalLiveIE(CBSNewsLiveBaseIE):
_VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?P<id>{CBSNewsBaseIE._LOCALE_RE})/live/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://www.cbsnews.com/losangeles/live/',
'info_dict': {
'id': 'CBSN-LA',
'ext': 'mp4',
'title': str,
'description': r're:KCBS/CBSN_LA.CRISPIN.\w+.RUNDOWN \w+ \w+',
'thumbnail': r're:^https?://.*\.jpg$',
'live_status': 'is_live',
},
'params': {'skip_download': 'm3u8'},
}]
def _get_id(self, url):
return format_field(self._LOCALES, self._match_id(url), 'CBSN-%s')
class CBSNewsLiveIE(CBSNewsLiveBaseIE):
IE_NAME = 'cbsnews:live'
IE_DESC = 'CBS News Livestream'
_VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://www.cbsnews.com/live/',
'info_dict': {
'id': 'CBSN-US',
'ext': 'mp4',
'title': str,
'description': r're:\w+ \w+ CRISPIN RUNDOWN',
'thumbnail': r're:^https?://.*\.jpg$',
'live_status': 'is_live',
},
'params': {'skip_download': 'm3u8'},
}]
def _get_id(self, url):
return 'CBSN-US'
class CBSNewsLiveVideoIE(InfoExtractor):
@@ -111,7 +411,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[^/?#]+)'
# Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
_TEST = {
_TESTS = [{
'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
'info_dict': {
'id': 'clinton-sanders-prepare-to-face-off-in-nh',
@@ -120,7 +420,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
'duration': 334,
},
'skip': 'Video gone',
}
}]
def _real_extract(self, url):
display_id = self._match_id(url)
@@ -131,13 +431,13 @@ class CBSNewsLiveVideoIE(InfoExtractor):
'dvr_slug': display_id,
})
formats = self._extract_akamai_formats(video_info['url'], display_id)
return {
'id': display_id,
'display_id': display_id,
'title': video_info['headline'],
'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
'duration': parse_duration(video_info.get('segmentDur')),
'formats': formats,
'formats': self._extract_akamai_formats(video_info['url'], display_id),
**traverse_obj(video_info, {
'title': 'headline',
'thumbnail': ('thumbnail_url_hd', {url_or_none}),
'duration': ('segmentDur', {parse_duration}),
}),
}
yt_dlp/extractor/comedycentral.py
+4 -1
View File
@@ -2,7 +2,7 @@ from .mtv import MTVServicesInfoExtractor
class ComedyCentralIE(MTVServicesInfoExtractor):
_VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist)/(?P<id>[0-9a-z]{6})'
_VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist|movies)/(?P<id>[0-9a-z]{6})'
_FEED_URL = 'http://comedycentral.com/feeds/mrss/'
_TESTS = [{
@@ -25,6 +25,9 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
}, {
'url': 'https://www.cc.com/collection-playlist/cosnej/stand-up-specials/t6vtjb',
'only_matching': True,
}, {
'url': 'https://www.cc.com/movies/tkp406/a-cluesterfuenke-christmas',
'only_matching': True,
}]
yt_dlp/extractor/common.py
+12 -3
View File
@@ -314,6 +314,11 @@ class InfoExtractor:
* "author" - human-readable name of the comment author
* "author_id" - user ID of the comment author
* "author_thumbnail" - The thumbnail of the comment author
* "author_url" - The url to the comment author's page
* "author_is_verified" - Whether the author is verified
on the platform
* "author_is_uploader" - Whether the comment is made by
the video uploader
* "id" - Comment ID
* "html" - Comment as HTML
* "text" - Plain text of the comment
@@ -325,8 +330,8 @@ class InfoExtractor:
* "dislike_count" - Number of negative ratings of the comment
* "is_favorited" - Whether the comment is marked as
favorite by the video uploader
* "author_is_uploader" - Whether the comment is made by
the video uploader
* "is_pinned" - Whether the comment is pinned to
the top of the comments
age_limit: Age restriction for the video, as an integer (years)
webpage_url: The URL to the video webpage, if given to yt-dlp it
should allow to get the same result again. (It will be set
@@ -350,6 +355,10 @@ class InfoExtractor:
* "start_time" - The start time of the chapter in seconds
* "end_time" - The end time of the chapter in seconds
* "title" (optional, string)
heatmap: A list of dictionaries, with the following entries:
* "start_time" - The start time of the data point in seconds
* "end_time" - The end time of the data point in seconds
* "value" - The normalized value of the data point (float between 0 and 1)
playable_in_embed: Whether this video is allowed to play in embedded
players on other sites. Can be True (=always allowed),
False (=never allowed), None (=unknown), or a string
@@ -3455,7 +3464,7 @@ class InfoExtractor:
def _get_cookies(self, url):
""" Return a http.cookies.SimpleCookie with the cookies for the url """
return LenientSimpleCookie(self._downloader._calc_cookies(url))
return LenientSimpleCookie(self._downloader.cookiejar.get_cookie_header(url))
def _apply_first_set_cookie_header(self, url_handle, cookie):
"""
yt_dlp/extractor/crtvg.py
+34
View File
@@ -0,0 +1,34 @@
from .common import InfoExtractor
from ..utils import remove_end
class CrtvgIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?crtvg\.es/tvg/a-carta/[^/#?]+-(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.crtvg.es/tvg/a-carta/os-caimans-do-tea-5839623',
'md5': 'c0958d9ff90e4503a75544358758921d',
'info_dict': {
'id': '5839623',
'title': 'Os caimáns do Tea',
'ext': 'mp4',
'description': 'md5:f71cfba21ae564f0a6f415b31de1f842',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
},
'params': {'skip_download': 'm3u8'}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
video_url = self._search_regex(r'var\s+url\s*=\s*["\']([^"\']+)', webpage, 'video url')
formats = self._extract_m3u8_formats(video_url + '/playlist.m3u8', video_id, fatal=False)
formats.extend(self._extract_mpd_formats(video_url + '/manifest.mpd', video_id, fatal=False))
return {
'id': video_id,
'formats': formats,
'title': remove_end(self._html_search_meta(
['og:title', 'twitter:title'], webpage, 'title', default=None), ' | CRTVG'),
'description': self._html_search_meta('description', webpage, 'description', default=None),
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage, 'thumbnail', default=None),
}
yt_dlp/extractor/crunchyroll.py
+497 -195
View File
@@ -1,28 +1,37 @@
import base64
import urllib.parse
import urllib.error
from .common import InfoExtractor
from ..utils import (
ExtractorError,
float_or_none,
format_field,
int_or_none,
join_nonempty,
parse_age_limit,
parse_count,
parse_iso8601,
qualities,
remove_start,
time_seconds,
traverse_obj,
try_get,
url_or_none,
urlencode_postdata,
)
class CrunchyrollBaseIE(InfoExtractor):
_LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
_BASE_URL = 'https://www.crunchyroll.com'
_API_BASE = 'https://api.crunchyroll.com'
_NETRC_MACHINE = 'crunchyroll'
params = None
_AUTH_HEADERS = None
_API_ENDPOINT = None
_BASIC_AUTH = None
_QUERY = {}
@property
def is_logged_in(self):
return self._get_cookies(self._LOGIN_URL).get('etp_rt')
return self._get_cookies(self._BASE_URL).get('etp_rt')
def _perform_login(self, username, password):
if self.is_logged_in:
@@ -35,7 +44,7 @@ class CrunchyrollBaseIE(InfoExtractor):
'device_id': 'whatvalueshouldbeforweb',
'device_type': 'com.crunchyroll.static',
'access_token': 'giKq5eY27ny3cqz',
'referer': self._LOGIN_URL
'referer': f'{self._BASE_URL}/welcome/login'
})
if upsell_response['code'] != 'ok':
raise ExtractorError('Could not get session id')
@@ -43,149 +52,89 @@ class CrunchyrollBaseIE(InfoExtractor):
login_response = self._download_json(
f'{self._API_BASE}/login.1.json', None, 'Logging in',
data=urllib.parse.urlencode({
data=urlencode_postdata({
'account': username,
'password': password,
'session_id': session_id
}).encode('ascii'))
}))
if login_response['code'] != 'ok':
raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
if not self.is_logged_in:
raise ExtractorError('Login succeeded but did not set etp_rt cookie')
def _get_embedded_json(self, webpage, display_id):
initial_state = self._parse_json(self._search_regex(
r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'), display_id)
app_config = self._parse_json(self._search_regex(
r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'), display_id)
return initial_state, app_config
def _update_query(self, lang):
if lang in CrunchyrollBaseIE._QUERY:
return
def _get_params(self, lang):
if not CrunchyrollBaseIE.params:
if self._get_cookies(f'https://www.crunchyroll.com/{lang}').get('etp_rt'):
grant_type, key = 'etp_rt_cookie', 'accountAuthClientId'
else:
grant_type, key = 'client_id', 'anonClientId'
webpage = self._download_webpage(
f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})')
initial_state, app_config = self._get_embedded_json(self._download_webpage(
f'https://www.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
api_domain = app_config['cxApiParams']['apiDomain'].replace('beta.crunchyroll.com', 'www.crunchyroll.com')
initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None)
CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, {
'locale': ('localization', 'locale'),
}) or None
auth_response = self._download_json(
f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
headers={
'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii')
}, data=f'grant_type={grant_type}'.encode('ascii'))
policy_response = self._download_json(
f'{api_domain}/index/v2', None, note='Retrieving signed policy',
headers={
'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
})
cms = policy_response.get('cms_web')
bucket = cms['bucket']
params = {
'Policy': cms['policy'],
'Signature': cms['signature'],
'Key-Pair-Id': cms['key_pair_id']
}
locale = traverse_obj(initial_state, ('localization', 'locale'))
if locale:
params['locale'] = locale
CrunchyrollBaseIE.params = (api_domain, bucket, params)
return CrunchyrollBaseIE.params
if CrunchyrollBaseIE._BASIC_AUTH:
return
app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None)
cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId']
self.write_debug(f'Using cxApiParam={cx_api_param}')
CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
class CrunchyrollBetaIE(CrunchyrollBaseIE):
IE_NAME = 'crunchyroll'
_VALID_URL = r'''(?x)
https?://(?:beta|www)\.crunchyroll\.com/
(?P<lang>(?:\w{2}(?:-\w{2})?/)?)
watch/(?P<id>\w+)
(?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
_TESTS = [{
'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
'info_dict': {
'id': 'GY2P1Q98Y',
'ext': 'mp4',
'duration': 1380.241,
'timestamp': 1459632600,
'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
'title': 'World Trigger Episode 73 – To the Future',
'upload_date': '20160402',
'series': 'World Trigger',
'series_id': 'GR757DMKY',
'season': 'World Trigger',
'season_id': 'GR9P39NJ6',
'season_number': 1,
'episode': 'To the Future',
'episode_number': 73,
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
'chapters': 'count:2',
},
'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
}, {
'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
'info_dict': {
'id': 'GYE5WKQGR',
'ext': 'mp4',
'duration': 366.459,
'timestamp': 1476788400,
'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
'title': 'SHELTER Episode – Porter Robinson presents Shelter the Animation',
'upload_date': '20161018',
'series': 'SHELTER',
'series_id': 'GYGG09WWY',
'season': 'SHELTER',
'season_id': 'GR09MGK4R',
'season_number': 1,
'episode': 'Porter Robinson presents Shelter the Animation',
'episode_number': 0,
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
'chapters': 'count:0',
},
'params': {'skip_download': True},
'skip': 'Video is Premium only',
}, {
'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
'only_matching': True,
}, {
'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
'only_matching': True,
}]
def _update_auth(self):
if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
return
def _real_extract(self, url):
lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
api_domain, bucket, params = self._get_params(lang)
assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand'
grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id'
auth_response = self._download_json(
f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
episode_response = self._download_json(
f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
note='Retrieving episode metadata', query=params)
if episode_response.get('is_premium_only') and not bucket.endswith('crunchyroll'):
if self.is_logged_in:
raise ExtractorError('This video is for premium members only', expected=True)
else:
self.raise_login_required('This video is for premium members only')
CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
stream_response = self._download_json(
f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id,
note='Retrieving stream info', query=params)
get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}):
self._update_query(lang)
self._update_auth()
requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
hardsub_preference = qualities(requested_hardsubs[::-1])
if not endpoint.startswith('/'):
endpoint = f'/{endpoint}'
return self._download_json(
f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}',
headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query})
def _call_api(self, path, internal_id, lang, note='api', query={}):
if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'):
path = f'/content/v2/{self._API_ENDPOINT}/{path}'
try:
result = self._call_base_api(
path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
except ExtractorError as error:
if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
return None
raise
if not result:
raise ExtractorError(f'Unexpected response when downloading {note} JSON')
return result
def _extract_formats(self, stream_response, display_id=None):
requested_formats = self._configuration_arg('format') or ['adaptive_hls']
available_formats = {}
for stream_type, streams in get_streams('streams'):
for stream_type, streams in traverse_obj(
stream_response, (('streams', ('data', 0)), {dict.items}, ...)):
if stream_type not in requested_formats:
continue
for stream in streams.values():
if not stream.get('url'):
continue
for stream in traverse_obj(streams, lambda _, v: v['url']):
hardsub_lang = stream.get('hardsub_locale') or ''
format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
available_formats[hardsub_lang] = (stream_type, format_id, hardsub_lang, stream['url'])
requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
if '' in available_formats and 'all' not in requested_hardsubs:
full_format_langs = set(requested_hardsubs)
self.to_screen(
@@ -196,6 +145,8 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
else:
full_format_langs = set(map(str.lower, available_formats))
audio_locale = traverse_obj(stream_response, ((None, 'meta'), 'audio_locale'), get_all=False)
hardsub_preference = qualities(requested_hardsubs[::-1])
formats = []
for stream_type, format_id, hardsub_lang, stream_url in available_formats.values():
if stream_type.endswith('hls'):
@@ -214,63 +165,292 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
continue
for f in adaptive_formats:
if f.get('acodec') != 'none':
f['language'] = stream_response.get('audio_locale')
f['language'] = audio_locale
f['quality'] = hardsub_preference(hardsub_lang.lower())
formats.extend(adaptive_formats)
chapters = None
return formats
def _extract_subtitles(self, data):
subtitles = {}
for locale, subtitle in traverse_obj(data, ((None, 'meta'), 'subtitles', {dict.items}, ...)):
subtitles[locale] = [traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})]
return subtitles
class CrunchyrollCmsBaseIE(CrunchyrollBaseIE):
_API_ENDPOINT = 'cms'
_CMS_EXPIRY = None
def _call_cms_api_signed(self, path, internal_id, lang, note='api'):
if not CrunchyrollCmsBaseIE._CMS_EXPIRY or CrunchyrollCmsBaseIE._CMS_EXPIRY <= time_seconds():
response = self._call_base_api('index/v2', None, lang, 'Retrieving signed policy')['cms_web']
CrunchyrollCmsBaseIE._CMS_QUERY = {
'Policy': response['policy'],
'Signature': response['signature'],
'Key-Pair-Id': response['key_pair_id'],
}
CrunchyrollCmsBaseIE._CMS_BUCKET = response['bucket']
CrunchyrollCmsBaseIE._CMS_EXPIRY = parse_iso8601(response['expires']) - 10
if not path.startswith('/cms/v2'):
path = f'/cms/v2{CrunchyrollCmsBaseIE._CMS_BUCKET}/{path}'
return self._call_base_api(
path, internal_id, lang, f'Downloading {note} JSON (signed cms)', query=CrunchyrollCmsBaseIE._CMS_QUERY)
class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
IE_NAME = 'crunchyroll'
_VALID_URL = r'''(?x)
https?://(?:beta\.|www\.)?crunchyroll\.com/
(?P<lang>(?:\w{2}(?:-\w{2})?/)?)
watch/(?!concert|musicvideo)(?P<id>\w+)'''
_TESTS = [{
# Premium only
'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
'info_dict': {
'id': 'GY2P1Q98Y',
'ext': 'mp4',
'duration': 1380.241,
'timestamp': 1459632600,
'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
'title': 'World Trigger Episode 73 – To the Future',
'upload_date': '20160402',
'series': 'World Trigger',
'series_id': 'GR757DMKY',
'season': 'World Trigger',
'season_id': 'GR9P39NJ6',
'season_number': 1,
'episode': 'To the Future',
'episode_number': 73,
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'chapters': 'count:2',
'age_limit': 14,
'like_count': int,
'dislike_count': int,
},
'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
}, {
# Premium only
'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
'info_dict': {
'id': 'GYE5WKQGR',
'ext': 'mp4',
'duration': 366.459,
'timestamp': 1476788400,
'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
'title': 'SHELTER – Porter Robinson presents Shelter the Animation',
'upload_date': '20161018',
'series': 'SHELTER',
'series_id': 'GYGG09WWY',
'season': 'SHELTER',
'season_id': 'GR09MGK4R',
'season_number': 1,
'episode': 'Porter Robinson presents Shelter the Animation',
'episode_number': 0,
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'age_limit': 14,
'like_count': int,
'dislike_count': int,
},
'params': {'skip_download': True},
}, {
'url': 'https://www.crunchyroll.com/watch/GJWU2VKK3/cherry-blossom-meeting-and-a-coming-blizzard',
'info_dict': {
'id': 'GJWU2VKK3',
'ext': 'mp4',
'duration': 1420.054,
'description': 'md5:2d1c67c0ec6ae514d9c30b0b99a625cd',
'title': 'The Ice Guy and His Cool Female Colleague Episode 1 – Cherry Blossom Meeting and a Coming Blizzard',
'series': 'The Ice Guy and His Cool Female Colleague',
'series_id': 'GW4HM75NP',
'season': 'The Ice Guy and His Cool Female Colleague',
'season_id': 'GY9PC21VE',
'season_number': 1,
'episode': 'Cherry Blossom Meeting and a Coming Blizzard',
'episode_number': 1,
'chapters': 'count:2',
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'timestamp': 1672839000,
'upload_date': '20230104',
'age_limit': 14,
'like_count': int,
'dislike_count': int,
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.crunchyroll.com/watch/GM8F313NQ',
'info_dict': {
'id': 'GM8F313NQ',
'ext': 'mp4',
'title': 'Garakowa -Restore the World-',
'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
'duration': 3996.104,
'age_limit': 13,
'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.crunchyroll.com/watch/G62PEZ2E6',
'info_dict': {
'id': 'G62PEZ2E6',
'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
'age_limit': 13,
'duration': 65.138,
'title': 'Garakowa -Restore the World-',
},
'playlist_mincount': 5,
}, {
'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
'only_matching': True,
}, {
'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
'only_matching': True,
}]
# We want to support lazy playlist filtering and movie listings cannot be inside a playlist
_RETURN_TYPE = 'video'
def _real_extract(self, url):
lang, internal_id = self._match_valid_url(url).group('lang', 'id')
# We need to use unsigned API call to allow ratings query string
response = traverse_obj(self._call_api(
f'objects/{internal_id}', internal_id, lang, 'object info', {'ratings': 'true'}), ('data', 0, {dict}))
if not response:
raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
object_type = response.get('type')
if object_type == 'episode':
result = self._transform_episode_response(response)
elif object_type == 'movie':
result = self._transform_movie_response(response)
elif object_type == 'movie_listing':
first_movie_id = traverse_obj(response, ('movie_listing_metadata', 'first_movie_id'))
if not self._yes_playlist(internal_id, first_movie_id):
return self.url_result(f'{self._BASE_URL}/{lang}watch/{first_movie_id}', CrunchyrollBetaIE, first_movie_id)
def entries():
movies = self._call_api(f'movie_listings/{internal_id}/movies', internal_id, lang, 'movie list')
for movie_response in traverse_obj(movies, ('data', ...)):
yield self.url_result(
f'{self._BASE_URL}/{lang}watch/{movie_response["id"]}',
CrunchyrollBetaIE, **self._transform_movie_response(movie_response))
return self.playlist_result(entries(), **self._transform_movie_response(response))
else:
raise ExtractorError(f'Unknown object type {object_type}')
# There might be multiple audio languages for one object (`<object>_metadata.versions`),
# so we need to get the id from `streams_link` instead or we dont know which language to choose
streams_link = response.get('streams_link')
if not streams_link and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')):
message = f'This {object_type} is for premium members only'
if self.is_logged_in:
raise ExtractorError(message, expected=True)
self.raise_login_required(message)
# We need go from unsigned to signed api to avoid getting soft banned
stream_response = self._call_cms_api_signed(remove_start(
streams_link, '/content/v2/cms/'), internal_id, lang, 'stream info')
result['formats'] = self._extract_formats(stream_response, internal_id)
result['subtitles'] = self._extract_subtitles(stream_response)
# if no intro chapter is available, a 403 without usable data is returned
intro_chapter = self._download_json(f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
display_id, fatal=False, errnote=False)
intro_chapter = self._download_json(
f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
internal_id, note='Downloading chapter info', fatal=False, errnote=False)
if isinstance(intro_chapter, dict):
chapters = [{
result['chapters'] = [{
'title': 'Intro',
'start_time': float_or_none(intro_chapter.get('startTime')),
'end_time': float_or_none(intro_chapter.get('endTime'))
'end_time': float_or_none(intro_chapter.get('endTime')),
}]
def calculate_count(item):
return parse_count(''.join((item['displayed'], item.get('unit') or '')))
result.update(traverse_obj(response, ('rating', {
'like_count': ('up', {calculate_count}),
'dislike_count': ('down', {calculate_count}),
})))
return result
@staticmethod
def _transform_episode_response(data):
metadata = traverse_obj(data, (('episode_metadata', None), {dict}), get_all=False) or {}
return {
'id': internal_id,
'title': '%s Episode %s – %s' % (
episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
'duration': float_or_none(episode_response.get('duration_ms'), 1000),
'timestamp': parse_iso8601(episode_response.get('upload_date')),
'series': episode_response.get('series_title'),
'series_id': episode_response.get('series_id'),
'season': episode_response.get('season_title'),
'season_id': episode_response.get('season_id'),
'season_number': episode_response.get('season_number'),
'episode': episode_response.get('title'),
'episode_number': episode_response.get('sequence_number'),
'formats': formats,
'thumbnails': [{
'url': thumb.get('source'),
'width': thumb.get('width'),
'height': thumb.get('height'),
} for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
'subtitles': {
lang: [{
'url': subtitle_data.get('url'),
'ext': subtitle_data.get('format')
}] for lang, subtitle_data in get_streams('subtitles')
},
'chapters': chapters
'id': data['id'],
'title': ' \u2013 '.join((
('%s%s' % (
format_field(metadata, 'season_title'),
format_field(metadata, 'episode', ' Episode %s'))),
format_field(data, 'title'))),
**traverse_obj(data, {
'episode': ('title', {str}),
'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
'thumbnails': ('images', 'thumbnail', ..., ..., {
'url': ('source', {url_or_none}),
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
}),
}),
**traverse_obj(metadata, {
'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
'timestamp': ('upload_date', {parse_iso8601}),
'series': ('series_title', {str}),
'series_id': ('series_id', {str}),
'season': ('season_title', {str}),
'season_id': ('season_id', {str}),
'season_number': ('season_number', ({int}, {float_or_none})),
'episode_number': ('sequence_number', ({int}, {float_or_none})),
'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
'language': ('audio_locale', {str}),
}, get_all=False),
}
@staticmethod
def _transform_movie_response(data):
metadata = traverse_obj(data, (('movie_metadata', 'movie_listing_metadata', None), {dict}), get_all=False) or {}
return {
'id': data['id'],
**traverse_obj(data, {
'title': ('title', {str}),
'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
'thumbnails': ('images', 'thumbnail', ..., ..., {
'url': ('source', {url_or_none}),
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
}),
}),
**traverse_obj(metadata, {
'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
}),
}
class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
class CrunchyrollBetaShowIE(CrunchyrollCmsBaseIE):
IE_NAME = 'crunchyroll:playlist'
_VALID_URL = r'''(?x)
https?://(?:beta|www)\.crunchyroll\.com/
https?://(?:beta\.|www\.)?crunchyroll\.com/
(?P<lang>(?:\w{2}(?:-\w{2})?/)?)
series/(?P<id>\w+)
(?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
series/(?P<id>\w+)'''
_TESTS = [{
'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
'info_dict': {
'id': 'GY19NQ2QR',
'title': 'Girl Friend BETA',
'description': 'md5:99c1b22ee30a74b536a8277ced8eb750',
# XXX: `thumbnail` does not get set from `thumbnails` in playlist
# 'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'age_limit': 14,
},
'playlist_mincount': 10,
}, {
@@ -279,41 +459,163 @@ class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
}]
def _real_extract(self, url):
lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
api_domain, bucket, params = self._get_params(lang)
series_response = self._download_json(
f'{api_domain}/cms/v2{bucket}/series/{internal_id}', display_id,
note='Retrieving series metadata', query=params)
seasons_response = self._download_json(
f'{api_domain}/cms/v2{bucket}/seasons?series_id={internal_id}', display_id,
note='Retrieving season list', query=params)
lang, internal_id = self._match_valid_url(url).group('lang', 'id')
def entries():
for season in seasons_response['items']:
episodes_response = self._download_json(
f'{api_domain}/cms/v2{bucket}/episodes?season_id={season["id"]}', display_id,
note=f'Retrieving episode list for {season.get("slug_title")}', query=params)
for episode in episodes_response['items']:
episode_id = episode['id']
episode_display_id = episode['slug_title']
yield {
'_type': 'url',
'url': f'https://www.crunchyroll.com/{lang}watch/{episode_id}/{episode_display_id}',
'ie_key': CrunchyrollBetaIE.ie_key(),
'id': episode_id,
'title': '%s Episode %s – %s' % (episode.get('season_title'), episode.get('episode'), episode.get('title')),
'description': try_get(episode, lambda x: x['description'].replace(r'\r\n', '\n')),
'duration': float_or_none(episode.get('duration_ms'), 1000),
'series': episode.get('series_title'),
'series_id': episode.get('series_id'),
'season': episode.get('season_title'),
'season_id': episode.get('season_id'),
'season_number': episode.get('season_number'),
'episode': episode.get('title'),
'episode_number': episode.get('sequence_number'),
'language': episode.get('audio_locale'),
}
seasons_response = self._call_cms_api_signed(f'seasons?series_id={internal_id}', internal_id, lang, 'seasons')
for season in traverse_obj(seasons_response, ('items', ..., {dict})):
episodes_response = self._call_cms_api_signed(
f'episodes?season_id={season["id"]}', season["id"], lang, 'episode list')
for episode_response in traverse_obj(episodes_response, ('items', ..., {dict})):
yield self.url_result(
f'{self._BASE_URL}/{lang}watch/{episode_response["id"]}',
CrunchyrollBetaIE, **CrunchyrollBetaIE._transform_episode_response(episode_response))
return self.playlist_result(entries(), internal_id, series_response.get('title'))
return self.playlist_result(
entries(), internal_id,
**traverse_obj(self._call_api(f'series/{internal_id}', internal_id, lang, 'series'), ('data', 0, {
'title': ('title', {str}),
'description': ('description', {lambda x: x.replace(r'\r\n', '\n')}),
'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
'thumbnails': ('images', ..., ..., ..., {
'url': ('source', {url_or_none}),
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
})
})))
class CrunchyrollMusicIE(CrunchyrollBaseIE):
IE_NAME = 'crunchyroll:music'
_VALID_URL = r'''(?x)
https?://(?:www\.)?crunchyroll\.com/
(?P<lang>(?:\w{2}(?:-\w{2})?/)?)
watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})'''
_TESTS = [{
'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C',
'info_dict': {
'ext': 'mp4',
'id': 'MV88BB7F2C',
'display_id': 'crossing-field',
'title': 'Crossing Field',
'track': 'Crossing Field',
'artist': 'LiSA',
'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'genre': ['Anime'],
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135',
'info_dict': {
'ext': 'mp4',
'id': 'MC2E2AC135',
'display_id': 'live-is-smile-always-364joker-at-yokohama-arena',
'title': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
'track': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
'artist': 'LiSA',
'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
'description': 'md5:747444e7e6300907b7a43f0a0503072e',
'genre': ['J-Pop'],
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
'only_matching': True,
}, {
'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena',
'only_matching': True,
}]
_API_ENDPOINT = 'music'
def _real_extract(self, url):
lang, internal_id, object_type = self._match_valid_url(url).group('lang', 'id', 'type')
path, name = {
'concert': ('concerts', 'concert info'),
'musicvideo': ('music_videos', 'music video info'),
}[object_type]
response = traverse_obj(self._call_api(f'{path}/{internal_id}', internal_id, lang, name), ('data', 0, {dict}))
if not response:
raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
streams_link = response.get('streams_link')
if not streams_link and response.get('isPremiumOnly'):
message = f'This {response.get("type") or "media"} is for premium members only'
if self.is_logged_in:
raise ExtractorError(message, expected=True)
self.raise_login_required(message)
result = self._transform_music_response(response)
stream_response = self._call_api(streams_link, internal_id, lang, 'stream info')
result['formats'] = self._extract_formats(stream_response, internal_id)
return result
@staticmethod
def _transform_music_response(data):
return {
'id': data['id'],
**traverse_obj(data, {
'display_id': 'slug',
'title': 'title',
'track': 'title',
'artist': ('artist', 'name'),
'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n') or None}),
'thumbnails': ('images', ..., ..., {
'url': ('source', {url_or_none}),
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
}),
'genre': ('genres', ..., 'displayValue'),
'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
}),
}
class CrunchyrollArtistIE(CrunchyrollBaseIE):
IE_NAME = 'crunchyroll:artist'
_VALID_URL = r'''(?x)
https?://(?:www\.)?crunchyroll\.com/
(?P<lang>(?:\w{2}(?:-\w{2})?/)?)
artist/(?P<id>\w{10})'''
_TESTS = [{
'url': 'https://www.crunchyroll.com/artist/MA179CB50D',
'info_dict': {
'id': 'MA179CB50D',
'title': 'LiSA',
'genre': ['J-Pop', 'Anime', 'Rock'],
'description': 'md5:16d87de61a55c3f7d6c454b73285938e',
},
'playlist_mincount': 83,
}, {
'url': 'https://www.crunchyroll.com/artist/MA179CB50D/lisa',
'only_matching': True,
}]
_API_ENDPOINT = 'music'
def _real_extract(self, url):
lang, internal_id = self._match_valid_url(url).group('lang', 'id')
response = traverse_obj(self._call_api(
f'artists/{internal_id}', internal_id, lang, 'artist info'), ('data', 0))
def entries():
for attribute, path in [('concerts', 'concert'), ('videos', 'musicvideo')]:
for internal_id in traverse_obj(response, (attribute, ...)):
yield self.url_result(f'{self._BASE_URL}/watch/{path}/{internal_id}', CrunchyrollMusicIE, internal_id)
return self.playlist_result(entries(), **self._transform_artist_response(response))
@staticmethod
def _transform_artist_response(data):
return {
'id': data['id'],
**traverse_obj(data, {
'title': 'name',
'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
'thumbnails': ('images', ..., ..., {
'url': ('source', {url_or_none}),
'width': ('width', {int_or_none}),
'height': ('height', {int_or_none}),
}),
'genre': ('genres', ..., 'displayValue'),
}),
}
yt_dlp/extractor/dacast.py
+158
View File
@@ -0,0 +1,158 @@
import hashlib
import re
import time
import urllib.error
from .common import InfoExtractor
from ..utils import (
ExtractorError,
classproperty,
float_or_none,
traverse_obj,
url_or_none,
)
class DacastBaseIE(InfoExtractor):
_URL_TYPE = None
@classproperty
def _VALID_URL(cls):
return fr'https?://iframe\.dacast\.com/{cls._URL_TYPE}/(?P<user_id>[\w-]+)/(?P<id>[\w-]+)'
@classproperty
def _EMBED_REGEX(cls):
return [rf'<iframe[^>]+\bsrc=["\'](?P<url>{cls._VALID_URL})']
_API_INFO_URL = 'https://playback.dacast.com/content/info'
@classmethod
def _get_url_from_id(cls, content_id):
user_id, media_id = content_id.split(f'-{cls._URL_TYPE}-')
return f'https://iframe.dacast.com/{cls._URL_TYPE}/{user_id}/{media_id}'
@classmethod
def _extract_embed_urls(cls, url, webpage):
yield from super()._extract_embed_urls(url, webpage)
for content_id in re.findall(
rf'<script[^>]+\bsrc=["\']https://player\.dacast\.com/js/player\.js\?contentId=([\w-]+-{cls._URL_TYPE}-[\w-]+)["\']', webpage):
yield cls._get_url_from_id(content_id)
class DacastVODIE(DacastBaseIE):
_URL_TYPE = 'vod'
_TESTS = [{
'url': 'https://iframe.dacast.com/vod/acae82153ef4d7a7344ae4eaa86af534/1c6143e3-5a06-371d-8695-19b96ea49090',
'info_dict': {
'id': '1c6143e3-5a06-371d-8695-19b96ea49090',
'ext': 'mp4',
'uploader_id': 'acae82153ef4d7a7344ae4eaa86af534',
'title': '2_4||Adnexal mass characterisation: O-RADS US and MRI||N. Bharwani, London/UK',
'thumbnail': 'https://universe-files.dacast.com/26137208-5858-65c1-5e9a-9d6b6bd2b6c2',
},
'params': {'skip_download': 'm3u8'},
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.dacast.com/support/knowledgebase/how-can-i-embed-a-video-on-my-website/',
'info_dict': {
'id': 'b6674869-f08a-23c5-1d7b-81f5309e1a90',
'ext': 'mp4',
'title': '4-HowToEmbedVideo.mp4',
'uploader_id': '3b67c4a9-3886-4eb1-d0eb-39b23b14bef3',
'thumbnail': 'https://universe-files.dacast.com/d26ab48f-a52a-8783-c42e-a90290ba06b6.png',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://gist.githubusercontent.com/bashonly/4ad249ef2910346fbdf3809b220f11ee/raw/87349778d4af1a80b1fcc3beb9c88108de5858f5/dacast_embeds.html',
'info_dict': {
'id': 'e7df418e-a83b-7a7f-7b5e-1a667981e8fa',
'ext': 'mp4',
'title': 'Evening Service 2-5-23',
'uploader_id': '943bb1ab3c03695ba85330d92d6d226e',
'thumbnail': 'https://universe-files.dacast.com/337472b3-e92c-2ea4-7eb7-5700da477f67',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
query = {'contentId': f'{user_id}-vod-{video_id}', 'provider': 'universe'}
info = self._download_json(self._API_INFO_URL, video_id, query=query, fatal=False)
access = self._download_json(
'https://playback.dacast.com/content/access', video_id,
note='Downloading access JSON', query=query, expected_status=403)
error = access.get('error')
if error in ('Broadcaster has been blocked', 'Content is offline'):
raise ExtractorError(error, expected=True)
elif error:
raise ExtractorError(f'Dacast API says "{error}"')
hls_url = access['hls']
hls_aes = {}
if 'DRM_EXT' in hls_url:
self.report_drm(video_id)
elif '/uspaes/' in hls_url:
# From https://player.dacast.com/js/player.js
ts = int(time.time())
signature = hashlib.sha1(
f'{10413792000 - ts}{ts}YfaKtquEEpDeusCKbvYszIEZnWmBcSvw').digest().hex()
hls_aes['uri'] = f'https://keys.dacast.com/uspaes/{video_id}.key?s={signature}&ts={ts}'
for retry in self.RetryManager():
try:
formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
except ExtractorError as e:
# CDN will randomly respond with 403
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
retry.error = e
continue
raise
return {
'id': video_id,
'uploader_id': user_id,
'formats': formats,
'hls_aes': hls_aes or None,
**traverse_obj(info, ('contentInfo', {
'title': 'title',
'duration': ('duration', {float_or_none}),
'thumbnail': ('thumbnailUrl', {url_or_none}),
})),
}
class DacastPlaylistIE(DacastBaseIE):
_URL_TYPE = 'playlist'
_TESTS = [{
'url': 'https://iframe.dacast.com/playlist/943bb1ab3c03695ba85330d92d6d226e/b632eb053cac17a9c9a02bcfc827f2d8',
'playlist_mincount': 28,
'info_dict': {
'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
'title': 'Archive Sermons',
},
}]
_WEBPAGE_TESTS = [{
'url': 'https://gist.githubusercontent.com/bashonly/7efb606f49f3c6e07ea0327de5a661d1/raw/05a16eac830245ea301fb0a585023bec71e6093c/dacast_playlist_embed.html',
'playlist_mincount': 28,
'info_dict': {
'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
'title': 'Archive Sermons',
},
}]
def _real_extract(self, url):
user_id, playlist_id = self._match_valid_url(url).group('user_id', 'id')
info = self._download_json(
self._API_INFO_URL, playlist_id, note='Downloading playlist JSON', query={
'contentId': f'{user_id}-playlist-{playlist_id}',
'provider': 'universe',
})['contentInfo']
def entries(info):
for video in traverse_obj(info, ('features', 'playlist', 'contents', lambda _, v: v['id'])):
yield self.url_result(
DacastVODIE._get_url_from_id(video['id']), DacastVODIE, video['id'], video.get('title'))
return self.playlist_result(entries(info), playlist_id, info.get('title'))
yt_dlp/extractor/daftsex.py
+18 -9
View File
@@ -1,6 +1,7 @@
from .common import InfoExtractor
from ..compat import compat_b64decode
from ..utils import (
ExtractorError,
int_or_none,
js_to_json,
parse_count,
@@ -12,21 +13,24 @@ from ..utils import (
class DaftsexIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?daftsex\.com/watch/(?P<id>-?\d+_\d+)'
_VALID_URL = r'https?://(?:www\.)?daft\.sex/watch/(?P<id>-?\d+_\d+)'
_TESTS = [{
'url': 'https://daftsex.com/watch/-35370899_456246186',
'md5': 'd95135e6cea2d905bea20dbe82cda64a',
'url': 'https://daft.sex/watch/-35370899_456246186',
'md5': '64c04ef7b4c7b04b308f3b0c78efe7cd',
'info_dict': {
'id': '-35370899_456246186',
'ext': 'mp4',
'title': 'just relaxing',
'description': 'just relaxing - Watch video Watch video in high quality',
'description': 'just relaxing – Watch video Watch video in high quality',
'upload_date': '20201113',
'timestamp': 1605261911,
'thumbnail': r're:https://[^/]+/impf/-43BuMDIawmBGr3GLcZ93CYwWf2PBv_tVWoS1A/dnu41DnARU4\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=6af2c26ff4a45e55334189301c867384&type=video_thumb',
'thumbnail': r're:^https?://.*\.jpg$',
'age_limit': 18,
'duration': 15.0,
'view_count': int
},
}, {
'url': 'https://daftsex.com/watch/-156601359_456242791',
'url': 'https://daft.sex/watch/-156601359_456242791',
'info_dict': {
'id': '-156601359_456242791',
'ext': 'mp4',
@@ -36,6 +40,7 @@ class DaftsexIE(InfoExtractor):
'timestamp': 1600250735,
'thumbnail': 'https://psv153-1.crazycloud.ru/videos/-156601359/456242791/thumb.jpg?extra=i3D32KaBbBFf9TqDRMAVmQ',
},
'skip': 'deleted / private'
}]
def _real_extract(self, url):
@@ -60,7 +65,7 @@ class DaftsexIE(InfoExtractor):
webpage, 'player color', fatal=False) or ''
embed_page = self._download_webpage(
'https://daxab.com/player/%s?color=%s' % (player_hash, player_color),
'https://dxb.to/player/%s?color=%s' % (player_hash, player_color),
video_id, headers={'Referer': url})
video_params = self._parse_json(
self._search_regex(
@@ -94,15 +99,19 @@ class DaftsexIE(InfoExtractor):
'age_limit': 18,
}
item = self._download_json(
items = self._download_json(
f'{server_domain}/method/video.get/{video_id}', video_id,
headers={'Referer': url}, query={
'token': video_params['video']['access_token'],
'videos': video_id,
'ckey': video_params['c_key'],
'credentials': video_params['video']['credentials'],
})['response']['items'][0]
})['response']['items']
if not items:
raise ExtractorError('Video is not available', video_id=video_id, expected=True)
item = items[0]
formats = []
for f_id, f_url in item.get('files', {}).items():
if f_id == 'external':
yt_dlp/extractor/digitalconcerthall.py
+20 -7
View File
@@ -11,7 +11,7 @@ from ..utils import (
class DigitalConcertHallIE(InfoExtractor):
IE_DESC = 'DigitalConcertHall extractor'
_VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/concert/(?P<id>[0-9]+)'
_VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/(?P<type>film|concert)/(?P<id>[0-9]+)'
_OAUTH_URL = 'https://api.digitalconcerthall.com/v2/oauth2/token'
_ACCESS_TOKEN = None
_NETRC_MACHINE = 'digitalconcerthall'
@@ -40,6 +40,19 @@ class DigitalConcertHallIE(InfoExtractor):
},
'params': {'skip_download': 'm3u8'},
'playlist_count': 3,
}, {
'url': 'https://www.digitalconcerthall.com/en/film/388',
'info_dict': {
'id': '388',
'ext': 'mp4',
'title': 'The Berliner Philharmoniker and Frank Peter Zimmermann',
'description': 'md5:cfe25a7044fa4be13743e5089b5b5eb2',
'thumbnail': r're:^https?://images.digitalconcerthall.com/cms/thumbnails.*\.jpg$',
'upload_date': '20220714',
'timestamp': 1657785600,
'album_artist': 'Frank Peter Zimmermann / Benedikt von Bernstorff / Jakob von Bernstorff',
},
'params': {'skip_download': 'm3u8'},
}]
def _perform_login(self, username, password):
@@ -75,7 +88,7 @@ class DigitalConcertHallIE(InfoExtractor):
if not self._ACCESS_TOKEN:
self.raise_login_required(method='password')
def _entries(self, items, language, **kwargs):
def _entries(self, items, language, type_, **kwargs):
for item in items:
video_id = item['id']
stream_info = self._download_json(
@@ -103,11 +116,11 @@ class DigitalConcertHallIE(InfoExtractor):
'start_time': chapter.get('time'),
'end_time': try_get(chapter, lambda x: x['time'] + x['duration']),
'title': chapter.get('text'),
} for chapter in item['cuepoints']] if item.get('cuepoints') else None,
} for chapter in item['cuepoints']] if item.get('cuepoints') and type_ == 'concert' else None,
}
def _real_extract(self, url):
language, video_id = self._match_valid_url(url).group('language', 'id')
language, type_, video_id = self._match_valid_url(url).group('language', 'type', 'id')
if not language:
language = 'en'
@@ -120,18 +133,18 @@ class DigitalConcertHallIE(InfoExtractor):
}]
vid_info = self._download_json(
f'https://api.digitalconcerthall.com/v2/concert/{video_id}', video_id, headers={
f'https://api.digitalconcerthall.com/v2/{type_}/{video_id}', video_id, headers={
'Accept': 'application/json',
'Accept-Language': language
})
album_artist = ' / '.join(traverse_obj(vid_info, ('_links', 'artist', ..., 'name')) or '')
videos = [vid_info] if type_ == 'film' else traverse_obj(vid_info, ('_embedded', ..., ...))
return {
'_type': 'playlist',
'id': video_id,
'title': vid_info.get('title'),
'entries': self._entries(traverse_obj(vid_info, ('_embedded', ..., ...)), language,
thumbnails=thumbnails, album_artist=album_artist),
'entries': self._entries(videos, language, thumbnails=thumbnails, album_artist=album_artist, type_=type_),
'thumbnails': thumbnails,
'album_artist': album_artist,
}
yt_dlp/extractor/elevensports.py
+59
View File
@@ -0,0 +1,59 @@
from .common import InfoExtractor
from ..utils import (
parse_iso8601,
traverse_obj,
url_or_none,
)
class ElevenSportsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?elevensports\.com/view/event/(?P<id>\w+)'
_TESTS = [{
'url': 'https://elevensports.com/view/event/clf46yr3kenn80jgrqsjmwefk',
'md5': 'c0958d9ff90e4503a75544358758921d',
'info_dict': {
'id': 'clf46yr3kenn80jgrqsjmwefk',
'title': 'Cleveland SC vs Lionsbridge FC',
'ext': 'mp4',
'description': 'md5:03b5238d6549f4ea1fddadf69b5e0b58',
'upload_date': '20230323',
'timestamp': 1679612400,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
},
'params': {'skip_download': 'm3u8'}
}, {
'url': 'https://elevensports.com/view/event/clhpyd53b06160jez74qhgkmf',
'md5': 'c0958d9ff90e4503a75544358758921d',
'info_dict': {
'id': 'clhpyd53b06160jez74qhgkmf',
'title': 'AJNLF vs ARRAF',
'ext': 'mp4',
'description': 'md5:c8c5e75c78f37c6d15cd6c475e43a8c1',
'upload_date': '20230521',
'timestamp': 1684684800,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
},
'params': {'skip_download': 'm3u8'}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
event_id = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['event']['mclsEventId']
event_data = self._download_json(
f'https://mcls-api.mycujoo.tv/bff/events/v1beta1/{event_id}', video_id,
headers={'Authorization': 'Bearer FBVKACGN37JQC5SFA0OVK8KKSIOP153G'})
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
event_data['streams'][0]['full_url'], video_id, 'mp4', m3u8_id='hls')
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
**traverse_obj(event_data, {
'title': ('title', {str}),
'description': ('description', {str}),
'timestamp': ('start_time', {parse_iso8601}),
'thumbnail': ('thumbnail_url', {url_or_none}),
}),
}
yt_dlp/extractor/europa.py
+41 -41
View File
@@ -6,6 +6,7 @@ from ..utils import (
parse_iso8601,
parse_qs,
qualities,
traverse_obj,
unified_strdate,
xpath_text
)
@@ -92,42 +93,17 @@ class EuropaIE(InfoExtractor):
class EuroParlWebstreamIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://(?:multimedia|webstreaming)\.europarl\.europa\.eu/[^/#?]+/
(?:embed/embed\.html\?event=|(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
https?://multimedia\.europarl\.europa\.eu/[^/#?]+/
(?:(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
'''
_TESTS = [{
'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
'info_dict': {
'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
'ext': 'mp4',
'release_timestamp': 1663137900,
'title': 'Plenary session',
'release_date': '20220914',
},
'params': {
'skip_download': True,
}
}, {
'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/eu-cop27-un-climate-change-conference-in-sharm-el-sheikh-egypt-ep-delegation-meets-with-ngo-represen_20221114-1600-SPECIAL-OTHER',
'info_dict': {
'id': 'a8428de8-b9cd-6a2e-11e4-3805d9c9ff5c',
'ext': 'mp4',
'release_timestamp': 1668434400,
'release_date': '20221114',
'title': 'md5:d3550280c33cc70e0678652e3d52c028',
},
'params': {
'skip_download': True,
}
}, {
# embed webpage
'url': 'https://webstreaming.europarl.europa.eu/ep/embed/embed.html?event=20220914-0900-PLENARY&language=en&autoplay=true&logo=true',
'info_dict': {
'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
'ext': 'mp4',
'title': 'Plenary session',
'release_timestamp': 1663139069,
'release_date': '20220914',
'release_timestamp': 1663137900,
},
'params': {
'skip_download': True,
@@ -144,30 +120,54 @@ class EuroParlWebstreamIE(InfoExtractor):
'live_status': 'is_live',
},
'skip': 'not live anymore'
}, {
'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
'info_dict': {
'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
'ext': 'mp4',
'release_date': '20230301',
'title': 'Committee on Culture and Education',
'release_timestamp': 1677666641,
}
}, {
# live stream
'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
'info_dict': {
'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
'ext': 'mp4',
'release_date': '20230524',
'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
'release_timestamp': 1684911541,
'live_status': 'is_live',
},
'skip': 'Not live anymore'
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
json_info = self._download_json(
'https://vis-api.vuplay.co.uk/event/external', display_id,
'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
query={
'player_key': 'europarl|718f822c-a48c-4841-9947-c9cb9bb1743c',
'external_id': display_id,
'api-version': 1.0,
'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
'externalReference': display_id
})
formats, subtitles = self._extract_mpd_formats_and_subtitles(json_info['streaming_url'], display_id)
fmts, subs = self._extract_m3u8_formats_and_subtitles(
json_info['streaming_url'].replace('.mpd', '.m3u8'), display_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
formats, subtitles = [], {}
for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
formats.extend(fmt)
self._merge_subtitles(subs, target=subtitles)
return {
'id': json_info['id'],
'title': json_info.get('title'),
'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
'formats': formats,
'subtitles': subtitles,
'release_timestamp': parse_iso8601(json_info.get('published_start')),
'is_live': 'LIVE' in json_info.get('state', '')
'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live'
}
yt_dlp/extractor/eurosport.py
+27 -1
View File
@@ -3,7 +3,7 @@ from ..utils import traverse_obj
class EurosportIE(InfoExtractor):
_VALID_URL = r'https?://www\.eurosport\.com/\w+/[\w-]+/\d+/[\w-]+_(?P<id>vid\d+)'
_VALID_URL = r'https?://www\.eurosport\.com/\w+/(?:[\w-]+/[\d-]+/)?[\w-]+_(?P<id>vid\d+)'
_TESTS = [{
'url': 'https://www.eurosport.com/tennis/roland-garros/2022/highlights-rafael-nadal-brushes-aside-caper-ruud-to-win-record-extending-14th-french-open-title_vid1694147/video.shtml',
'info_dict': {
@@ -44,6 +44,32 @@ class EurosportIE(InfoExtractor):
'description': 'md5:32bbe3a773ac132c57fb1e8cca4b7c71',
'upload_date': '20220727',
}
}, {
'url': 'https://www.eurosport.com/football/champions-league/2022-2023/pep-guardiola-emotionally-destroyed-after-manchester-city-win-over-bayern-munich-in-champions-league_vid1896254/video.shtml',
'info_dict': {
'id': '3096477',
'ext': 'mp4',
'title': 'md5:82edc17370124c7a19b3cf518517583b',
'duration': 84.0,
'description': 'md5:b3f44ef7f5b5b95b24a273b163083feb',
'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/04/12/3682873-74947393-2560-1440.jpg',
'timestamp': 1681292028,
'upload_date': '20230412',
'display_id': 'vid1896254',
}
}, {
'url': 'https://www.eurosport.com/football/last-year-s-semi-final-pain-was-still-there-pep-guardiola-after-man-city-reach-cl-final_vid1914115/video.shtml',
'info_dict': {
'id': '3149108',
'ext': 'mp4',
'title': '\'Last year\'s semi-final pain was still there\' - Pep Guardiola after Man City reach CL final',
'description': 'md5:89ef142fe0170a66abab77fac2955d8e',
'display_id': 'vid1914115',
'timestamp': 1684403618,
'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/05/18/3707254-75435008-2560-1440.jpg',
'duration': 105.0,
'upload_date': '20230518',
}
}]
_TOKEN = None
yt_dlp/extractor/facebook.py
+9 -5
View File
@@ -390,7 +390,10 @@ class FacebookIE(InfoExtractor):
k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
title = get_first(media, ('title', 'text'))
description = get_first(media, ('creation_story', 'comet_sections', 'message', 'story', 'message', 'text'))
uploader_data = get_first(media, 'owner') or get_first(post, ('node', 'actors', ...)) or {}
uploader_data = (
get_first(media, ('owner', {dict}))
or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
or get_first(post, ('node', 'actors', ..., {dict})) or {})
page_title = title or self._html_search_regex((
r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
@@ -415,16 +418,17 @@ class FacebookIE(InfoExtractor):
# in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
if thumbnail and not re.search(r'\.(?:jpg|png)', thumbnail):
thumbnail = None
view_count = parse_count(self._search_regex(
r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
default=None))
info_dict = {
'description': description,
'uploader': uploader,
'uploader_id': uploader_data.get('id'),
'timestamp': timestamp,
'thumbnail': thumbnail,
'view_count': view_count,
'view_count': parse_count(self._search_regex(
(r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)',),
webpage, 'view count', default=None)),
'concurrent_view_count': get_first(post, (
('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
}
info_json_ld = self._search_json_ld(webpage, video_id, default={})
yt_dlp/extractor/idolplus.py
+115
View File
@@ -0,0 +1,115 @@
from .common import InfoExtractor
from ..utils import traverse_obj, try_call, url_or_none
class IdolPlusIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?idolplus\.com/z[us]/(?:concert/|contents/?\?(?:[^#]+&)?albumId=)(?P<id>\w+)'
_TESTS = [{
'url': 'https://idolplus.com/zs/contents?albumId=M012077298PPV00',
'md5': '2ace3f4661c943a2f7e79f0b88cea1e7',
'info_dict': {
'id': 'M012077298PPV00',
'ext': 'mp4',
'title': '[MultiCam] Aegyo on Top of Aegyo (IZ*ONE EATING TRIP)',
'release_date': '20200707',
'formats': 'count:65',
},
'params': {'format': '532-KIM_MINJU'},
}, {
'url': 'https://idolplus.com/zs/contents?albumId=M01232H058PPV00&catId=E9TX5',
'info_dict': {
'id': 'M01232H058PPV00',
'ext': 'mp4',
'title': 'YENA (CIRCLE CHART MUSIC AWARDS 2022 RED CARPET)',
'release_date': '20230218',
'formats': 'count:5',
},
'params': {'skip_download': 'm3u8'},
}, {
# live stream
'url': 'https://idolplus.com/zu/contents?albumId=M012323174PPV00',
'info_dict': {
'id': 'M012323174PPV00',
'ext': 'mp4',
'title': 'Hanteo Music Awards 2022 DAY2',
'release_date': '20230211',
'formats': 'count:5',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://idolplus.com/zs/concert/M012323039PPV00',
'info_dict': {
'id': 'M012323039PPV00',
'ext': 'mp4',
'title': 'CIRCLE CHART MUSIC AWARDS 2022',
'release_date': '20230218',
'formats': 'count:5',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
data_list = traverse_obj(self._download_json(
'https://idolplus.com/api/zs/viewdata/ruleset/build', video_id,
headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
'rulesetId': 'contents',
'albumId': video_id,
'distribute': 'PRD',
'loggedIn': 'false',
'region': 'zs',
'countryGroup': '00010',
'lang': 'en',
'saId': '999999999998',
}), ('data', 'viewData', ...))
player_data = {}
while data_list:
player_data = data_list.pop()
if traverse_obj(player_data, 'type') == 'player':
break
elif traverse_obj(player_data, ('dataList', ...)):
data_list += player_data['dataList']
formats = self._extract_m3u8_formats(traverse_obj(player_data, (
'vodPlayerList', 'vodProfile', 0, 'vodServer', 0, 'video_url', {url_or_none})), video_id)
subtitles = {}
for caption in traverse_obj(player_data, ('vodPlayerList', 'caption')) or []:
subtitles.setdefault(caption.get('lang') or 'und', []).append({
'url': caption.get('smi_url'),
'ext': 'vtt',
})
# Add member multicams as alternative formats
if (traverse_obj(player_data, ('detail', 'has_cuesheet')) == 'Y'
and traverse_obj(player_data, ('detail', 'is_omni_member')) == 'Y'):
cuesheet = traverse_obj(self._download_json(
'https://idolplus.com/gapi/contents/v1.0/content/cuesheet', video_id,
'Downloading JSON metadata for member multicams',
headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
'ALBUM_ID': video_id,
'COUNTRY_GRP': '00010',
'LANG': 'en',
'SA_ID': '999999999998',
'COUNTRY_CODE': 'KR',
}), ('data', 'cuesheet_item', 0))
for member in traverse_obj(cuesheet, ('members', ...)):
index = try_call(lambda: int(member['omni_view_index']) - 1)
member_video_url = traverse_obj(cuesheet, ('omni_view', index, 'cdn_url', 0, 'url', {url_or_none}))
if not member_video_url:
continue
member_formats = self._extract_m3u8_formats(
member_video_url, video_id, note=f'Downloading m3u8 for multicam {member["name"]}')
for mf in member_formats:
mf['format_id'] = f'{mf["format_id"]}-{member["name"].replace(" ", "_")}'
formats.extend(member_formats)
return {
'id': video_id,
'title': traverse_obj(player_data, ('detail', 'albumName')),
'formats': formats,
'subtitles': subtitles,
'release_date': traverse_obj(player_data, ('detail', 'broadcastDate')),
}
yt_dlp/extractor/iwara.py
+83 -5
View File
@@ -1,6 +1,7 @@
import functools
import urllib.parse
import hashlib
import json
from .common import InfoExtractor
from ..utils import (
@@ -14,7 +15,49 @@ from ..utils import (
)
class IwaraIE(InfoExtractor):
# https://github.com/yt-dlp/yt-dlp/issues/6671
class IwaraBaseIE(InfoExtractor):
_USERTOKEN = None
_MEDIATOKEN = None
_NETRC_MACHINE = 'iwara'
def _get_user_token(self, invalidate=False):
if not invalidate and self._USERTOKEN:
return self._USERTOKEN
username, password = self._get_login_info()
IwaraBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
if not IwaraBaseIE._USERTOKEN or invalidate:
IwaraBaseIE._USERTOKEN = self._download_json(
'https://api.iwara.tv/user/login', None, note='Logging in',
data=json.dumps({
'email': username,
'password': password
}).encode('utf-8'),
headers={
'Content-Type': 'application/json'
})['token']
self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
return self._USERTOKEN
def _get_media_token(self, invalidate=False):
if not invalidate and self._MEDIATOKEN:
return self._MEDIATOKEN
IwaraBaseIE._MEDIATOKEN = self._download_json(
'https://api.iwara.tv/user/token', None, note='Fetching media token',
data=b'', # Need to have some data here, even if it's empty
headers={
'Authorization': f'Bearer {self._get_user_token()}',
'Content-Type': 'application/json'
})['accessToken']
return self._MEDIATOKEN
class IwaraIE(IwaraBaseIE):
IE_NAME = 'iwara'
_VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos?/(?P<id>[a-zA-Z0-9]+)'
_TESTS = [{
@@ -56,6 +99,26 @@ class IwaraIE(InfoExtractor):
'timestamp': 1678732213,
'modified_timestamp': 1679110271,
},
}, {
'url': 'https://iwara.tv/video/blggmfno8ghl725bg',
'info_dict': {
'id': 'blggmfno8ghl725bg',
'ext': 'mp4',
'age_limit': 18,
'title': 'お外でおしっこしちゃう猫耳ロリメイド',
'description': 'md5:0342ba9bf6db09edbbb28729657c3611',
'uploader': 'Fe_Kurosabi',
'uploader_id': 'fekurosabi',
'tags': [
'pee'
],
'like_count': 192,
'view_count': 12119,
'comment_count': 0,
'timestamp': 1598880567,
'modified_timestamp': 1598908995,
'availability': 'needs_auth',
},
}]
def _extract_formats(self, video_id, fileurl):
@@ -79,12 +142,18 @@ class IwaraIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
username, password = self._get_login_info()
headers = {
'Authorization': f'Bearer {self._get_media_token()}',
} if username and password else None
video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True, headers=headers)
errmsg = video_data.get('message')
# at this point we can actually get uploaded user info, but do we need it?
if errmsg == 'errors.privateVideo':
self.raise_login_required('Private video. Login if you have permissions to watch')
elif errmsg:
elif errmsg == 'errors.notFound' and not username:
self.raise_login_required('Video may need login to view')
elif errmsg: # None if success
raise ExtractorError(f'Iwara says: {errmsg}')
if not video_data.get('fileUrl'):
@@ -112,8 +181,17 @@ class IwaraIE(InfoExtractor):
'formats': list(self._extract_formats(video_id, video_data.get('fileUrl'))),
}
def _perform_login(self, username, password):
if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
self.write_debug('Skipping logging in')
return
class IwaraUserIE(InfoExtractor):
IwaraBaseIE._USERTOKEN = self._get_user_token(True)
self._get_media_token(True)
self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
class IwaraUserIE(IwaraBaseIE):
_VALID_URL = r'https?://(?:www\.)?iwara\.tv/profile/(?P<id>[^/?#&]+)'
IE_NAME = 'iwara:user'
_PER_PAGE = 32
@@ -165,7 +243,7 @@ class IwaraUserIE(InfoExtractor):
playlist_id, traverse_obj(user_info, ('user', 'name')))
class IwaraPlaylistIE(InfoExtractor):
class IwaraPlaylistIE(IwaraBaseIE):
# the ID is an UUID but I don't think it's necessary to write concrete regex
_VALID_URL = r'https?://(?:www\.)?iwara\.tv/playlist/(?P<id>[0-9a-f-]+)'
IE_NAME = 'iwara:playlist'
yt_dlp/extractor/jstream.py
+73
View File
@@ -0,0 +1,73 @@
import base64
import re
import json
from .common import InfoExtractor
from ..utils import (
float_or_none,
js_to_json,
remove_start,
)
class JStreamIE(InfoExtractor):
# group "id" only exists for compliance, not directly used in requests
# also all components are mandatory
_VALID_URL = r'jstream:(?P<host>www\d+):(?P<id>(?P<publisher>[a-z0-9]+):(?P<mid>\d+))'
_TESTS = [{
'url': 'jstream:www50:eqd638pvwx:752',
'info_dict': {
'id': 'eqd638pvwx:752',
'ext': 'mp4',
'title': '阪神淡路大震災 激震の記録2020年版　解説動画',
'duration': 672,
'thumbnail': r're:https?://eqd638pvwx\.eq\.webcdn\.stream\.ne\.jp/.+\.jpg',
},
}]
def _parse_jsonp(self, callback, string, video_id):
return self._search_json(rf'\s*{re.escape(callback)}\s*\(', string, callback, video_id)
def _find_formats(self, video_id, movie_list_hls, host, publisher, subtitles):
for value in movie_list_hls:
text = value.get('text') or ''
if not text.startswith('auto'):
continue
m3u8_id = remove_start(remove_start(text, 'auto'), '_') or None
fmts, subs = self._extract_m3u8_formats_and_subtitles(
f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/{value.get("url")}', video_id, 'mp4', m3u8_id=m3u8_id)
self._merge_subtitles(subs, target=subtitles)
yield from fmts
def _real_extract(self, url):
host, publisher, mid, video_id = self._match_valid_url(url).group('host', 'publisher', 'mid', 'id')
video_info_jsonp = self._download_webpage(
f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/eq_meta/v1/{mid}.jsonp',
video_id, 'Requesting video info')
video_info = self._parse_jsonp('metaDataResult', video_info_jsonp, video_id)['movie']
subtitles = {}
formats = list(self._find_formats(video_id, video_info.get('movie_list_hls'), host, publisher, subtitles))
self._remove_duplicate_formats(formats)
return {
'id': video_id,
'title': video_info.get('title'),
'duration': float_or_none(video_info.get('duration')),
'thumbnail': video_info.get('thumbnail_url'),
'formats': formats,
'subtitles': subtitles,
}
@classmethod
def _extract_embed_urls(cls, url, webpage):
# check for eligiblity of webpage
# https://support.eq.stream.co.jp/hc/ja/articles/115008388147-%E3%83%97%E3%83%AC%E3%82%A4%E3%83%A4%E3%83%BCAPI%E3%81%AE%E3%82%B5%E3%83%B3%E3%83%97%E3%83%AB%E3%82%B3%E3%83%BC%E3%83%89
script_tag = re.search(r'<script\s*[^>]+?src="https://ssl-cache\.stream\.ne\.jp/(?P<host>www\d+)/(?P<publisher>[a-z0-9]+)/[^"]+?/if\.js"', webpage)
if not script_tag:
return
host, publisher = script_tag.groups()
for m in re.finditer(r'(?s)PlayerFactoryIF\.create\(\s*({[^\}]+?})\s*\)\s*;', webpage):
# TODO: using json.loads here as InfoExtractor._parse_json is not classmethod
info = json.loads(js_to_json(m.group(1)))
mid = base64.b64decode(info.get('m')).decode()
yield f'jstream:{host}:{publisher}:{mid}'
yt_dlp/extractor/ketnet.py
-70
View File
@@ -1,70 +0,0 @@
from .canvas import CanvasIE
from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import (
int_or_none,
parse_iso8601,
)
class KetnetIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
_TESTS = [{
'url': 'https://www.ketnet.be/kijken/n/nachtwacht/3/nachtwacht-s3a1-de-greystook',
'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
'info_dict': {
'id': 'pbs-pub-aef8b526-115e-4006-aa24-e59ff6c6ef6f$vid-ddb815bf-c8e7-467b-8879-6bad7a32cebd',
'ext': 'mp4',
'title': 'Nachtwacht - Reeks 3: Aflevering 1',
'description': 'De Nachtwacht krijgt te maken met een parasiet',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1468.02,
'timestamp': 1609225200,
'upload_date': '20201229',
'series': 'Nachtwacht',
'season': 'Reeks 3',
'episode': 'De Greystook',
'episode_number': 1,
},
'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
}, {
'url': 'https://www.ketnet.be/themas/karrewiet/jaaroverzicht-20200/karrewiet-het-jaar-van-black-mamba',
'only_matching': True,
}]
def _real_extract(self, url):
display_id = self._match_id(url)
video = self._download_json(
'https://senior-bff.ketnet.be/graphql', display_id, query={
'query': '''{
video(id: "content/ketnet/nl/%s.model.json") {
description
episodeNr
imageUrl
mediaReference
programTitle
publicationDate
seasonTitle
subtitleVideodetail
titleVideodetail
}
}''' % display_id,
})['data']['video']
mz_id = compat_urllib_parse_unquote(video['mediaReference'])
return {
'_type': 'url_transparent',
'id': mz_id,
'title': video['titleVideodetail'],
'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/' + mz_id,
'thumbnail': video.get('imageUrl'),
'description': video.get('description'),
'timestamp': parse_iso8601(video.get('publicationDate')),
'series': video.get('programTitle'),
'season': video.get('seasonTitle'),
'episode': video.get('subtitleVideodetail'),
'episode_number': int_or_none(video.get('episodeNr')),
'ie_key': CanvasIE.ie_key(),
}
yt_dlp/extractor/litv.py
+2 -2
View File
@@ -4,8 +4,8 @@ from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
traverse_obj,
smuggle_url,
traverse_obj,
unsmuggle_url,
)
@@ -113,7 +113,7 @@ class LiTVIE(InfoExtractor):
entry_protocol='m3u8_native', m3u8_id='hls')
for a_format in formats:
# LiTV HLS segments doesn't like compressions
a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True
a_format.setdefault('http_headers', {})['Accept-Encoding'] = 'identity'
title = program_info['title'] + program_info.get('secondaryMark', '')
description = program_info.get('description')
yt_dlp/extractor/livestream.py
+60 -36
View File
@@ -1,33 +1,36 @@
import re
import itertools
import re
from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urlparse,
)
from ..compat import compat_str, compat_urlparse
from ..utils import (
find_xpath_attr,
xpath_attr,
xpath_with_ns,
xpath_text,
orderedSet,
update_url_query,
int_or_none,
float_or_none,
parse_iso8601,
determine_ext,
find_xpath_attr,
float_or_none,
int_or_none,
orderedSet,
parse_iso8601,
traverse_obj,
update_url_query,
xpath_attr,
xpath_text,
xpath_with_ns,
)
class LivestreamIE(InfoExtractor):
IE_NAME = 'livestream'
_VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
_VALID_URL = r'''(?x)
https?://(?:new\.)?livestream\.com/
(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))
(?:/events/(?P<event_id>\d+)|/(?P<event_name>[^/]+))?
(?:/videos/(?P<id>\d+))?
'''
_EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"']
_TESTS = [{
'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
'md5': '53274c76ba7754fb0e8d072716f2292b',
'md5': '7876c5f5dc3e711b6b73acce4aac1527',
'info_dict': {
'id': '4719370',
'ext': 'mp4',
@@ -37,22 +40,37 @@ class LivestreamIE(InfoExtractor):
'duration': 5968.0,
'like_count': int,
'view_count': int,
'comment_count': int,
'thumbnail': r're:^http://.*\.jpg$'
}
}, {
'url': 'http://new.livestream.com/tedx/cityenglish',
'url': 'https://livestream.com/coheedandcambria/websterhall',
'info_dict': {
'title': 'TEDCity2.0 (English)',
'id': '2245590',
'id': '1585861',
'title': 'Live From Webster Hall'
},
'playlist_mincount': 1,
}, {
'url': 'https://livestream.com/dayananda/events/7954027',
'info_dict': {
'title': 'Live from Mevo',
'id': '7954027',
},
'playlist_mincount': 4,
}, {
'url': 'http://new.livestream.com/chess24/tatasteelchess',
'url': 'https://livestream.com/accounts/82',
'info_dict': {
'title': 'Tata Steel Chess',
'id': '3705884',
},
'playlist_mincount': 60,
'id': '253978',
'view_count': int,
'title': 'trsr',
'comment_count': int,
'like_count': int,
'upload_date': '20120306',
'timestamp': 1331042383,
'thumbnail': 'http://img.new.livestream.com/videos/0000000000000372/cacbeed6-fb68-4b5e-ad9c-e148124e68a9_640x427.jpg',
'duration': 15.332,
'ext': 'mp4'
}
}, {
'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
'only_matching': True,
@@ -179,7 +197,7 @@ class LivestreamIE(InfoExtractor):
'is_live': is_live,
}
def _extract_event(self, event_data):
def _generate_event_playlist(self, event_data):
event_id = compat_str(event_data['id'])
account_id = compat_str(event_data['owner_account_id'])
feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
@@ -189,7 +207,6 @@ class LivestreamIE(InfoExtractor):
return self._extract_stream_info(stream_info)
last_video = None
entries = []
for i in itertools.count(1):
if last_video is None:
info_url = feed_root_url
@@ -197,31 +214,38 @@ class LivestreamIE(InfoExtractor):
info_url = '{root}?&id={id}&newer=-1&type=video'.format(
root=feed_root_url, id=last_video)
videos_info = self._download_json(
info_url, event_id, 'Downloading page {0}'.format(i))['data']
info_url, event_id, f'Downloading page {i}')['data']
videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
if not videos_info:
break
for v in videos_info:
v_id = compat_str(v['id'])
entries.append(self.url_result(
'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
'Livestream', v_id, v.get('caption')))
yield self.url_result(
f'http://livestream.com/accounts/{account_id}/events/{event_id}/videos/{v_id}',
LivestreamIE, v_id, v.get('caption'))
last_video = videos_info[-1]['id']
return self.playlist_result(entries, event_id, event_data['full_name'])
def _real_extract(self, url):
mobj = self._match_valid_url(url)
video_id = mobj.group('id')
event = mobj.group('event_id') or mobj.group('event_name')
account = mobj.group('account_id') or mobj.group('account_name')
api_url = self._API_URL_TEMPLATE % (account, event)
api_url = f'http://livestream.com/api/accounts/{account}'
if video_id:
video_data = self._download_json(
api_url + '/videos/%s' % video_id, video_id)
f'{api_url}/events/{event}/videos/{video_id}', video_id)
return self._extract_video_info(video_data)
else:
event_data = self._download_json(api_url, video_id)
return self._extract_event(event_data)
elif event:
event_data = self._download_json(f'{api_url}/events/{event}', None)
return self.playlist_result(
self._generate_event_playlist(event_data), str(event_data['id']), event_data['full_name'])
account_data = self._download_json(api_url, None)
items = traverse_obj(account_data, (('upcoming_events', 'past_events'), 'data', ...))
return self.playlist_result(
itertools.chain.from_iterable(map(self._generate_event_playlist, items)),
account_data.get('id'), account_data.get('full_name'))
# The original version of Livestream uses a different system
yt_dlp/extractor/mzaalo.py
+92
View File
@@ -0,0 +1,92 @@
from .common import InfoExtractor
from ..utils import (
parse_age_limit,
parse_duration,
traverse_obj,
url_or_none,
)
class MzaaloIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?mzaalo\.com/play/(?P<type>movie|original|clip)/(?P<id>[a-fA-F0-9-]+)/[\w-]+'
_TESTS = [{
# Movies
'url': 'https://www.mzaalo.com/play/movie/c0958d9f-f90e-4503-a755-44358758921d/Jamun',
'info_dict': {
'id': 'c0958d9f-f90e-4503-a755-44358758921d',
'title': 'Jamun',
'ext': 'mp4',
'description': 'md5:24fe9ebb9bbe5b36f7b54b90ab1e2f31',
'thumbnails': 'count:15',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 5527.0,
'language': 'hin',
'categories': ['Drama'],
'age_limit': 13,
},
'params': {'skip_download': 'm3u8'}
}, {
# Shows
'url': 'https://www.mzaalo.com/play/original/93d42b2b-f373-4c2d-bca4-997412cb069d/Modi-Season-2-CM-TO-PM/Episode-1:Decision,-Not-Promises',
'info_dict': {
'id': '93d42b2b-f373-4c2d-bca4-997412cb069d',
'title': 'Episode 1:Decision, Not Promises',
'ext': 'mp4',
'description': 'md5:16f76058432a54774fbb2561a1955652',
'thumbnails': 'count:22',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 2040.0,
'language': 'hin',
'categories': ['Drama'],
'age_limit': 13,
},
'params': {'skip_download': 'm3u8'}
}, {
# Streams/Clips
'url': 'https://www.mzaalo.com/play/clip/83cdbcb5-400a-42f1-a1d2-459053cfbda5/Manto-Ki-Kahaaniya',
'info_dict': {
'id': '83cdbcb5-400a-42f1-a1d2-459053cfbda5',
'title': 'Manto Ki Kahaaniya',
'ext': 'mp4',
'description': 'md5:c3c5f1d05f0fd1bfcb05b673d1cc9f2f',
'thumbnails': 'count:3',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 1937.0,
'language': 'hin',
},
'params': {'skip_download': 'm3u8'}
}]
def _real_extract(self, url):
video_id, type_ = self._match_valid_url(url).group('id', 'type')
path = (f'partner/streamurl?&assetId={video_id}&getClipDetails=YES' if type_ == 'clip'
else f'api/v2/player/details?assetType={type_.upper()}&assetId={video_id}')
data = self._download_json(
f'https://production.mzaalo.com/platform/{path}', video_id, headers={
'Ocp-Apim-Subscription-Key': '1d0caac2702049b89a305929fdf4cbae',
})['data']
formats = self._extract_m3u8_formats(data['streamURL'], video_id)
subtitles = {}
for subs_lang, subs_url in traverse_obj(data, ('subtitles', {dict.items}, ...)):
if url_or_none(subs_url):
subtitles[subs_lang] = [{'url': subs_url, 'ext': 'vtt'}]
lang = traverse_obj(data, ('language', {str.lower}))
for f in formats:
f['language'] = lang
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
**traverse_obj(data, {
'title': ('title', {str}),
'description': ('description', {str}),
'duration': ('duration', {parse_duration}),
'age_limit': ('maturity_rating', {parse_age_limit}),
'thumbnails': ('images', ..., {'url': {url_or_none}}),
'categories': ('genre', ..., {str}),
}),
}
yt_dlp/extractor/naver.py
+1 -1
View File
@@ -21,7 +21,7 @@ from ..utils import (
class NaverBaseIE(InfoExtractor):
_CAPTION_EXT_RE = r'\.(?:ttml|vtt)'
@staticmethod # NB: Used in VLiveWebArchiveIE
@staticmethod # NB: Used in VLiveWebArchiveIE, WeverseIE
def process_subtitles(vod_data, process_url):
ret = {'subtitles': {}, 'automatic_captions': {}}
for caption in traverse_obj(vod_data, ('captions', 'list', ...)):
yt_dlp/extractor/nekohacker.py
+217
View File
@@ -0,0 +1,217 @@
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
determine_ext,
extract_attributes,
get_element_by_class,
get_element_text_and_html_by_tag,
parse_duration,
traverse_obj,
try_call,
url_or_none,
)
class NekoHackerIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
_TESTS = [{
'url': 'https://nekohacker.com/nekoverse/',
'info_dict': {
'id': 'nekoverse',
'title': 'Nekoverse',
},
'playlist': [
{
'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
'md5': '44223701ebedba0467ebda4cc07fb3aa',
'info_dict': {
'id': '1712',
'ext': 'mp3',
'title': 'Spaceship',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20221101',
'album': 'Nekoverse',
'artist': 'Neko Hacker',
'track': 'Spaceship',
'track_number': 1,
'duration': 195.0
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2022/11/02-City-Runner.mp3',
'md5': '8f853c71719389d32bbbd3f1a87b3f08',
'info_dict': {
'id': '1713',
'ext': 'mp3',
'title': 'City Runner',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20221101',
'album': 'Nekoverse',
'artist': 'Neko Hacker',
'track': 'City Runner',
'track_number': 2,
'duration': 148.0
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2022/11/03-Nature-Talk.mp3',
'md5': '5a8a8ae852720cee4c0ac95c7d1a7450',
'info_dict': {
'id': '1714',
'ext': 'mp3',
'title': 'Nature Talk',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20221101',
'album': 'Nekoverse',
'artist': 'Neko Hacker',
'track': 'Nature Talk',
'track_number': 3,
'duration': 174.0
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2022/11/04-Crystal-World.mp3',
'md5': 'd8e59a48061764e50d92386a294abd50',
'info_dict': {
'id': '1715',
'ext': 'mp3',
'title': 'Crystal World',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20221101',
'album': 'Nekoverse',
'artist': 'Neko Hacker',
'track': 'Crystal World',
'track_number': 4,
'duration': 199.0
}
}
]
}, {
'url': 'https://nekohacker.com/susume/',
'info_dict': {
'id': 'susume',
'title': '進め！むじなカンパニー',
},
'playlist': [
{
'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-feat.-六科なじむ-CV_-日高里菜-割戶真友-CV_-金元寿子-軽井沢ユキ-CV_-上坂すみれ-出稼ぎガルシア-CV_-金子彩花-.mp3',
'md5': 'fb13f008aa81f26ba48f91fd2d6186ce',
'info_dict': {
'id': '711',
'ext': 'mp3',
'title': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20210115',
'album': '進め！むじなカンパニー',
'artist': 'Neko Hacker',
'track': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
'track_number': 1,
'duration': None
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-feat.-六科なじむ-CV_-日高里菜-.mp3',
'md5': '028803f70241df512b7764e73396fdd1',
'info_dict': {
'id': '709',
'ext': 'mp3',
'title': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20210115',
'album': '進め！むじなカンパニー',
'artist': 'Neko Hacker',
'track': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
'track_number': 2,
'duration': None
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-instrumental.mp3',
'md5': 'adde9e9a16e1da5e602b579c247d0fb9',
'info_dict': {
'id': '710',
'ext': 'mp3',
'title': '進め！むじなカンパニー (instrumental)',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20210115',
'album': '進め！むじなカンパニー',
'artist': 'Neko Hacker',
'track': '進め！むじなカンパニー (instrumental)',
'track_number': 3,
'duration': None
}
},
{
'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-instrumental.mp3',
'md5': 'ebb0443039cf5f9ff7fd557ed9b23599',
'info_dict': {
'id': '712',
'ext': 'mp3',
'title': 'むじな de なじむ (instrumental)',
'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
'vcodec': 'none',
'acodec': 'mp3',
'release_date': '20210115',
'album': '進め！むじなカンパニー',
'artist': 'Neko Hacker',
'track': 'むじな de なじむ (instrumental)',
'track_number': 4,
'duration': None
}
}
]
}]
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
playlist = get_element_by_class('playlist', webpage)
if not playlist:
iframe = try_call(lambda: get_element_text_and_html_by_tag('iframe', webpage)[1]) or ''
iframe_src = url_or_none(extract_attributes(iframe).get('src'))
if not iframe_src:
raise ExtractorError('No playlist or embed found in webpage')
elif re.match(r'https?://(?:\w+\.)?spotify\.com/', iframe_src):
raise ExtractorError('Spotify embeds are not supported', expected=True)
return self.url_result(url, 'Generic')
entries = []
for track_number, track in enumerate(re.findall(r'(<li[^>]+data-audiopath[^>]+>)', playlist), 1):
entry = traverse_obj(extract_attributes(track), {
'url': ('data-audiopath', {url_or_none}),
'ext': ('data-audiopath', {determine_ext}),
'id': 'data-trackid',
'title': 'data-tracktitle',
'track': 'data-tracktitle',
'album': 'data-albumtitle',
'duration': ('data-tracktime', {parse_duration}),
'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
'thumbnail': ('data-albumart', {url_or_none}),
})
entries.append({
**entry,
'track_number': track_number,
'artist': 'Neko Hacker',
'vcodec': 'none',
'acodec': 'mp3' if entry['ext'] == 'mp3' else None,
})
return self.playlist_result(entries, playlist_id, traverse_obj(entries, (0, 'album')))
yt_dlp/extractor/nhk.py
+28 -8
View File
@@ -67,7 +67,7 @@ class NhkBaseIE(InfoExtractor):
info.update({
'_type': 'url_transparent',
'ie_key': 'Piksel',
'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + vod_id,
'url': 'https://movie-s.nhk.or.jp/v/refid/nhkworld/prefid/' + vod_id,
'id': vod_id,
})
else:
@@ -94,6 +94,19 @@ class NhkVodIE(NhkBaseIE):
# Content available only for a limited period of time. Visit
# https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
_TESTS = [{
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2061601/',
'info_dict': {
'id': 'yd8322ch',
'ext': 'mp4',
'description': 'md5:109c8b05d67a62d0592f2b445d2cd898',
'title': 'GRAND SUMO Highlights - [Recap] May Tournament Day 1 (Opening Day)',
'upload_date': '20230514',
'timestamp': 1684083791,
'series': 'GRAND SUMO Highlights',
'episode': '[Recap] May Tournament Day 1 (Opening Day)',
'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1684084443/4028649.jpg?w=1920&h=1080',
},
}, {
# video clip
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
@@ -104,6 +117,9 @@ class NhkVodIE(NhkBaseIE):
'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
'timestamp': 1565965194,
'upload_date': '20190816',
'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1567086278/3715195.jpg?w=1920&h=1080',
'series': 'Dining with the Chef',
'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
},
}, {
# audio clip
@@ -114,10 +130,7 @@ class NhkVodIE(NhkBaseIE):
'title': "Japan's Top Inventions - Miniature Video Cameras",
'description': 'md5:07ea722bdbbb4936fdd360b6a480c25b',
},
'params': {
# m3u8 download
'skip_download': True,
},
'skip': '404 Not Found',
}, {
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
'only_matching': True,
@@ -133,7 +146,6 @@ class NhkVodIE(NhkBaseIE):
}, {
# video, alphabetic character in ID #29670
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
'only_matching': True,
'info_dict': {
'id': 'qfjay6cg',
'ext': 'mp4',
@@ -142,7 +154,8 @@ class NhkVodIE(NhkBaseIE):
'thumbnail': r're:^https?:/(/[a-z0-9.-]+)+\.jpg\?w=1920&h=1080$',
'upload_date': '20210615',
'timestamp': 1623722008,
}
},
'skip': '404 Not Found',
}]
def _real_extract(self, url):
@@ -153,12 +166,19 @@ class NhkVodProgramIE(NhkBaseIE):
_VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
_TESTS = [{
# video program episodes
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/sumo',
'info_dict': {
'id': 'sumo',
'title': 'GRAND SUMO Highlights',
},
'playlist_mincount': 12,
}, {
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
'info_dict': {
'id': 'japanrailway',
'title': 'Japan Railway Journal',
},
'playlist_mincount': 1,
'playlist_mincount': 12,
}, {
# video program clips
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway/?type=clip',
yt_dlp/extractor/niconico.py
+163
View File
@@ -5,13 +5,17 @@ import json
import re
import time
from urllib.parse import urlparse
from .common import InfoExtractor, SearchInfoExtractor
from ..compat import (
compat_HTTPError,
)
from ..dependencies import websockets
from ..utils import (
ExtractorError,
OnDemandPagedList,
WebSocketsWrapper,
bug_reports_message,
clean_html,
float_or_none,
@@ -895,3 +899,162 @@ class NiconicoUserIE(InfoExtractor):
def _real_extract(self, url):
list_id = self._match_id(url)
return self.playlist_result(self._entries(list_id), list_id, ie=NiconicoIE.ie_key())
class NiconicoLiveIE(InfoExtractor):
IE_NAME = 'niconico:live'
IE_DESC = 'ニコニコ生放送'
_VALID_URL = r'https?://(?:sp\.)?live2?\.nicovideo\.jp/(?:watch|gate)/(?P<id>lv\d+)'
_TESTS = [{
'note': 'this test case includes invisible characters for title, pasting them as-is',
'url': 'https://live.nicovideo.jp/watch/lv339533123',
'info_dict': {
'id': 'lv339533123',
'title': '激辛ペヤング食べます‪( ;ᯅ; )‬（歌枠オーディション参加中）',
'view_count': 1526,
'comment_count': 1772,
'description': '初めましてもかって言います❕\nのんびり自由に適当に暮らしてます',
'uploader': 'もか',
'channel': 'ゲストさんのコミュニティ',
'channel_id': 'co5776900',
'channel_url': 'https://com.nicovideo.jp/community/co5776900',
'timestamp': 1670677328,
'is_live': True,
},
'skip': 'livestream',
}, {
'url': 'https://live2.nicovideo.jp/watch/lv339533123',
'only_matching': True,
}, {
'url': 'https://sp.live.nicovideo.jp/watch/lv339533123',
'only_matching': True,
}, {
'url': 'https://sp.live2.nicovideo.jp/watch/lv339533123',
'only_matching': True,
}]
_KNOWN_LATENCY = ('high', 'low')
def _real_extract(self, url):
if not websockets:
raise ExtractorError('websockets library is not available. Please install it.', expected=True)
video_id = self._match_id(url)
webpage, urlh = self._download_webpage_handle(f'https://live.nicovideo.jp/watch/{video_id}', video_id)
embedded_data = self._parse_json(unescapeHTML(self._search_regex(
r'<script\s+id="embedded-data"\s*data-props="(.+?)"', webpage, 'embedded data')), video_id)
ws_url = traverse_obj(embedded_data, ('site', 'relive', 'webSocketUrl'))
if not ws_url:
raise ExtractorError('The live hasn\'t started yet or already ended.', expected=True)
ws_url = update_url_query(ws_url, {
'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
})
hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
if latency not in self._KNOWN_LATENCY:
latency = 'high'
ws = WebSocketsWrapper(ws_url, {
'Cookies': str_or_none(cookies) or '',
'Origin': f'https://{hostname}',
'Accept': '*/*',
'User-Agent': self.get_param('http_headers')['User-Agent'],
})
self.write_debug('[debug] Sending HLS server request')
ws.send(json.dumps({
'type': 'startWatching',
'data': {
'stream': {
'quality': 'abr',
'protocol': 'hls+fmp4',
'latency': latency,
'chasePlay': False
},
'room': {
'protocol': 'webSocket',
'commentable': True
},
'reconnect': False,
}
}))
while True:
recv = ws.recv()
if not recv:
continue
data = json.loads(recv)
if not isinstance(data, dict):
continue
if data.get('type') == 'stream':
m3u8_url = data['data']['uri']
qualities = data['data']['availableQualities']
break
elif data.get('type') == 'disconnect':
self.write_debug(recv)
raise ExtractorError('Disconnected at middle of extraction')
elif data.get('type') == 'error':
self.write_debug(recv)
message = traverse_obj(data, ('body', 'code')) or recv
raise ExtractorError(message)
elif self.get_param('verbose', False):
if len(recv) > 100:
recv = recv[:100] + '...'
self.write_debug('Server said: %s' % recv)
title = traverse_obj(embedded_data, ('program', 'title')) or self._html_search_meta(
('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
raw_thumbs = traverse_obj(embedded_data, ('program', 'thumbnail')) or {}
thumbnails = []
for name, value in raw_thumbs.items():
if not isinstance(value, dict):
thumbnails.append({
'id': name,
'url': value,
**parse_resolution(value, lenient=True),
})
continue
for k, img_url in value.items():
res = parse_resolution(k, lenient=True) or parse_resolution(img_url, lenient=True)
width, height = res.get('width'), res.get('height')
thumbnails.append({
'id': f'{name}_{width}x{height}',
'url': img_url,
**res,
})
formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', live=True)
for fmt, q in zip(formats, reversed(qualities[1:])):
fmt.update({
'format_id': q,
'protocol': 'niconico_live',
'ws': ws,
'video_id': video_id,
'cookies': cookies,
'live_latency': latency,
'origin': hostname,
})
return {
'id': video_id,
'title': title,
**traverse_obj(embedded_data, {
'view_count': ('program', 'statistics', 'watchCount'),
'comment_count': ('program', 'statistics', 'commentCount'),
'uploader': ('program', 'supplier', 'name'),
'channel': ('socialGroup', 'name'),
'channel_id': ('socialGroup', 'id'),
'channel_url': ('socialGroup', 'socialGroupPageUrl'),
}),
'description': clean_html(traverse_obj(embedded_data, ('program', 'description'))),
'timestamp': int_or_none(traverse_obj(embedded_data, ('program', 'openTime'))),
'is_live': True,
'thumbnails': thumbnails,
'formats': formats,
}
yt_dlp/extractor/owncloud.py
+80
View File
@@ -0,0 +1,80 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
ExtractorError,
determine_ext,
url_or_none,
urlencode_postdata,
)
class OwnCloudIE(InfoExtractor):
_INSTANCES_RE = '|'.join((
r'(?:[^\.]+\.)?sciebo\.de',
r'cloud\.uni-koblenz-landau\.de',
))
_VALID_URL = rf'https?://(?:{_INSTANCES_RE})/s/(?P<id>[\w.-]+)'
_TESTS = [
{
'url': 'https://ruhr-uni-bochum.sciebo.de/s/wWhqZzh9jTumVFN',
'info_dict': {
'id': 'wWhqZzh9jTumVFN',
'ext': 'mp4',
'title': 'CmvpJST.mp4',
},
},
{
'url': 'https://ruhr-uni-bochum.sciebo.de/s/WNDuFu0XuFtmm3f',
'info_dict': {
'id': 'WNDuFu0XuFtmm3f',
'ext': 'mp4',
'title': 'CmvpJST.mp4',
},
'params': {
'videopassword': '12345',
},
},
]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage, urlh = self._download_webpage_handle(url, video_id)
if re.search(r'<label[^>]+for="password"', webpage):
webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
hidden_inputs = self._hidden_inputs(webpage)
title = hidden_inputs.get('filename')
parsed_url = urllib.parse.urlparse(url)
return {
'id': video_id,
'title': title,
'url': url_or_none(hidden_inputs.get('downloadURL')) or parsed_url._replace(
path=urllib.parse.urljoin(parsed_url.path, 'download')).geturl(),
'ext': determine_ext(title),
}
def _verify_video_password(self, webpage, url, video_id):
password = self.get_param('videopassword')
if password is None:
raise ExtractorError(
'This video is protected by a password, use the --video-password option',
expected=True)
validation_response = self._download_webpage(
url, video_id, 'Validating Password', 'Wrong password?',
data=urlencode_postdata({
'requesttoken': self._hidden_inputs(webpage)['requesttoken'],
'password': password,
}))
if re.search(r'<label[^>]+for="password"', validation_response):
warning = self._search_regex(
r'<div[^>]+class="warning">([^<]*)</div>', validation_response,
'warning', default='The password is wrong')
raise ExtractorError(f'Opening the video failed, {self.IE_NAME} said: {warning!r}', expected=True)
return validation_response
yt_dlp/extractor/piksel.py
+9 -7
View File
@@ -7,8 +7,10 @@ from ..utils import (
int_or_none,
join_nonempty,
parse_iso8601,
traverse_obj,
try_get,
unescapeHTML,
urljoin,
)
@@ -63,11 +65,11 @@ class PikselIE(InfoExtractor):
}
]
def _call_api(self, app_token, resource, display_id, query, fatal=True):
response = (self._download_json(
'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
display_id, query=query, fatal=fatal) or {}).get('response')
failure = try_get(response, lambda x: x['failure']['reason'])
def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.com', fatal=True):
url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5')
response = traverse_obj(
self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {}
failure = traverse_obj(response, ('failure', 'reason')) if response else 'Empty response from API'
if failure:
if fatal:
raise ExtractorError(failure, expected=True)
@@ -83,7 +85,7 @@ class PikselIE(InfoExtractor):
], webpage, 'app token')
query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
program = self._call_api(
app_token, 'program', display_id, query)['WsProgramResponse']['program']
app_token, 'program', display_id, query, url)['WsProgramResponse']['program']
video_id = program['uuid']
video_data = program['asset']
title = video_data['title']
@@ -129,7 +131,7 @@ class PikselIE(InfoExtractor):
process_asset_files(try_get(self._call_api(
app_token, 'asset_file', display_id, {
'assetid': asset_id,
}, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
}, url, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
m3u8_url = dict_get(video_data, [
'm3u8iPadURL',
yt_dlp/extractor/playsuisse.py
+64 -24
View File
@@ -5,10 +5,16 @@ from ..utils import int_or_none, traverse_obj
class PlaySuisseIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/watch/(?P<id>[0-9]+)'
_VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/(?:watch|detail)/(?:[^#]*[?&]episodeId=)?(?P<id>[0-9]+)'
_TESTS = [
{
# Old URL
'url': 'https://www.playsuisse.ch/watch/763211/0',
'only_matching': True,
},
{
# episode in a series
'url': 'https://www.playsuisse.ch/watch/763182?episodeId=763211',
'md5': '82df2a470b2dfa60c2d33772a8a60cf8',
'info_dict': {
'id': '763211',
@@ -21,11 +27,11 @@ class PlaySuisseIE(InfoExtractor):
'season_number': 1,
'episode': 'Knochen',
'episode_number': 1,
'thumbnail': 'md5:9260abe0c0ec9b69914d0a10d54c5878'
'thumbnail': 're:https://playsuisse-img.akamaized.net/',
}
},
{
'url': 'https://www.playsuisse.ch/watch/808675/0',
}, {
# film
'url': 'https://www.playsuisse.ch/watch/808675',
'md5': '818b94c1d2d7c4beef953f12cb8f3e75',
'info_dict': {
'id': '808675',
@@ -33,26 +39,60 @@ class PlaySuisseIE(InfoExtractor):
'title': 'Der Läufer',
'description': 'md5:9f61265c7e6dcc3e046137a792b275fd',
'duration': 5280,
'episode': 'Der Läufer',
'thumbnail': 'md5:44af7d65ee02bbba4576b131868bb783'
'thumbnail': 're:https://playsuisse-img.akamaized.net/',
}
},
{
'url': 'https://www.playsuisse.ch/watch/817193/0',
'md5': '1d6c066f92cd7fffd8b28a53526d6b59',
}, {
# series (treated as a playlist)
'url': 'https://www.playsuisse.ch/detail/1115687',
'info_dict': {
'id': '817193',
'ext': 'mp4',
'title': 'Die Einweihungsparty',
'description': 'md5:91ebf04d3a42cb3ab70666acf750a930',
'duration': 1380,
'series': 'Nr. 47',
'season': 'Season 1',
'season_number': 1,
'episode': 'Die Einweihungsparty',
'episode_number': 1,
'thumbnail': 'md5:637585fb106e3a4bcd991958924c7e44'
}
'description': 'md5:e4a2ae29a8895823045b5c3145a02aa3',
'id': '1115687',
'series': 'They all came out to Montreux',
'title': 'They all came out to Montreux',
},
'playlist': [{
'info_dict': {
'description': 'md5:f2462744834b959a31adc6292380cda2',
'duration': 3180,
'episode': 'Folge 1',
'episode_number': 1,
'id': '1112663',
'season': 'Season 1',
'season_number': 1,
'series': 'They all came out to Montreux',
'thumbnail': 're:https://playsuisse-img.akamaized.net/',
'title': 'Folge 1',
'ext': 'mp4'
},
}, {
'info_dict': {
'description': 'md5:9dfd308699fe850d3bce12dc1bad9b27',
'duration': 2935,
'episode': 'Folge 2',
'episode_number': 2,
'id': '1112661',
'season': 'Season 1',
'season_number': 1,
'series': 'They all came out to Montreux',
'thumbnail': 're:https://playsuisse-img.akamaized.net/',
'title': 'Folge 2',
'ext': 'mp4'
},
}, {
'info_dict': {
'description': 'md5:14a93a3356b2492a8f786ab2227ef602',
'duration': 2994,
'episode': 'Folge 3',
'episode_number': 3,
'id': '1112664',
'season': 'Season 1',
'season_number': 1,
'series': 'They all came out to Montreux',
'thumbnail': 're:https://playsuisse-img.akamaized.net/',
'title': 'Folge 3',
'ext': 'mp4'
}
}],
}
]
@@ -142,6 +182,6 @@ class PlaySuisseIE(InfoExtractor):
'subtitles': subtitles,
'series': media_data.get('seriesName'),
'season_number': int_or_none(media_data.get('seasonNumber')),
'episode': media_data.get('name'),
'episode': media_data.get('name') if media_data.get('episodeNumber') else None,
'episode_number': int_or_none(media_data.get('episodeNumber')),
}
yt_dlp/extractor/polskieradio.py
+137 -71
View File
@@ -2,26 +2,24 @@ import itertools
import json
import math
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse_unquote,
compat_urlparse
)
from ..compat import compat_str
from ..utils import (
determine_ext,
extract_attributes,
ExtractorError,
InAdvancePagedList,
determine_ext,
extract_attributes,
int_or_none,
js_to_json,
parse_iso8601,
strip_or_none,
traverse_obj,
unified_timestamp,
unescapeHTML,
unified_timestamp,
url_or_none,
urljoin,
)
@@ -44,7 +42,7 @@ class PolskieRadioBaseExtractor(InfoExtractor):
'duration': int_or_none(media.get('length')),
'vcodec': 'none' if media.get('provider') == 'audio' else None,
})
entry_title = compat_urllib_parse_unquote(media['desc'])
entry_title = urllib.parse.unquote(media['desc'])
if entry_title:
entry['title'] = entry_title
yield entry
@@ -130,10 +128,11 @@ class PolskieRadioLegacyIE(PolskieRadioBaseExtractor):
return self.playlist_result(entries, playlist_id, title, description)
class PolskieRadioIE(InfoExtractor):
# new next.js sites, excluding radiokierowcow.pl
_VALID_URL = r'https?://(?:[^/]+\.)?polskieradio(?:24)?\.pl/artykul/(?P<id>\d+)'
class PolskieRadioIE(PolskieRadioBaseExtractor):
# new next.js sites
_VALID_URL = r'https?://(?:[^/]+\.)?(?:polskieradio(?:24)?|radiokierowcow)\.pl/artykul/(?P<id>\d+)'
_TESTS = [{
# articleData, attachments
'url': 'https://jedynka.polskieradio.pl/artykul/1587943',
'info_dict': {
'id': '1587943',
@@ -148,6 +147,31 @@ class PolskieRadioIE(InfoExtractor):
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
},
}],
}, {
# post, legacy html players
'url': 'https://trojka.polskieradio.pl/artykul/2589163,Czy-wciaz-otrzymujemy-zdjecia-z-sond-Voyager',
'info_dict': {
'id': '2589163',
'title': 'Czy wciąż otrzymujemy zdjęcia z sond Voyager?',
'description': 'md5:cf1a7f348d63a2db9c0d7a63d1669473',
},
'playlist': [{
'info_dict': {
'id': '2577880',
'ext': 'mp3',
'title': 'md5:a57d10a0c02abd34dd675cb33707ad5a',
'duration': 321,
},
}],
}, {
# data, legacy
'url': 'https://radiokierowcow.pl/artykul/2694529',
'info_dict': {
'id': '2694529',
'title': 'Zielona fala reliktem przeszłości?',
'description': 'md5:f20a9a7ed9cb58916c54add94eae3bc0',
},
'playlist_count': 3,
}, {
'url': 'https://trojka.polskieradio.pl/artykul/1632955',
'only_matching': True,
@@ -166,7 +190,8 @@ class PolskieRadioIE(InfoExtractor):
webpage = self._download_webpage(url, playlist_id)
article_data = traverse_obj(
self._search_nextjs_data(webpage, playlist_id), ('props', 'pageProps', 'data', 'articleData'))
self._search_nextjs_data(webpage, playlist_id), (
'props', 'pageProps', (('data', 'articleData'), 'post', 'data')), get_all=False)
title = strip_or_none(article_data['title'])
@@ -178,7 +203,13 @@ class PolskieRadioIE(InfoExtractor):
'id': self._search_regex(
r'([a-f\d]{8}-(?:[a-f\d]{4}-){3}[a-f\d]{12})', entry['file'], 'entry id'),
'title': strip_or_none(entry.get('description')) or title,
} for entry in article_data.get('attachments') or () if entry['fileType'] in ('Audio', )]
} for entry in article_data.get('attachments') or () if entry.get('fileType') in ('Audio', )]
if not entries:
# some legacy articles have no json attachments, but players in body
entries = self._extract_webpage_player_entries(article_data['content'], playlist_id, {
'title': title,
})
return self.playlist_result(entries, playlist_id, title, description)
@@ -214,6 +245,15 @@ class PolskieRadioAuditionIE(InfoExtractor):
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 722,
}, {
# some articles were "promoted to main page" and thus link to old frontend
'url': 'https://trojka.polskieradio.pl/audycja/305',
'info_dict': {
'id': '305',
'title': 'Co w mowie piszczy?',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_count': 1523,
}]
def _call_lp3(self, path, query, video_id, note):
@@ -254,7 +294,6 @@ class PolskieRadioAuditionIE(InfoExtractor):
for article in page['data']:
yield {
'_type': 'url_transparent',
'ie_key': PolskieRadioIE.ie_key(),
'id': str(article['id']),
'url': article['url'],
'title': article.get('shortTitle'),
@@ -282,11 +321,8 @@ class PolskieRadioAuditionIE(InfoExtractor):
class PolskieRadioCategoryIE(InfoExtractor):
# legacy sites
IE_NAME = 'polskieradio:category'
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/(?:\d+(?:,[^/]+)?/|[^/]+/Tag)(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
'only_matching': True
}, {
'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
'info_dict': {
'id': '4143',
@@ -300,6 +336,36 @@ class PolskieRadioCategoryIE(InfoExtractor):
'title': 'Muzyka',
},
'playlist_mincount': 61
}, {
# billennium tabs
'url': 'https://www.polskieradio.pl/8/2385',
'info_dict': {
'id': '2385',
'title': 'Droga przez mąkę',
},
'playlist_mincount': 111,
}, {
'url': 'https://www.polskieradio.pl/10/4930',
'info_dict': {
'id': '4930',
'title': 'Teraz K-pop!',
},
'playlist_mincount': 392,
}, {
# post back pages, audio content directly without articles
'url': 'https://www.polskieradio.pl/8,dwojka/7376,nowa-mowa',
'info_dict': {
'id': '7376',
'title': 'Nowa mowa',
},
'playlist_mincount': 244,
}, {
'url': 'https://www.polskieradio.pl/Krzysztof-Dziuba/Tag175458',
'info_dict': {
'id': '175458',
'title': 'Krzysztof Dziuba',
},
'playlist_mincount': 420,
}, {
'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
'only_matching': True,
@@ -311,25 +377,61 @@ class PolskieRadioCategoryIE(InfoExtractor):
def _entries(self, url, page, category_id):
content = page
is_billennium_tabs = 'onclick="TB_LoadTab(' in page
is_post_back = 'onclick="__doPostBack(' in page
pagination = page if is_billennium_tabs else None
for page_num in itertools.count(2):
for a_entry, entry_id in re.findall(
r'(?s)<article[^>]+>.*?(<a[^>]+href=["\']/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
r'(?s)<article[^>]+>.*?(<a[^>]+href=["\'](?:(?:https?)?://[^/]+)?/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
content):
entry = extract_attributes(a_entry)
href = entry.get('href')
if not href:
continue
yield self.url_result(
compat_urlparse.urljoin(url, href), PolskieRadioLegacyIE,
entry_id, entry.get('title'))
mobj = re.search(
r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
content)
if not mobj:
break
next_url = compat_urlparse.urljoin(url, mobj.group('url'))
content = self._download_webpage(
next_url, category_id, 'Downloading page %s' % page_num)
if entry.get('href'):
yield self.url_result(
urljoin(url, entry['href']), PolskieRadioLegacyIE, entry_id, entry.get('title'))
for a_entry in re.findall(r'<span data-media=({[^ ]+})', content):
yield traverse_obj(self._parse_json(a_entry, category_id), {
'url': 'file',
'id': 'uid',
'duration': 'length',
'title': ('title', {urllib.parse.unquote}),
'description': ('desc', {urllib.parse.unquote}),
})
if is_billennium_tabs:
params = self._search_json(
r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+onclick=["\']TB_LoadTab\(',
pagination, 'next page params', category_id, default=None, close_objects=1,
contains_pattern='.+', transform_source=lambda x: '[%s' % js_to_json(unescapeHTML(x)))
if not params:
break
tab_content = self._download_json(
'https://www.polskieradio.pl/CMS/TemplateBoxesManagement/TemplateBoxTabContent.aspx/GetTabContent',
category_id, f'Downloading page {page_num}', headers={'content-type': 'application/json'},
data=json.dumps(dict(zip((
'boxInstanceId', 'tabId', 'categoryType', 'sectionId', 'categoryId', 'pagerMode',
'subjectIds', 'tagIndexId', 'queryString', 'name', 'openArticlesInParentTemplate',
'idSectionFromUrl', 'maxDocumentAge', 'showCategoryForArticle', 'pageNumber'
), params))).encode())['d']
content, pagination = tab_content['Content'], tab_content.get('PagerContent')
elif is_post_back:
target = self._search_regex(
r'onclick=(?:["\'])__doPostBack\((?P<q1>["\'])(?P<target>[\w$]+)(?P=q1)\s*,\s*(?P<q2>["\'])Next(?P=q2)',
content, 'pagination postback target', group='target', default=None)
if not target:
break
content = self._download_webpage(
url, category_id, f'Downloading page {page_num}',
data=urllib.parse.urlencode({
**self._hidden_inputs(content),
'__EVENTTARGET': target,
'__EVENTARGUMENT': 'Next',
}).encode())
else:
next_url = urljoin(url, self._search_regex(
r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
content, 'next page url', group='url', default=None))
if not next_url:
break
content = self._download_webpage(next_url, category_id, f'Downloading page {page_num}')
def _real_extract(self, url):
category_id = self._match_id(url)
@@ -337,7 +439,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
if PolskieRadioAuditionIE.suitable(urlh.url):
return self.url_result(urlh.url, PolskieRadioAuditionIE, category_id)
title = self._html_search_regex(
r'<title>([^<]+) - [^<]+ - [^<]+</title>',
r'<title>([^<]+)(?: - [^<]+ - [^<]+| w [Pp]olskie[Rr]adio\.pl\s*)</title>',
webpage, 'title', fatal=False)
return self.playlist_result(
self._entries(url, webpage, category_id),
@@ -506,39 +608,3 @@ class PolskieRadioPodcastIE(PolskieRadioPodcastBaseExtractor):
'Content-Type': 'application/json',
})
return self._parse_episode(data[0])
class PolskieRadioRadioKierowcowIE(PolskieRadioBaseExtractor):
_VALID_URL = r'https?://(?:www\.)?radiokierowcow\.pl/artykul/(?P<id>[0-9]+)'
IE_NAME = 'polskieradio:kierowcow'
_TESTS = [{
'url': 'https://radiokierowcow.pl/artykul/2694529',
'info_dict': {
'id': '2694529',
'title': 'Zielona fala reliktem przeszłości?',
'description': 'md5:343950a8717c9818fdfd4bd2b8ca9ff2',
},
'playlist_count': 3,
}]
def _real_extract(self, url):
media_id = self._match_id(url)
webpage = self._download_webpage(url, media_id)
nextjs_build = self._search_nextjs_data(webpage, media_id)['buildId']
article = self._download_json(
f'https://radiokierowcow.pl/_next/data/{nextjs_build}/artykul/{media_id}.json?articleId={media_id}',
media_id)
data = article['pageProps']['data']
title = data['title']
entries = self._extract_webpage_player_entries(data['content'], media_id, {
'title': title,
})
return {
'_type': 'playlist',
'id': media_id,
'entries': entries,
'title': title,
'description': data.get('lead'),
}
yt_dlp/extractor/rai.py
+269 -306
View File
@@ -1,19 +1,12 @@
import re
from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urlparse,
)
from ..utils import (
clean_html,
determine_ext,
ExtractorError,
filter_dict,
find_xpath_attr,
fix_xml_ampersands,
GeoRestrictedError,
HEADRequest,
int_or_none,
join_nonempty,
parse_duration,
@@ -35,82 +28,70 @@ class RaiBaseIE(InfoExtractor):
_GEO_BYPASS = False
def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
def fix_cdata(s):
# remove \r\n\t before and after <![CDATA[ ]]> to avoid
# polluted text with xpath_text
s = re.sub(r'(\]\]>)[\r\n\t]+(</)', '\\1\\2', s)
return re.sub(r'(>)[\r\n\t]+(<!\[CDATA\[)', '\\1\\2', s)
if not re.match(r'https?://', relinker_url):
return {'formats': [{'url': relinker_url}]}
# set User-Agent to generic 'Rai' to avoid quality filtering from
# the media server and get the maximum qualities available
relinker = self._download_xml(
relinker_url, video_id, note='Downloading XML metadata',
transform_source=fix_cdata, query={'output': 64},
headers={**self.geo_verification_headers(), 'User-Agent': 'Rai'})
if xpath_text(relinker, './license_url', default='{}') != '{}':
self.report_drm(video_id)
is_live = xpath_text(relinker, './is_live', default='N') == 'Y'
duration = parse_duration(xpath_text(relinker, './duration', default=None))
media_url = xpath_text(relinker, './url[@type="content"]', default=None)
if not media_url:
self.raise_no_formats('The relinker returned no media url')
# geo flag is a bit unreliable and not properly set all the time
geoprotection = xpath_text(relinker, './geoprotection', default='N') == 'Y'
ext = determine_ext(media_url)
formats = []
geoprotection = None
is_live = None
duration = None
for platform in ('mon', 'flash', 'native'):
relinker = self._download_xml(
relinker_url, video_id,
note=f'Downloading XML metadata for platform {platform}',
transform_source=fix_xml_ampersands,
query={'output': 45, 'pl': platform},
headers=self.geo_verification_headers())
if ext == 'mp3':
formats.append({
'url': media_url,
'vcodec': 'none',
'acodec': 'mp3',
'format_id': 'https-mp3',
})
elif ext == 'm3u8' or 'format=m3u8' in media_url:
formats.extend(self._extract_m3u8_formats(
media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
elif ext == 'f4m':
# very likely no longer needed. Cannot find any url that uses it.
manifest_url = update_url_query(
media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
{'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
formats.extend(self._extract_f4m_formats(
manifest_url, video_id, f4m_id='hds', fatal=False))
elif ext == 'mp4':
bitrate = int_or_none(xpath_text(relinker, './bitrate'))
formats.append({
'url': media_url,
'tbr': bitrate if bitrate > 0 else None,
'format_id': join_nonempty('https', bitrate, delim='-'),
})
else:
raise ExtractorError('Unrecognized media file found')
if xpath_text(relinker, './license_url', default='{}') != '{}':
self.report_drm(video_id)
if not geoprotection:
geoprotection = xpath_text(
relinker, './geoprotection', default=None) == 'Y'
if not is_live:
is_live = xpath_text(
relinker, './is_live', default=None) == 'Y'
if not duration:
duration = parse_duration(xpath_text(
relinker, './duration', default=None))
url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
if url_elem is None:
continue
media_url = url_elem.text
# This does not imply geo restriction (e.g.
# http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
if '/video_no_available.mp4' in media_url:
continue
ext = determine_ext(media_url)
if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
continue
if ext == 'mp3':
formats.append({
'url': media_url,
'vcodec': 'none',
'acodec': 'mp3',
'format_id': 'http-mp3',
})
break
elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
formats.extend(self._extract_m3u8_formats(
media_url, video_id, 'mp4', 'm3u8_native',
m3u8_id='hls', fatal=False))
elif ext == 'f4m' or platform == 'flash':
manifest_url = update_url_query(
media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
{'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
formats.extend(self._extract_f4m_formats(
manifest_url, video_id, f4m_id='hds', fatal=False))
else:
bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
formats.append({
'url': media_url,
'tbr': bitrate if bitrate > 0 else None,
'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
})
if not formats and geoprotection is True:
if (not formats and geoprotection is True) or '/video_no_available.mp4' in media_url:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
if not audio_only:
formats.extend(self._create_http_urls(relinker_url, formats))
if not audio_only and not is_live:
formats.extend(self._create_http_urls(media_url, relinker_url, formats))
return filter_dict({
'is_live': is_live,
@@ -118,38 +99,31 @@ class RaiBaseIE(InfoExtractor):
'formats': formats,
})
def _create_http_urls(self, relinker_url, fmts):
_RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
def _create_http_urls(self, manifest_url, relinker_url, fmts):
_MANIFEST_REG = r'/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8'
_MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
_QUALITY = {
# tbr: w, h
'250': [352, 198],
'400': [512, 288],
'700': [512, 288],
'800': [700, 394],
'1200': [736, 414],
'1800': [1024, 576],
'2400': [1280, 720],
'3200': [1440, 810],
'3600': [1440, 810],
'5000': [1920, 1080],
'10000': [1920, 1080],
250: [352, 198],
400: [512, 288],
600: [512, 288],
700: [512, 288],
800: [700, 394],
1200: [736, 414],
1500: [920, 518],
1800: [1024, 576],
2400: [1280, 720],
3200: [1440, 810],
3600: [1440, 810],
5000: [1920, 1080],
10000: [1920, 1080],
}
def test_url(url):
resp = self._request_webpage(
HEADRequest(url), None, headers={'User-Agent': 'Rai'},
fatal=False, errnote=False, note=False)
if resp is False:
def percentage(number, target, pc=20, roof=125):
'''check if the target is in the range of number +/- percent'''
if not number or number < 0:
return False
if resp.code == 200:
return False if resp.url == url else resp.url
return None
# filter out audio-only formats
fmts = [f for f in fmts if not f.get('vcodec') == 'none']
return abs(target - number) < min(float(number) * float(pc) / 100.0, roof)
def get_format_info(tbr):
import math
@@ -157,67 +131,78 @@ class RaiBaseIE(InfoExtractor):
if len(fmts) == 1 and not br:
br = fmts[0].get('tbr')
if br and br > 300:
tbr = compat_str(math.floor(br / 100) * 100)
tbr = math.floor(br / 100) * 100
else:
tbr = '250'
tbr = 250
# try extracting info from available m3u8 formats
format_copy = None
format_copy = [None, None]
for f in fmts:
if f.get('tbr'):
br_limit = math.floor(br / 100)
if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
format_copy = f.copy()
if percentage(tbr, f['tbr']):
format_copy[0] = f.copy()
if [f.get('width'), f.get('height')] == _QUALITY.get(tbr):
format_copy[1] = f.copy()
format_copy[1]['tbr'] = tbr
# prefer format with similar bitrate because there might be
# multiple video with the same resolution but different bitrate
format_copy = format_copy[0] or format_copy[1] or {}
return {
'format_id': f'https-{tbr}',
'width': format_copy.get('width'),
'height': format_copy.get('height'),
'tbr': format_copy.get('tbr'),
'vcodec': format_copy.get('vcodec'),
'acodec': format_copy.get('acodec'),
'fps': format_copy.get('fps'),
'format_id': f'https-{tbr}',
} if format_copy else {
'format_id': f'https-{tbr}',
'width': _QUALITY[tbr][0],
'height': _QUALITY[tbr][1],
'format_id': f'https-{tbr}',
'tbr': int(tbr),
'tbr': tbr,
'vcodec': 'avc1',
'acodec': 'mp4a',
'fps': 25,
}
loc = test_url(_MP4_TMPL % (relinker_url, '*'))
if not isinstance(loc, compat_str):
return []
# filter out single-stream formats
fmts = [f for f in fmts
if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none']
mobj = re.match(
_RELINKER_REG,
test_url(relinker_url) or '')
mobj = re.search(_MANIFEST_REG, manifest_url)
if not mobj:
return []
available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
available_qualities = [i for i in available_qualities if i]
formats = []
for q in available_qualities:
fmt = {
for q in filter(None, available_qualities):
self.write_debug(f'Creating https format for quality {q}')
formats.append({
'url': _MP4_TMPL % (relinker_url, q),
'protocol': 'https',
'ext': 'mp4',
**get_format_info(q)
}
formats.append(fmt)
})
return formats
@staticmethod
def _get_thumbnails_list(thumbs, url):
return [{
'url': urljoin(url, thumb_url),
} for thumb_url in (thumbs or {}).values() if thumb_url]
@staticmethod
def _extract_subtitles(url, video_data):
STL_EXT = 'stl'
SRT_EXT = 'srt'
subtitles = {}
subtitles_array = video_data.get('subtitlesArray') or []
subtitles_array = video_data.get('subtitlesArray') or video_data.get('subtitleList') or []
for k in ('subtitles', 'subtitlesUrl'):
subtitles_array.append({'url': video_data.get(k)})
for subtitle in subtitles_array:
sub_url = subtitle.get('url')
if sub_url and isinstance(sub_url, compat_str):
if sub_url and isinstance(sub_url, str):
sub_lang = subtitle.get('language') or 'it'
sub_url = urljoin(url, sub_url)
sub_ext = determine_ext(sub_url, SRT_EXT)
@@ -236,7 +221,7 @@ class RaiBaseIE(InfoExtractor):
class RaiPlayIE(RaiBaseIE):
_VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
_TESTS = [{
'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
'url': 'https://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
'md5': '8970abf8caf8aef4696e7b1f2adfc696',
'info_dict': {
'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
@@ -244,22 +229,20 @@ class RaiPlayIE(RaiBaseIE):
'title': 'Report del 07/04/2014',
'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Rai Gulp',
'thumbnail': r're:^https?://www\.raiplay\.it/.+\.jpg',
'uploader': 'Rai 3',
'creator': 'Rai 3',
'duration': 6160,
'series': 'Report',
'season': '2013/14',
'subtitles': {
'it': 'count:4',
},
'subtitles': {'it': 'count:4'},
'release_year': 2022,
'episode': 'Espresso nel caffè - 07/04/2014',
'timestamp': 1396919880,
'upload_date': '20140408',
'formats': 'count:4',
},
'params': {
'skip_download': True,
},
'params': {'skip_download': True},
}, {
# 1080p direct mp4 url
'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
@@ -270,8 +253,9 @@ class RaiPlayIE(RaiBaseIE):
'title': 'Blanca - S1E1 - Senza occhi',
'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Rai 1',
'thumbnail': r're:^https://www\.raiplay\.it/dl/img/.+\.jpg',
'uploader': 'Rai Premium',
'creator': 'Rai Fiction',
'duration': 6493,
'series': 'Blanca',
'season': 'Season 1',
@@ -281,6 +265,30 @@ class RaiPlayIE(RaiBaseIE):
'episode': 'Senza occhi',
'timestamp': 1637318940,
'upload_date': '20211119',
'formats': 'count:12',
},
'params': {'skip_download': True},
'expected_warnings': ['Video not available. Likely due to geo-restriction.']
}, {
# 1500 quality
'url': 'https://www.raiplay.it/video/2012/09/S1E11---Tutto-cio-che-luccica-0cab3323-732e-45d6-8e86-7704acab6598.html',
'md5': 'a634d20e8ab2d43724c273563f6bf87a',
'info_dict': {
'id': '0cab3323-732e-45d6-8e86-7704acab6598',
'ext': 'mp4',
'title': 'Mia and Me - S1E11 - Tutto ciò che luccica',
'alt_title': 'St 1 Ep 11 - Mia and Me - Tutto ciò che luccica',
'description': 'md5:4969e594184b1920c4c1f2b704da9dea',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Rai Gulp',
'series': 'Mia and Me',
'season': 'Season 1',
'episode_number': 11,
'release_year': 2015,
'season_number': 1,
'episode': 'Tutto ciò che luccica',
'timestamp': 1348495020,
'upload_date': '20120924',
},
}, {
'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
@@ -299,57 +307,40 @@ class RaiPlayIE(RaiBaseIE):
base, video_id = self._match_valid_url(url).groups()
media = self._download_json(
base + '.json', video_id, 'Downloading video JSON')
f'{base}.json', video_id, 'Downloading video JSON')
if not self.get_param('allow_unplayable_formats'):
if try_get(
media,
(lambda x: x['rights_management']['rights']['drm'],
lambda x: x['program_info']['rights_management']['rights']['drm']),
dict):
if traverse_obj(media, (('program_info', None), 'rights_management', 'rights', 'drm')):
self.report_drm(video_id)
title = media['name']
video = media['video']
relinker_info = self._extract_relinker_info(video['content_url'], video_id)
thumbnails = []
for _, value in media.get('images', {}).items():
if value:
thumbnails.append({
'url': urljoin(url, value),
})
date_published = media.get('date_published')
time_published = media.get('time_published')
if date_published and time_published:
date_published += ' ' + time_published
subtitles = self._extract_subtitles(url, video)
program_info = media.get('program_info') or {}
date_published = join_nonempty(
media.get('date_published'), media.get('time_published'), delim=' ')
season = media.get('season')
alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
return {
'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
'display_id': video_id,
'title': title,
'title': media.get('name'),
'alt_title': strip_or_none(alt_title or None),
'description': media.get('description'),
'uploader': strip_or_none(media.get('channel') or None),
'creator': strip_or_none(media.get('editor') or None),
'uploader': strip_or_none(
traverse_obj(media, ('program_info', 'channel'))
or media.get('channel') or None),
'creator': strip_or_none(
traverse_obj(media, ('program_info', 'editor'))
or media.get('editor') or None),
'duration': parse_duration(video.get('duration')),
'timestamp': unified_timestamp(date_published),
'thumbnails': thumbnails,
'series': program_info.get('name'),
'thumbnails': self._get_thumbnails_list(media.get('images'), url),
'series': traverse_obj(media, ('program_info', 'name')),
'season_number': int_or_none(season),
'season': season if (season and not season.isdigit()) else None,
'episode': media.get('episode_title'),
'episode_number': int_or_none(media.get('episode')),
'subtitles': subtitles,
'subtitles': self._extract_subtitles(url, video),
'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
**relinker_info
}
@@ -371,38 +362,39 @@ class RaiPlayLiveIE(RaiPlayIE): # XXX: Do not subclass from concrete IE
'live_status': 'is_live',
'upload_date': '20090502',
'timestamp': 1241276220,
'formats': 'count:3',
},
'params': {
'skip_download': True,
},
'params': {'skip_download': True},
}]
class RaiPlayPlaylistIE(InfoExtractor):
_VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
_TESTS = [{
# entire series episodes + extras...
'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
'info_dict': {
'id': 'nondirloalmiocapo',
'title': 'Non dirlo al mio capo',
'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
},
'playlist_mincount': 12,
'playlist_mincount': 30,
}, {
# single season
'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
'info_dict': {
'id': 'nondirloalmiocapo',
'title': 'Non dirlo al mio capo - Stagione 2',
'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
},
'playlist_mincount': 12,
'playlist_count': 12,
}]
def _real_extract(self, url):
base, playlist_id, extra_id = self._match_valid_url(url).groups()
program = self._download_json(
base + '.json', playlist_id, 'Downloading program JSON')
f'{base}.json', playlist_id, 'Downloading program JSON')
if extra_id:
extra_id = extra_id.upper().rstrip('/')
@@ -450,7 +442,7 @@ class RaiPlaySoundIE(RaiBaseIE):
'title': 'Il Ruggito del Coniglio del 10/12/2021',
'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
'thumbnail': r're:^https?://.*\.jpg$',
'thumbnail': r're:^https?://.+\.jpg$',
'uploader': 'rai radio 2',
'duration': 5685,
'series': 'Il Ruggito del Coniglio',
@@ -459,9 +451,7 @@ class RaiPlaySoundIE(RaiBaseIE):
'timestamp': 1638346620,
'upload_date': '20211201',
},
'params': {
'skip_download': True,
},
'params': {'skip_download': True},
}]
def _real_extract(self, url):
@@ -480,9 +470,6 @@ class RaiPlaySoundIE(RaiBaseIE):
lambda x: x['live']['create_date']))
podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
thumbnails = [{
'url': urljoin(url, thumb_url),
} for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
return {
**info,
@@ -494,7 +481,7 @@ class RaiPlaySoundIE(RaiBaseIE):
'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
'timestamp': unified_timestamp(date_published),
'thumbnails': thumbnails,
'thumbnails': self._get_thumbnails_list(podcast_info.get('images'), url),
'series': podcast_info.get('title'),
'season_number': int_or_none(media.get('season')),
'episode': media.get('episode_title'),
@@ -512,30 +499,30 @@ class RaiPlaySoundLiveIE(RaiPlaySoundIE): # XXX: Do not subclass from concrete
'display_id': 'radio2',
'ext': 'mp4',
'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
'thumbnail': r're:^https://www\.raiplaysound\.it/dl/img/.+\.png',
'uploader': 'rai radio 2',
'series': 'Rai Radio 2',
'creator': 'raiplaysound',
'is_live': True,
'live_status': 'is_live',
},
'params': {
'skip_download': 'live',
},
'params': {'skip_download': True},
}]
class RaiPlaySoundPlaylistIE(InfoExtractor):
_VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
_TESTS = [{
# entire show
'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
'info_dict': {
'id': 'ilruggitodelconiglio',
'title': 'Il Ruggito del Coniglio',
'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
'description': 'md5:48cff6972435964284614d70474132e6',
},
'playlist_mincount': 65,
}, {
# single season
'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
'info_dict': {
'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
@@ -568,22 +555,19 @@ class RaiPlaySoundPlaylistIE(InfoExtractor):
class RaiIE(RaiBaseIE):
_VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
_TESTS = [{
# var uniquename = "ContentItem-..."
# data-id="ContentItem-..."
'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
'info_dict': {
'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
'ext': 'mp4',
'title': 'TG PRIMO TEMPO',
'thumbnail': r're:^https?://.*\.jpg$',
'thumbnail': r're:^https?://.*\.jpg',
'duration': 1758,
'upload_date': '20140612',
},
'skip': 'This content is available only in Italy',
'params': {'skip_download': True},
'expected_warnings': ['Video not available. Likely due to geo-restriction.']
}, {
# with ContentItem in og:url
'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
'md5': '06345bd97c932f19ffb129973d07a020',
'info_dict': {
'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
'ext': 'mp4',
@@ -592,123 +576,51 @@ class RaiIE(RaiBaseIE):
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 2214,
'upload_date': '20161103'
}
},
'params': {'skip_download': True},
}, {
# Direct MMS URL
# Direct MMS: Media URL no longer works.
'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
'only_matching': True,
}]
def _extract_from_content_id(self, content_id, url):
def _real_extract(self, url):
content_id = self._match_id(url)
media = self._download_json(
f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
content_id, 'Downloading video JSON')
content_id, 'Downloading video JSON', fatal=False, expected_status=404)
title = media['name'].strip()
if media is None:
return None
media_type = media['type']
if 'Audio' in media_type:
if 'Audio' in media['type']:
relinker_info = {
'formats': [{
'format_id': media.get('formatoAudio'),
'format_id': join_nonempty('https', media.get('formatoAudio'), delim='-'),
'url': media['audioUrl'],
'ext': media.get('formatoAudio'),
'vcodec': 'none',
'acodec': media.get('formatoAudio'),
}]
}
elif 'Video' in media_type:
elif 'Video' in media['type']:
relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
else:
raise ExtractorError('not a media file')
thumbnails = []
for image_type in ('image', 'image_medium', 'image_300'):
thumbnail_url = media.get(image_type)
if thumbnail_url:
thumbnails.append({
'url': compat_urlparse.urljoin(url, thumbnail_url),
})
subtitles = self._extract_subtitles(url, media)
thumbnails = self._get_thumbnails_list(
{image_type: media.get(image_type) for image_type in (
'image', 'image_medium', 'image_300')}, url)
return {
'id': content_id,
'title': title,
'description': strip_or_none(media.get('desc') or None),
'title': strip_or_none(media.get('name') or media.get('title')),
'description': strip_or_none(media.get('desc')) or None,
'thumbnails': thumbnails,
'uploader': strip_or_none(media.get('author') or None),
'uploader': strip_or_none(media.get('author')) or None,
'upload_date': unified_strdate(media.get('date')),
'duration': parse_duration(media.get('length')),
'subtitles': subtitles,
**relinker_info
}
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
content_item_id = None
content_item_url = self._html_search_meta(
('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
'twitter:player', 'jsonlink'), webpage, default=None)
if content_item_url:
content_item_id = self._search_regex(
rf'ContentItem-({self._UUID_RE})', content_item_url,
'content item id', default=None)
if not content_item_id:
content_item_id = self._search_regex(
rf'''(?x)
(?:
(?:initEdizione|drawMediaRaiTV)\(|
<(?:[^>]+\bdata-id|var\s+uniquename)=|
<iframe[^>]+\bsrc=
)
(["\'])
(?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
''',
webpage, 'content item id', default=None, group='id')
content_item_ids = set()
if content_item_id:
content_item_ids.add(content_item_id)
if video_id not in content_item_ids:
content_item_ids.add(video_id)
for content_item_id in content_item_ids:
try:
return self._extract_from_content_id(content_item_id, url)
except GeoRestrictedError:
raise
except ExtractorError:
pass
relinker_url = self._proto_relative_url(self._search_regex(
r'''(?x)
(?:
var\s+videoURL|
mediaInfo\.mediaUri
)\s*=\s*
([\'"])
(?P<url>
(?:https?:)?
//mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
(?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
''',
webpage, 'relinker URL', group='url'))
relinker_info = self._extract_relinker_info(
urljoin(url, relinker_url), video_id)
title = self._search_regex(
r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
webpage, 'title', group='title',
default=None) or self._og_search_title(webpage)
return {
'id': video_id,
'title': title,
'subtitles': self._extract_subtitles(url, media),
**relinker_info
}
@@ -726,7 +638,8 @@ class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE
'duration': 1589,
'upload_date': '20220529',
'uploader': 'rainews',
}
},
'params': {'skip_download': True},
}, {
# old content with fallback method to extract media urls
'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
@@ -739,12 +652,14 @@ class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE
'duration': 833,
'upload_date': '20161103'
},
'params': {'skip_download': True},
'expected_warnings': ['unable to extract player_data'],
}, {
# iframe + drm
'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
'only_matching': True,
}]
_PLAYER_TAG = 'news'
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -752,8 +667,8 @@ class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE
webpage = self._download_webpage(url, video_id)
player_data = self._search_json(
r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
transform_source=clean_html, fatal=False)
rf'<rai{self._PLAYER_TAG}-player\s*data=\'', webpage, 'player_data', video_id,
transform_source=clean_html, default={})
track_info = player_data.get('track_info')
relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
@@ -770,16 +685,36 @@ class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE
return {
'id': video_id,
'title': track_info.get('title') or self._og_search_title(webpage),
'title': player_data.get('title') or track_info.get('title') or self._og_search_title(webpage),
'upload_date': unified_strdate(track_info.get('date')),
'uploader': strip_or_none(track_info.get('editor') or None),
**relinker_info
}
class RaiSudtirolIE(RaiBaseIE):
_VALID_URL = r'https?://raisudtirol\.rai\.it/.+?media=(?P<id>[TP]tv\d+)'
class RaiCulturaIE(RaiNewsIE): # XXX: Do not subclass from concrete IE
_VALID_URL = rf'https?://(www\.)?raicultura\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
_EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
_TESTS = [{
'url': 'https://www.raicultura.it/letteratura/articoli/2018/12/Alberto-Asor-Rosa-Letteratura-e-potere-05ba8775-82b5-45c5-a89d-dd955fbde1fb.html',
'info_dict': {
'id': '05ba8775-82b5-45c5-a89d-dd955fbde1fb',
'ext': 'mp4',
'title': 'Alberto Asor Rosa: Letteratura e potere',
'duration': 1756,
'upload_date': '20181206',
'uploader': 'raicultura',
'formats': 'count:2',
},
'params': {'skip_download': True},
}]
_PLAYER_TAG = 'cultura'
class RaiSudtirolIE(RaiBaseIE):
_VALID_URL = r'https?://raisudtirol\.rai\.it/.+media=(?P<id>\w+)'
_TESTS = [{
# mp4 file
'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
'info_dict': {
'id': 'Ptv1619729460',
@@ -787,34 +722,62 @@ class RaiSudtirolIE(RaiBaseIE):
'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
'series': 'Euro: trasmisciun d\'economia',
'upload_date': '20210429',
'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+?\.jpg',
'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+\.jpg',
'uploader': 'raisudtirol',
}
'formats': 'count:1',
},
'params': {'skip_download': True},
}, {
# m3u manifest
'url': 'https://raisudtirol.rai.it/it/kidsplayer.php?lang=it&media=GUGGUG_P1.smil',
'info_dict': {
'id': 'GUGGUG_P1',
'ext': 'mp4',
'title': 'GUGGUG! La Prospettiva - Die Perspektive',
'uploader': 'raisudtirol',
'formats': 'count:6',
},
'params': {'skip_download': True},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
video_date = self._html_search_regex(r'<span class="med_data">(.+?)</span>', webpage, 'video_date', fatal=False)
video_title = self._html_search_regex(r'<span class="med_title">(.+?)</span>', webpage, 'video_title', fatal=False)
video_url = self._html_search_regex(r'sources:\s*\[\{file:\s*"(.+?)"\}\]', webpage, 'video_url')
video_thumb = self._html_search_regex(r'image: \'(.+?)\'', webpage, 'video_thumb', fatal=False)
video_date = self._html_search_regex(
r'<span class="med_data">(.+?)</span>', webpage, 'video_date', default=None)
video_title = self._html_search_regex([
r'<span class="med_title">(.+?)</span>', r'title: \'(.+?)\','],
webpage, 'video_title', default=None)
video_url = self._html_search_regex([
r'sources:\s*\[\{file:\s*"(.+?)"\}\]',
r'<source\s+src="(.+?)"\s+type="application/x-mpegURL"'],
webpage, 'video_url', default=None)
return {
'id': video_id,
'title': join_nonempty(video_title, video_date, delim=' - '),
'series': video_title,
'upload_date': unified_strdate(video_date),
'thumbnail': urljoin('https://raisudtirol.rai.it/', video_thumb),
'uploader': 'raisudtirol',
'formats': [{
ext = determine_ext(video_url)
if ext == 'm3u8':
formats = self._extract_m3u8_formats(video_url, video_id)
elif ext == 'mp4':
formats = [{
'format_id': 'https-mp4',
'url': self._proto_relative_url(video_url),
'width': 1024,
'height': 576,
'fps': 25,
'vcodec': 'h264',
'acodec': 'aac',
}],
'vcodec': 'avc1',
'acodec': 'mp4a',
}]
else:
formats = []
self.raise_no_formats(f'Unrecognized media file: {video_url}')
return {
'id': video_id,
'title': join_nonempty(video_title, video_date, delim=' - '),
'series': video_title if video_date else None,
'upload_date': unified_strdate(video_date),
'thumbnail': urljoin('https://raisudtirol.rai.it/', self._html_search_regex(
r'image: \'(.+?)\'', webpage, 'video_thumb', default=None)),
'uploader': 'raisudtirol',
'formats': formats,
}
yt_dlp/extractor/recurbate.py
+43
View File
@@ -0,0 +1,43 @@
import urllib.error
from .common import InfoExtractor
from ..utils import ExtractorError, merge_dicts
class RecurbateIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?recurbate\.com/play\.php\?video=(?P<id>\d+)'
_TESTS = [{
'url': 'https://recurbate.com/play.php?video=39161415',
'md5': 'dd2b4ec57aa3e3572cb5cf0997fca99f',
'info_dict': {
'id': '39161415',
'ext': 'mp4',
'description': 'md5:db48d09e4d93fc715f47fd3d6b7edd51',
'title': 'Performer zsnicole33 show on 2022-10-25 20:23, Chaturbate Archive – Recurbate',
'age_limit': 18,
},
'skip': 'Website require membership.',
}]
def _real_extract(self, url):
SUBSCRIPTION_MISSING_MESSAGE = 'This video is only available for registered users; Set your authenticated browser user agent via the --user-agent parameter.'
video_id = self._match_id(url)
try:
webpage = self._download_webpage(url, video_id)
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
raise
token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
video_url = f'https://recurbate.com/api/get.php?video={video_id}&token={token}'
video_webpage = self._download_webpage(video_url, video_id)
if video_webpage == 'shall_subscribe':
self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
entries = self._parse_html5_media_entries(video_url, video_webpage, video_id)
return merge_dicts({
'id': video_id,
'title': self._html_extract_title(webpage, 'title'),
'description': self._og_search_description(webpage),
'age_limit': self._rta_search(webpage),
}, entries[0])
yt_dlp/extractor/rottentomatoes.py
+66 -16
View File
@@ -1,30 +1,80 @@
from .common import InfoExtractor
from .internetvideoarchive import InternetVideoArchiveIE
from ..utils import (
ExtractorError,
clean_html,
float_or_none,
get_element_by_class,
join_nonempty,
traverse_obj,
url_or_none,
)
class RottenTomatoesIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
_VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/(?P<playlist>[^/]+)(?:/(?P<tr>trailers)(?:/(?P<id>\w+))?)?'
_TEST = {
_TESTS = [{
'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
'info_dict': {
'id': '11028566',
'ext': 'mp4',
'title': 'Toy Story 3',
'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.'
},
}
'skip': 'No longer available',
}, {
'url': 'https://www.rottentomatoes.com/m/toy_story_3/trailers/VycaVoBKhGuk',
'info_dict': {
'id': 'VycaVoBKhGuk',
'ext': 'mp4',
'title': 'Toy Story 3: Trailer 2',
'description': '',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 149.941
},
}, {
'url': 'http://www.rottentomatoes.com/m/toy_story_3',
'info_dict': {
'id': 'toy_story_3',
'title': 'Toy Story 3',
},
'playlist_mincount': 4,
}, {
'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers',
'info_dict': {
'id': 'toy_story_3-trailers',
},
'playlist_mincount': 5,
}]
def _extract_videos(self, data, display_id):
for video in traverse_obj(data, (lambda _, v: v['publicId'] and v['file'] and v['type'] == 'hls')):
yield {
'formats': self._extract_m3u8_formats(
video['file'], display_id, 'mp4', m3u8_id='hls', fatal=False),
**traverse_obj(video, {
'id': 'publicId',
'title': 'title',
'description': 'description',
'duration': ('durationInSeconds', {float_or_none}),
'thumbnail': ('image', {url_or_none}),
}),
}
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
iva_id = self._search_regex(r'publishedid=(\d+)', webpage, 'internet video archive id')
playlist_id, trailers, video_id = self._match_valid_url(url).group('playlist', 'tr', 'id')
playlist_id = join_nonempty(playlist_id, trailers)
webpage = self._download_webpage(url, playlist_id)
data = self._search_json(
r'<script[^>]+\bid=["\'](?:heroV|v)ideos["\'][^>]*>', webpage,
'data', playlist_id, contains_pattern=r'\[{(?s:.+)}\]')
return {
'_type': 'url_transparent',
'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?domain=www.videodetective.com&customerid=69249&playerid=641&publishedid=' + iva_id,
'ie_key': InternetVideoArchiveIE.ie_key(),
'id': video_id,
'title': self._og_search_title(webpage),
}
if video_id:
video_data = traverse_obj(data, lambda _, v: v['publicId'] == video_id)
if not video_data:
raise ExtractorError('Unable to extract video from webpage')
return next(self._extract_videos(video_data, video_id))
return self.playlist_result(
self._extract_videos(data, playlist_id), playlist_id,
clean_html(get_element_by_class('scoreboard__title', webpage)))
yt_dlp/extractor/tvplay.py
+1 -212
View File
@@ -30,10 +30,7 @@ class TVPlayIE(InfoExtractor):
(?:
tvplay(?:\.skaties)?\.lv(?:/parraides)?|
(?:tv3play|play\.tv3)\.lt(?:/programos)?|
tv3play(?:\.tv3)?\.ee/sisu|
(?:tv(?:3|6|8|10)play)\.se/program|
(?:(?:tv3play|viasat4play|tv6play)\.no|(?:tv3play)\.dk)/programmer|
play\.nova(?:tv)?\.bg/programi
tv3play(?:\.tv3)?\.ee/sisu
)
/(?:[^/]+/)+
)
@@ -92,117 +89,6 @@ class TVPlayIE(InfoExtractor):
'skip_download': True,
},
},
{
'url': 'http://www.tv3play.se/program/husraddarna/395385?autostart=true',
'info_dict': {
'id': '395385',
'ext': 'mp4',
'title': 'Husräddarna S02E07',
'description': 'md5:f210c6c89f42d4fc39faa551be813777',
'duration': 2574,
'timestamp': 1400596321,
'upload_date': '20140520',
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://www.tv6play.se/program/den-sista-dokusapan/266636?autostart=true',
'info_dict': {
'id': '266636',
'ext': 'mp4',
'title': 'Den sista dokusåpan S01E08',
'description': 'md5:295be39c872520221b933830f660b110',
'duration': 1492,
'timestamp': 1330522854,
'upload_date': '20120229',
'age_limit': 18,
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://www.tv8play.se/program/antikjakten/282756?autostart=true',
'info_dict': {
'id': '282756',
'ext': 'mp4',
'title': 'Antikjakten S01E10',
'description': 'md5:1b201169beabd97e20c5ad0ad67b13b8',
'duration': 2646,
'timestamp': 1348575868,
'upload_date': '20120925',
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://www.tv3play.no/programmer/anna-anka-soker-assistent/230898?autostart=true',
'info_dict': {
'id': '230898',
'ext': 'mp4',
'title': 'Anna Anka søker assistent - Ep. 8',
'description': 'md5:f80916bf5bbe1c5f760d127f8dd71474',
'duration': 2656,
'timestamp': 1277720005,
'upload_date': '20100628',
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://www.viasat4play.no/programmer/budbringerne/21873?autostart=true',
'info_dict': {
'id': '21873',
'ext': 'mp4',
'title': 'Budbringerne program 10',
'description': 'md5:4db78dc4ec8a85bb04fd322a3ee5092d',
'duration': 1297,
'timestamp': 1254205102,
'upload_date': '20090929',
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://www.tv6play.no/programmer/hotelinspektor-alex-polizzi/361883?autostart=true',
'info_dict': {
'id': '361883',
'ext': 'mp4',
'title': 'Hotelinspektør Alex Polizzi - Ep. 10',
'description': 'md5:3ecf808db9ec96c862c8ecb3a7fdaf81',
'duration': 2594,
'timestamp': 1393236292,
'upload_date': '20140224',
},
'params': {
'skip_download': True,
},
},
{
'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
'info_dict': {
'id': '624952',
'ext': 'flv',
'title': 'Здравей, България (12.06.2015 г.) ',
'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
'duration': 8838,
'timestamp': 1434100372,
'upload_date': '20150612',
},
'params': {
# rtmp download
'skip_download': True,
},
},
{
'url': 'https://play.nova.bg/programi/zdravei-bulgariya/764300?autostart=true',
'only_matching': True,
},
{
'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
'only_matching': True,
@@ -327,103 +213,6 @@ class TVPlayIE(InfoExtractor):
}
class ViafreeIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
(?:www\.)?
viafree\.(?P<country>dk|no|se|fi)
/(?P<id>(?:program(?:mer)?|ohjelmat)?/(?:[^/]+/)+[^/?#&]+)
'''
_TESTS = [{
'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
'info_dict': {
'id': '757786',
'ext': 'mp4',
'title': 'Det beste vorspielet - Sesong 2 - Episode 1',
'description': 'md5:b632cb848331404ccacd8cd03e83b4c3',
'series': 'Det beste vorspielet',
'season_number': 2,
'duration': 1116,
'timestamp': 1471200600,
'upload_date': '20160814',
},
'params': {
'skip_download': True,
},
}, {
'url': 'https://www.viafree.dk/programmer/humor/comedy-central-roast-of-charlie-sheen/film/1047660',
'info_dict': {
'id': '1047660',
'ext': 'mp4',
'title': 'Comedy Central Roast of Charlie Sheen - Comedy Central Roast of Charlie Sheen',
'description': 'md5:ec956d941ae9fd7c65a48fd64951dc6d',
'series': 'Comedy Central Roast of Charlie Sheen',
'season_number': 1,
'duration': 3747,
'timestamp': 1608246060,
'upload_date': '20201217'
},
'params': {
'skip_download': True
}
}, {
# with relatedClips
'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-1',
'only_matching': True,
}, {
# Different og:image URL schema
'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
'only_matching': True,
}, {
'url': 'http://www.viafree.se/program/livsstil/husraddarna/sasong-2/avsnitt-2',
'only_matching': True,
}, {
'url': 'http://www.viafree.dk/programmer/reality/paradise-hotel/saeson-7/episode-5',
'only_matching': True,
}, {
'url': 'http://www.viafree.se/program/underhallning/i-like-radio-live/sasong-1/676869',
'only_matching': True,
}, {
'url': 'https://www.viafree.fi/ohjelmat/entertainment/amazing-makeovers/kausi-7/jakso-2',
'only_matching': True,
}]
_GEO_BYPASS = False
def _real_extract(self, url):
country, path = self._match_valid_url(url).groups()
content = self._download_json(
'https://viafree-content.mtg-api.com/viafree-content/v1/%s/path/%s' % (country, path), path)
program = content['_embedded']['viafreeBlocks'][0]['_embedded']['program']
guid = program['guid']
meta = content['meta']
title = meta['title']
try:
stream_href = self._download_json(
program['_links']['streamLink']['href'], guid,
headers=self.geo_verification_headers())['embedded']['prioritizedStreams'][0]['links']['stream']['href']
except ExtractorError as e:
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
self.raise_geo_restricted(countries=[country])
raise
formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_href, guid, 'mp4')
episode = program.get('episode') or {}
return {
'id': guid,
'title': title,
'thumbnail': meta.get('image'),
'description': meta.get('description'),
'series': episode.get('seriesTitle'),
'subtitles': subtitles,
'episode_number': int_or_none(episode.get('episodeNumber')),
'season_number': int_or_none(episode.get('seasonNumber')),
'duration': int_or_none(try_get(program, lambda x: x['video']['duration']['milliseconds']), 1000),
'timestamp': parse_iso8601(try_get(program, lambda x: x['availability']['start'])),
'formats': formats,
}
class TVPlayHomeIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://
yt_dlp/extractor/twitch.py
+14 -3
View File
@@ -41,7 +41,6 @@ class TwitchBaseIE(InfoExtractor):
_USHER_BASE = 'https://usher.ttvnw.net'
_LOGIN_FORM_URL = 'https://www.twitch.tv/login'
_LOGIN_POST_URL = 'https://passport.twitch.tv/login'
_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
_NETRC_MACHINE = 'twitch'
_OPERATION_HASHES = {
@@ -58,6 +57,11 @@ class TwitchBaseIE(InfoExtractor):
'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
}
@property
def _CLIENT_ID(self):
return self._configuration_arg(
'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key=TwitchStreamIE, casesense=True)[0]
def _perform_login(self, username, password):
def fail(message):
raise ExtractorError(
@@ -194,7 +198,8 @@ class TwitchVodIE(TwitchBaseIE):
https?://
(?:
(?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
player\.twitch\.tv/\?.*?\bvideo=v?
player\.twitch\.tv/\?.*?\bvideo=v?|
www\.twitch\.tv/[^/]+/schedule\?vodID=
)
(?P<id>\d+)
'''
@@ -363,6 +368,9 @@ class TwitchVodIE(TwitchBaseIE):
'skip_download': True
},
'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
}, {
'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
'only_matching': True,
}]
def _download_info(self, item_id):
@@ -1075,7 +1083,7 @@ class TwitchClipsIE(TwitchBaseIE):
https?://
(?:
clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
(?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
(?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/)?clip/
)
(?P<id>[^/?#&]+)
'''
@@ -1111,6 +1119,9 @@ class TwitchClipsIE(TwitchBaseIE):
}, {
'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
'only_matching': True,
}, {
'url': 'https://m.twitch.tv/clip/FaintLightGullWholeWheat',
'only_matching': True,
}]
def _real_extract(self, url):
yt_dlp/extractor/twitter.py
+3
View File
@@ -705,6 +705,7 @@ class TwitterIE(TwitterBaseIE):
'uploader': r're:Monique Camarra.+?',
'uploader_id': 'MoniqueCamarra',
'live_status': 'was_live',
'release_timestamp': 1658417414,
'description': 'md5:acce559345fd49f129c20dbcda3f1201',
'timestamp': 1658407771464,
},
@@ -1327,6 +1328,8 @@ class TwitterSpacesIE(TwitterBaseIE):
'uploader_id': traverse_obj(
metadata, ('creator_results', 'result', 'legacy', 'screen_name')),
'live_status': live_status,
'release_timestamp': try_call(
lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
'timestamp': metadata.get('created_at'),
'formats': formats,
}
yt_dlp/extractor/unsupported.py
+3 -2
View File
@@ -131,8 +131,9 @@ class KnownPiracyIE(UnsupportedInfoExtractor):
URLS = (
r'dood\.(?:to|watch|so|pm|wf|re)',
# Sites youtube-dl supports, but we won't
r'https://viewsb\.com',
r'https://filemoon\.sx',
r'viewsb\.com',
r'filemoon\.sx',
r'hentai\.animestigma\.com',
)
_TESTS = [{
yt_dlp/extractor/vrt.py
+378 -35
View File
@@ -1,45 +1,137 @@
from .common import InfoExtractor
import functools
import json
import time
import urllib.error
import urllib.parse
from .gigya import GigyaBaseIE
from ..utils import (
ExtractorError,
clean_html,
extract_attributes,
float_or_none,
get_element_by_class,
get_element_html_by_class,
int_or_none,
join_nonempty,
jwt_encode_hs256,
make_archive_id,
parse_age_limit,
parse_iso8601,
str_or_none,
strip_or_none,
unified_timestamp,
traverse_obj,
url_or_none,
urlencode_postdata,
)
class VRTIE(InfoExtractor):
class VRTBaseIE(GigyaBaseIE):
_GEO_BYPASS = False
_PLAYER_INFO = {
'platform': 'desktop',
'app': {
'type': 'browser',
'name': 'Chrome',
},
'device': 'undefined (undefined)',
'os': {
'name': 'Windows',
'version': 'x86_64'
},
'player': {
'name': 'VRT web player',
'version': '2.7.4-prod-2023-04-19T06:05:45'
}
}
# From https://player.vrt.be/vrtnws/js/main.js & https://player.vrt.be/ketnet/js/main.fd1de01a40a1e3d842ea.js
_JWT_KEY_ID = '0-0Fp51UZykfaiCJrfTE3+oMI8zvDteYfPtR+2n1R+z8w='
_JWT_SIGNING_KEY = '2a9251d782700769fb856da5725daf38661874ca6f80ae7dc2b05ec1a81a24ae'
def _extract_formats_and_subtitles(self, data, video_id):
if traverse_obj(data, 'drm'):
self.report_drm(video_id)
formats, subtitles = [], {}
for target in traverse_obj(data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type'])):
format_type = target['type'].upper()
format_url = target['url']
if format_type in ('HLS', 'HLS_AES'):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif format_type == 'HDS':
formats.extend(self._extract_f4m_formats(
format_url, video_id, f4m_id=format_type, fatal=False))
elif format_type == 'MPEG_DASH':
fmts, subs = self._extract_mpd_formats_and_subtitles(
format_url, video_id, mpd_id=format_type, fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif format_type == 'HSS':
fmts, subs = self._extract_ism_formats_and_subtitles(
format_url, video_id, ism_id='mss', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
formats.append({
'format_id': format_type,
'url': format_url,
})
for sub in traverse_obj(data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED')):
subtitles.setdefault('nl', []).append({'url': sub['url']})
return formats, subtitles
def _call_api(self, video_id, client='null', id_token=None, version='v2'):
player_info = {'exp': (round(time.time(), 3) + 900), **self._PLAYER_INFO}
player_token = self._download_json(
'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2/tokens',
video_id, 'Downloading player token', headers={
**self.geo_verification_headers(),
'Content-Type': 'application/json',
}, data=json.dumps({
'identityToken': id_token or {},
'playerInfo': jwt_encode_hs256(player_info, self._JWT_SIGNING_KEY, headers={
'kid': self._JWT_KEY_ID
}).decode()
}, separators=(',', ':')).encode())['vrtPlayerToken']
return self._download_json(
f'https://media-services-public.vrt.be/media-aggregator/{version}/media-items/{video_id}',
video_id, 'Downloading API JSON', query={
'vrtPlayerToken': player_token,
'client': client,
}, expected_status=400)
class VRTIE(VRTBaseIE):
IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
_VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
_TESTS = [{
'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/',
'md5': 'e1663accf5cf13f375f3cd0d10476669',
'info_dict': {
'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd',
'ext': 'mp4',
'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand',
'description': 'Op maandagavond 15 april ging een deel van het dakgebinte van de Parijse kathedraal in vlammen op.',
'timestamp': 1557924660,
'upload_date': '20190515',
'description': 'md5:6fd85f999b2d1841aa5568f4bf02c3ff',
'duration': 31.2,
'thumbnail': 'https://images.vrt.be/orig/2019/05/15/2d914d61-7710-11e9-abcc-02b7b76bf47f.jpg',
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/',
'md5': '910bba927566e9ab992278f647eb4b75',
'info_dict': {
'id': 'pbs-pub-f2c86a46-8138-413a-a4b9-a0015a16ce2c$vid-1f112b31-e58e-4379-908d-aca6d80f8818',
'ext': 'mp4',
'title': 'De Belgian Cats zijn klaar voor het EK mét Ann Wauters',
'timestamp': 1557923760,
'upload_date': '20190515',
'title': 'De Belgian Cats zijn klaar voor het EK',
'description': 'Video: De Belgian Cats zijn klaar voor het EK mét Ann Wauters | basketbal, sport in het journaal',
'duration': 115.17,
'thumbnail': 'https://images.vrt.be/orig/2019/05/15/11c0dba3-770e-11e9-abcc-02b7b76bf47f.jpg',
},
}, {
'url': 'https://www.vrt.be/vrtnws/en/2019/05/15/belgium_s-eurovision-entry-falls-at-the-first-hurdle/',
'only_matching': True,
}, {
'url': 'https://www.vrt.be/vrtnws/de/2019/05/15/aus-fuer-eliott-im-halbfinale-des-eurosongfestivals/',
'only_matching': True,
'params': {'skip_download': 'm3u8'},
}]
_CLIENT_MAP = {
'vrt.be/vrtnws': 'vrtnieuws',
@@ -49,34 +141,285 @@ class VRTIE(InfoExtractor):
def _real_extract(self, url):
site, display_id = self._match_valid_url(url).groups()
webpage = self._download_webpage(url, display_id)
attrs = extract_attributes(self._search_regex(
r'(<[^>]+class="vrtvideo( [^"]*)?"[^>]*>)', webpage, 'vrt video'))
attrs = extract_attributes(get_element_html_by_class('vrtvideo', webpage) or '')
asset_id = attrs['data-video-id']
publication_id = attrs.get('data-publication-id')
asset_id = attrs.get('data-video-id') or attrs['data-videoid']
publication_id = traverse_obj(attrs, 'data-publication-id', 'data-publicationid')
if publication_id:
asset_id = publication_id + '$' + asset_id
client = attrs.get('data-client-code') or self._CLIENT_MAP[site]
asset_id = f'{publication_id}${asset_id}'
client = traverse_obj(attrs, 'data-client-code', 'data-client') or self._CLIENT_MAP[site]
data = self._call_api(asset_id, client)
formats, subtitles = self._extract_formats_and_subtitles(data, asset_id)
title = strip_or_none(get_element_by_class(
'vrt-title', webpage) or self._html_search_meta(
['og:title', 'twitter:title', 'name'], webpage))
description = self._html_search_meta(
['og:description', 'twitter:description', 'description'], webpage)
if description == '…':
description = None
timestamp = unified_timestamp(self._html_search_meta(
'article:published_time', webpage))
return {
'_type': 'url_transparent',
'id': asset_id,
'display_id': display_id,
'title': title,
'formats': formats,
'subtitles': subtitles,
'description': description,
'thumbnail': attrs.get('data-posterimage'),
'timestamp': timestamp,
'thumbnail': url_or_none(attrs.get('data-posterimage')),
'duration': float_or_none(attrs.get('data-duration'), 1000),
'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (client, asset_id),
'ie_key': 'Canvas',
'_old_archive_ids': [make_archive_id('Canvas', asset_id)],
**traverse_obj(data, {
'title': ('title', {str}),
'description': ('shortDescription', {str}),
'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
'thumbnail': ('posterImageUrl', {url_or_none}),
}),
}
class VrtNUIE(VRTBaseIE):
IE_DESC = 'VRT MAX'
_VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
_TESTS = [{
# CONTENT_IS_AGE_RESTRICTED
'url': 'https://www.vrt.be/vrtnu/a-z/de-ideale-wereld/2023-vj/de-ideale-wereld-d20230116/',
'info_dict': {
'id': 'pbs-pub-855b00a8-6ce2-4032-ac4f-1fcf3ae78524$vid-d2243aa1-ec46-4e34-a55b-92568459906f',
'ext': 'mp4',
'title': 'Tom Waes',
'description': 'Satirisch actualiteitenmagazine met Ella Leyers. Tom Waes is te gast.',
'timestamp': 1673905125,
'release_timestamp': 1673905125,
'series': 'De ideale wereld',
'season_id': '1672830988794',
'episode': 'Aflevering 1',
'episode_number': 1,
'episode_id': '1672830988861',
'display_id': 'de-ideale-wereld-d20230116',
'channel': 'VRT',
'duration': 1939.0,
'thumbnail': 'https://images.vrt.be/orig/2023/01/10/1bb39cb3-9115-11ed-b07d-02b7b76bf47f.jpg',
'release_date': '20230116',
'upload_date': '20230116',
'age_limit': 12,
},
}, {
'url': 'https://www.vrt.be/vrtnu/a-z/buurman--wat-doet-u-nu-/6/buurman--wat-doet-u-nu--s6-trailer/',
'info_dict': {
'id': 'pbs-pub-ad4050eb-d9e5-48c2-9ec8-b6c355032361$vid-0465537a-34a8-4617-8352-4d8d983b4eee',
'ext': 'mp4',
'title': 'Trailer seizoen 6 \'Buurman, wat doet u nu?\'',
'description': 'md5:197424726c61384b4e5c519f16c0cf02',
'timestamp': 1652940000,
'release_timestamp': 1652940000,
'series': 'Buurman, wat doet u nu?',
'season': 'Seizoen 6',
'season_number': 6,
'season_id': '1652344200907',
'episode': 'Aflevering 0',
'episode_number': 0,
'episode_id': '1652951873524',
'display_id': 'buurman--wat-doet-u-nu--s6-trailer',
'channel': 'VRT',
'duration': 33.13,
'thumbnail': 'https://images.vrt.be/orig/2022/05/23/3c234d21-da83-11ec-b07d-02b7b76bf47f.jpg',
'release_date': '20220519',
'upload_date': '20220519',
},
'params': {'skip_download': 'm3u8'},
}]
_NETRC_MACHINE = 'vrtnu'
_authenticated = False
def _perform_login(self, username, password):
auth_info = self._gigya_login({
'APIKey': '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy',
'targetEnv': 'jssdk',
'loginID': username,
'password': password,
'authMode': 'cookie',
})
if auth_info.get('errorDetails'):
raise ExtractorError(f'Unable to login. VrtNU said: {auth_info["errorDetails"]}', expected=True)
# Sometimes authentication fails for no good reason, retry
for retry in self.RetryManager():
if retry.attempt > 1:
self._sleep(1, None)
try:
self._request_webpage(
'https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token',
errnote='Could not get XSRF Token', query={
'provider': 'site',
'destination': 'https://www.vrt.be/vrtnu/',
})
self._request_webpage(
'https://login.vrt.be/perform_login', None,
note='Performing login', errnote='Login failed',
query={'client_id': 'vrtnu-site'}, data=urlencode_postdata({
'UID': auth_info['UID'],
'UIDSignature': auth_info['UIDSignature'],
'signatureTimestamp': auth_info['signatureTimestamp'],
'_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
}))
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
retry.error = e
continue
raise
self._authenticated = True
def _real_extract(self, url):
display_id = self._match_id(url)
parsed_url = urllib.parse.urlparse(url)
details = self._download_json(
f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip("/")}.model.json',
display_id, 'Downloading asset JSON', 'Unable to download asset JSON')['details']
watch_info = traverse_obj(details, (
'actions', lambda _, v: v['type'] == 'watch-episode', {dict}), get_all=False) or {}
video_id = join_nonempty(
'episodePublicationId', 'episodeVideoId', delim='$', from_dict=watch_info)
if '$' not in video_id:
raise ExtractorError('Unable to extract video ID')
vrtnutoken = self._download_json(
'https://token.vrt.be/refreshtoken', video_id, note='Retrieving vrtnutoken',
errnote='Token refresh failed')['vrtnutoken'] if self._authenticated else None
video_info = self._call_api(video_id, 'vrtnu-web@PROD', vrtnutoken)
if 'title' not in video_info:
code = video_info.get('code')
if code in ('AUTHENTICATION_REQUIRED', 'CONTENT_IS_AGE_RESTRICTED'):
self.raise_login_required(code, method='password')
elif code in ('INVALID_LOCATION', 'CONTENT_AVAILABLE_ONLY_IN_BE'):
self.raise_geo_restricted(countries=['BE'])
elif code == 'CONTENT_AVAILABLE_ONLY_FOR_BE_RESIDENTS_AND_EXPATS':
if not self._authenticated:
self.raise_login_required(code, method='password')
self.raise_geo_restricted(countries=['BE'])
raise ExtractorError(code, expected=True)
formats, subtitles = self._extract_formats_and_subtitles(video_info, video_id)
return {
**traverse_obj(details, {
'title': 'title',
'description': ('description', {clean_html}),
'timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
'release_timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
'series': ('data', 'program', 'title'),
'season': ('data', 'season', 'title', 'value'),
'season_number': ('data', 'season', 'title', 'raw', {int_or_none}),
'season_id': ('data', 'season', 'id', {str_or_none}),
'episode': ('data', 'episode', 'number', 'value', {str_or_none}),
'episode_number': ('data', 'episode', 'number', 'raw', {int_or_none}),
'episode_id': ('data', 'episode', 'id', {str_or_none}),
'age_limit': ('data', 'episode', 'age', 'raw', {parse_age_limit}),
}),
'id': video_id,
'display_id': display_id,
'channel': 'VRT',
'formats': formats,
'duration': float_or_none(video_info.get('duration'), 1000),
'thumbnail': url_or_none(video_info.get('posterImageUrl')),
'subtitles': subtitles,
'_old_archive_ids': [make_archive_id('Canvas', video_id)],
}
class KetnetIE(VRTBaseIE):
_VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
_TESTS = [{
'url': 'https://www.ketnet.be/kijken/m/meisjes/6/meisjes-s6a5',
'info_dict': {
'id': 'pbs-pub-39f8351c-a0a0-43e6-8394-205d597d6162$vid-5e306921-a9aa-4fa9-9f39-5b82c8f1028e',
'ext': 'mp4',
'title': 'Meisjes',
'episode': 'Reeks 6: Week 5',
'season': 'Reeks 6',
'series': 'Meisjes',
'timestamp': 1685251800,
'upload_date': '20230528',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
display_id = self._match_id(url)
video = self._download_json(
'https://senior-bff.ketnet.be/graphql', display_id, query={
'query': '''{
video(id: "content/ketnet/nl/%s.model.json") {
description
episodeNr
imageUrl
mediaReference
programTitle
publicationDate
seasonTitle
subtitleVideodetail
titleVideodetail
}
}''' % display_id,
})['data']['video']
video_id = urllib.parse.unquote(video['mediaReference'])
data = self._call_api(video_id, 'ketnet@PROD', version='v1')
formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'_old_archive_ids': [make_archive_id('Canvas', video_id)],
**traverse_obj(video, {
'title': ('titleVideodetail', {str}),
'description': ('description', {str}),
'thumbnail': ('thumbnail', {url_or_none}),
'timestamp': ('publicationDate', {parse_iso8601}),
'series': ('programTitle', {str}),
'season': ('seasonTitle', {str}),
'episode': ('subtitleVideodetail', {str}),
'episode_number': ('episodeNr', {int_or_none}),
}),
}
class DagelijkseKostIE(VRTBaseIE):
IE_DESC = 'dagelijksekost.een.be'
_VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
'info_dict': {
'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
'ext': 'mp4',
'title': 'Hachis parmentier met witloof',
'description': 'md5:9960478392d87f63567b5b117688cdc5',
'display_id': 'hachis-parmentier-met-witloof',
},
'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
video_id = self._html_search_regex(
r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
data = self._call_api(video_id, 'dako@prod', version='v1')
formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
'display_id': display_id,
'title': strip_or_none(get_element_by_class(
'dish-metadata__title', webpage) or self._html_search_meta('twitter:title', webpage)),
'description': clean_html(get_element_by_class(
'dish-description', webpage)) or self._html_search_meta(
['description', 'twitter:description', 'og:description'], webpage),
'_old_archive_ids': [make_archive_id('Canvas', video_id)],
}
yt_dlp/extractor/weverse.py
+607
View File
@@ -0,0 +1,607 @@
import base64
import hashlib
import hmac
import itertools
import json
import re
import time
import urllib.error
import urllib.parse
import uuid
from .common import InfoExtractor
from .naver import NaverBaseIE
from .youtube import YoutubeIE
from ..utils import (
ExtractorError,
UserNotLive,
float_or_none,
int_or_none,
str_or_none,
traverse_obj,
try_call,
update_url_query,
url_or_none,
)
class WeverseBaseIE(InfoExtractor):
_NETRC_MACHINE = 'weverse'
_ACCOUNT_API_BASE = 'https://accountapi.weverse.io/web/api/v2'
_API_HEADERS = {
'Referer': 'https://weverse.io/',
'WEV-device-Id': str(uuid.uuid4()),
}
def _perform_login(self, username, password):
if self._API_HEADERS.get('Authorization'):
return
headers = {
'x-acc-app-secret': '5419526f1c624b38b10787e5c10b2a7a',
'x-acc-app-version': '2.2.6',
'x-acc-language': 'en',
'x-acc-service-id': 'weverse',
'x-acc-trace-id': str(uuid.uuid4()),
'x-clog-user-device-id': str(uuid.uuid4()),
}
check_username = self._download_json(
f'{self._ACCOUNT_API_BASE}/signup/email/status', None,
note='Checking username', query={'email': username}, headers=headers)
if not check_username.get('hasPassword'):
raise ExtractorError('Invalid username provided', expected=True)
headers['content-type'] = 'application/json'
try:
auth = self._download_json(
f'{self._ACCOUNT_API_BASE}/auth/token/by-credentials', None, data=json.dumps({
'email': username,
'password': password,
}, separators=(',', ':')).encode(), headers=headers, note='Logging in')
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
raise ExtractorError('Invalid password provided', expected=True)
raise
WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {auth["accessToken"]}'
def _real_initialize(self):
if self._API_HEADERS.get('Authorization'):
return
token = try_call(lambda: self._get_cookies('https://weverse.io/')['we2_access_token'].value)
if not token:
self.raise_login_required()
WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {token}'
def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
# Ref: https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/2488.a09b41ff.chunk.js
# From https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/main.e206f7c1.js:
key = b'1b9cb6378d959b45714bec49971ade22e6e24e42'
api_path = update_url_query(ep, {
'appId': 'be4d79eb8fc7bd008ee82c8ec4ff6fd4',
'language': 'en',
'platform': 'WEB',
'wpf': 'pc',
})
wmsgpad = int(time.time() * 1000)
wmd = base64.b64encode(hmac.HMAC(
key, f'{api_path[:255]}{wmsgpad}'.encode(), digestmod=hashlib.sha1).digest()).decode()
headers = {'Content-Type': 'application/json'} if data else {}
try:
return self._download_json(
f'https://global.apis.naver.com/weverse/wevweb{api_path}', video_id, note=note,
data=data, headers={**self._API_HEADERS, **headers}, query={
'wmsgpad': wmsgpad,
'wmd': wmd,
})
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
self.raise_login_required(
'Session token has expired. Log in again or refresh cookies in browser')
elif isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
raise ExtractorError('Your account does not have access to this content', expected=True)
raise
def _call_post_api(self, video_id):
return self._call_api(f'/post/v1.0/post-{video_id}?fieldSet=postV1', video_id)
def _get_community_id(self, channel):
return str(self._call_api(
f'/community/v1.0/communityIdUrlPathByUrlPathArtistCode?keyword={channel}',
channel, note='Fetching community ID')['communityId'])
def _get_formats(self, data, video_id):
formats = traverse_obj(data, ('videos', 'list', lambda _, v: url_or_none(v['source']), {
'url': 'source',
'width': ('encodingOption', 'width', {int_or_none}),
'height': ('encodingOption', 'height', {int_or_none}),
'vcodec': 'type',
'vbr': ('bitrate', 'video', {int_or_none}),
'abr': ('bitrate', 'audio', {int_or_none}),
'filesize': ('size', {int_or_none}),
'format_id': ('encodingOption', 'id', {str_or_none}),
}))
for stream in traverse_obj(data, ('streams', lambda _, v: v['type'] == 'HLS' and url_or_none(v['source']))):
query = {}
for param in traverse_obj(stream, ('keys', lambda _, v: v['type'] == 'param' and v['name'])):
query[param['name']] = param.get('value', '')
fmts = self._extract_m3u8_formats(
stream['source'], video_id, 'mp4', m3u8_id='hls', fatal=False, query=query)
if query:
for fmt in fmts:
fmt['url'] = update_url_query(fmt['url'], query)
fmt['extra_param_to_segment_url'] = urllib.parse.urlencode(query)
formats.extend(fmts)
return formats
def _get_subs(self, caption_url):
subs_ext_re = r'\.(?:ttml|vtt)'
replace_ext = lambda x, y: re.sub(subs_ext_re, y, x)
if re.search(subs_ext_re, caption_url):
return [replace_ext(caption_url, '.ttml'), replace_ext(caption_url, '.vtt')]
return [caption_url]
def _parse_post_meta(self, metadata):
return traverse_obj(metadata, {
'title': ((('extension', 'mediaInfo', 'title'), 'title'), {str}),
'description': ((('extension', 'mediaInfo', 'body'), 'body'), {str}),
'uploader': ('author', 'profileName', {str}),
'uploader_id': ('author', 'memberId', {str}),
'creator': ('community', 'communityName', {str}),
'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
'duration': ('extension', 'video', 'playTime', {float_or_none}),
'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
'release_timestamp': ('extension', 'video', 'onAirStartAt', {lambda x: int_or_none(x, 1000)}),
'thumbnail': ('extension', (('mediaInfo', 'thumbnail', 'url'), ('video', 'thumb')), {url_or_none}),
'view_count': ('extension', 'video', 'playCount', {int_or_none}),
'like_count': ('extension', 'video', 'likeCount', {int_or_none}),
'comment_count': ('commentCount', {int_or_none}),
}, get_all=False)
def _extract_availability(self, data):
return self._availability(**traverse_obj(data, ((('extension', 'video'), None), {
'needs_premium': 'paid',
'needs_subscription': 'membershipOnly',
}), get_all=False, expected_type=bool), needs_auth=True)
def _extract_live_status(self, data):
data = traverse_obj(data, ('extension', 'video', {dict})) or {}
if data.get('type') == 'LIVE':
return traverse_obj({
'ONAIR': 'is_live',
'DONE': 'post_live',
'STANDBY': 'is_upcoming',
'DELAY': 'is_upcoming',
}, (data.get('status'), {str})) or 'is_live'
return 'was_live' if data.get('liveToVod') else 'not_live'
class WeverseIE(WeverseBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/live/(?P<id>[\d-]+)'
_TESTS = [{
'url': 'https://weverse.io/billlie/live/0-107323480',
'md5': '1fa849f00181eef9100d3c8254c47979',
'info_dict': {
'id': '0-107323480',
'ext': 'mp4',
'title': '행복한 평이루💜',
'description': '',
'uploader': 'Billlie',
'uploader_id': '5ae14aed7b7cdc65fa87c41fe06cc936',
'channel': 'billlie',
'channel_id': '72',
'channel_url': 'https://weverse.io/billlie',
'creator': 'Billlie',
'timestamp': 1666262062,
'upload_date': '20221020',
'release_timestamp': 1666262058,
'release_date': '20221020',
'duration': 3102,
'thumbnail': r're:^https?://.*\.jpe?g$',
'view_count': int,
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
'live_status': 'was_live',
},
}, {
'url': 'https://weverse.io/lesserafim/live/2-102331763',
'md5': 'e46125c08b13a6c8c1f4565035cca987',
'info_dict': {
'id': '2-102331763',
'ext': 'mp4',
'title': '🎂김채원 생신🎂',
'description': '🎂김채원 생신🎂',
'uploader': 'LE SSERAFIM ',
'uploader_id': 'd26ddc1e258488a0a2b795218d14d59d',
'channel': 'lesserafim',
'channel_id': '47',
'channel_url': 'https://weverse.io/lesserafim',
'creator': 'LE SSERAFIM',
'timestamp': 1659353400,
'upload_date': '20220801',
'release_timestamp': 1659353400,
'release_date': '20220801',
'duration': 3006,
'thumbnail': r're:^https?://.*\.jpe?g$',
'view_count': int,
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
'live_status': 'was_live',
'subtitles': {
'id_ID': 'count:2',
'en_US': 'count:2',
'es_ES': 'count:2',
'vi_VN': 'count:2',
'th_TH': 'count:2',
'zh_CN': 'count:2',
'zh_TW': 'count:2',
'ja_JP': 'count:2',
'ko_KR': 'count:2',
},
},
}, {
'url': 'https://weverse.io/treasure/live/2-117230416',
'info_dict': {
'id': '2-117230416',
'ext': 'mp4',
'title': r're:스껄도려님 첫 스무살 생파🦋',
'description': '',
'uploader': 'TREASURE',
'uploader_id': '77eabbc449ca37f7970054a136f60082',
'channel': 'treasure',
'channel_id': '20',
'channel_url': 'https://weverse.io/treasure',
'creator': 'TREASURE',
'timestamp': 1680667651,
'upload_date': '20230405',
'release_timestamp': 1680667639,
'release_date': '20230405',
'thumbnail': r're:^https?://.*\.jpe?g$',
'view_count': int,
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
'live_status': 'is_live',
},
'skip': 'Livestream has ended',
}]
def _real_extract(self, url):
channel, video_id = self._match_valid_url(url).group('artist', 'id')
post = self._call_post_api(video_id)
api_video_id = post['extension']['video']['videoId']
availability = self._extract_availability(post)
live_status = self._extract_live_status(post)
video_info, formats = {}, []
if live_status == 'is_upcoming':
self.raise_no_formats('Livestream has not yet started', expected=True)
elif live_status == 'is_live':
video_info = self._call_api(
f'/video/v1.0/lives/{api_video_id}/playInfo?preview.format=json&preview.version=v2',
video_id, note='Downloading live JSON')
playback = self._parse_json(video_info['lipPlayback'], video_id)
m3u8_url = traverse_obj(playback, (
'media', lambda _, v: v['protocol'] == 'HLS', 'path', {url_or_none}), get_all=False)
formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', live=True)
elif live_status == 'post_live':
if availability in ('premium_only', 'subscriber_only'):
self.report_drm(video_id)
self.raise_no_formats(
'Livestream has ended and downloadable VOD is not available', expected=True)
else:
infra_video_id = post['extension']['video']['infraVideoId']
in_key = self._call_api(
f'/video/v1.0/vod/{api_video_id}/inKey?preview=false', video_id,
data=b'{}', note='Downloading VOD API key')['inKey']
video_info = self._download_json(
f'https://global.apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{infra_video_id}',
video_id, note='Downloading VOD JSON', query={
'key': in_key,
'sid': traverse_obj(post, ('extension', 'video', 'serviceId')) or '2070',
'pid': str(uuid.uuid4()),
'nonce': int(time.time() * 1000),
'devt': 'html5_pc',
'prv': 'Y' if post.get('membershipOnly') else 'N',
'aup': 'N',
'stpb': 'N',
'cpl': 'en',
'env': 'prod',
'lc': 'en',
'adi': '[{"adSystem":"null"}]',
'adu': '/',
})
formats = self._get_formats(video_info, video_id)
has_drm = traverse_obj(video_info, ('meta', 'provider', 'name', {str.lower})) == 'drm'
if has_drm and formats:
self.report_warning(
'Requested content is DRM-protected, only a 30-second preview is available', video_id)
elif has_drm and not formats:
self.report_drm(video_id)
return {
'id': video_id,
'channel': channel,
'channel_url': f'https://weverse.io/{channel}',
'formats': formats,
'availability': availability,
'live_status': live_status,
**self._parse_post_meta(post),
**NaverBaseIE.process_subtitles(video_info, self._get_subs),
}
class WeverseMediaIE(WeverseBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/media/(?P<id>[\d-]+)'
_TESTS = [{
'url': 'https://weverse.io/billlie/media/4-116372884',
'md5': '8efc9cfd61b2f25209eb1a5326314d28',
'info_dict': {
'id': 'e-C9wLSQs6o',
'ext': 'mp4',
'title': 'Billlie | \'EUNOIA\' Performance Video (heartbeat ver.)',
'description': 'md5:6181caaf2a2397bca913ffe368c104e5',
'channel': 'Billlie',
'channel_id': 'UCyc9sUCxELTDK9vELO5Fzeg',
'channel_url': 'https://www.youtube.com/channel/UCyc9sUCxELTDK9vELO5Fzeg',
'uploader': 'Billlie',
'uploader_id': '@Billlie',
'uploader_url': 'http://www.youtube.com/@Billlie',
'upload_date': '20230403',
'duration': 211,
'age_limit': 0,
'playable_in_embed': True,
'live_status': 'not_live',
'availability': 'public',
'view_count': int,
'comment_count': int,
'like_count': int,
'channel_follower_count': int,
'thumbnail': 'https://i.ytimg.com/vi/e-C9wLSQs6o/maxresdefault.jpg',
'categories': ['Entertainment'],
'tags': 'count:7',
},
}, {
'url': 'https://weverse.io/billlie/media/3-102914520',
'md5': '031551fcbd716bc4f080cb6174a43d8a',
'info_dict': {
'id': '3-102914520',
'ext': 'mp4',
'title': 'From. SUHYEON🌸',
'description': 'Billlie 멤버별 독점 영상 공개💙💜',
'uploader': 'Billlie_official',
'uploader_id': 'f569c6e92f7eaffef0a395037dcaa54f',
'channel': 'billlie',
'channel_id': '72',
'channel_url': 'https://weverse.io/billlie',
'creator': 'Billlie',
'timestamp': 1662174000,
'upload_date': '20220903',
'release_timestamp': 1662174000,
'release_date': '20220903',
'duration': 17.0,
'thumbnail': r're:^https?://.*\.jpe?g$',
'view_count': int,
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
'live_status': 'not_live',
},
}]
def _real_extract(self, url):
channel, video_id = self._match_valid_url(url).group('artist', 'id')
post = self._call_post_api(video_id)
media_type = traverse_obj(post, ('extension', 'mediaInfo', 'mediaType', {str.lower}))
youtube_id = traverse_obj(post, ('extension', 'youtube', 'youtubeVideoId', {str}))
if media_type == 'vod':
return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)
elif media_type == 'youtube' and youtube_id:
return self.url_result(youtube_id, YoutubeIE)
elif media_type == 'image':
self.raise_no_formats('No video content found in webpage', expected=True)
elif media_type:
raise ExtractorError(f'Unsupported media type "{media_type}"')
self.raise_no_formats('No video content found in webpage')
class WeverseMomentIE(WeverseBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/moment/(?P<uid>[\da-f]+)/post/(?P<id>[\d-]+)'
_TESTS = [{
'url': 'https://weverse.io/secretnumber/moment/66a07e164b56a696ee71c99315ffe27b/post/1-117229444',
'md5': '87733ac19a54081b7dfc2442036d282b',
'info_dict': {
'id': '1-117229444',
'ext': 'mp4',
'title': '今日もめっちゃいい天気☀️🌤️',
'uploader': '레아',
'uploader_id': '66a07e164b56a696ee71c99315ffe27b',
'channel': 'secretnumber',
'channel_id': '56',
'creator': 'SECRET NUMBER',
'duration': 10,
'upload_date': '20230405',
'timestamp': 1680653968,
'thumbnail': r're:^https?://.*\.jpe?g$',
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
},
'skip': 'Moment has expired',
}]
def _real_extract(self, url):
channel, uploader_id, video_id = self._match_valid_url(url).group('artist', 'uid', 'id')
post = self._call_post_api(video_id)
api_video_id = post['extension']['moment']['video']['videoId']
video_info = self._call_api(
f'/cvideo/v1.0/cvideo-{api_video_id}/playInfo?videoId={api_video_id}', video_id,
note='Downloading moment JSON')['playInfo']
return {
'id': video_id,
'channel': channel,
'uploader_id': uploader_id,
'formats': self._get_formats(video_info, video_id),
'availability': self._extract_availability(post),
**traverse_obj(post, {
'title': ((('extension', 'moment', 'body'), 'body'), {str}),
'uploader': ('author', 'profileName', {str}),
'creator': (('community', 'author'), 'communityName', {str}),
'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
'duration': ('extension', 'moment', 'video', 'uploadInfo', 'playTime', {float_or_none}),
'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
'thumbnail': ('extension', 'moment', 'video', 'uploadInfo', 'imageUrl', {url_or_none}),
'like_count': ('emotionCount', {int_or_none}),
'comment_count': ('commentCount', {int_or_none}),
}, get_all=False),
**NaverBaseIE.process_subtitles(video_info, self._get_subs),
}
class WeverseTabBaseIE(WeverseBaseIE):
_ENDPOINT = None
_PATH = None
_QUERY = {}
_RESULT_IE = None
def _entries(self, channel_id, channel, first_page):
query = self._QUERY.copy()
for page in itertools.count(1):
posts = first_page if page == 1 else self._call_api(
update_url_query(self._ENDPOINT % channel_id, query), channel,
note=f'Downloading {self._PATH} tab page {page}')
for post in traverse_obj(posts, ('data', lambda _, v: v['postId'])):
yield self.url_result(
f'https://weverse.io/{channel}/{self._PATH}/{post["postId"]}',
self._RESULT_IE, post['postId'], **self._parse_post_meta(post),
channel=channel, channel_url=f'https://weverse.io/{channel}',
availability=self._extract_availability(post),
live_status=self._extract_live_status(post))
query['after'] = traverse_obj(posts, ('paging', 'nextParams', 'after', {str}))
if not query['after']:
break
def _real_extract(self, url):
channel = self._match_id(url)
channel_id = self._get_community_id(channel)
first_page = self._call_api(
update_url_query(self._ENDPOINT % channel_id, self._QUERY), channel,
note=f'Downloading {self._PATH} tab page 1')
return self.playlist_result(
self._entries(channel_id, channel, first_page), f'{channel}-{self._PATH}',
**traverse_obj(first_page, ('data', ..., {
'playlist_title': ('community', 'communityName', {str}),
'thumbnail': ('author', 'profileImageUrl', {url_or_none}),
}), get_all=False))
class WeverseLiveTabIE(WeverseTabBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/live/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://weverse.io/billlie/live/',
'playlist_mincount': 55,
'info_dict': {
'id': 'billlie-live',
'title': 'Billlie',
'thumbnail': r're:^https?://.*\.jpe?g$',
},
}]
_ENDPOINT = '/post/v1.0/community-%s/liveTabPosts'
_PATH = 'live'
_QUERY = {'fieldSet': 'postsV1'}
_RESULT_IE = WeverseIE
class WeverseMediaTabIE(WeverseTabBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/media(?:/|/all|/new)?(?:[?#]|$)'
_TESTS = [{
'url': 'https://weverse.io/billlie/media/',
'playlist_mincount': 231,
'info_dict': {
'id': 'billlie-media',
'title': 'Billlie',
'thumbnail': r're:^https?://.*\.jpe?g$',
},
}, {
'url': 'https://weverse.io/lesserafim/media/all',
'only_matching': True,
}, {
'url': 'https://weverse.io/lesserafim/media/new',
'only_matching': True,
}]
_ENDPOINT = '/media/v1.0/community-%s/more'
_PATH = 'media'
_QUERY = {'fieldSet': 'postsV1', 'filterType': 'RECENT'}
_RESULT_IE = WeverseMediaIE
class WeverseLiveIE(WeverseBaseIE):
_VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://weverse.io/purplekiss',
'info_dict': {
'id': '3-116560493',
'ext': 'mp4',
'title': r're:모하냥🫶🏻',
'description': '내일은 금요일~><',
'uploader': '채인',
'uploader_id': '1ffb1d9d904d6b3db2783f876eb9229d',
'channel': 'purplekiss',
'channel_id': '35',
'channel_url': 'https://weverse.io/purplekiss',
'creator': 'PURPLE KISS',
'timestamp': 1680780892,
'upload_date': '20230406',
'release_timestamp': 1680780883,
'release_date': '20230406',
'thumbnail': 'https://weverse-live.pstatic.net/v1.0/live/62044/thumb',
'view_count': int,
'like_count': int,
'comment_count': int,
'availability': 'needs_auth',
'live_status': 'is_live',
},
'skip': 'Livestream has ended',
}, {
'url': 'https://weverse.io/billlie/',
'only_matching': True,
}]
def _real_extract(self, url):
channel = self._match_id(url)
channel_id = self._get_community_id(channel)
video_id = traverse_obj(
self._call_api(update_url_query(f'/post/v1.0/community-{channel_id}/liveTab', {
'debugMessage': 'true',
'fields': 'onAirLivePosts.fieldSet(postsV1).limit(10),reservedLivePosts.fieldSet(postsV1).limit(10)',
}), channel, note='Downloading live JSON'), (
('onAirLivePosts', 'reservedLivePosts'), 'data',
lambda _, v: self._extract_live_status(v) in ('is_live', 'is_upcoming'), 'postId', {str}),
get_all=False)
if not video_id:
raise UserNotLive(video_id=channel)
return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)
yt_dlp/extractor/weyyak.py
+86
View File
@@ -0,0 +1,86 @@
from .common import InfoExtractor
from ..utils import (
float_or_none,
int_or_none,
parse_age_limit,
traverse_obj,
unified_timestamp,
url_or_none,
)
class WeyyakIE(InfoExtractor):
_VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/(?:player/)?(?P<type>episode|movie)/(?P<id>\d+)'
_TESTS = [
{
'url': 'https://weyyak.com/en/player/episode/1341952/Ribat-Al-Hob-Episode49',
'md5': '0caf55c1a615531c8fe60f146ae46849',
'info_dict': {
'id': '1341952',
'ext': 'mp4',
'title': 'Ribat Al Hob',
'duration': 2771,
'alt_title': 'رباط الحب',
'season': 'Season 1',
'season_number': 1,
'episode': 'Episode 49',
'episode_number': 49,
'timestamp': 1485907200,
'upload_date': '20170201',
'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
'categories': ['Drama', 'Thrillers', 'Romance'],
'tags': 'count:8',
},
},
{
'url': 'https://weyyak.com/en/movie/233255/8-Seconds',
'md5': 'fe740ae0f63e4d1c8a7fc147a410c564',
'info_dict': {
'id': '233255',
'ext': 'mp4',
'title': '8 Seconds',
'duration': 6490,
'alt_title': '8 ثواني',
'description': 'md5:45b83a155c30b49950624c7e99600b9d',
'age_limit': 15,
'release_year': 2015,
'timestamp': 1683106031,
'upload_date': '20230503',
'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
'categories': ['Drama', 'Social'],
'cast': ['Ceylin Adiyaman', 'Esra Inal'],
},
},
]
def _real_extract(self, url):
video_id, lang, type_ = self._match_valid_url(url).group('id', 'lang', 'type')
path = 'episode/' if type_ == 'episode' else 'contents/moviedetails?contentkey='
data = self._download_json(
f'https://msapifo-prod-me.weyyak.z5.com/v1/{lang}/{path}{video_id}', video_id)['data']
m3u8_url = self._download_json(
f'https://api-weyyak.akamaized.net/get_info/{data["video_id"]}',
video_id, 'Extracting video details')['url_video']
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id)
return {
'id': video_id,
'formats': formats,
'subtitles': subtitles,
**traverse_obj(data, {
'title': ('title', {str}),
'alt_title': ('translated_title', {str}),
'description': ('synopsis', {str}),
'duration': ('length', {float_or_none}),
'age_limit': ('age_rating', {parse_age_limit}),
'season_number': ('season_number', {int_or_none}),
'episode_number': ('episode_number', {int_or_none}),
'thumbnail': ('imagery', 'thumbnail', {url_or_none}),
'categories': ('genres', ..., {str}),
'tags': ('tags', ..., {str}),
'cast': (('main_actor', 'main_actress'), {str}),
'timestamp': ('insertedAt', {unified_timestamp}),
'release_year': ('production_year', {int_or_none}),
}),
}
yt_dlp/extractor/wrestleuniverse.py
+102 -35
View File
@@ -2,6 +2,7 @@ import base64
import binascii
import json
import time
import uuid
from .common import InfoExtractor
from ..dependencies import Cryptodome
@@ -12,30 +13,95 @@ from ..utils import (
traverse_obj,
try_call,
url_or_none,
urlencode_postdata,
)
class WrestleUniverseBaseIE(InfoExtractor):
_NETRC_MACHINE = 'wrestleuniverse'
_VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
_API_PATH = None
_TOKEN = None
_REAL_TOKEN = None
_TOKEN_EXPIRY = None
_REFRESH_TOKEN = None
_DEVICE_ID = None
_LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
_LOGIN_HEADERS = {
'Accept': '*/*',
'Content-Type': 'application/json',
'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
'Referer': 'https://www.wrestle-universe.com/',
'Origin': 'https://www.wrestle-universe.com',
}
def _get_token_cookie(self):
if not self._TOKEN or not self._TOKEN_EXPIRY:
self._TOKEN = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
if not self._TOKEN:
@property
def _TOKEN(self):
if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
if not token and not self._REFRESH_TOKEN:
self.raise_login_required()
expiry = traverse_obj(jwt_decode_hs256(self._TOKEN), ('exp', {int_or_none}))
if not expiry:
raise ExtractorError('There was a problem with the token cookie')
self._TOKEN_EXPIRY = expiry
self._REAL_TOKEN = token
if self._TOKEN_EXPIRY <= int(time.time()):
raise ExtractorError(
'Expired token. Refresh your cookies in browser and try again', expected=True)
if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
if not self._REFRESH_TOKEN:
raise ExtractorError(
'Expired token. Refresh your cookies in browser and try again', expected=True)
self._refresh_token()
return self._TOKEN
return self._REAL_TOKEN
@_TOKEN.setter
def _TOKEN(self, value):
self._REAL_TOKEN = value
expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
if not expiry:
raise ExtractorError('There was a problem with the auth token')
self._TOKEN_EXPIRY = expiry
def _perform_login(self, username, password):
login = self._download_json(
'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
'returnSecureToken': True,
'email': username,
'password': password,
}, separators=(',', ':')).encode())
self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
if not self._REFRESH_TOKEN:
self.report_warning('No refresh token was granted')
self._TOKEN = traverse_obj(login, ('idToken', {str}))
def _real_initialize(self):
if WrestleUniverseBaseIE._DEVICE_ID:
return
WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
if not WrestleUniverseBaseIE._DEVICE_ID:
WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
if WrestleUniverseBaseIE._DEVICE_ID:
return
WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
def _refresh_token(self):
refresh = self._download_json(
'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
query=self._LOGIN_QUERY, data=urlencode_postdata({
'grant_type': 'refresh_token',
'refresh_token': self._REFRESH_TOKEN,
}), headers={
**self._LOGIN_HEADERS,
'Content-Type': 'application/x-www-form-urlencoded',
})
if traverse_obj(refresh, ('refresh_token', {str})):
self._REFRESH_TOKEN = refresh['refresh_token']
token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
if not token:
raise ExtractorError('No auth token returned from refresh request')
self._TOKEN = token
def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
headers = {'CA-CID': ''}
@@ -43,7 +109,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
headers['Content-Type'] = 'application/json;charset=utf-8'
data = json.dumps(data, separators=(',', ':')).encode()
if auth:
headers['Authorization'] = f'Bearer {self._get_token_cookie()}'
headers['Authorization'] = f'Bearer {self._TOKEN}'
return self._download_json(
f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
@@ -65,7 +131,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
api_json = self._call_api(video_id, param, msg, data={
# 'deviceId' (random uuid4 generated at login) is not required yet
'deviceId': self._DEVICE_ID,
'token': token,
**data,
}, query=query, fatal=fatal)
@@ -105,7 +171,7 @@ class WrestleUniverseVODIE(WrestleUniverseBaseIE):
'upload_date': '20230129',
'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
'chapters': 'count:7',
'cast': 'count:18',
'cast': 'count:21',
},
'params': {
'skip_download': 'm3u8',
@@ -169,6 +235,7 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
'params': {
'skip_download': 'm3u8',
},
'skip': 'No longer available',
}, {
'note': 'unencrypted HLS',
'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
@@ -196,14 +263,17 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
lang, video_id = self._match_valid_url(url).group('lang', 'id')
metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
info = traverse_obj(metadata, {
'title': ('displayName', {str}),
'description': ('description', {str}),
'channel': ('labels', 'group', {str}),
'location': ('labels', 'venue', {str}),
'timestamp': ('startTime', {int_or_none}),
'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
})
info = {
'id': video_id,
**traverse_obj(metadata, {
'title': ('displayName', {str}),
'description': ('description', {str}),
'channel': ('labels', 'group', {str}),
'location': ('labels', 'venue', {str}),
'timestamp': ('startTime', {int_or_none}),
'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
}),
}
ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
if info.get('timestamp') and ended_time:
@@ -211,23 +281,20 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
video_data, decrypt = self._call_encrypted_api(
video_id, ':watchArchive', 'watch archive', data={'method': 1})
formats = self._get_formats(video_data, (
info['formats'] = self._get_formats(video_data, (
('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
for f in formats:
for f in info['formats']:
# bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
if f.get('tbr'):
f['tbr'] = int(f['tbr'] / 2.5)
hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
if not hls_aes_key and traverse_obj(video_data, ('hls', 'encryptType', {int}), default=0) > 0:
self.report_warning('HLS AES-128 key was not found in API response')
return {
'id': video_id,
'formats': formats,
'hls_aes': {
if hls_aes_key:
info['hls_aes'] = {
'key': hls_aes_key,
'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
},
**info,
}
elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
self.report_warning('HLS AES-128 key was not found in API response')
return info
yt_dlp/extractor/wykop.py
+268
View File
@@ -0,0 +1,268 @@
import json
import urllib.error
from .common import InfoExtractor
from ..utils import (
ExtractorError,
format_field,
parse_iso8601,
traverse_obj,
url_or_none,
)
class WykopBaseExtractor(InfoExtractor):
def _get_token(self, force_refresh=False):
if not force_refresh:
maybe_cached = self.cache.load('wykop', 'bearer')
if maybe_cached:
return maybe_cached
new_token = traverse_obj(
self._do_call_api('auth', None, 'Downloading anonymous auth token', data={
# hardcoded in frontend
'key': 'w53947240748',
'secret': 'd537d9e0a7adc1510842059ae5316419',
}), ('data', 'token'))
self.cache.store('wykop', 'bearer', new_token)
return new_token
def _do_call_api(self, path, video_id, note='Downloading JSON metadata', data=None, headers={}):
if data:
data = json.dumps({'data': data}).encode()
headers['Content-Type'] = 'application/json'
return self._download_json(
f'https://wykop.pl/api/v3/{path}', video_id,
note=note, data=data, headers=headers)
def _call_api(self, path, video_id, note='Downloading JSON metadata'):
token = self._get_token()
for retrying in range(2):
try:
return self._do_call_api(path, video_id, note, headers={'Authorization': f'Bearer {token}'})
except ExtractorError as e:
if not retrying and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
token = self._get_token(True)
continue
raise
def _common_data_extract(self, data):
author = traverse_obj(data, ('author', 'username'), expected_type=str)
return {
'_type': 'url_transparent',
'display_id': data.get('slug'),
'url': traverse_obj(data,
('media', 'embed', 'url'), # what gets an iframe embed
('source', 'url'), # clickable url (dig only)
expected_type=url_or_none),
'thumbnail': traverse_obj(
data, ('media', 'photo', 'url'), ('media', 'embed', 'thumbnail'), expected_type=url_or_none),
'uploader': author,
'uploader_id': author,
'uploader_url': format_field(author, None, 'https://wykop.pl/ludzie/%s'),
'timestamp': parse_iso8601(data.get('created_at'), delimiter=' '), # time it got submitted
'like_count': traverse_obj(data, ('votes', 'up'), expected_type=int),
'dislike_count': traverse_obj(data, ('votes', 'down'), expected_type=int),
'comment_count': traverse_obj(data, ('comments', 'count'), expected_type=int),
'age_limit': 18 if data.get('adult') else 0,
'tags': data.get('tags'),
}
class WykopDigIE(WykopBaseExtractor):
IE_NAME = 'wykop:dig'
_VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<id>\d+)'
_TESTS = [{
'url': 'https://wykop.pl/link/6912923/najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
'info_dict': {
'id': 'rlSTBvViflc',
'ext': 'mp4',
'title': 'Najbardziej zrzędliwy kot na świecie I Frozen Planet II I BBC Earth',
'display_id': 'najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
'description': 'md5:ac0f87dea1cdcb6b0c53f3612a095c87',
'tags': ['zwierzaczki', 'koty', 'smiesznykotek', 'humor', 'rozrywka', 'ciekawostki'],
'age_limit': 0,
'timestamp': 1669154480,
'release_timestamp': 1669194241,
'release_date': '20221123',
'uploader': 'starnak',
'uploader_id': 'starnak',
'uploader_url': 'https://wykop.pl/ludzie/starnak',
'like_count': int,
'dislike_count': int,
'comment_count': int,
'thumbnail': r're:https?://wykop\.pl/cdn/.+',
'view_count': int,
'channel': 'BBC Earth',
'channel_id': 'UCwmZiChSryoWQCZMIQezgTg',
'channel_url': 'https://www.youtube.com/channel/UCwmZiChSryoWQCZMIQezgTg',
'categories': ['Pets & Animals'],
'upload_date': '20220923',
'duration': 191,
'channel_follower_count': int,
'availability': 'public',
'live_status': 'not_live',
'playable_in_embed': True,
},
}]
@classmethod
def suitable(cls, url):
return cls._match_valid_url(url) and not WykopDigCommentIE.suitable(url)
def _real_extract(self, url):
video_id = self._match_id(url)
data = self._call_api(f'links/{video_id}', video_id)['data']
return {
**self._common_data_extract(data),
'id': video_id,
'title': data['title'],
'description': data.get('description'),
# time it got "digged" to the homepage
'release_timestamp': parse_iso8601(data.get('published_at'), delimiter=' '),
}
class WykopDigCommentIE(WykopBaseExtractor):
IE_NAME = 'wykop:dig:comment'
_VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<dig_id>\d+)/[^/]+/komentarz/(?P<id>\d+)'
_TESTS = [{
'url': 'https://wykop.pl/link/6992589/strollowal-oszusta-przez-ponad-24-minuty-udawal-naiwniaka-i-nagral-rozmowe/komentarz/114540527/podobna-sytuacja-ponizej-ciekawa-dyskusja-z-oszustem-na-sam-koniec-sam-bylem-w-biurze-swiadkiem-podobnej-rozmowy-niemal-zakonczonej-sukcesem-bandyty-g',
'info_dict': {
'id': 'u6tEi2FmKZY',
'ext': 'mp4',
'title': 'md5:e7c741c5baa7ed6478000caf72865577',
'display_id': 'md5:45b2d12bd0e262d09cc7cf7abc8412db',
'description': 'md5:bcec7983429f9c0630f9deb9d3d1ba5e',
'timestamp': 1674476945,
'uploader': 'Bartholomew',
'uploader_id': 'Bartholomew',
'uploader_url': 'https://wykop.pl/ludzie/Bartholomew',
'thumbnail': r're:https?://wykop\.pl/cdn/.+',
'tags': [],
'availability': 'public',
'duration': 1838,
'upload_date': '20230117',
'categories': ['Entertainment'],
'view_count': int,
'like_count': int,
'dislike_count': int,
'comment_count': int,
'channel_follower_count': int,
'playable_in_embed': True,
'live_status': 'not_live',
'age_limit': 0,
'chapters': 'count:3',
'channel': 'Poszukiwacze Okazji',
'channel_id': 'UCzzvJDZThwv06dR4xmzrZBw',
'channel_url': 'https://www.youtube.com/channel/UCzzvJDZThwv06dR4xmzrZBw',
},
}]
def _real_extract(self, url):
dig_id, comment_id = self._search_regex(self._VALID_URL, url, 'dig and comment ids', group=('dig_id', 'id'))
data = self._call_api(f'links/{dig_id}/comments/{comment_id}', comment_id)['data']
return {
**self._common_data_extract(data),
'id': comment_id,
'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
'description': data.get('content'),
}
class WykopPostIE(WykopBaseExtractor):
IE_NAME = 'wykop:post'
_VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<id>\d+)'
_TESTS = [{
'url': 'https://wykop.pl/wpis/68893343/kot-koty-smiesznykotek',
'info_dict': {
'id': 'PL8JMjiUPHUhwc9ZlKa_5IFeBwBV8Xe7jI',
'title': 'PawelW124 - #kot #koty #smiesznykotek',
'description': '#kot #koty #smiesznykotek',
'display_id': 'kot-koty-smiesznykotek',
'tags': ['kot', 'koty', 'smiesznykotek'],
'uploader': 'PawelW124',
'uploader_id': 'PawelW124',
'uploader_url': 'https://wykop.pl/ludzie/PawelW124',
'timestamp': 1668938142,
'age_limit': 0,
'like_count': int,
'dislike_count': int,
'thumbnail': r're:https?://wykop\.pl/cdn/.+',
'comment_count': int,
'channel': 'Revan',
'channel_id': 'UCW9T_-uZoiI7ROARQdTDyOw',
'channel_url': 'https://www.youtube.com/channel/UCW9T_-uZoiI7ROARQdTDyOw',
'upload_date': '20221120',
'modified_date': '20220814',
'availability': 'public',
'view_count': int,
},
'playlist_mincount': 15,
'params': {
'flat_playlist': True,
}
}]
@classmethod
def suitable(cls, url):
return cls._match_valid_url(url) and not WykopPostCommentIE.suitable(url)
def _real_extract(self, url):
video_id = self._match_id(url)
data = self._call_api(f'entries/{video_id}', video_id)['data']
return {
**self._common_data_extract(data),
'id': video_id,
'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
'description': data.get('content'),
}
class WykopPostCommentIE(WykopBaseExtractor):
IE_NAME = 'wykop:post:comment'
_VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<post_id>\d+)/[^/#]+#(?P<id>\d+)'
_TESTS = [{
'url': 'https://wykop.pl/wpis/70084873/test-test-test#249303979',
'info_dict': {
'id': 'confusedquickarmyant',
'ext': 'mp4',
'title': 'tpap - treść komentarza',
'display_id': 'tresc-komentarza',
'description': 'treść komentarza',
'uploader': 'tpap',
'uploader_id': 'tpap',
'uploader_url': 'https://wykop.pl/ludzie/tpap',
'timestamp': 1675349470,
'upload_date': '20230202',
'tags': [],
'duration': 2.12,
'age_limit': 0,
'categories': [],
'view_count': int,
'like_count': int,
'dislike_count': int,
'thumbnail': r're:https?://wykop\.pl/cdn/.+',
},
}]
def _real_extract(self, url):
post_id, comment_id = self._search_regex(self._VALID_URL, url, 'post and comment ids', group=('post_id', 'id'))
data = self._call_api(f'entries/{post_id}/comments/{comment_id}', comment_id)['data']
return {
**self._common_data_extract(data),
'id': comment_id,
'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
'description': data.get('content'),
}
yt_dlp/extractor/youtube.py
+162 -55
View File
@@ -66,7 +66,6 @@ from ..utils import (
variadic,
)
STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
# any clients starting with _ cannot be explicitly requested by the user
INNERTUBE_CLIENTS = {
@@ -894,9 +893,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
def extract_relative_time(relative_time_text):
"""
Extracts a relative time from string and converts to dt object
e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
"""
mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
# XXX: this could be moved to a general function in utils.py
# The relative time text strings are roughly the same as what
# Javascript's Intl.RelativeTimeFormat function generates.
# See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
mobj = re.search(
r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
relative_time_text)
if mobj:
start = mobj.group('start')
if start:
@@ -1039,6 +1045,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
else self._get_count({'simpleText': view_count_text}))
view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
or self._get_text(reel_header_renderer, 'channelTitleText'))
channel_handle = traverse_obj(renderer, (
'shortBylineText', 'runs', ..., 'navigationEndpoint',
(('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
expected_type=self.handle_from_url, get_all=False)
return {
'_type': 'url',
'ie_key': YoutubeIE.ie_key(),
@@ -1048,9 +1061,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
'description': description,
'duration': duration,
'channel_id': channel_id,
'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
or self._get_text(reel_header_renderer, 'channelTitleText')),
'channel': channel,
'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
'uploader': channel,
'uploader_id': channel_handle,
'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
'timestamp': (self._parse_time_text(time_text)
if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
@@ -1274,6 +1289,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'Philipp Hagemeister',
'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
'uploader_id': '@PhilippHagemeister',
'heatmap': 'count:100',
}
},
{
@@ -1427,6 +1443,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'uploader': 'FlyingKitty',
'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
'uploader_id': '@FlyingKitty900',
'comment_count': int,
},
},
{
@@ -3023,17 +3040,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
# Obsolete patterns
r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
jscode, 'Initial JS player signature function name', group='sig')
@@ -3277,42 +3291,66 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
chapter_time, chapter_title, duration)
for contents in content_list)), [])
def _extract_heatmap_from_player_overlay(self, data):
content_list = traverse_obj(data, (
'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer', 'decoratedPlayerBarRenderer', 'playerBar',
'multiMarkersPlayerBarRenderer', 'markersMap', ..., 'value', 'heatmap', 'heatmapRenderer', 'heatMarkers', {list}))
return next(filter(None, (
traverse_obj(contents, (..., 'heatMarkerRenderer', {
'start_time': ('timeRangeStartMillis', {functools.partial(float_or_none, scale=1000)}),
'end_time': {lambda x: (x['timeRangeStartMillis'] + x['markerDurationMillis']) / 1000},
'value': ('heatMarkerIntensityScoreNormalized', {float_or_none}),
})) for contents in content_list)), None)
def _extract_comment(self, comment_renderer, parent=None):
comment_id = comment_renderer.get('commentId')
if not comment_id:
return
text = self._get_text(comment_renderer, 'contentText')
info = {
'id': comment_id,
'text': self._get_text(comment_renderer, 'contentText'),
'like_count': self._get_count(comment_renderer, 'voteCount'),
'author_id': traverse_obj(comment_renderer, ('authorEndpoint', 'browseEndpoint', 'browseId', {self.ucid_or_none})),
'author': self._get_text(comment_renderer, 'authorText'),
'author_thumbnail': traverse_obj(comment_renderer, ('authorThumbnail', 'thumbnails', -1, 'url', {url_or_none})),
'parent': parent or 'root',
}
# Timestamp is an estimate calculated from the current time and time_text
time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
timestamp = self._parse_time_text(time_text)
author = self._get_text(comment_renderer, 'authorText')
author_id = try_get(comment_renderer,
lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
lambda x: x['likeCount']), str)) or 0
author_thumbnail = try_get(comment_renderer,
lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
is_favorited = 'creatorHeart' in (try_get(
comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
return {
'id': comment_id,
'text': text,
info.update({
# FIXME: non-standard, but we need a way of showing that it is an estimate.
'_time_text': time_text,
'timestamp': timestamp,
'time_text': time_text,
'like_count': votes,
'is_favorited': is_favorited,
'author': author,
'author_id': author_id,
'author_thumbnail': author_thumbnail,
'author_is_uploader': author_is_uploader,
'parent': parent or 'root'
}
})
info['author_url'] = urljoin(
'https://www.youtube.com', traverse_obj(comment_renderer, ('authorEndpoint', (
('browseEndpoint', 'canonicalBaseUrl'), ('commandMetadata', 'webCommandMetadata', 'url'))),
expected_type=str, get_all=False))
author_is_uploader = traverse_obj(comment_renderer, 'authorIsChannelOwner')
if author_is_uploader is not None:
info['author_is_uploader'] = author_is_uploader
comment_abr = traverse_obj(
comment_renderer, ('actionsButtons', 'commentActionButtonsRenderer'), expected_type=dict)
if comment_abr is not None:
info['is_favorited'] = 'creatorHeart' in comment_abr
comment_ab_icontype = traverse_obj(
comment_renderer, ('authorCommentBadge', 'authorCommentBadgeRenderer', 'icon', 'iconType'))
if comment_ab_icontype is not None:
info['author_is_verified'] = comment_ab_icontype in ('CHECK_CIRCLE_THICK', 'OFFICIAL_ARTIST_BADGE')
is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
if is_pinned:
info['is_pinned'] = True
return info
def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
@@ -3325,7 +3363,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
expected_comment_count = self._get_count(
comments_header_renderer, 'countText', 'commentsCount')
if expected_comment_count:
if expected_comment_count is not None:
tracker['est_total'] = expected_comment_count
self.to_screen(f'Downloading ~{expected_comment_count} comments')
comment_sort_index = int(get_single_config_arg('comment_sort') != 'top') # 1 = new, 0 = top
@@ -3360,14 +3398,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
comment = self._extract_comment(comment_renderer, parent)
if not comment:
continue
is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
comment_id = comment['id']
if is_pinned:
if comment.get('is_pinned'):
tracker['pinned_comment_ids'].add(comment_id)
# Sometimes YouTube may break and give us infinite looping comments.
# See: https://github.com/yt-dlp/yt-dlp/issues/6290
if comment_id in tracker['seen_comment_ids']:
if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
if comment_id in tracker['pinned_comment_ids'] and not comment.get('is_pinned'):
# Pinned comments may appear a second time in newest first sort
# See: https://github.com/yt-dlp/yt-dlp/issues/6712
continue
@@ -3396,7 +3433,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if not tracker:
tracker = dict(
running_total=0,
est_total=0,
est_total=None,
current_page_thread=0,
total_parent_comments=0,
total_reply_comments=0,
@@ -3429,11 +3466,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
is_forced_continuation = True
continuation_items_path = (
'onResponseReceivedEndpoints', ..., ('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems')
for page_num in itertools.count(0):
if not continuation:
break
headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
comment_prog_str = f"({tracker['running_total']}/~{tracker['est_total']})"
if page_num == 0:
if is_first_continuation:
note_prefix = 'Downloading comment section API JSON'
@@ -3444,11 +3483,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
' ' if parent else '', ' replies' if parent else '',
page_num, comment_prog_str)
# Do a deep check for incomplete data as sometimes YouTube may return no comments for a continuation
# Ignore check if YouTube says the comment count is 0.
check_get_keys = None
if not is_forced_continuation and not (tracker['est_total'] == 0 and tracker['running_total'] == 0):
check_get_keys = [[*continuation_items_path, ..., (
'commentsHeaderRenderer' if is_first_continuation else ('commentThreadRenderer', 'commentRenderer'))]]
try:
response = self._extract_response(
item_id=None, query=continuation,
ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
check_get_keys=check_get_keys)
except ExtractorError as e:
# Ignore incomplete data error for replies if retries didn't work.
# This is to allow any other parent comments and comment threads to be downloaded.
@@ -3460,15 +3506,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
else:
raise
is_forced_continuation = False
continuation_contents = traverse_obj(
response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
continuation = None
for continuation_section in continuation_contents:
continuation_items = traverse_obj(
continuation_section,
(('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
get_all=False, expected_type=list) or []
for continuation_items in traverse_obj(response, continuation_items_path, expected_type=list, default=[]):
if is_first_continuation:
continuation = extract_header(continuation_items)
is_first_continuation = False
@@ -4349,6 +4388,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
or self._extract_chapters_from_description(video_description, duration)
or None)
info['heatmap'] = self._extract_heatmap_from_player_overlay(initial_data)
contents = traverse_obj(
initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
expected_type=list, default=[])
@@ -4611,8 +4652,11 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
def _music_reponsive_list_entry(self, renderer):
video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
if video_id:
title = traverse_obj(renderer, (
'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
'text', 'runs', 0, 'text'))
return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
ie=YoutubeIE.ie_key(), video_id=video_id)
ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
if playlist_id:
video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
@@ -4671,11 +4715,19 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
def _rich_entries(self, rich_grid_renderer):
renderer = traverse_obj(
rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
rich_grid_renderer,
('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer')), get_all=False) or {}
video_id = renderer.get('videoId')
if not video_id:
if video_id:
yield self._extract_video(renderer)
return
playlist_id = renderer.get('playlistId')
if playlist_id:
yield self.url_result(
f'https://www.youtube.com/playlist?list={playlist_id}',
ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
video_title=self._get_text(renderer, 'title'))
return
yield self._extract_video(renderer)
def _video_entry(self, video_renderer):
video_id = video_renderer.get('videoId')
@@ -4904,7 +4956,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
if metadata_renderer:
channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
('channelUrl', {self.ucid_from_url}))
('channelUrl', {self.ucid_from_url}))
info.update({
'channel': metadata_renderer.get('title'),
'channel_id': channel_id,
@@ -5861,7 +5913,25 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'uploader_id': '@colethedj1894',
'uploader': 'colethedj',
},
'playlist': [{
'info_dict': {
'title': 'youtube-dl test video "\'/\\ä↭𝕐',
'id': 'BaW_jenozKc',
'_type': 'url',
'ie_key': 'Youtube',
'duration': 10,
'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
'view_count': int,
'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
'channel': 'Philipp Hagemeister',
'uploader_id': '@PhilippHagemeister',
'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
'uploader': 'Philipp Hagemeister',
}
}],
'playlist_count': 1,
'params': {'extract_flat': True},
}, {
'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
'url': 'https://www.youtube.com/feed/recommended',
@@ -6162,6 +6232,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'channel_url': str,
'concurrent_view_count': int,
'channel': str,
'uploader': str,
'uploader_url': str,
'uploader_id': str
}
}],
'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -6217,6 +6290,40 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'uploader': '3Blue1Brown',
},
'playlist_count': 0,
}, {
# Podcasts tab, with rich entry playlistRenderers
'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
'info_dict': {
'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
'title': '99 Percent Invisible - Podcasts',
'uploader': '99 Percent Invisible',
'channel_follower_count': int,
'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
'tags': [],
'channel': '99 Percent Invisible',
'uploader_id': '@99percentinvisiblepodcast',
},
'playlist_count': 1,
}, {
# Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
'url': 'https://www.youtube.com/@AHimitsu/releases',
'info_dict': {
'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
'channel': 'A Himitsu',
'uploader_url': 'https://www.youtube.com/@AHimitsu',
'title': 'A Himitsu - Releases',
'uploader_id': '@AHimitsu',
'uploader': 'A Himitsu',
'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
'tags': 'count:16',
'description': 'I make music',
'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
'channel_follower_count': int,
},
'playlist_mincount': 10,
}]
@classmethod
yt_dlp/extractor/zingmp3.py
+71 -30
View File
@@ -1,16 +1,11 @@
import functools
import hashlib
import hmac
import itertools
import json
import urllib.parse
from .common import InfoExtractor
from ..utils import (
OnDemandPagedList,
int_or_none,
traverse_obj,
urljoin,
)
from ..utils import int_or_none, traverse_obj, try_call, urljoin
class ZingMp3BaseIE(InfoExtractor):
@@ -37,6 +32,7 @@ class ZingMp3BaseIE(InfoExtractor):
'info-artist': '/api/v2/page/get/artist',
'user-list-song': '/api/v2/song/get/list',
'user-list-video': '/api/v2/video/get/list',
'hub': '/api/v2/page/get/hub-detail',
}
def _api_url(self, url_type, params):
@@ -46,9 +42,9 @@ class ZingMp3BaseIE(InfoExtractor):
''.join(f'{k}={v}' for k, v in sorted(params.items())).encode()).hexdigest()
data = {
**params,
'apiKey': '88265e23d4284f25963e6eedac8fbfa3',
'sig': hmac.new(
b'2aa2d1c561e809b267f3638c4a307aab', f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
'apiKey': 'X5BM3w8N7MKozC0B85o4KMlzLZKhV00y',
'sig': hmac.new(b'acOrvUS15XRW2o9JksiK1KgQ6Vbds8ZW',
f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
}
return f'{self._DOMAIN}{api_slug}?{urllib.parse.urlencode(data)}'
@@ -67,6 +63,19 @@ class ZingMp3BaseIE(InfoExtractor):
for url in traverse_obj(items, (..., 'link')) or []:
yield self.url_result(urljoin(self._DOMAIN, url))
def _fetch_page(self, id_, url_type, page):
raise NotImplementedError('This method must be implemented by subclasses')
def _paged_list(self, _id, url_type):
count = 0
for page in itertools.count(1):
data = self._fetch_page(_id, url_type, page)
entries = list(self._parse_items(data.get('items')))
count += len(entries)
yield from entries
if not data.get('hasMore') or try_call(lambda: count > data['total']):
break
class ZingMp3IE(ZingMp3BaseIE):
_VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed'
@@ -166,8 +175,11 @@ class ZingMp3IE(ZingMp3BaseIE):
'height': int_or_none(res),
})
if not formats and item.get('msg') == 'Sorry, this content is not available in your country.':
self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
if not formats:
if item.get('msg') == 'Sorry, this content is not available in your country.':
self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
else:
self.raise_no_formats('The song is only for VIP accounts.')
lyric = item.get('lyric') or self._call_api('lyric', {'id': item_id}, fatal=False).get('file')
@@ -200,7 +212,7 @@ class ZingMp3AlbumIE(ZingMp3BaseIE):
'id': 'ZWZAEZZD',
'title': 'Những Bài Hát Hay Nhất Của Mr. Siro',
},
'playlist_mincount': 49,
'playlist_mincount': 20,
}, {
'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
'only_matching': True,
@@ -305,22 +317,20 @@ class ZingMp3ChartMusicVideoIE(ZingMp3BaseIE):
'id': 'IWZ9Z086',
'title': 'the-loai-video_Khong-Loi',
},
'playlist_mincount': 10,
'playlist_mincount': 1,
}]
def _fetch_page(self, song_id, url_type, page):
return self._parse_items(self._call_api(url_type, {
return self._call_api(url_type, {
'id': song_id,
'type': 'genre',
'page': page + 1,
'page': page,
'count': self._PER_PAGE
}).get('items'))
})
def _real_extract(self, url):
song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
return self.playlist_result(
OnDemandPagedList(functools.partial(self._fetch_page, song_id, url_type), self._PER_PAGE),
song_id, f'{url_type}_{regions}')
return self.playlist_result(self._paged_list(song_id, url_type), song_id, f'{url_type}_{regions}')
class ZingMp3UserIE(ZingMp3BaseIE):
@@ -331,7 +341,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
'info_dict': {
'id': 'IWZ98609',
'title': 'Mr. Siro - bai-hat',
'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
},
'playlist_mincount': 91,
}, {
@@ -339,7 +349,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
'info_dict': {
'id': 'IWZ98609',
'title': 'Mr. Siro - album',
'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
},
'playlist_mincount': 3,
}, {
@@ -347,7 +357,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
'info_dict': {
'id': 'IWZ98609',
'title': 'Mr. Siro - single',
'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
},
'playlist_mincount': 20,
}, {
@@ -355,19 +365,19 @@ class ZingMp3UserIE(ZingMp3BaseIE):
'info_dict': {
'id': 'IWZ98609',
'title': 'Mr. Siro - video',
'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
},
'playlist_mincount': 15,
}]
def _fetch_page(self, user_id, url_type, page):
url_type = 'user-list-song' if url_type == 'bai-hat' else 'user-list-video'
return self._parse_items(self._call_api(url_type, {
return self._call_api(url_type, {
'id': user_id,
'type': 'artist',
'page': page + 1,
'page': page,
'count': self._PER_PAGE
}, query={'sort': 'new', 'sectionId': 'aSong'}).get('items'))
})
def _real_extract(self, url):
user_alias, url_type = self._match_valid_url(url).group('user', 'type')
@@ -376,10 +386,41 @@ class ZingMp3UserIE(ZingMp3BaseIE):
user_info = self._call_api('info-artist', {}, user_alias, query={'alias': user_alias})
if url_type in ('bai-hat', 'video'):
entries = OnDemandPagedList(
functools.partial(self._fetch_page, user_info['id'], url_type), self._PER_PAGE)
entries = self._paged_list(user_info['id'], url_type)
else:
entries = self._parse_items(traverse_obj(user_info, (
'sections', lambda _, v: v['link'] == f'/{user_alias}/{url_type}', 'items', ...)))
'sections',
lambda _, v: v['sectionId'] == 'aAlbum' if url_type == 'album' else v['sectionId'] == 'aSingle',
'items', ...)))
return self.playlist_result(
entries, user_info['id'], f'{user_info.get("name")} - {url_type}', user_info.get('biography'))
class ZingMp3HubIE(ZingMp3BaseIE):
IE_NAME = 'zingmp3:hub'
_VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>hub)/(?P<regions>[^/]+)/(?P<id>[^\.]+)'
_TESTS = [{
'url': 'https://zingmp3.vn/hub/Nhac-Moi/IWZ9Z0CA.html',
'info_dict': {
'id': 'IWZ9Z0CA',
'title': 'Nhạc Mới',
'description': 'md5:1cc31b68a6f746427b07b2756c22a558',
},
'playlist_mincount': 20,
}, {
'url': 'https://zingmp3.vn/hub/Nhac-Viet/IWZ9Z087.html',
'info_dict': {
'id': 'IWZ9Z087',
'title': 'Nhạc Việt',
'description': 'md5:acc976c8bdde64d5c6ee4a92c39f7a77',
},
'playlist_mincount': 30,
}]
def _real_extract(self, url):
song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
hub_detail = self._call_api(url_type, {'id': song_id})
entries = self._parse_items(traverse_obj(hub_detail, (
'sections', lambda _, v: v['sectionId'] == 'hub', 'items', ...)))
return self.playlist_result(
entries, song_id, hub_detail.get('title'), hub_detail.get('description'))