artem/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2026-09-13 06:24:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into valid-urls-proposal

Browse Source
This commit is contained in:
nixxo
2023-01-12 21:50:40 +01:00
parent a73ef8c776 87ebab0615
commit 1c3a9489ca
98 changed files with 4408 additions and 827 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yt_dlp/extractor/_extractors.py
+40 -2
View File
@@ -79,6 +79,7 @@ from .agora import (
)
from .airmozilla import AirMozillaIE
from .airtv import AirTVIE
from .aitube import AitubeKZVideoIE
from .aljazeera import AlJazeeraIE
from .alphaporno import AlphaPornoIE
from .amara import AmaraIE
@@ -87,7 +88,10 @@ from .alura import (
AluraCourseIE
)
from .amcnetworks import AMCNetworksIE
from .amazon import AmazonStoreIE
from .amazon import (
AmazonStoreIE,
AmazonReviewsIE,
)
from .amazonminitv import (
AmazonMiniTVIE,
AmazonMiniTVSeasonIE,
@@ -184,6 +188,10 @@ from .bbc import (
from .beeg import BeegIE
from .behindkink import BehindKinkIE
from .bellmedia import BellMediaIE
from .beatbump import (
BeatBumpVideoIE,
BeatBumpPlaylistIE,
)
from .beatport import BeatportIE
from .berufetv import BerufeTVIE
from .bet import BetIE
@@ -467,6 +475,8 @@ from .drtuber import DrTuberIE
from .drtv import (
DRTVIE,
DRTVLiveIE,
DRTVSeasonIE,
DRTVSeriesIE,
)
from .dtube import DTubeIE
from .dvtv import DVTVIE
@@ -827,6 +837,7 @@ from .jwplatform import JWPlatformIE
from .kakao import KakaoIE
from .kaltura import KalturaIE
from .kanal2 import Kanal2IE
from .kankanews import KankaNewsIE
from .karaoketv import KaraoketvIE
from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
@@ -836,6 +847,10 @@ from .khanacademy import (
KhanAcademyIE,
KhanAcademyUnitIE,
)
from .kick import (
KickIE,
KickVODIE,
)
from .kicker import KickerIE
from .kickstarter import KickStarterIE
from .kinja import KinjaEmbedIE
@@ -1155,6 +1170,7 @@ from .neteasemusic import (
from .netverse import (
NetverseIE,
NetversePlaylistIE,
NetverseSearchIE,
)
from .newgrounds import (
NewgroundsIE,
@@ -1405,6 +1421,8 @@ from .pokergo import (
from .polsatgo import PolsatGoIE
from .polskieradio import (
PolskieRadioIE,
PolskieRadioLegacyIE,
PolskieRadioAuditionIE,
PolskieRadioCategoryIE,
PolskieRadioPlayerIE,
PolskieRadioPodcastIE,
@@ -1535,7 +1553,10 @@ from .rokfin import (
)
from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE
from .rottentomatoes import RottenTomatoesIE
from .rozhlas import RozhlasIE
from .rozhlas import (
RozhlasIE,
RozhlasVltavaIE,
)
from .rte import RteIE, RteRadioIE
from .rtlnl import (
RtlNlIE,
@@ -1693,6 +1714,7 @@ from .soundcloud import (
SoundcloudSetIE,
SoundcloudRelatedIE,
SoundcloudUserIE,
SoundcloudUserPermalinkIE,
SoundcloudTrackStationIE,
SoundcloudPlaylistIE,
SoundcloudSearchIE,
@@ -1854,6 +1876,11 @@ from .theweatherchannel import TheWeatherChannelIE
from .thisamericanlife import ThisAmericanLifeIE
from .thisav import ThisAVIE
from .thisoldhouse import ThisOldHouseIE
from .thisvid import (
ThisVidIE,
ThisVidMemberIE,
ThisVidPlaylistIE,
)
from .threespeak import (
ThreeSpeakIE,
ThreeSpeakUserIE,
@@ -1866,6 +1893,7 @@ from .tiktok import (
TikTokEffectIE,
TikTokTagIE,
TikTokVMIE,
TikTokLiveIE,
DouyinIE,
)
from .tinypic import TinyPicIE
@@ -1903,6 +1931,7 @@ from .trovo import (
TrovoChannelVodIE,
TrovoChannelClipIE,
)
from .trtcocuk import TrtCocukVideoIE
from .trueid import TrueIDIE
from .trunews import TruNewsIE
from .truth import TruthIE
@@ -2086,6 +2115,13 @@ from .videocampus_sachsen import (
)
from .videodetective import VideoDetectiveIE
from .videofyme import VideofyMeIE
from .videoken import (
VideoKenIE,
VideoKenPlayerIE,
VideoKenPlaylistIE,
VideoKenCategoryIE,
VideoKenTopicIE,
)
from .videomore import (
VideomoreIE,
VideomoreVideoIE,
@@ -2153,6 +2189,7 @@ from .voicy import (
VoicyIE,
VoicyChannelIE,
)
from .volejtv import VolejTVIE
from .voot import (
VootIE,
VootSeriesIE,
@@ -2235,6 +2272,7 @@ from .wsj import (
WSJArticleIE,
)
from .wwe import WWEIE
from .xanimu import XanimuIE
from .xbef import XBefIE
from .xboxclips import XboxClipsIE
from .xfileshare import XFileShareIE
yt_dlp/extractor/adn.py
+1 -1
View File
@@ -168,7 +168,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
}, data=b'')['token']
links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
self._K = ''.join(random.choices('0123456789abcdef', k=16))
message = bytes_to_intlist(json.dumps({
'k': self._K,
't': token,
yt_dlp/extractor/aitube.py
+60
View File
@@ -0,0 +1,60 @@
from .common import InfoExtractor
from ..utils import int_or_none, merge_dicts
class AitubeKZVideoIE(InfoExtractor):
_VALID_URL = r'https?://aitube\.kz/(?:video|embed/)\?(?:[^\?]+)?id=(?P<id>[\w-]+)'
_TESTS = [{
# id paramater as first parameter
'url': 'https://aitube.kz/video?id=9291d29b-c038-49a1-ad42-3da2051d353c&playlistId=d55b1f5f-ef2a-4f23-b646-2a86275b86b7&season=1',
'info_dict': {
'id': '9291d29b-c038-49a1-ad42-3da2051d353c',
'ext': 'mp4',
'duration': 2174.0,
'channel_id': '94962f73-013b-432c-8853-1bd78ca860fe',
'like_count': int,
'channel': 'ASTANA TV',
'comment_count': int,
'view_count': int,
'description': 'Смотреть любимые сериалы и видео, поделиться видео и сериалами с друзьями и близкими',
'thumbnail': 'https://cdn.static02.aitube.kz/kz.aitudala.aitube.staticaccess/files/ddf2a2ff-bee3-409b-b5f2-2a8202bba75b',
'upload_date': '20221102',
'timestamp': 1667370519,
'title': 'Ангел хранитель 1 серия',
'channel_follower_count': int,
}
}, {
# embed url
'url': 'https://aitube.kz/embed/?id=9291d29b-c038-49a1-ad42-3da2051d353c',
'only_matching': True,
}, {
# id parameter is not as first paramater
'url': 'https://aitube.kz/video?season=1&id=9291d29b-c038-49a1-ad42-3da2051d353c&playlistId=d55b1f5f-ef2a-4f23-b646-2a86275b86b7',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
nextjs_data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['videoInfo']
json_ld_data = self._search_json_ld(webpage, video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
f'https://api-http.aitube.kz/kz.aitudala.aitube.staticaccess/video/{video_id}/video', video_id)
return merge_dicts({
'id': video_id,
'title': nextjs_data.get('title') or self._html_search_meta(['name', 'og:title'], webpage),
'description': nextjs_data.get('description'),
'formats': formats,
'subtitles': subtitles,
'view_count': (nextjs_data.get('viewCount')
or int_or_none(self._html_search_meta('ya:ovs:views_total', webpage))),
'like_count': nextjs_data.get('likeCount'),
'channel': nextjs_data.get('channelTitle'),
'channel_id': nextjs_data.get('channelId'),
'thumbnail': nextjs_data.get('coverUrl'),
'comment_count': nextjs_data.get('commentCount'),
'channel_follower_count': int_or_none(nextjs_data.get('channelSubscriberCount')),
}, json_ld_data)
yt_dlp/extractor/amazon.py
+109 -7
View File
@@ -1,5 +1,17 @@
import re
from .common import InfoExtractor
from ..utils import ExtractorError, int_or_none
from ..utils import (
ExtractorError,
clean_html,
float_or_none,
get_element_by_attribute,
get_element_by_class,
int_or_none,
js_to_json,
traverse_obj,
url_or_none,
)
class AmazonStoreIE(InfoExtractor):
@@ -9,7 +21,7 @@ class AmazonStoreIE(InfoExtractor):
'url': 'https://www.amazon.co.uk/dp/B098XNCHLD/',
'info_dict': {
'id': 'B098XNCHLD',
'title': 'md5:dae240564cbb2642170c02f7f0d7e472',
'title': str,
},
'playlist_mincount': 1,
'playlist': [{
@@ -20,28 +32,32 @@ class AmazonStoreIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 34,
},
}]
}],
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.in/Sony-WH-1000XM4-Cancelling-Headphones-Bluetooth/dp/B0863TXGM3',
'info_dict': {
'id': 'B0863TXGM3',
'title': 'md5:d1d3352428f8f015706c84b31e132169',
'title': str,
},
'playlist_mincount': 4,
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.com/dp/B0845NXCXF/',
'info_dict': {
'id': 'B0845NXCXF',
'title': 'md5:f3fa12779bf62ddb6a6ec86a360a858e',
'title': str,
},
'playlist-mincount': 1,
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.es/Samsung-Smartphone-s-AMOLED-Quad-c%C3%A1mara-espa%C3%B1ola/dp/B08WX337PQ',
'info_dict': {
'id': 'B08WX337PQ',
'title': 'md5:f3fa12779bf62ddb6a6ec86a360a858e',
'title': str,
},
'playlist_mincount': 1,
'expected_warnings': ['Unable to extract data'],
}]
def _real_extract(self, url):
@@ -52,7 +68,7 @@ class AmazonStoreIE(InfoExtractor):
try:
data_json = self._search_json(
r'var\s?obj\s?=\s?jQuery\.parseJSON\(\'', webpage, 'data', id,
transform_source=lambda x: x.replace(R'\\u', R'\u'))
transform_source=js_to_json)
except ExtractorError as e:
retry.error = e
@@ -66,3 +82,89 @@ class AmazonStoreIE(InfoExtractor):
'width': int_or_none(video.get('videoWidth')),
} for video in (data_json.get('videos') or []) if video.get('isVideo') and video.get('url')]
return self.playlist_result(entries, playlist_id=id, playlist_title=data_json.get('title'))
class AmazonReviewsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?amazon\.(?:[a-z]{2,3})(?:\.[a-z]{2})?/gp/customer-reviews/(?P<id>[^/&#$?]+)'
_TESTS = [{
'url': 'https://www.amazon.com/gp/customer-reviews/R10VE9VUSY19L3/ref=cm_cr_arp_d_rvw_ttl',
'info_dict': {
'id': 'R10VE9VUSY19L3',
'ext': 'mp4',
'title': 'Get squad #Suspicious',
'description': 'md5:7012695052f440a1e064e402d87e0afb',
'uploader': 'Kimberly Cronkright',
'average_rating': 1.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}, {
'url': 'https://www.amazon.com/gp/customer-reviews/R10VE9VUSY19L3/ref=cm_cr_arp_d_rvw_ttl?language=es_US',
'info_dict': {
'id': 'R10VE9VUSY19L3',
'ext': 'mp4',
'title': 'Get squad #Suspicious',
'description': 'md5:7012695052f440a1e064e402d87e0afb',
'uploader': 'Kimberly Cronkright',
'average_rating': 1.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}, {
'url': 'https://www.amazon.in/gp/customer-reviews/RV1CO8JN5VGXV/',
'info_dict': {
'id': 'RV1CO8JN5VGXV',
'ext': 'mp4',
'title': 'Not sure about its durability',
'description': 'md5:1a252c106357f0a3109ebf37d2e87494',
'uploader': 'Shoaib Gulzar',
'average_rating': 2.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}]
def _real_extract(self, url):
video_id = self._match_id(url)
for retry in self.RetryManager():
webpage = self._download_webpage(url, video_id)
review_body = get_element_by_attribute('data-hook', 'review-body', webpage)
if not review_body:
retry.error = ExtractorError('Review body was not found in webpage', expected=True)
formats, subtitles = [], {}
manifest_url = self._search_regex(
r'data-video-url="([^"]+)"', review_body, 'm3u8 url', default=None)
if url_or_none(manifest_url):
fmts, subtitles = self._extract_m3u8_formats_and_subtitles(
manifest_url, video_id, 'mp4', fatal=False)
formats.extend(fmts)
video_url = self._search_regex(
r'<input[^>]+\bvalue="([^"]+)"[^>]+\bclass="video-url"', review_body, 'mp4 url', default=None)
if url_or_none(video_url):
formats.append({
'url': video_url,
'ext': 'mp4',
'format_id': 'http-mp4',
})
if not formats:
self.raise_no_formats('No video found for this customer review', expected=True)
return {
'id': video_id,
'title': (clean_html(get_element_by_attribute('data-hook', 'review-title', webpage))
or self._html_extract_title(webpage)),
'description': clean_html(traverse_obj(re.findall(
r'<span(?:\s+class="cr-original-review-content")?>(.+?)</span>', review_body), -1)),
'uploader': clean_html(get_element_by_class('a-profile-name', webpage)),
'average_rating': float_or_none(clean_html(get_element_by_attribute(
'data-hook', 'review-star-rating', webpage) or '').partition(' ')[0]),
'thumbnail': self._search_regex(
r'data-thumbnail-url="([^"]+)"', review_body, 'thumbnail', default=None),
'formats': formats,
'subtitles': subtitles,
}
yt_dlp/extractor/ard.py
+11 -8
View File
@@ -46,6 +46,9 @@ class ARDMediathekBaseIE(InfoExtractor):
subtitles['de'] = [{
'ext': 'ttml',
'url': subtitle_url,
}, {
'ext': 'vtt',
'url': subtitle_url.replace('/ebutt/', '/webvtt/') + '.vtt',
}]
return {
@@ -286,16 +289,16 @@ class ARDMediathekIE(ARDMediathekBaseIE):
class ARDIE(InfoExtractor):
_VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
_TESTS = [{
# available till 7.01.2022
'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
# available till 7.12.2023
'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
'md5': 'a438f671e87a7eba04000336a119ccc4',
'info_dict': {
'id': 'maischberger-die-woche-video100',
'display_id': 'maischberger-die-woche-video100',
'id': 'maischberger-video-424',
'display_id': 'maischberger-video-424',
'ext': 'mp4',
'duration': 3687.0,
'title': 'maischberger. die woche vom 7. Januar 2021',
'upload_date': '20210107',
'duration': 4452.0,
'title': 'maischberger am 07.12.2022',
'upload_date': '20221207',
'thumbnail': r're:^https?://.*\.jpg$',
},
}, {
yt_dlp/extractor/arte.py
+20 -4
View File
@@ -65,6 +65,21 @@ class ArteTVIE(ArteTVBaseIE):
}, {
'url': 'https://api.arte.tv/api/player/v2/config/de/LIVE',
'only_matching': True,
}, {
'url': 'https://www.arte.tv/de/videos/110203-006-A/zaz/',
'info_dict': {
'id': '110203-006-A',
'chapters': 'count:16',
'description': 'md5:cf592f1df52fe52007e3f8eac813c084',
'alt_title': 'Zaz',
'title': 'Baloise Session 2022',
'timestamp': 1668445200,
'duration': 4054,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/ubQjmVCGyRx3hmBuZEK9QZ/940x530',
'upload_date': '20221114',
'ext': 'mp4',
},
'expected_warnings': ['geo restricted']
}]
_GEO_BYPASS = True
@@ -180,10 +195,6 @@ class ArteTVIE(ArteTVBaseIE):
else:
self.report_warning(f'Skipping stream with unknown protocol {stream["protocol"]}')
# TODO: chapters from stream['segments']?
# The JS also looks for chapters in config['data']['attributes']['chapters'],
# but I am yet to find a video having those
formats.extend(secondary_formats)
self._remove_duplicate_formats(formats)
@@ -205,6 +216,11 @@ class ArteTVIE(ArteTVBaseIE):
{'url': image['url'], 'id': image.get('caption')}
for image in metadata.get('images') or [] if url_or_none(image.get('url'))
],
# TODO: chapters may also be in stream['segments']?
'chapters': traverse_obj(config, ('data', 'attributes', 'chapters', 'elements', ..., {
'start_time': 'startTime',
'title': 'title',
})) or None,
}
yt_dlp/extractor/bandcamp.py
+45 -3
View File
@@ -29,11 +29,18 @@ class BandcampIE(InfoExtractor):
'info_dict': {
'id': '1812978515',
'ext': 'mp3',
'title': "youtube-dl \"'/\\ä↭ - youtube-dl \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭",
'title': 'youtube-dl "\'/\\ä↭ - youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
'duration': 9.8485,
'uploader': 'youtube-dl "\'/\\ä↭',
'uploader': 'youtube-dl "\'/\\ä↭',
'upload_date': '20121129',
'timestamp': 1354224127,
'track': 'youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
'album_artist': 'youtube-dl "\'/\\ä↭',
'track_id': '1812978515',
'artist': 'youtube-dl "\'/\\ä↭',
'uploader_url': 'https://youtube-dl.bandcamp.com',
'uploader_id': 'youtube-dl',
'thumbnail': 'https://f4.bcbits.com/img/a3216802731_5.jpg',
},
'_skip': 'There is a limit of 200 free downloads / month for the test song'
}, {
@@ -41,7 +48,8 @@ class BandcampIE(InfoExtractor):
'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
'info_dict': {
'id': '2650410135',
'ext': 'aiff',
'ext': 'm4a',
'acodec': r're:[fa]lac',
'title': 'Ben Prunty - Lanius (Battle)',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Ben Prunty',
@@ -54,7 +62,10 @@ class BandcampIE(InfoExtractor):
'track_number': 1,
'track_id': '2650410135',
'artist': 'Ben Prunty',
'album_artist': 'Ben Prunty',
'album': 'FTL: Advanced Edition Soundtrack',
'uploader_url': 'https://benprunty.bandcamp.com',
'uploader_id': 'benprunty',
},
}, {
# no free download, mp3 128
@@ -75,7 +86,34 @@ class BandcampIE(InfoExtractor):
'track_number': 5,
'track_id': '2584466013',
'artist': 'Mastodon',
'album_artist': 'Mastodon',
'album': 'Call of the Mastodon',
'uploader_url': 'https://relapsealumni.bandcamp.com',
'uploader_id': 'relapsealumni',
},
}, {
# track from compilation album (artist/album_artist difference)
'url': 'https://diskotopia.bandcamp.com/track/safehouse',
'md5': '19c5337bca1428afa54129f86a2f6a69',
'info_dict': {
'id': '1978174799',
'ext': 'mp3',
'title': 'submerse - submerse - Safehouse',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'submerse',
'timestamp': 1480779297,
'upload_date': '20161203',
'release_timestamp': 1481068800,
'release_date': '20161207',
'duration': 154.066,
'track': 'submerse - Safehouse',
'track_number': 3,
'track_id': '1978174799',
'artist': 'submerse',
'album_artist': 'Diskotopia',
'album': 'DSK F/W 2016-2017 Free Compilation',
'uploader_url': 'https://diskotopia.bandcamp.com',
'uploader_id': 'diskotopia',
},
}]
@@ -121,6 +159,9 @@ class BandcampIE(InfoExtractor):
embed = self._extract_data_attr(webpage, title, 'embed', False)
current = tralbum.get('current') or {}
artist = embed.get('artist') or current.get('artist') or tralbum.get('artist')
album_artist = self._html_search_regex(
r'<h3 class="albumTitle">[\S\s]*?by\s*<span>\s*<a href="[^>]+">\s*([^>]+?)\s*</a>',
webpage, 'album artist', fatal=False)
timestamp = unified_timestamp(
current.get('publish_date') or tralbum.get('album_publish_date'))
@@ -205,6 +246,7 @@ class BandcampIE(InfoExtractor):
'track_id': track_id,
'artist': artist,
'album': embed.get('album_title'),
'album_artist': album_artist,
'formats': formats,
}
yt_dlp/extractor/beatbump.py
+101
View File
@@ -0,0 +1,101 @@
from .common import InfoExtractor
from .youtube import YoutubeIE, YoutubeTabIE
class BeatBumpVideoIE(InfoExtractor):
_VALID_URL = r'https://beatbump\.ml/listen\?id=(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs',
'md5': '5ff3fff41d3935b9810a9731e485fe66',
'info_dict': {
'id': 'MgNrAu2pzNs',
'ext': 'mp4',
'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
'artist': 'Stephen',
'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
'upload_date': '20190312',
'categories': ['Music'],
'playable_in_embed': True,
'duration': 169,
'like_count': int,
'alt_title': 'Voyeur Girl',
'view_count': int,
'track': 'Voyeur Girl',
'uploader': 'Stephen - Topic',
'title': 'Voyeur Girl',
'channel_follower_count': int,
'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
'age_limit': 0,
'availability': 'public',
'live_status': 'not_live',
'album': 'it\'s too much love to know my dear',
'channel': 'Stephen',
'comment_count': int,
'description': 'md5:7ae382a65843d6df2685993e90a8628f',
'tags': 'count:11',
'creator': 'Stephen',
'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
}
}]
def _real_extract(self, url):
id_ = self._match_id(url)
return self.url_result(f'https://music.youtube.com/watch?v={id_}', YoutubeIE, id_)
class BeatBumpPlaylistIE(InfoExtractor):
_VALID_URL = r'https://beatbump\.ml/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE',
'playlist_count': 50,
'info_dict': {
'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
'availability': 'unlisted',
'view_count': int,
'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
'description': '',
'tags': [],
'modified_date': '20221223',
}
}, {
'url': 'https://beatbump.ml/artist/UC_aEa8K-EOJ3D6gOs7HcyNg',
'playlist_mincount': 1,
'params': {'flatplaylist': True},
'info_dict': {
'id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'uploader_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'channel_follower_count': int,
'title': 'NoCopyrightSounds - Videos',
'uploader': 'NoCopyrightSounds',
'description': 'md5:cd4fd53d81d363d05eee6c1b478b491a',
'channel': 'NoCopyrightSounds',
'tags': 'count:12',
'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
},
}, {
'url': 'https://beatbump.ml/playlist/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
'playlist_mincount': 1,
'params': {'flatplaylist': True},
'info_dict': {
'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
'view_count': int,
'channel_url': 'https://www.youtube.com/@NoCopyrightSounds',
'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'title': 'NCS : All Releases 💿',
'uploader': 'NoCopyrightSounds',
'availability': 'public',
'channel': 'NoCopyrightSounds',
'tags': [],
'modified_date': '20221225',
'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
}
}]
def _real_extract(self, url):
id_ = self._match_id(url)
return self.url_result(f'https://music.youtube.com/browse/{id_}', YoutubeTabIE, id_)
yt_dlp/extractor/bilibili.py
+84 -22
View File
@@ -16,13 +16,16 @@ from ..utils import (
format_field,
int_or_none,
make_archive_id,
merge_dicts,
mimetype2ext,
parse_count,
parse_qs,
qualities,
smuggle_url,
srt_subtitles_timecode,
str_or_none,
traverse_obj,
unsmuggle_url,
url_or_none,
urlencode_postdata,
)
@@ -303,7 +306,8 @@ class BiliBiliIE(BilibiliBaseIE):
getter=lambda entry: f'https://www.bilibili.com/video/{video_id}?p={entry["page"]}')
if is_anthology:
title += f' p{part_id:02d} {traverse_obj(page_list_json, ((part_id or 1) - 1, "part")) or ""}'
part_id = part_id or 1
title += f' p{part_id:02d} {traverse_obj(page_list_json, (part_id - 1, "part")) or ""}'
aid = video_data.get('aid')
old_video_id = format_field(aid, None, f'%s_part{part_id or 1}')
@@ -880,16 +884,12 @@ class BiliIntlBaseIE(InfoExtractor):
return formats
def _extract_video_info(self, video_data, *, ep_id=None, aid=None):
def _parse_video_metadata(self, video_data):
return {
'id': ep_id or aid,
'title': video_data.get('title_display') or video_data.get('title'),
'thumbnail': video_data.get('cover'),
'episode_number': int_or_none(self._search_regex(
r'^E(\d+)(?:$| - )', video_data.get('title_display') or '', 'episode number', default=None)),
'formats': self._get_formats(ep_id=ep_id, aid=aid),
'subtitles': self._get_subtitles(ep_id=ep_id, aid=aid),
'extractor_key': BiliIntlIE.ie_key(),
}
def _perform_login(self, username, password):
@@ -935,6 +935,10 @@ class BiliIntlIE(BiliIntlBaseIE):
'title': 'E2 - The First Night',
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'episode_number': 2,
'upload_date': '20201009',
'episode': 'Episode 2',
'timestamp': 1602259500,
'description': 'md5:297b5a17155eb645e14a14b385ab547e',
}
}, {
# Non-Bstation page
@@ -945,6 +949,10 @@ class BiliIntlIE(BiliIntlBaseIE):
'title': 'E3 - Who?',
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'episode_number': 3,
'description': 'md5:e1a775e71a35c43f141484715470ad09',
'episode': 'Episode 3',
'upload_date': '20211219',
'timestamp': 1639928700,
}
}, {
# Subtitle with empty content
@@ -957,6 +965,17 @@ class BiliIntlIE(BiliIntlBaseIE):
'episode_number': 140,
},
'skip': 'According to the copyright owner\'s request, you may only watch the video after you log in.'
}, {
'url': 'https://www.bilibili.tv/en/video/2041863208',
'info_dict': {
'id': '2041863208',
'ext': 'mp4',
'timestamp': 1670874843,
'description': 'Scheduled for April 2023.\nStudio: ufotable',
'thumbnail': r're:https?://pic[-\.]bstarstatic.+/ugc/.+\.jpg$',
'upload_date': '20221212',
'title': 'Kimetsu no Yaiba Season 3 Official Trailer - Bstation',
}
}, {
'url': 'https://www.biliintl.com/en/play/34613/341736',
'only_matching': True,
@@ -974,42 +993,78 @@ class BiliIntlIE(BiliIntlBaseIE):
'only_matching': True,
}]
def _real_extract(self, url):
season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
video_id = ep_id or aid
def _make_url(video_id, series_id=None):
if series_id:
return f'https://www.bilibili.tv/en/play/{series_id}/{video_id}'
return f'https://www.bilibili.tv/en/video/{video_id}'
def _extract_video_metadata(self, url, video_id, season_id):
url, smuggled_data = unsmuggle_url(url, {})
if smuggled_data.get('title'):
return smuggled_data
webpage = self._download_webpage(url, video_id)
# Bstation layout
initial_data = (
self._search_json(r'window\.__INITIAL_(?:DATA|STATE)__\s*=', webpage, 'preload state', video_id, default={})
or self._search_nuxt_data(webpage, video_id, '__initialState', fatal=False, traverse=None))
video_data = traverse_obj(
initial_data, ('OgvVideo', 'epDetail'), ('UgcVideo', 'videoData'), ('ugc', 'archive'), expected_type=dict)
initial_data, ('OgvVideo', 'epDetail'), ('UgcVideo', 'videoData'), ('ugc', 'archive'), expected_type=dict) or {}
if season_id and not video_data:
# Non-Bstation layout, read through episode list
season_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={season_id}&platform=web', video_id)
video_data = traverse_obj(season_json,
('sections', ..., 'episodes', lambda _, v: str(v['episode_id']) == ep_id),
expected_type=dict, get_all=False)
return self._extract_video_info(video_data or {}, ep_id=ep_id, aid=aid)
video_data = traverse_obj(season_json, (
'sections', ..., 'episodes', lambda _, v: str(v['episode_id']) == video_id
), expected_type=dict, get_all=False)
# XXX: webpage metadata may not accurate, it just used to not crash when video_data not found
return merge_dicts(
self._parse_video_metadata(video_data), self._search_json_ld(webpage, video_id), {
'title': self._html_search_meta('og:title', webpage),
'description': self._html_search_meta('og:description', webpage)
})
def _real_extract(self, url):
season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
video_id = ep_id or aid
return {
'id': video_id,
**self._extract_video_metadata(url, video_id, season_id),
'formats': self._get_formats(ep_id=ep_id, aid=aid),
'subtitles': self.extract_subtitles(ep_id=ep_id, aid=aid),
}
class BiliIntlSeriesIE(BiliIntlBaseIE):
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-zA-Z]{2}/)?play/(?P<id>\d+)/?(?:[?#]|$)'
IE_NAME = 'biliIntl:series'
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-zA-Z]{2}/)?(?:play|media)/(?P<id>\d+)/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://www.bilibili.tv/en/play/34613',
'playlist_mincount': 15,
'info_dict': {
'id': '34613',
'title': 'Fly Me to the Moon',
'description': 'md5:a861ee1c4dc0acfad85f557cc42ac627',
'categories': ['Romance', 'Comedy', 'Slice of life'],
'title': 'TONIKAWA: Over the Moon For You',
'description': 'md5:297b5a17155eb645e14a14b385ab547e',
'categories': ['Slice of life', 'Comedy', 'Romance'],
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'view_count': int,
},
'params': {
'skip_download': True,
},
}, {
'url': 'https://www.bilibili.tv/en/media/1048837',
'info_dict': {
'id': '1048837',
'title': 'SPY×FAMILY',
'description': 'md5:b4434eb1a9a97ad2bccb779514b89f17',
'categories': ['Adventure', 'Action', 'Comedy'],
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.jpg$',
'view_count': int,
},
'playlist_mincount': 25,
}, {
'url': 'https://www.biliintl.com/en/play/34613',
'only_matching': True,
@@ -1020,9 +1075,12 @@ class BiliIntlSeriesIE(BiliIntlBaseIE):
def _entries(self, series_id):
series_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={series_id}&platform=web', series_id)
for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict, default=[]):
episode_id = str(episode.get('episode_id'))
yield self._extract_video_info(episode, ep_id=episode_id)
for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict):
episode_id = str(episode['episode_id'])
yield self.url_result(smuggle_url(
BiliIntlIE._make_url(episode_id, series_id),
self._parse_video_metadata(episode)
), BiliIntlIE, episode_id)
def _real_extract(self, url):
series_id = self._match_id(url)
@@ -1034,7 +1092,7 @@ class BiliIntlSeriesIE(BiliIntlBaseIE):
class BiliLiveIE(InfoExtractor):
_VALID_URL = r'https?://live.bilibili.com/(?P<id>\d+)'
_VALID_URL = r'https?://live.bilibili.com/(?:blanc/)?(?P<id>\d+)'
_TESTS = [{
'url': 'https://live.bilibili.com/196',
@@ -1050,6 +1108,9 @@ class BiliLiveIE(InfoExtractor):
}, {
'url': 'https://live.bilibili.com/196?broadcast_type=0&is_room_feed=1?spm_id_from=333.999.space_home.strengthen_live_card.click',
'only_matching': True
}, {
'url': 'https://live.bilibili.com/blanc/196',
'only_matching': True
}]
_FORMATS = {
@@ -1111,6 +1172,7 @@ class BiliLiveIE(InfoExtractor):
'thumbnail': room_data.get('user_cover'),
'timestamp': stream_data.get('live_time'),
'formats': formats,
'is_live': True,
'http_headers': {
'Referer': url,
},
yt_dlp/extractor/cda.py
+40 -7
View File
@@ -4,6 +4,7 @@ import datetime
import hashlib
import hmac
import json
import random
import re
from .common import InfoExtractor
@@ -27,11 +28,10 @@ class CDAIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
_NETRC_MACHINE = 'cdapl'
_BASE_URL = 'http://www.cda.pl/'
_BASE_URL = 'https://www.cda.pl'
_BASE_API_URL = 'https://api.cda.pl'
_API_HEADERS = {
'Accept': 'application/vnd.cda.public+json',
'User-Agent': 'pl.cda 1.0 (version 1.2.88 build 15306; Android 9; Xiaomi Redmi 3S)',
}
# hardcoded in the app
_LOGIN_REQUEST_AUTH = 'Basic YzU3YzBlZDUtYTIzOC00MWQwLWI2NjQtNmZmMWMxY2Y2YzVlOklBTm95QlhRRVR6U09MV1hnV3MwMW0xT2VyNWJNZzV4clRNTXhpNGZJUGVGZ0lWUlo5UGVYTDhtUGZaR1U1U3Q'
@@ -101,6 +101,38 @@ class CDAIE(InfoExtractor):
}, **kwargs)
def _perform_login(self, username, password):
app_version = random.choice((
'1.2.88 build 15306',
'1.2.174 build 18469',
))
android_version = random.randrange(8, 14)
phone_model = random.choice((
# x-kom.pl top selling Android smartphones, as of 2022-12-26
# https://www.x-kom.pl/g-4/c/1590-smartfony-i-telefony.html?f201-system-operacyjny=61322-android
'ASUS ZenFone 8',
'Motorola edge 20 5G',
'Motorola edge 30 neo 5G',
'Motorola moto g22',
'OnePlus Nord 2T 5G',
'Samsung Galaxy A32 SM‑A325F',
'Samsung Galaxy M13',
'Samsung Galaxy S20 FE 5G',
'Xiaomi 11T',
'Xiaomi POCO M4 Pro',
'Xiaomi Redmi 10',
'Xiaomi Redmi 10C',
'Xiaomi Redmi 9C NFC',
'Xiaomi Redmi Note 10 Pro',
'Xiaomi Redmi Note 11 Pro',
'Xiaomi Redmi Note 11',
'Xiaomi Redmi Note 11S 5G',
'Xiaomi Redmi Note 11S',
'realme 10',
'realme 9 Pro+',
'vivo Y33s',
))
self._API_HEADERS['User-Agent'] = f'pl.cda 1.0 (version {app_version}; Android {android_version}; {phone_model})'
cached_bearer = self.cache.load(self._BEARER_CACHE, username) or {}
if cached_bearer.get('valid_until', 0) > datetime.datetime.now().timestamp() + 5:
self._API_HEADERS['Authorization'] = f'Bearer {cached_bearer["token"]}'
@@ -138,9 +170,6 @@ class CDAIE(InfoExtractor):
meta = self._download_json(
f'{self._BASE_API_URL}/video/{video_id}', video_id, headers=self._API_HEADERS)['video']
if meta.get('premium') and not meta.get('premium_free'):
self.report_drm(video_id)
uploader = traverse_obj(meta, 'author', 'login')
formats = [{
@@ -151,6 +180,10 @@ class CDAIE(InfoExtractor):
'filesize': quality.get('length'),
} for quality in meta['qualities'] if quality.get('file')]
if meta.get('premium') and not meta.get('premium_free') and not formats:
raise ExtractorError(
'Video requires CDA Premium - subscription needed', expected=True)
return {
'id': video_id,
'title': meta.get('title'),
@@ -167,10 +200,10 @@ class CDAIE(InfoExtractor):
def _web_extract(self, video_id, url):
self._set_cookie('cda.pl', 'cda.player', 'html5')
webpage = self._download_webpage(
self._BASE_URL + '/video/' + video_id, video_id)
f'{self._BASE_URL}/video/{video_id}/vfilm', video_id)
if 'Ten film jest dostępny dla użytkowników premium' in webpage:
raise ExtractorError('This video is only available for premium users.', expected=True)
self.raise_login_required('This video is only available for premium users')
if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
self.raise_geo_restricted()
yt_dlp/extractor/ciscowebex.py
+25 -5
View File
@@ -1,5 +1,6 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
try_get,
unified_timestamp,
@@ -38,11 +39,30 @@ class CiscoWebexIE(InfoExtractor):
siteurl = mobj.group('siteurl_1') or mobj.group('siteurl_2')
video_id = mobj.group('id')
stream = self._download_json(
password = self.get_param('videopassword')
headers = {'Accept': 'application/json'}
if password:
headers['accessPwd'] = password
stream, urlh = self._download_json_handle(
'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
video_id, fatal=False, query={'siteurl': siteurl})
if not stream:
self.raise_login_required(method='cookies')
video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
if urlh.status == 403:
if stream['code'] == 53004:
self.raise_login_required()
if stream['code'] == 53005:
if password:
raise ExtractorError('Wrong password', expected=True)
raise ExtractorError(
'This video is protected by a password, use the --video-password option', expected=True)
raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
if urlh.status == 429:
self.raise_login_required(
f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
method='cookies')
video_id = stream.get('recordUUID') or video_id
@@ -78,7 +98,7 @@ class CiscoWebexIE(InfoExtractor):
'title': stream['recordName'],
'description': stream.get('description'),
'uploader': stream.get('ownerDisplayName'),
'uploader_id': stream.get('ownerUserName') or stream.get('ownerId'), # mail or id
'uploader_id': stream.get('ownerUserName') or stream.get('ownerId'),
'timestamp': unified_timestamp(stream.get('createTime')),
'duration': int_or_none(stream.get('duration'), 1000),
'webpage_url': 'https://%s.webex.com/recordingservice/sites/%s/recording/playback/%s' % (subdomain, siteurl, video_id),
yt_dlp/extractor/common.py
+66 -28
View File
@@ -1267,10 +1267,9 @@ class InfoExtractor:
Like _search_regex, but strips HTML tags and unescapes entities.
"""
res = self._search_regex(pattern, string, name, default, fatal, flags, group)
if res:
return clean_html(res).strip()
else:
return res
if isinstance(res, tuple):
return tuple(map(clean_html, res))
return clean_html(res)
def _get_netrc_login_info(self, netrc_machine=None):
username = None
@@ -1401,10 +1400,16 @@ class InfoExtractor:
# And then there are the jokers who advertise that they use RTA, but actually don't.
AGE_LIMIT_MARKERS = [
r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
r'>[^<]*you acknowledge you are at least (\d+) years old',
r'>\s*(?:18\s+U(?:\.S\.C\.|SC)\s+)?(?:§+\s*)?2257\b',
]
if any(re.search(marker, html) for marker in AGE_LIMIT_MARKERS):
return 18
return 0
age_limit = 0
for marker in AGE_LIMIT_MARKERS:
mobj = re.search(marker, html)
if mobj:
age_limit = max(age_limit, int(traverse_obj(mobj, 1, default=18)))
return age_limit
def _media_rating_search(self, html):
# See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
@@ -1764,6 +1769,9 @@ class InfoExtractor:
def _extract_f4m_formats(self, manifest_url, video_id, preference=None, quality=None, f4m_id=None,
transform_source=lambda s: fix_xml_ampersands(s).strip(),
fatal=True, m3u8_id=None, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
manifest_url, video_id, 'Downloading f4m manifest',
'Unable to download f4m manifest',
@@ -1913,6 +1921,9 @@ class InfoExtractor:
errnote=None, fatal=True, live=False, data=None, headers={},
query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
if not m3u8_url:
if errnote is not False:
errnote = errnote or 'Failed to obtain m3u8 URL'
@@ -2192,6 +2203,9 @@ class InfoExtractor:
return '/'.join(out)
def _extract_smil_formats_and_subtitles(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
if res is False:
assert not fatal
@@ -2467,6 +2481,10 @@ class InfoExtractor:
def _extract_mpd_formats_and_subtitles(
self, mpd_url, video_id, mpd_id=None, note=None, errnote=None,
fatal=True, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
mpd_url, video_id,
note='Downloading MPD manifest' if note is None else note,
@@ -2836,6 +2854,9 @@ class InfoExtractor:
return fmts
def _extract_ism_formats_and_subtitles(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
ism_url, video_id,
note='Downloading ISM manifest' if note is None else note,
@@ -3205,7 +3226,7 @@ class InfoExtractor:
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
mobj = re.search(
r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
webpage)
if mobj:
try:
@@ -3226,19 +3247,20 @@ class InfoExtractor:
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
# JWPlayer backward compatibility: flattened playlists
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
if 'playlist' not in jwplayer_data:
jwplayer_data = {'playlist': [jwplayer_data]}
entries = []
if not isinstance(jwplayer_data, dict):
return entries
# JWPlayer backward compatibility: single playlist item
playlist_items = jwplayer_data.get('playlist')
# JWPlayer backward compatibility: single playlist item/flattened playlists
# https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
if not isinstance(jwplayer_data['playlist'], list):
jwplayer_data['playlist'] = [jwplayer_data['playlist']]
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
if not isinstance(playlist_items, list):
playlist_items = (playlist_items or jwplayer_data, )
for video_data in jwplayer_data['playlist']:
for video_data in playlist_items:
if not isinstance(video_data, dict):
continue
# JWPlayer backward compatibility: flattened sources
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
if 'sources' not in video_data:
@@ -3276,6 +3298,13 @@ class InfoExtractor:
'timestamp': int_or_none(video_data.get('pubdate')),
'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
'subtitles': subtitles,
'alt_title': clean_html(video_data.get('subtitle')), # attributes used e.g. by Tele5 ...
'genre': clean_html(video_data.get('genre')),
'channel': clean_html(dict_get(video_data, ('category', 'channel'))),
'season_number': int_or_none(video_data.get('season')),
'episode_number': int_or_none(video_data.get('episode')),
'release_year': int_or_none(video_data.get('releasedate')),
'age_limit': int_or_none(video_data.get('age_restriction')),
}
# https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
@@ -3293,7 +3322,7 @@ class InfoExtractor:
def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
urls = []
urls = set()
formats = []
for source in jwplayer_sources_data:
if not isinstance(source, dict):
@@ -3302,14 +3331,14 @@ class InfoExtractor:
base_url, self._proto_relative_url(source.get('file')))
if not source_url or source_url in urls:
continue
urls.append(source_url)
urls.add(source_url)
source_type = source.get('type') or ''
ext = mimetype2ext(source_type) or determine_ext(source_url)
if source_type == 'hls' or ext == 'm3u8':
if source_type == 'hls' or ext == 'm3u8' or 'format=m3u8-aapl' in source_url:
formats.extend(self._extract_m3u8_formats(
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
m3u8_id=m3u8_id, fatal=False))
elif source_type == 'dash' or ext == 'mpd':
elif source_type == 'dash' or ext == 'mpd' or 'format=mpd-time-csf' in source_url:
formats.extend(self._extract_mpd_formats(
source_url, video_id, mpd_id=mpd_id, fatal=False))
elif ext == 'smil':
@@ -3324,13 +3353,12 @@ class InfoExtractor:
'ext': ext,
})
else:
format_id = str_or_none(source.get('label'))
height = int_or_none(source.get('height'))
if height is None:
if height is None and format_id:
# Often no height is provided but there is a label in
# format like "1080p", "720p SD", or 1080.
height = int_or_none(self._search_regex(
r'^(\d{3,4})[pP]?(?:\b|$)', str(source.get('label') or ''),
'height', default=None))
height = parse_resolution(format_id).get('height')
a_format = {
'url': source_url,
'width': int_or_none(source.get('width')),
@@ -3338,6 +3366,7 @@ class InfoExtractor:
'tbr': int_or_none(source.get('bitrate'), scale=1000),
'filesize': int_or_none(source.get('filesize')),
'ext': ext,
'format_id': format_id
}
if source_url.startswith('rtmp'):
a_format['ext'] = 'flv'
@@ -3431,13 +3460,17 @@ class InfoExtractor:
continue
t['name'] = cls.ie_key()
yield t
if getattr(cls, '__wrapped__', None):
yield from cls.__wrapped__.get_testcases(include_onlymatching)
@classmethod
def get_webpage_testcases(cls):
tests = vars(cls).get('_WEBPAGE_TESTS', [])
for t in tests:
t['name'] = cls.ie_key()
return tests
yield t
if getattr(cls, '__wrapped__', None):
yield from cls.__wrapped__.get_webpage_testcases()
@classproperty(cache=True)
def age_limit(cls):
@@ -3483,7 +3516,7 @@ class InfoExtractor:
elif cls.IE_DESC:
desc += f' {cls.IE_DESC}'
if cls.SEARCH_KEY:
desc += f'; "{cls.SEARCH_KEY}:" prefix'
desc += f'{";" if cls.IE_DESC else ""} "{cls.SEARCH_KEY}:" prefix'
if search_examples:
_COUNTS = ('', '5', '10', 'all')
desc += f' (e.g. "{cls.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(search_examples)}")'
@@ -3699,10 +3732,12 @@ class InfoExtractor:
if plugin_name:
mro = inspect.getmro(cls)
super_class = cls.__wrapped__ = mro[mro.index(cls) + 1]
cls.IE_NAME, cls.ie_key = f'{super_class.IE_NAME}+{plugin_name}', super_class.ie_key
cls.PLUGIN_NAME, cls.ie_key = plugin_name, super_class.ie_key
cls.IE_NAME = f'{super_class.IE_NAME}+{plugin_name}'
while getattr(super_class, '__wrapped__', None):
super_class = super_class.__wrapped__
setattr(sys.modules[super_class.__module__], super_class.__name__, cls)
_PLUGIN_OVERRIDES[super_class].append(cls)
return super().__init_subclass__(**kwargs)
@@ -3759,3 +3794,6 @@ class UnsupportedURLIE(InfoExtractor):
def _real_extract(self, url):
raise UnsupportedError(url)
_PLUGIN_OVERRIDES = collections.defaultdict(list)
yt_dlp/extractor/crunchyroll.py
+3 -2
View File
@@ -182,7 +182,7 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
self.to_screen(
'To get all formats of a hardsub language, use '
'"--extractor-args crunchyrollbeta:hardsub=<language_code or all>". '
'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta for more info',
'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info',
only_once=True)
else:
full_format_langs = set(map(str.lower, available_formats))
@@ -291,7 +291,8 @@ class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
'season_id': episode.get('season_id'),
'season_number': episode.get('season_number'),
'episode': episode.get('title'),
'episode_number': episode.get('sequence_number')
'episode_number': episode.get('sequence_number'),
'language': episode.get('audio_locale'),
}
return self.playlist_result(entries(), internal_id, series_response.get('title'))
yt_dlp/extractor/curiositystream.py
+6 -2
View File
@@ -1,4 +1,5 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import compat_str
@@ -23,7 +24,7 @@ class CuriosityStreamBaseIE(InfoExtractor):
auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
if auth_cookie:
self.write_debug('Obtained auth_token cookie')
self._auth_token = auth_cookie.value
self._auth_token = urllib.parse.unquote(auth_cookie.value)
if self._auth_token:
headers['X-Auth-Token'] = self._auth_token
result = self._download_json(
@@ -54,8 +55,11 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
'channel': 'Curiosity Stream',
'categories': ['Technology', 'Interview'],
'average_rating': 96.79,
'average_rating': float,
'series_id': '2',
'thumbnail': r're:https://img.curiositystream.com/.+\.jpg',
'tags': [],
'duration': 158
},
'params': {
# m3u8 download
yt_dlp/extractor/discovery.py
+1 -1
View File
@@ -78,7 +78,7 @@ class DiscoveryIE(DiscoveryGoBaseIE):
'Downloading token JSON metadata', query={
'authRel': 'authorization',
'client_id': '3020a40c2356a645b4b4',
'nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
'nonce': ''.join(random.choices(string.ascii_letters, k=32)),
'redirectUri': 'https://www.discovery.com/',
})['access_token']
yt_dlp/extractor/drtv.py
+102 -7
View File
@@ -2,22 +2,24 @@ import binascii
import hashlib
import re
from .common import InfoExtractor
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
from ..compat import compat_urllib_parse_unquote
from ..utils import (
ExtractorError,
int_or_none,
float_or_none,
int_or_none,
mimetype2ext,
str_or_none,
traverse_obj,
try_get,
unified_timestamp,
update_url_query,
url_or_none,
)
SERIES_API = 'https://production-cdn.dr-massive.com/api/page?device=web_browser&item_detail_expand=all&lang=da&max_list_prefetch=3&path=%s'
class DRTVIE(InfoExtractor):
_VALID_URL = r'''(?x)
@@ -141,13 +143,13 @@ class DRTVIE(InfoExtractor):
}]
def _real_extract(self, url):
video_id = self._match_id(url)
raw_video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
webpage = self._download_webpage(url, raw_video_id)
if '>Programmet er ikke længere tilgængeligt' in webpage:
raise ExtractorError(
'Video %s is not available' % video_id, expected=True)
'Video %s is not available' % raw_video_id, expected=True)
video_id = self._search_regex(
(r'data-(?:material-identifier|episode-slug)="([^"]+)"',
@@ -182,6 +184,10 @@ class DRTVIE(InfoExtractor):
data = self._download_json(
programcard_url, video_id, 'Downloading video JSON', query=query)
supplementary_data = self._download_json(
SERIES_API % f'/episode/{raw_video_id}', raw_video_id,
default={}) if re.search(r'_\d+$', raw_video_id) else {}
title = str_or_none(data.get('Title')) or re.sub(
r'\s*\|\s*(?:TV\s*\|\s*DR|DRTV)$', '',
self._og_search_title(webpage))
@@ -313,8 +319,8 @@ class DRTVIE(InfoExtractor):
'season': str_or_none(data.get('SeasonTitle')),
'season_number': int_or_none(data.get('SeasonNumber')),
'season_id': str_or_none(data.get('SeasonUrn')),
'episode': str_or_none(data.get('EpisodeTitle')),
'episode_number': int_or_none(data.get('EpisodeNumber')),
'episode': traverse_obj(supplementary_data, ('entries', 0, 'item', 'contextualTitle')) or str_or_none(data.get('EpisodeTitle')),
'episode_number': traverse_obj(supplementary_data, ('entries', 0, 'item', 'episodeNumber')) or int_or_none(data.get('EpisodeNumber')),
'release_year': int_or_none(data.get('ProductionYear')),
}
@@ -372,3 +378,92 @@ class DRTVLiveIE(InfoExtractor):
'formats': formats,
'is_live': True,
}
class DRTVSeasonIE(InfoExtractor):
IE_NAME = 'drtv:season'
_VALID_URL = r'https?://(?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/saeson/(?P<display_id>[\w-]+)_(?P<id>\d+)'
_GEO_COUNTRIES = ['DK']
_TESTS = [{
'url': 'https://www.dr.dk/drtv/saeson/frank-and-kastaniegaarden_9008',
'info_dict': {
'id': '9008',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 8
}, {
'url': 'https://www.dr.dk/drtv/saeson/frank-and-kastaniegaarden_8761',
'info_dict': {
'id': '8761',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 19
}]
def _real_extract(self, url):
display_id, season_id = self._match_valid_url(url).group('display_id', 'id')
data = self._download_json(SERIES_API % f'/saeson/{display_id}_{season_id}', display_id)
entries = [{
'_type': 'url',
'url': f'https://www.dr.dk/drtv{episode["path"]}',
'ie_key': DRTVIE.ie_key(),
'title': episode.get('title'),
'episode': episode.get('episodeName'),
'description': episode.get('shortDescription'),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber')),
'episode_number': episode.get('episodeNumber'),
} for episode in traverse_obj(data, ('entries', 0, 'item', 'episodes', 'items'))]
return {
'_type': 'playlist',
'id': season_id,
'display_id': display_id,
'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'entries': entries,
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
}
class DRTVSeriesIE(InfoExtractor):
IE_NAME = 'drtv:series'
_VALID_URL = r'https?://(?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/serie/(?P<display_id>[\w-]+)_(?P<id>\d+)'
_GEO_COUNTRIES = ['DK']
_TESTS = [{
'url': 'https://www.dr.dk/drtv/serie/frank-and-kastaniegaarden_6954',
'info_dict': {
'id': '6954',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 15
}]
def _real_extract(self, url):
display_id, series_id = self._match_valid_url(url).group('display_id', 'id')
data = self._download_json(SERIES_API % f'/serie/{display_id}_{series_id}', display_id)
entries = [{
'_type': 'url',
'url': f'https://www.dr.dk/drtv{season.get("path")}',
'ie_key': DRTVSeasonIE.ie_key(),
'title': season.get('title'),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
} for season in traverse_obj(data, ('entries', 0, 'item', 'show', 'seasons', 'items'))]
return {
'_type': 'playlist',
'id': series_id,
'display_id': display_id,
'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'entries': entries
}
yt_dlp/extractor/embedly.py
+63 -7
View File
@@ -1,24 +1,80 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from .youtube import YoutubeTabIE
from ..utils import parse_qs, smuggle_url, traverse_obj
class EmbedlyIE(InfoExtractor):
_VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
_VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?(?:src|url)=(?:[^#&]+)'
_TESTS = [{
'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
'info_dict': {
'id': 'UUGLim4T2loE5rwCMdpCIPVg',
'modified_date': '20221225',
'view_count': int,
'uploader_url': 'https://www.youtube.com/@TraciHinesMusic',
'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'uploader': 'TraciJHines',
'channel_url': 'https://www.youtube.com/@TraciHinesMusic',
'channel': 'TraciJHines',
'availability': 'public',
'uploader_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'description': '',
'tags': [],
'title': 'Uploads from TraciJHines',
},
'playlist_mincount': 10,
}, {
'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
'params': {'noplaylist': True},
'info_dict': {
'id': 'SU4fj_aEMVw',
'ext': 'mp4',
'title': 'I\'m on Patreon!',
'age_limit': 0,
'categories': ['Entertainment'],
'thumbnail': 'https://i.ytimg.com/vi_webp/SU4fj_aEMVw/maxresdefault.webp',
'live_status': 'not_live',
'playable_in_embed': True,
'channel': 'TraciJHines',
'uploader_id': 'TraciJHines',
'channel_url': 'https://www.youtube.com/channel/UCGLim4T2loE5rwCMdpCIPVg',
'uploader_url': 'http://www.youtube.com/user/TraciJHines',
'upload_date': '20150211',
'duration': 282,
'availability': 'public',
'channel_follower_count': int,
'tags': 'count:39',
'view_count': int,
'comment_count': int,
'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'like_count': int,
'uploader': 'TraciJHines',
'description': 'md5:8af6425f50bd46fbf29f3db0fc3a8364',
'chapters': list,
},
}, {
'url': 'https://cdn.embedly.com/widgets/media.html?src=https://player.vimeo.com/video/1234567?h=abcdefgh',
'only_matching': True,
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
# Bypass suitable check
def _extract_from_webpage(cls, url, webpage):
# Bypass "ie=cls" and suitable check
for mobj in re.finditer(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage):
yield mobj.group('url')
yield cls.url_result(mobj.group('url'))
for mobj in re.finditer(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage):
yield urllib.parse.unquote(mobj.group('url'))
yield cls.url_result(urllib.parse.unquote(mobj.group('url')))
def _real_extract(self, url):
return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))
qs = parse_qs(url)
src = urllib.parse.unquote(traverse_obj(qs, ('url', 0)) or '')
if src and YoutubeTabIE.suitable(src):
return self.url_result(src, YoutubeTabIE)
return self.url_result(smuggle_url(
urllib.parse.unquote(traverse_obj(qs, ('src', 0), ('url', 0))),
{'http_headers': {'Referer': url}}))
yt_dlp/extractor/extractors.py
+4 -2
View File
@@ -1,10 +1,10 @@
import contextlib
import os
from ..utils import load_plugins
from ..plugins import load_plugins
# NB: Must be before other imports so that plugins can be correctly injected
_PLUGIN_CLASSES = load_plugins('extractor', 'IE', {})
_PLUGIN_CLASSES = load_plugins('extractor', 'IE')
_LAZY_LOADER = False
if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
@@ -24,3 +24,5 @@ if not _LAZY_LOADER:
globals().update(_PLUGIN_CLASSES)
_ALL_CLASSES[:0] = _PLUGIN_CLASSES.values()
from .common import _PLUGIN_OVERRIDES # noqa: F401
yt_dlp/extractor/fifa.py
+6 -17
View File
@@ -17,8 +17,10 @@ class FifaIE(InfoExtractor):
'description': 'md5:f4520d0ee80529c8ba4134a7d692ff8b',
'ext': 'mp4',
'categories': ['FIFA Tournaments'],
'thumbnail': 'https://digitalhub.fifa.com/transform/fa6f0b3e-a2e9-4cf7-9f32-53c57bcb7360/2006_Final_ITA_FRA',
'thumbnail': 'https://digitalhub.fifa.com/transform/135e2656-3a51-407b-8810-6c34bec5b59b/FMR_2006_Italy_France_Final_Hero',
'duration': 8165,
'release_timestamp': 1152403200,
'release_date': '20060709',
},
'params': {'skip_download': 'm3u8'},
}, {
@@ -54,7 +56,7 @@ class FifaIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
preconnect_link = self._search_regex(
r'<link[^>]+rel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
r'<link\b[^>]+\brel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
video_details = self._download_json(
f'{preconnect_link}/sections/videoDetails/{video_id}', video_id, 'Downloading Video Details', fatal=False)
@@ -62,22 +64,9 @@ class FifaIE(InfoExtractor):
preplay_parameters = self._download_json(
f'{preconnect_link}/videoPlayerData/{video_id}', video_id, 'Downloading Preplay Parameters')['preplayParameters']
cid = preplay_parameters['contentId']
content_data = self._download_json(
f'https://content.uplynk.com/preplay/{cid}/multiple.json', video_id, 'Downloading Content Data', query={
'v': preplay_parameters['preplayAPIVersion'],
'tc': preplay_parameters['tokenCheckAlgorithmVersion'],
'rn': preplay_parameters['randomNumber'],
'exp': preplay_parameters['tokenExpirationDate'],
'ct': preplay_parameters['contentType'],
'cid': cid,
'mbtracks': preplay_parameters['tracksAssetNumber'],
'ad': preplay_parameters['adConfiguration'],
'ad.preroll': int(preplay_parameters['adPreroll']),
'ad.cmsid': preplay_parameters['adCMSSourceId'],
'ad.vid': preplay_parameters['adSourceVideoID'],
'sig': preplay_parameters['signature'],
})
'https://content.uplynk.com/preplay/{contentId}/multiple.json?{queryStr}&sig={signature}'.format(**preplay_parameters),
video_id, 'Downloading Content Data')
formats, subtitles = self._extract_m3u8_formats_and_subtitles(content_data['playURL'], video_id)
yt_dlp/extractor/funimation.py
+1 -1
View File
@@ -210,7 +210,7 @@ class FunimationIE(FunimationBaseIE):
page = self._download_json(
'https://www.funimation.com/api/showexperience/%s/' % experience_id,
display_id, headers=headers, expected_status=403, query={
'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
'pinst_id': ''.join(random.choices(string.digits + string.ascii_letters, k=8)),
}, note=f'Downloading {format_name} JSON')
sources = page.get('items') or []
if not sources:
yt_dlp/extractor/generic.py
+145 -109
View File
@@ -32,6 +32,7 @@ from ..utils import (
unified_timestamp,
unsmuggle_url,
url_or_none,
urljoin,
variadic,
xpath_attr,
xpath_text,
@@ -1867,11 +1868,13 @@ class GenericIE(InfoExtractor):
'display_id': 'kelis-4th-of-july',
'ext': 'mp4',
'title': 'Kelis - 4th Of July',
'thumbnail': 'https://kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
'description': 'Kelis - 4th Of July',
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
},
'params': {
'skip_download': True,
},
'expected_warnings': ['Untested major version'],
}, {
# KVS Player
'url': 'https://www.kvs-demo.com/embed/105/',
@@ -1880,35 +1883,12 @@ class GenericIE(InfoExtractor):
'display_id': 'kelis-4th-of-july',
'ext': 'mp4',
'title': 'Kelis - 4th Of July / Embed Player',
'thumbnail': 'https://kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
},
'params': {
'skip_download': True,
},
}, {
# KVS Player
'url': 'https://thisvid.com/videos/french-boy-pantsed/',
'md5': '3397979512c682f6b85b3b04989df224',
'info_dict': {
'id': '2400174',
'display_id': 'french-boy-pantsed',
'ext': 'mp4',
'title': 'French Boy Pantsed - ThisVid.com',
'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
}
}, {
# KVS Player
'url': 'https://thisvid.com/embed/2400174/',
'md5': '3397979512c682f6b85b3b04989df224',
'info_dict': {
'id': '2400174',
'display_id': 'french-boy-pantsed',
'ext': 'mp4',
'title': 'French Boy Pantsed - ThisVid.com',
'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
}
}, {
# KVS Player
'url': 'https://youix.com/video/leningrad-zoj/',
'md5': '94f96ba95706dc3880812b27b7d8a2b8',
'info_dict': {
@@ -1916,8 +1896,8 @@ class GenericIE(InfoExtractor):
'display_id': 'leningrad-zoj',
'ext': 'mp4',
'title': 'Клип: Ленинград - ЗОЖ скачать, смотреть онлайн | Youix.com',
'thumbnail': 'https://youix.com/contents/videos_screenshots/18000/18485/preview_480x320_youix_com.mp4.jpg',
}
'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
},
}, {
# KVS Player
'url': 'https://youix.com/embed/18485',
@@ -1927,19 +1907,20 @@ class GenericIE(InfoExtractor):
'display_id': 'leningrad-zoj',
'ext': 'mp4',
'title': 'Ленинград - ЗОЖ',
'thumbnail': 'https://youix.com/contents/videos_screenshots/18000/18485/preview_480x320_youix_com.mp4.jpg',
}
'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
},
}, {
# KVS Player
'url': 'https://bogmedia.org/videos/21217/40-nochey-40-nights-2016/',
'md5': '94166bdb26b4cb1fb9214319a629fc51',
'info_dict': {
'id': '21217',
'display_id': '40-nochey-40-nights-2016',
'display_id': '40-nochey-2016',
'ext': 'mp4',
'title': '40 ночей (2016) - BogMedia.org',
'description': 'md5:4e6d7d622636eb7948275432eb256dc3',
'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
}
},
},
{
# KVS Player (for sites that serve kt_player.js via non-https urls)
@@ -1949,9 +1930,9 @@ class GenericIE(InfoExtractor):
'id': '389508',
'display_id': 'syren-de-mer-onlyfans-05-07-2020have-a-happy-safe-holiday5f014e68a220979bdb8cd-source',
'ext': 'mp4',
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
'thumbnail': 'http://www.camhub.world/contents/videos_screenshots/389000/389508/preview.mp4.jpg',
}
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
'thumbnail': r're:https?://www\.camhub\.world/contents/videos_screenshots/389000/389508/preview\.mp4\.jpg',
},
},
{
# Reddit-hosted video that will redirect and be processed by RedditIE
@@ -2154,7 +2135,52 @@ class GenericIE(InfoExtractor):
'age_limit': 0,
'direct': True,
}
}
},
{
'note': 'server returns data in brotli compression by default if `accept-encoding: *` is specified.',
'url': 'https://www.extra.cz/cauky-lidi-70-dil-babis-predstavil-pohadky-prymulanek-nebo-andrejovy-nove-saty-ac867',
'info_dict': {
'id': 'cauky-lidi-70-dil-babis-predstavil-pohadky-prymulanek-nebo-andrejovy-nove-saty-ac867',
'ext': 'mp4',
'title': 'čauky lidi 70 finall',
'description': 'čauky lidi 70 finall',
'thumbnail': 'h',
'upload_date': '20220606',
'timestamp': 1654513791,
'duration': 318.0,
'direct': True,
'age_limit': 0,
},
},
{
'note': 'JW Player embed with unicode-escape sequences in URL',
'url': 'https://www.medici.tv/en/concerts/lahav-shani-mozart-mahler-israel-philharmonic-abu-dhabi-classics',
'info_dict': {
'id': 'm',
'ext': 'mp4',
'title': 'Lahav Shani conducts the Israel Philharmonic\'s first-ever concert in Abu Dhabi',
'description': 'Mahler\'s ',
'uploader': 'www.medici.tv',
'age_limit': 0,
'thumbnail': r're:^https?://.+\.jpg',
},
'params': {
'skip_download': True,
},
},
{
'url': 'https://shooshtime.com/videos/284002/just-out-of-the-shower-joi/',
'md5': 'e2f0a4c329f7986280b7328e24036d60',
'info_dict': {
'id': '284002',
'display_id': 'just-out-of-the-shower-joi',
'ext': 'mp4',
'title': 'Just Out Of The Shower JOI - Shooshtime',
'thumbnail': 'https://i.shoosh.co/contents/videos_screenshots/284000/284002/preview.mp4.jpg',
'height': 720,
'age_limit': 18,
},
},
]
def report_following_redirect(self, new_url):
@@ -2220,43 +2246,87 @@ class GenericIE(InfoExtractor):
'entries': entries,
}
def _kvs_getrealurl(self, video_url, license_code):
@classmethod
def _kvs_get_real_url(cls, video_url, license_code):
if not video_url.startswith('function/0/'):
return video_url # not obfuscated
url_path, _, url_query = video_url.partition('?')
urlparts = url_path.split('/')[2:]
license = self._kvs_getlicensetoken(license_code)
newmagic = urlparts[5][:32]
parsed = urllib.parse.urlparse(video_url[len('function/0/'):])
license = cls._kvs_get_license_token(license_code)
urlparts = parsed.path.split('/')
for o in range(len(newmagic) - 1, -1, -1):
new = ''
l = (o + sum(int(n) for n in license[o:])) % 32
HASH_LENGTH = 32
hash = urlparts[3][:HASH_LENGTH]
indices = list(range(HASH_LENGTH))
for i in range(0, len(newmagic)):
if i == o:
new += newmagic[l]
elif i == l:
new += newmagic[o]
else:
new += newmagic[i]
newmagic = new
# Swap indices of hash according to the destination calculated from the license token
accum = 0
for src in reversed(range(HASH_LENGTH)):
accum += license[src]
dest = (src + accum) % HASH_LENGTH
indices[src], indices[dest] = indices[dest], indices[src]
urlparts[5] = newmagic + urlparts[5][32:]
return '/'.join(urlparts) + '?' + url_query
urlparts[3] = ''.join(hash[index] for index in indices) + urlparts[3][HASH_LENGTH:]
return urllib.parse.urlunparse(parsed._replace(path='/'.join(urlparts)))
def _kvs_getlicensetoken(self, license):
modlicense = license.replace('$', '').replace('0', '1')
center = int(len(modlicense) / 2)
@staticmethod
def _kvs_get_license_token(license):
license = license.replace('$', '')
license_values = [int(char) for char in license]
modlicense = license.replace('0', '1')
center = len(modlicense) // 2
fronthalf = int(modlicense[:center + 1])
backhalf = int(modlicense[center:])
modlicense = str(4 * abs(fronthalf - backhalf))[:center + 1]
modlicense = str(4 * abs(fronthalf - backhalf))
retval = ''
for o in range(0, center + 1):
for i in range(1, 5):
retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
return retval
return [
(license_values[index + offset] + current) % 10
for index, current in enumerate(map(int, modlicense))
for offset in range(4)
]
def _extract_kvs(self, url, webpage, video_id):
flashvars = self._search_json(
r'(?s:<script\b[^>]*>.*?var\s+flashvars\s*=)',
webpage, 'flashvars', video_id, transform_source=js_to_json)
# extract the part after the last / as the display_id from the
# canonical URL.
display_id = self._search_regex(
r'(?:<link href="https?://[^"]+/(.+?)/?" rel="canonical"\s*/?>'
r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
webpage, 'display_id', fatal=False)
title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
thumbnail = flashvars['preview_url']
if thumbnail.startswith('//'):
protocol, _, _ = url.partition('/')
thumbnail = protocol + thumbnail
url_keys = list(filter(re.compile(r'^video_(?:url|alt_url\d*)$').match, flashvars.keys()))
formats = []
for key in url_keys:
if '/get_file/' not in flashvars[key]:
continue
format_id = flashvars.get(f'{key}_text', key)
formats.append({
'url': urljoin(url, self._kvs_get_real_url(flashvars[key], flashvars['license_code'])),
'format_id': format_id,
'ext': 'mp4',
**(parse_resolution(format_id) or parse_resolution(flashvars[key])),
'http_headers': {'Referer': url},
})
if not formats[-1].get('height'):
formats[-1]['quality'] = 1
return {
'id': flashvars['video_id'],
'display_id': display_id,
'title': title,
'thumbnail': thumbnail,
'formats': formats,
}
def _real_extract(self, url):
if url.startswith('//'):
@@ -2312,7 +2382,7 @@ class GenericIE(InfoExtractor):
# It may probably better to solve this by checking Content-Type for application/octet-stream
# after a HEAD request, but not sure if we can rely on this.
full_response = self._request_webpage(url, video_id, headers={
'Accept-Encoding': '*',
'Accept-Encoding': 'identity',
**smuggled_data.get('http_headers', {})
})
new_url = full_response.geturl()
@@ -2565,6 +2635,17 @@ class GenericIE(InfoExtractor):
self.report_detected('video.js embed')
return [{'formats': formats, 'subtitles': subtitles}]
# Look for generic KVS player (before json-ld bc of some urls that break otherwise)
found = self._search_regex((
r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:\S+?/)+kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,',
), webpage, 'KVS player', group='ver', default=False)
if found:
self.report_detected('KWS Player')
if found.split('.')[0] not in ('4', '5', '6'):
self.report_warning(f'Untested major version ({found}) in player engine - download may fail.')
return [self._extract_kvs(url, webpage, video_id)]
# Looking for http://schema.org/VideoObject
json_ld = self._search_json_ld(webpage, video_id, default={})
if json_ld.get('url') not in (url, None):
@@ -2607,52 +2688,6 @@ class GenericIE(InfoExtractor):
['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
if found:
self.report_detected('JW Player embed')
if not found:
# Look for generic KVS player
found = re.search(r'<script [^>]*?src="https?://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
if found:
self.report_detected('KWS Player')
if found.group('maj_ver') not in ['4', '5']:
self.report_warning('Untested major version (%s) in player engine--Download may fail.' % found.group('ver'))
flashvars = re.search(r'(?ms)<script.*?>.*?var\s+flashvars\s*=\s*(\{.*?\});.*?</script>', webpage)
flashvars = self._parse_json(flashvars.group(1), video_id, transform_source=js_to_json)
# extract the part after the last / as the display_id from the
# canonical URL.
display_id = self._search_regex(
r'(?:<link href="https?://[^"]+/(.+?)/?" rel="canonical"\s*/?>'
r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
webpage, 'display_id', fatal=False
)
title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
thumbnail = flashvars['preview_url']
if thumbnail.startswith('//'):
protocol, _, _ = url.partition('/')
thumbnail = protocol + thumbnail
url_keys = list(filter(re.compile(r'video_url|video_alt_url\d*').fullmatch, flashvars.keys()))
formats = []
for key in url_keys:
if '/get_file/' not in flashvars[key]:
continue
format_id = flashvars.get(f'{key}_text', key)
formats.append({
'url': self._kvs_getrealurl(flashvars[key], flashvars['license_code']),
'format_id': format_id,
'ext': 'mp4',
**(parse_resolution(format_id) or parse_resolution(flashvars[key]))
})
if not formats[-1].get('height'):
formats[-1]['quality'] = 1
return [{
'id': flashvars['video_id'],
'display_id': display_id,
'title': title,
'thumbnail': thumbnail,
'formats': formats,
}]
if not found:
# Broaden the search a little bit
found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
@@ -2733,6 +2768,7 @@ class GenericIE(InfoExtractor):
entries = []
for video_url in orderedSet(found):
video_url = video_url.encode().decode('unicode-escape')
video_url = unescapeHTML(video_url)
video_url = video_url.replace('\\/', '/')
video_url = urllib.parse.urljoin(url, video_url)
yt_dlp/extractor/iqiyi.py
+12 -9
View File
@@ -527,11 +527,14 @@ class IqIE(InfoExtractor):
webpack_js_url = self._proto_relative_url(self._search_regex(
r'<script src="((?:https?)?//stc.iqiyipic.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
webpack_map1, webpack_map2 = [self._parse_json(js_map, video_id, transform_source=js_to_json) for js_map in self._search_regex(
r'\(({[^}]*})\[\w+\][^\)]*\)\s*\+\s*["\']\.["\']\s*\+\s*({[^}]*})\[\w+\]\+["\']\.js', webpack_js, 'JS locations', group=(1, 2))]
for module_index in reversed(list(webpack_map2.keys())):
webpack_map = self._search_json(
r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\da-f]+["\']\s*,?\s*)+}',
end_pattern=r'\[\w+\]\+["\']\.js', transform_source=js_to_json)
for module_index in reversed(webpack_map):
module_js = self._download_webpage(
f'https://stc.iqiyipic.com/_next/static/chunks/{webpack_map1.get(module_index, module_index)}.{webpack_map2[module_index]}.js',
f'https://stc.iqiyipic.com/_next/static/chunks/{module_index}.{webpack_map[module_index]}.js',
video_id, note=f'Downloading #{module_index} module JS', errnote='Unable to download module JS', fatal=False) or ''
if 'vms request' in module_js:
self.cache.store('iq', 'player_js', module_js)
@@ -543,11 +546,11 @@ class IqIE(InfoExtractor):
self._extract_vms_player_js(webpage, video_id), 'signature function')
def _update_bid_tags(self, webpage, video_id):
extracted_bid_tags = self._parse_json(
self._search_regex(
r'arguments\[1\][^,]*,\s*function\s*\([^\)]*\)\s*{\s*"use strict";?\s*var \w=({.+}})\s*,\s*\w\s*=\s*{\s*getNewVd',
self._extract_vms_player_js(webpage, video_id), 'video tags', default=''),
video_id, transform_source=js_to_json, fatal=False)
extracted_bid_tags = self._search_json(
r'function\s*\([^)]*\)\s*\{\s*"use strict";?\s*var \w\s*=\s*',
self._extract_vms_player_js(webpage, video_id), 'video tags', video_id,
contains_pattern=r'{\s*\d+\s*:\s*\{\s*nbid\s*:.+}\s*}',
end_pattern=r'\s*,\s*\w\s*=\s*\{\s*getNewVd', fatal=False, transform_source=js_to_json)
if not extracted_bid_tags:
return
self._BID_TAGS = {
yt_dlp/extractor/joj.py
+19 -7
View File
@@ -23,9 +23,19 @@ class JojIE(InfoExtractor):
'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
'ext': 'mp4',
'title': 'NOVÉ BÝVANIE',
'thumbnail': r're:^https?://.*\.jpg$',
'thumbnail': r're:^https?://.*?$',
'duration': 3118,
}
}, {
'url': 'https://media.joj.sk/embed/CSM0Na0l0p1',
'info_dict': {
'id': 'CSM0Na0l0p1',
'ext': 'mp4',
'height': 576,
'title': 'Extrémne rodiny 2 - POKRAČOVANIE (2012/04/09 21:30:00)',
'duration': 3937,
'thumbnail': r're:^https?://.*?$',
}
}, {
'url': 'https://media.joj.sk/embed/9i1cxv',
'only_matching': True,
@@ -43,10 +53,10 @@ class JojIE(InfoExtractor):
webpage = self._download_webpage(
'https://media.joj.sk/embed/%s' % video_id, video_id)
title = self._search_regex(
(r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
r'<title>(?P<title>[^<]+)'), webpage, 'title',
default=None, group='title') or self._og_search_title(webpage)
title = (self._search_json(r'videoTitle\s*:', webpage, 'title', video_id,
contains_pattern=r'["\'].+["\']', default=None)
or self._html_extract_title(webpage, default=None)
or self._og_search_title(webpage))
bitrates = self._parse_json(
self._search_regex(
@@ -58,11 +68,13 @@ class JojIE(InfoExtractor):
for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
if isinstance(format_url, compat_str):
height = self._search_regex(
r'(\d+)[pP]\.', format_url, 'height', default=None)
r'(\d+)[pP]|(pal)\.', format_url, 'height', default=None)
if height == 'pal':
height = 576
formats.append({
'url': format_url,
'format_id': format_field(height, None, '%sp'),
'height': int(height),
'height': int_or_none(height),
})
if not formats:
playlist = self._download_xml(
yt_dlp/extractor/kankanews.py
+48
View File
@@ -0,0 +1,48 @@
import time
import random
import string
import hashlib
import urllib.parse
from .common import InfoExtractor
class KankaNewsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?kankanews\.com/a/\d+\-\d+\-\d+/(?P<id>\d+)\.shtml'
_TESTS = [{
'url': 'https://www.kankanews.com/a/2022-11-08/00310276054.shtml?appid=1088227',
'md5': '05e126513c74b1258d657452a6f4eef9',
'info_dict': {
'id': '4485057',
'url': 'http://mediaplay.kksmg.com/2022/11/08/h264_450k_mp4_1a388ad771e0e4cc28b0da44d245054e_ncm.mp4',
'ext': 'mp4',
'title': '视频｜第23个中国记者节，我们在进博切蛋糕',
'thumbnail': r're:^https?://.*\.jpg*',
}
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
video_id = self._search_regex(r'omsid\s*=\s*"(\d+)"', webpage, 'video id')
params = {
'nonce': ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)),
'omsid': video_id,
'platform': 'pc',
'timestamp': int(time.time()),
'version': '1.0',
}
params['sign'] = hashlib.md5((hashlib.md5((
urllib.parse.urlencode(params) + '&28c8edde3d61a0411511d3b1866f0636'
).encode()).hexdigest()).encode()).hexdigest()
meta = self._download_json('https://api-app.kankanews.com/kankan/pc/getvideo',
video_id, query=params)['result']['video']
return {
'id': video_id,
'url': meta['videourl'],
'title': self._search_regex(r'g\.title\s*=\s*"([^"]+)"', webpage, 'title'),
'thumbnail': meta.get('titlepic'),
}
yt_dlp/extractor/kick.py
+127
View File
@@ -0,0 +1,127 @@
from .common import InfoExtractor
from ..utils import (
HEADRequest,
UserNotLive,
float_or_none,
merge_dicts,
str_or_none,
traverse_obj,
unified_timestamp,
url_or_none,
)
class KickBaseIE(InfoExtractor):
def _real_initialize(self):
self._request_webpage(HEADRequest('https://kick.com/'), None, 'Setting up session')
xsrf_token = self._get_cookies('https://kick.com/').get('XSRF-TOKEN')
if not xsrf_token:
self.write_debug('kick.com did not set XSRF-TOKEN cookie')
KickBaseIE._API_HEADERS = {
'Authorization': f'Bearer {xsrf_token.value}',
'X-XSRF-TOKEN': xsrf_token.value,
} if xsrf_token else {}
def _call_api(self, path, display_id, note='Downloading API JSON', headers={}, **kwargs):
return self._download_json(
f'https://kick.com/api/v1/{path}', display_id, note=note,
headers=merge_dicts(headers, self._API_HEADERS), **kwargs)
class KickIE(KickBaseIE):
_VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w_]+)'
_TESTS = [{
'url': 'https://kick.com/yuppy',
'info_dict': {
'id': '6cde1-kickrp-joe-flemmingskick-info-heremust-knowmust-see21',
'ext': 'mp4',
'title': str,
'description': str,
'channel': 'yuppy',
'channel_id': '33538',
'uploader': 'Yuppy',
'uploader_id': '33793',
'upload_date': str,
'live_status': 'is_live',
'timestamp': int,
'thumbnail': r're:^https?://.*\.jpg',
'categories': list,
},
'skip': 'livestream',
}, {
'url': 'https://kick.com/kmack710',
'only_matching': True,
}]
def _real_extract(self, url):
channel = self._match_id(url)
response = self._call_api(f'channels/{channel}', channel)
if not traverse_obj(response, 'livestream', expected_type=dict):
raise UserNotLive(video_id=channel)
return {
'id': str(traverse_obj(
response, ('livestream', ('slug', 'id')), get_all=False, default=channel)),
'formats': self._extract_m3u8_formats(
response['playback_url'], channel, 'mp4', live=True),
'title': traverse_obj(
response, ('livestream', ('session_title', 'slug')), get_all=False, default=''),
'description': traverse_obj(response, ('user', 'bio')),
'channel': channel,
'channel_id': str_or_none(traverse_obj(response, 'id', ('livestream', 'channel_id'))),
'uploader': traverse_obj(response, 'name', ('user', 'username')),
'uploader_id': str_or_none(traverse_obj(response, 'user_id', ('user', 'id'))),
'is_live': True,
'timestamp': unified_timestamp(traverse_obj(response, ('livestream', 'created_at'))),
'thumbnail': traverse_obj(
response, ('livestream', 'thumbnail', 'url'), expected_type=url_or_none),
'categories': traverse_obj(response, ('recent_categories', ..., 'name')),
}
class KickVODIE(KickBaseIE):
_VALID_URL = r'https?://(?:www\.)?kick\.com/video/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
_TESTS = [{
'url': 'https://kick.com/video/54244b5e-050a-4df4-a013-b2433dafbe35',
'md5': '73691206a6a49db25c5aa1588e6538fc',
'info_dict': {
'id': '54244b5e-050a-4df4-a013-b2433dafbe35',
'ext': 'mp4',
'title': 'Making 710-carBoosting. Kinda No Pixel inspired. !guilded - !links',
'description': 'md5:a0d3546bf7955d0a8252ffe0fd6f518f',
'channel': 'kmack710',
'channel_id': '16278',
'uploader': 'Kmack710',
'uploader_id': '16412',
'upload_date': '20221206',
'timestamp': 1670318289,
'duration': 40104.0,
'thumbnail': r're:^https?://.*\.jpg',
'categories': ['Grand Theft Auto V'],
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
response = self._call_api(f'video/{video_id}', video_id)
return {
'id': video_id,
'formats': self._extract_m3u8_formats(response['source'], video_id, 'mp4'),
'title': traverse_obj(
response, ('livestream', ('session_title', 'slug')), get_all=False, default=''),
'description': traverse_obj(response, ('livestream', 'channel', 'user', 'bio')),
'channel': traverse_obj(response, ('livestream', 'channel', 'slug')),
'channel_id': str_or_none(traverse_obj(response, ('livestream', 'channel', 'id'))),
'uploader': traverse_obj(response, ('livestream', 'channel', 'user', 'username')),
'uploader_id': str_or_none(traverse_obj(response, ('livestream', 'channel', 'user_id'))),
'timestamp': unified_timestamp(response.get('created_at')),
'duration': float_or_none(traverse_obj(response, ('livestream', 'duration')), scale=1000),
'thumbnail': traverse_obj(
response, ('livestream', 'thumbnail'), expected_type=url_or_none),
'categories': traverse_obj(response, ('livestream', 'categories', ..., 'name')),
}
yt_dlp/extractor/linuxacademy.py
+2 -3
View File
@@ -75,9 +75,8 @@ class LinuxAcademyIE(InfoExtractor):
def _perform_login(self, username, password):
def random_string():
return ''.join([
random.choice('0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~')
for _ in range(32)])
return ''.join(random.choices(
'0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~', k=32))
webpage, urlh = self._download_webpage_handle(
self._AUTHORIZE_URL, None, 'Downloading authorize page', query={
yt_dlp/extractor/mixch.py
+4 -6
View File
@@ -1,8 +1,5 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
traverse_obj,
)
from ..utils import UserNotLive, traverse_obj
class MixchIE(InfoExtractor):
@@ -33,7 +30,7 @@ class MixchIE(InfoExtractor):
initial_js_state = self._parse_json(self._search_regex(
r'(?m)^\s*window\.__INITIAL_JS_STATE__\s*=\s*(\{.+?\});\s*$', webpage, 'initial JS state'), video_id)
if not initial_js_state.get('liveInfo'):
raise ExtractorError('Livestream has ended.', expected=True)
raise UserNotLive(video_id=video_id)
return {
'id': video_id,
@@ -45,7 +42,8 @@ class MixchIE(InfoExtractor):
'uploader_id': video_id,
'formats': [{
'format_id': 'hls',
'url': traverse_obj(initial_js_state, ('liveInfo', 'hls')) or 'https://d1hd0ww6piyb43.cloudfront.net/hls/torte_%s.m3u8' % video_id,
'url': (traverse_obj(initial_js_state, ('liveInfo', 'hls'))
or f'https://d1hd0ww6piyb43.cloudfront.net/hls/torte_{video_id}.m3u8'),
'ext': 'mp4',
'protocol': 'm3u8',
}],
yt_dlp/extractor/nbc.py
+1
View File
@@ -136,6 +136,7 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
query = {
'mbr': 'true',
'manifest': 'm3u',
'switch': 'HLSServiceSecure',
}
video_id = video_data['mpxGuid']
tp_path = 'NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id)
yt_dlp/extractor/netverse.py
+29 -1
View File
@@ -1,6 +1,6 @@
import itertools
from .common import InfoExtractor
from .common import InfoExtractor, SearchInfoExtractor
from .dailymotion import DailymotionIE
from ..utils import smuggle_url, traverse_obj
@@ -251,3 +251,31 @@ class NetversePlaylistIE(NetverseBaseIE):
self.parse_playlist(playlist_data['response'], playlist_id),
traverse_obj(playlist_data, ('response', 'webseries_info', 'slug')),
traverse_obj(playlist_data, ('response', 'webseries_info', 'title')))
class NetverseSearchIE(SearchInfoExtractor):
_SEARCH_KEY = 'netsearch'
_TESTS = [{
'url': 'netsearch10:tetangga',
'info_dict': {
'id': 'tetangga',
'title': 'tetangga',
},
'playlist_count': 10,
}]
def _search_results(self, query):
last_page = None
for i in itertools.count(1):
search_data = self._download_json(
'https://api.netverse.id/search/elastic/search', query,
query={'q': query, 'page': i}, note=f'Downloading page {i}')
videos = traverse_obj(search_data, ('response', 'data', ...))
for video in videos:
yield self.url_result(f'https://netverse.id/video/{video["slug"]}', NetverseIE)
last_page = last_page or traverse_obj(search_data, ('response', 'lastpage'))
if not videos or i >= (last_page or 0):
break
yt_dlp/extractor/odnoklassniki.py
+33
View File
@@ -11,6 +11,7 @@ from ..utils import (
int_or_none,
qualities,
smuggle_url,
traverse_obj,
unescapeHTML,
unified_strdate,
unsmuggle_url,
@@ -153,6 +154,26 @@ class OdnoklassnikiIE(InfoExtractor):
'title': 'Быковское крещение',
'duration': 3038.181,
},
'skip': 'HTTP Error 400',
}, {
'note': 'subtitles',
'url': 'https://ok.ru/video/4249587550747',
'info_dict': {
'id': '4249587550747',
'ext': 'mp4',
'title': 'Small Country An African Childhood (2020) (1080p) +subtitle',
'uploader': 'Sunflower Movies',
'uploader_id': '595802161179',
'upload_date': '20220816',
'duration': 6728,
'age_limit': 0,
'thumbnail': r're:^https?://i\.mycdn\.me/videoPreview\?.+',
'like_count': int,
'subtitles': dict,
},
'params': {
'skip_download': True,
},
}, {
'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
'only_matching': True,
@@ -202,6 +223,7 @@ class OdnoklassnikiIE(InfoExtractor):
'like_count': 0,
'duration': 10444,
},
'skip': 'Site no longer embeds',
}]
@classmethod
@@ -294,6 +316,16 @@ class OdnoklassnikiIE(InfoExtractor):
like_count = int_or_none(metadata.get('likeCount'))
subtitles = {}
for sub in traverse_obj(metadata, ('movie', 'subtitleTracks', ...), expected_type=dict):
sub_url = sub.get('url')
if not sub_url:
continue
subtitles.setdefault(sub.get('language') or 'en', []).append({
'url': sub_url,
'ext': 'vtt',
})
info = {
'id': video_id,
'title': title,
@@ -305,6 +337,7 @@ class OdnoklassnikiIE(InfoExtractor):
'like_count': like_count,
'age_limit': age_limit,
'start_time': start_time,
'subtitles': subtitles,
}
# pladform
yt_dlp/extractor/peekvids.py
+152 -38
View File
@@ -1,71 +1,128 @@
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
get_element_by_class,
int_or_none,
merge_dicts,
url_or_none,
)
class PeekVidsIE(InfoExtractor):
class PeekVidsBaseIE(InfoExtractor):
def _real_extract(self, url):
domain, video_id = self._match_valid_url(url).group('domain', 'id')
webpage = self._download_webpage(url, video_id, expected_status=429)
if '>Rate Limit Exceeded' in webpage:
raise ExtractorError(
f'You are suspected as a bot. Wait, or pass the captcha on the site and provide cookies. {self._login_hint()}',
video_id=video_id, expected=True)
title = self._html_search_regex(r'(?s)<h1\b[^>]*>(.+?)</h1>', webpage, 'title')
display_id = video_id
video_id = self._search_regex(r'(?s)<video\b[^>]+\bdata-id\s*=\s*["\']?([\w-]+)', webpage, 'short video ID')
srcs = self._download_json(
f'https://www.{domain}/v-alt/{video_id}', video_id,
note='Downloading list of source files')
formats = []
for k, v in srcs.items():
f_url = url_or_none(v)
if not f_url:
continue
height = self._search_regex(r'^data-src(\d{3,})$', k, 'height', default=None)
if not height:
continue
formats.append({
'url': f_url,
'format_id': height,
'height': int_or_none(height),
})
if not formats:
formats = [{'url': url} for url in srcs.values()]
info = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={})
info.pop('url', None)
# may not have found the thumbnail if it was in a list in the ld+json
info.setdefault('thumbnail', self._og_search_thumbnail(webpage))
detail = (get_element_by_class('detail-video-block', webpage)
or get_element_by_class('detail-block', webpage) or '')
info['description'] = self._html_search_regex(
rf'(?s)(.+?)(?:{re.escape(info.get("description", ""))}\s*<|<ul\b)',
detail, 'description', default=None) or None
info['title'] = re.sub(r'\s*[,-][^,-]+$', '', info.get('title') or title) or self._generic_title(url)
def cat_tags(name, html):
l = self._html_search_regex(
rf'(?s)<span\b[^>]*>\s*{re.escape(name)}\s*:\s*</span>(.+?)</li>',
html, name, default='')
return list(filter(None, re.split(r'\s+', l)))
return merge_dicts({
'id': video_id,
'display_id': display_id,
'age_limit': 18,
'formats': formats,
'categories': cat_tags('Categories', detail),
'tags': cat_tags('Tags', detail),
'uploader': self._html_search_regex(r'[Uu]ploaded\s+by\s(.+?)"', webpage, 'uploader', default=None),
}, info)
class PeekVidsIE(PeekVidsBaseIE):
_VALID_URL = r'''(?x)
https?://(?:www\.)?peekvids\.com/
https?://(?:www\.)?(?P<domain>peekvids\.com)/
(?:(?:[^/?#]+/){2}|embed/?\?(?:[^#]*&)?v=)
(?P<id>[^/?&#]*)
'''
_TESTS = [{
'url': 'https://peekvids.com/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp/BSyLMbN0YCd',
'md5': 'a00940646c428e232407e3e62f0e8ef5',
'md5': '2ff6a357a9717dc9dc9894b51307e9a2',
'info_dict': {
'id': 'BSyLMbN0YCd',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp, SEXYhub',
'id': '1262717',
'display_id': 'BSyLMbN0YCd',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
'ext': 'mp4',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'Watch Dane Jones - Cute redhead with perfect tits with Mini Vamp (7 min), uploaded by SEXYhub.com',
'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
'timestamp': 1642579329,
'upload_date': '20220119',
'duration': 416,
'view_count': int,
'age_limit': 18,
'uploader': 'SEXYhub.com',
'categories': list,
'tags': list,
},
}]
_DOMAIN = 'www.peekvids.com'
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
short_video_id = self._html_search_regex(r'<video [^>]*data-id="(.+?)"', webpage, 'short video ID')
srcs = self._download_json(
f'https://{self._DOMAIN}/v-alt/{short_video_id}', video_id,
note='Downloading list of source files')
formats = [{
'url': url,
'ext': 'mp4',
'format_id': name[8:],
} for name, url in srcs.items() if len(name) > 8 and name.startswith('data-src')]
if not formats:
formats = [{'url': url} for url in srcs.values()]
info = self._search_json_ld(webpage, video_id, expected_type='VideoObject')
info.update({
'id': video_id,
'age_limit': 18,
'formats': formats,
})
return info
class PlayVidsIE(PeekVidsIE): # XXX: Do not subclass from concrete IE
_VALID_URL = r'https?://(?:www\.)?playvids\.com/(?:embed/|[^/]{2}/)?(?P<id>[^/?#]*)'
class PlayVidsIE(PeekVidsBaseIE):
_VALID_URL = r'https?://(?:www\.)?(?P<domain>playvids\.com)/(?:embed/|\w\w?/)?(?P<id>[^/?#]*)'
_TESTS = [{
'url': 'https://www.playvids.com/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
'md5': 'cd7dfd8a2e815a45402369c76e3c1825',
'md5': '2f12e50213dd65f142175da633c4564c',
'info_dict': {
'id': 'U3pBrYhsjXM',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp, SEXYhub',
'id': '1978030',
'display_id': 'U3pBrYhsjXM',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
'ext': 'mp4',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'Watch Dane Jones - Cute redhead with perfect tits with Mini Vamp video in HD, uploaded by SEXYhub.com',
'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
'timestamp': 1640435839,
'upload_date': '20211225',
'duration': 416,
'view_count': int,
'age_limit': 18,
'uploader': 'SEXYhub.com',
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/es/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
@@ -73,5 +130,62 @@ class PlayVidsIE(PeekVidsIE): # XXX: Do not subclass from concrete IE
}, {
'url': 'https://www.playvids.com/embed/U3pBrYhsjXM',
'only_matching': True,
}, {
'url': 'https://www.playvids.com/bKmGLe3IwjZ/sv/brazzers-800-phone-sex-madison-ivy-always-on-the-line',
'md5': 'e783986e596cafbf46411a174ab42ba6',
'info_dict': {
'id': '762385',
'display_id': 'bKmGLe3IwjZ',
'ext': 'mp4',
'title': 'Brazzers - 1 800 Phone Sex: Madison Ivy Always On The Line 6',
'description': 'md5:bdcd2db2b8ad85831a491d7c8605dcef',
'timestamp': 1516958544,
'upload_date': '20180126',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 480,
'uploader': 'Brazzers',
'age_limit': 18,
'view_count': int,
'age_limit': 18,
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/v/47iUho33toY',
'md5': 'b056b5049d34b648c1e86497cf4febce',
'info_dict': {
'id': '700621',
'display_id': '47iUho33toY',
'ext': 'mp4',
'title': 'KATEE OWEN STRIPTIASE IN SEXY RED LINGERIE',
'description': None,
'timestamp': 1507052209,
'upload_date': '20171003',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 332,
'uploader': 'Cacerenele',
'age_limit': 18,
'view_count': int,
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/z3_7iwWCmqt/sexy-teen-filipina-striptease-beautiful-pinay-bargirl-strips-and-dances',
'md5': 'efa09be9f031314b7b7e3bc6510cd0df',
'info_dict': {
'id': '1523518',
'display_id': 'z3_7iwWCmqt',
'ext': 'mp4',
'title': 'SEXY TEEN FILIPINA STRIPTEASE - Beautiful Pinay Bargirl Strips and Dances',
'description': None,
'timestamp': 1607470323,
'upload_date': '20201208',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 593,
'uploader': 'yorours',
'age_limit': 18,
'view_count': int,
'categories': list,
'tags': list,
},
}]
_DOMAIN = 'www.playvids.com'
yt_dlp/extractor/polskieradio.py
+165 -48
View File
@@ -10,6 +10,7 @@ from ..compat import (
compat_urlparse
)
from ..utils import (
determine_ext,
extract_attributes,
ExtractorError,
InAdvancePagedList,
@@ -17,6 +18,7 @@ from ..utils import (
js_to_json,
parse_iso8601,
strip_or_none,
traverse_obj,
unified_timestamp,
unescapeHTML,
url_or_none,
@@ -48,28 +50,11 @@ class PolskieRadioBaseExtractor(InfoExtractor):
yield entry
class PolskieRadioIE(PolskieRadioBaseExtractor):
_VALID_URL = r'https?://(?:www\.)?polskieradio(?:24)?\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
_TESTS = [{ # Old-style single broadcast.
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
'info_dict': {
'id': '1587943',
'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
},
'playlist': [{
'md5': '2984ee6ce9046d91fc233bc1a864a09a',
'info_dict': {
'id': '1540576',
'ext': 'mp3',
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
'timestamp': 1456594200,
'upload_date': '20160227',
'duration': 2364,
'thumbnail': r're:^https?://static\.prsa\.pl/images/.*\.jpg$'
},
}],
}, { # New-style single broadcast.
class PolskieRadioLegacyIE(PolskieRadioBaseExtractor):
# legacy sites
IE_NAME = 'polskieradio:legacy'
_VALID_URL = r'https?://(?:www\.)?polskieradio(?:24)?\.pl/\d+/\d+/[Aa]rtykul/(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.polskieradio.pl/8/2382/Artykul/2534482,Zagarysci-Poezja-jak-spoiwo',
'info_dict': {
'id': '2534482',
@@ -96,16 +81,6 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
'ext': 'mp3',
'title': 'Pogłos 29 października godz. 23:01',
},
}, {
'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
'only_matching': True,
}, {
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
'only_matching': True,
}, {
# with mp4 video
'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
'only_matching': True,
}, {
'url': 'https://polskieradio24.pl/130/4503/Artykul/2621876,Narusza-nasza-suwerennosc-Publicysci-o-uzaleznieniu-funduszy-UE-od-praworzadnosci',
'only_matching': True,
@@ -114,7 +89,9 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
webpage, urlh = self._download_webpage_handle(url, playlist_id)
if PolskieRadioIE.suitable(urlh.url):
return self.url_result(urlh.url, PolskieRadioIE, playlist_id)
content = self._search_regex(
r'(?s)<div[^>]+class="\s*this-article\s*"[^>]*>(.+?)<div[^>]+class="tags"[^>]*>',
@@ -153,23 +130,160 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
return self.playlist_result(entries, playlist_id, title, description)
class PolskieRadioCategoryIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
class PolskieRadioIE(InfoExtractor):
# new next.js sites, excluding radiokierowcow.pl
_VALID_URL = r'https?://(?:[^/]+\.)?polskieradio(?:24)?\.pl/artykul/(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
'url': 'https://jedynka.polskieradio.pl/artykul/1587943',
'info_dict': {
'id': '1587943',
'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
},
'playlist': [{
'md5': '2984ee6ce9046d91fc233bc1a864a09a',
'info_dict': {
'id': '7a85d429-5356-4def-a347-925e4ae7406b',
'ext': 'mp3',
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
},
}],
}, {
'url': 'https://trojka.polskieradio.pl/artykul/1632955',
'only_matching': True,
}, {
# with mp4 video
'url': 'https://trojka.polskieradio.pl/artykul/1634903',
'only_matching': True,
}, {
'url': 'https://jedynka.polskieradio.pl/artykul/3042436,Polityka-wschodnia-ojca-i-syna-Wladyslawa-Lokietka-i-Kazimierza-Wielkiego',
'only_matching': True,
}]
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
article_data = traverse_obj(
self._search_nextjs_data(webpage, playlist_id), ('props', 'pageProps', 'data', 'articleData'))
title = strip_or_none(article_data['title'])
description = strip_or_none(article_data.get('lead'))
entries = [{
'url': entry['file'],
'ext': determine_ext(entry.get('fileName')),
'id': self._search_regex(
r'([a-f\d]{8}-(?:[a-f\d]{4}-){3}[a-f\d]{12})', entry['file'], 'entry id'),
'title': strip_or_none(entry.get('description')) or title,
} for entry in article_data.get('attachments') or () if entry['fileType'] in ('Audio', )]
return self.playlist_result(entries, playlist_id, title, description)
class PolskieRadioAuditionIE(InfoExtractor):
# new next.js sites
IE_NAME = 'polskieradio:audition'
_VALID_URL = r'https?://(?:[^/]+\.)?polskieradio\.pl/audycj[ae]/(?P<id>\d+)'
_TESTS = [{
# articles, PR1
'url': 'https://jedynka.polskieradio.pl/audycje/5102',
'info_dict': {
'id': '5102',
'title': 'HISTORIA ŻYWA',
'title': 'Historia żywa',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 38,
}, {
'url': 'http://www.polskieradio.pl/7/4807',
# episodes, PR1
'url': 'https://jedynka.polskieradio.pl/audycje/5769',
'info_dict': {
'id': '4807',
'title': 'Vademecum 1050. rocznicy Chrztu Polski'
'id': '5769',
'title': 'AgroFakty',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 5
'playlist_mincount': 269,
}, {
# both episodes and articles, PR3
'url': 'https://trojka.polskieradio.pl/audycja/8906',
'info_dict': {
'id': '8906',
'title': 'Trójka budzi',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 722,
}]
def _call_lp3(self, path, query, video_id, note):
return self._download_json(
f'https://lp3test.polskieradio.pl/{path}', video_id, note,
query=query, headers={'x-api-key': '9bf6c5a2-a7d0-4980-9ed7-a3f7291f2a81'})
def _entries(self, playlist_id, has_episodes, has_articles):
for i in itertools.count(1) if has_episodes else []:
page = self._call_lp3(
'AudioArticle/GetListByCategoryId', {
'categoryId': playlist_id,
'PageSize': 10,
'skip': i,
'format': 400,
}, playlist_id, f'Downloading episode list page {i}')
if not traverse_obj(page, 'data'):
break
for episode in page['data']:
yield {
'id': str(episode['id']),
'url': episode['file'],
'title': episode.get('title'),
'duration': int_or_none(episode.get('duration')),
'timestamp': parse_iso8601(episode.get('datePublic')),
}
for i in itertools.count(1) if has_articles else []:
page = self._call_lp3(
'Article/GetListByCategoryId', {
'categoryId': playlist_id,
'PageSize': 9,
'skip': i,
'format': 400,
}, playlist_id, f'Downloading article list page {i}')
if not traverse_obj(page, 'data'):
break
for article in page['data']:
yield {
'_type': 'url_transparent',
'ie_key': PolskieRadioIE.ie_key(),
'id': str(article['id']),
'url': article['url'],
'title': article.get('shortTitle'),
'description': traverse_obj(article, ('description', 'lead')),
'timestamp': parse_iso8601(article.get('datePublic')),
}
def _real_extract(self, url):
playlist_id = self._match_id(url)
page_props = traverse_obj(
self._search_nextjs_data(self._download_webpage(url, playlist_id), playlist_id),
('props', 'pageProps', ('data', None)), get_all=False)
has_episodes = bool(traverse_obj(page_props, 'episodes', 'audios'))
has_articles = bool(traverse_obj(page_props, 'articles'))
return self.playlist_result(
self._entries(playlist_id, has_episodes, has_articles), playlist_id,
title=traverse_obj(page_props, ('details', 'name')),
description=traverse_obj(page_props, ('details', 'description', 'lead')),
thumbnail=traverse_obj(page_props, ('details', 'photo')))
class PolskieRadioCategoryIE(InfoExtractor):
# legacy sites
IE_NAME = 'polskieradio:category'
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
'only_matching': True
}, {
@@ -186,9 +300,6 @@ class PolskieRadioCategoryIE(InfoExtractor):
'title': 'Muzyka',
},
'playlist_mincount': 61
}, {
'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
'only_matching': True,
}, {
'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
'only_matching': True,
@@ -196,7 +307,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
@classmethod
def suitable(cls, url):
return False if PolskieRadioIE.suitable(url) else super(PolskieRadioCategoryIE, cls).suitable(url)
return False if PolskieRadioLegacyIE.suitable(url) else super().suitable(url)
def _entries(self, url, page, category_id):
content = page
@@ -209,7 +320,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
if not href:
continue
yield self.url_result(
compat_urlparse.urljoin(url, href), PolskieRadioIE.ie_key(),
compat_urlparse.urljoin(url, href), PolskieRadioLegacyIE,
entry_id, entry.get('title'))
mobj = re.search(
r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
@@ -222,7 +333,9 @@ class PolskieRadioCategoryIE(InfoExtractor):
def _real_extract(self, url):
category_id = self._match_id(url)
webpage = self._download_webpage(url, category_id)
webpage, urlh = self._download_webpage_handle(url, category_id)
if PolskieRadioAuditionIE.suitable(urlh.url):
return self.url_result(urlh.url, PolskieRadioAuditionIE, category_id)
title = self._html_search_regex(
r'<title>([^<]+) - [^<]+ - [^<]+</title>',
webpage, 'title', fatal=False)
@@ -358,7 +471,7 @@ class PolskieRadioPodcastListIE(PolskieRadioPodcastBaseExtractor):
'entries': InAdvancePagedList(
get_page, math.ceil(data['itemCount'] / self._PAGE_SIZE), self._PAGE_SIZE),
'id': str(data['id']),
'title': data['title'],
'title': data.get('title'),
'description': data.get('description'),
'uploader': data.get('announcer'),
}
@@ -374,6 +487,10 @@ class PolskieRadioPodcastIE(PolskieRadioPodcastBaseExtractor):
'ext': 'mp3',
'title': 'Theresa May rezygnuje. Co dalej z brexitem?',
'description': 'md5:e41c409a29d022b70ef0faa61dbded60',
'episode': 'Theresa May rezygnuje. Co dalej z brexitem?',
'duration': 2893,
'thumbnail': 'https://static.prsa.pl/images/58649376-c8a0-4ba2-a714-78b383285f5f.jpg',
'series': 'Raport o stanie świata',
},
}]
yt_dlp/extractor/reddit.py
+24 -1
View File
@@ -32,6 +32,7 @@ class RedditIE(InfoExtractor):
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'videos',
},
'params': {
'skip_download': True,
@@ -55,6 +56,7 @@ class RedditIE(InfoExtractor):
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'aww',
},
}, {
# videos embedded in reddit text post
@@ -64,6 +66,25 @@ class RedditIE(InfoExtractor):
'id': 'wzqkxp',
'title': 'md5:72d3d19402aa11eff5bd32fc96369b37',
},
}, {
# crossposted reddit-hosted media
'url': 'https://www.reddit.com/r/dumbfuckers_club/comments/zjjw82/cringe/',
'md5': '746180895c7b75a9d6b05341f507699a',
'info_dict': {
'id': 'a1oneun6pa5a1',
'ext': 'mp4',
'display_id': 'zjjw82',
'title': 'Cringe',
'uploader': 'Otaku-senpai69420',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'upload_date': '20221212',
'timestamp': 1670812309,
'duration': 16,
'like_count': int,
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
},
}, {
'url': 'https://www.reddit.com/r/videos/comments/6rrwyj',
'only_matching': True,
@@ -146,6 +167,7 @@ class RedditIE(InfoExtractor):
'thumbnails': thumbnails,
'timestamp': float_or_none(data.get('created_utc')),
'uploader': data.get('author'),
'channel_id': data.get('subreddit'),
'like_count': int_or_none(data.get('ups')),
'dislike_count': int_or_none(data.get('downs')),
'comment_count': int_or_none(data.get('num_comments')),
@@ -179,7 +201,8 @@ class RedditIE(InfoExtractor):
raise ExtractorError('No media found', expected=True)
# Check if media is hosted on reddit:
reddit_video = traverse_obj(data, (('media', 'secure_media'), 'reddit_video'), get_all=False)
reddit_video = traverse_obj(data, (
(None, ('crosspost_parent_list', ...)), ('secure_media', 'media'), 'reddit_video'), get_all=False)
if reddit_video:
playlist_urls = [
try_get(reddit_video, lambda x: unescapeHTML(x[y]))
yt_dlp/extractor/rozhlas.py
+136 -4
View File
@@ -1,8 +1,5 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
remove_start,
)
from ..utils import extract_attributes, int_or_none, remove_start, traverse_obj
class RozhlasIE(InfoExtractor):
@@ -45,3 +42,138 @@ class RozhlasIE(InfoExtractor):
'duration': duration,
'vcodec': 'none',
}
class RozhlasVltavaIE(InfoExtractor):
_VALID_URL = r'https?://(?:\w+\.rozhlas|english\.radio)\.cz/[\w-]+-(?P<id>\d+)'
_TESTS = [{
'url': 'https://wave.rozhlas.cz/papej-masicko-porcujeme-a-bilancujeme-filmy-a-serialy-ktere-letos-zabily-8891337',
'md5': 'ba2fdbc1242fc16771c7695d271ec355',
'info_dict': {
'id': 8891337,
'title': 'md5:21f99739d04ab49d8c189ec711eef4ec',
},
'playlist_count': 1,
'playlist': [{
'md5': 'ba2fdbc1242fc16771c7695d271ec355',
'info_dict': {
'id': '10520988',
'ext': 'mp3',
'title': 'Papej masíčko! Porcujeme a bilancujeme filmy a seriály, které to letos zabily',
'description': 'md5:1c6d29fb9564e1f17fc1bb83ae7da0bc',
'duration': 1574,
'artist': 'Aleš Stuchlý',
'channel_id': 'radio-wave',
},
}]
}, {
'url': 'https://wave.rozhlas.cz/poslechnete-si-neklid-podcastovy-thriller-o-vine-strachu-a-vztahu-ktery-zasel-8554744',
'info_dict': {
'id': 8554744,
'title': 'Poslechněte si Neklid. Podcastový thriller o vině, strachu a vztahu, který zašel příliš daleko',
},
'playlist_count': 5,
'playlist': [{
'md5': '93d4109cf8f40523699ae9c1d4600bdd',
'info_dict': {
'id': '9890713',
'ext': 'mp3',
'title': 'Neklid #1',
'description': '1. díl: Neklid: 1. díl',
'duration': 1025,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #1',
'chapter_number': 1,
},
}, {
'md5': 'e9763235be4a6dcf94bc8a5bac1ca126',
'info_dict': {
'id': '9890716',
'ext': 'mp3',
'title': 'Neklid #2',
'description': '2. díl: Neklid: 2. díl',
'duration': 768,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #2',
'chapter_number': 2,
},
}, {
'md5': '00b642ea94b78cc949ac84da09f87895',
'info_dict': {
'id': '9890722',
'ext': 'mp3',
'title': 'Neklid #3',
'description': '3. díl: Neklid: 3. díl',
'duration': 607,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #3',
'chapter_number': 3,
},
}, {
'md5': 'faef97b1b49da7df874740f118c19dea',
'info_dict': {
'id': '9890728',
'ext': 'mp3',
'title': 'Neklid #4',
'description': '4. díl: Neklid: 4. díl',
'duration': 621,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #4',
'chapter_number': 4,
},
}, {
'md5': '6e729fa39b647325b868d419c76f3efa',
'info_dict': {
'id': '9890734',
'ext': 'mp3',
'title': 'Neklid #5',
'description': '5. díl: Neklid: 5. díl',
'duration': 908,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #5',
'chapter_number': 5,
},
}]
}]
def _extract_video(self, entry):
chapter_number = int_or_none(traverse_obj(entry, ('meta', 'ga', 'contentSerialPart')))
return {
'id': entry['meta']['ga']['contentId'],
'title': traverse_obj(entry, ('meta', 'ga', 'contentName')),
'description': entry.get('title'),
'duration': entry.get('duration'),
'artist': traverse_obj(entry, ('meta', 'ga', 'contentAuthor')),
'channel_id': traverse_obj(entry, ('meta', 'ga', 'contentCreator')),
'chapter': traverse_obj(entry, ('meta', 'ga', 'contentNameShort')) if chapter_number else None,
'chapter_number': chapter_number,
'formats': [{
'url': audio_link['url'],
'ext': audio_link.get('variant'),
'format_id': audio_link.get('variant'),
'abr': audio_link.get('bitrate'),
'acodec': audio_link.get('variant'),
'vcodec': 'none',
} for audio_link in entry['audioLinks']],
}
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
# FIXME: Use get_element_text_and_html_by_tag when it accepts less strict html
data = self._parse_json(extract_attributes(self._search_regex(
r'(<div class="mujRozhlasPlayer" data-player=\'[^\']+\'>)',
webpage, 'player'))['data-player'], video_id)['data']
return {
'_type': 'playlist',
'id': data.get('embedId'),
'title': traverse_obj(data, ('series', 'title')),
'entries': map(self._extract_video, data['playlist']),
}
yt_dlp/extractor/slideslive.py
+362 -28
View File
@@ -1,16 +1,24 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
parse_qs,
smuggle_url,
traverse_obj,
unified_timestamp,
update_url_query,
url_or_none,
xpath_text,
)
class SlidesLiveIE(InfoExtractor):
_VALID_URL = r'https?://slideslive\.com/(?P<id>[0-9]+)'
_VALID_URL = r'https?://slideslive\.com/(?:embed/(?:presentation/)?)?(?P<id>[0-9]+)'
_TESTS = [{
# service_name = yoda
# service_name = yoda, only XML slides info
'url': 'https://slideslive.com/38902413/gcc-ia16-backend',
'info_dict': {
'id': '38902413',
@@ -19,12 +27,14 @@ class SlidesLiveIE(InfoExtractor):
'timestamp': 1648189972,
'upload_date': '20220325',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:42',
'chapters': 'count:41',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = yoda
# service_name = yoda, /v7/ slides
'url': 'https://slideslive.com/38935785',
'info_dict': {
'id': '38935785',
@@ -32,13 +42,15 @@ class SlidesLiveIE(InfoExtractor):
'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges',
'upload_date': '20211115',
'timestamp': 1636996003,
'thumbnail': r're:^https?://.*\.jpg',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:640',
'chapters': 'count:639',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = yoda
# service_name = yoda, /v1/ slides
'url': 'https://slideslive.com/38973182/how-should-a-machine-learning-researcher-think-about-ai-ethics',
'info_dict': {
'id': '38973182',
@@ -47,12 +59,14 @@ class SlidesLiveIE(InfoExtractor):
'upload_date': '20220201',
'thumbnail': r're:^https?://.*\.jpg',
'timestamp': 1643728135,
'thumbnails': 'count:3',
'chapters': 'count:2',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = youtube
# service_name = youtube, only XML slides info
'url': 'https://slideslive.com/38897546/special-metaprednaska-petra-ludwiga-hodnoty-pro-lepsi-spolecnost',
'md5': '8a79b5e3d700837f40bd2afca3c8fa01',
'info_dict': {
@@ -76,26 +90,253 @@ class SlidesLiveIE(InfoExtractor):
'comment_count': int,
'channel_follower_count': int,
'age_limit': 0,
'thumbnail': r're:^https?://.*\.jpg',
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:169',
'playable_in_embed': True,
'availability': 'unlisted',
'tags': [],
'categories': ['People & Blogs'],
'chapters': 'count:168',
},
}, {
# service_name = youtube
# embed-only presentation, only XML slides info
'url': 'https://slideslive.com/embed/presentation/38925850',
'info_dict': {
'id': '38925850',
'ext': 'mp4',
'title': 'Towards a Deep Network Architecture for Structured Smoothness',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:8',
'timestamp': 1629671508,
'upload_date': '20210822',
'chapters': 'count:7',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# embed-only presentation, only JSON slides info, /v5/ slides (.png)
'url': 'https://slideslive.com/38979920/',
'info_dict': {
'id': '38979920',
'ext': 'mp4',
'title': 'MoReL: Multi-omics Relational Learning',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:7',
'timestamp': 1654714970,
'upload_date': '20220608',
'chapters': 'count:6',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v2/ slides (.jpg)
'url': 'https://slideslive.com/38954074',
'info_dict': {
'id': '38954074',
'ext': 'mp4',
'title': 'Decentralized Attribution of Generative Models',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:16',
'timestamp': 1622806321,
'upload_date': '20210604',
'chapters': 'count:15',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v4/ slides (.png)
'url': 'https://slideslive.com/38979570/',
'info_dict': {
'id': '38979570',
'ext': 'mp4',
'title': 'Efficient Active Search for Combinatorial Optimization Problems',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:9',
'timestamp': 1654714896,
'upload_date': '20220608',
'chapters': 'count:8',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v10/ slides
'url': 'https://slideslive.com/embed/presentation/38979880?embed_parent_url=https%3A%2F%2Fedit.videoken.com%2F',
'info_dict': {
'id': '38979880',
'ext': 'mp4',
'title': 'The Representation Power of Neural Networks',
'timestamp': 1654714962,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:22',
'upload_date': '20220608',
'chapters': 'count:21',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v7/ slides, 2 video slides
'url': 'https://slideslive.com/embed/presentation/38979682?embed_container_origin=https%3A%2F%2Fedit.videoken.com',
'playlist_count': 3,
'info_dict': {
'id': '38979682-playlist',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models',
},
'playlist': [{
'info_dict': {
'id': '38979682',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models',
'timestamp': 1654714920,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:30',
'upload_date': '20220608',
'chapters': 'count:31',
},
}, {
'info_dict': {
'id': '38979682-021',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 021',
'duration': 3,
'timestamp': 1654714920,
'upload_date': '20220608',
},
}, {
'info_dict': {
'id': '38979682-024',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 024',
'duration': 4,
'timestamp': 1654714920,
'upload_date': '20220608',
},
}],
'params': {
'skip_download': 'm3u8',
},
}, {
# /v6/ slides, 1 video slide, edit.videoken.com embed
'url': 'https://slideslive.com/38979481/',
'playlist_count': 2,
'info_dict': {
'id': '38979481-playlist',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification',
},
'playlist': [{
'info_dict': {
'id': '38979481',
'ext': 'mp4',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification',
'timestamp': 1654714877,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:43',
'upload_date': '20220608',
'chapters': 'count:43',
},
}, {
'info_dict': {
'id': '38979481-013',
'ext': 'mp4',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification - Slide 013',
'duration': 3,
'timestamp': 1654714877,
'upload_date': '20220608',
},
}],
'params': {
'skip_download': 'm3u8',
},
}, {
# /v3/ slides, .jpg and .png, service_name = youtube
'url': 'https://slideslive.com/embed/38932460/',
'info_dict': {
'id': 'RTPdrgkyTiE',
'display_id': '38932460',
'ext': 'mp4',
'title': 'Active Learning for Hierarchical Multi-Label Classification',
'description': 'Watch full version of this video at https://slideslive.com/38932460.',
'channel': 'SlidesLive Videos - A',
'channel_id': 'UC62SdArr41t_-_fX40QCLRw',
'channel_url': 'https://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw',
'uploader': 'SlidesLive Videos - A',
'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
'uploader_url': 'http://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw',
'upload_date': '20200903',
'timestamp': 1602599092,
'duration': 942,
'age_limit': 0,
'live_status': 'not_live',
'playable_in_embed': True,
'availability': 'unlisted',
'categories': ['People & Blogs'],
'tags': [],
'channel_follower_count': int,
'like_count': int,
'view_count': int,
'thumbnail': r're:^https?://.*\.(?:jpg|png|webp)',
'thumbnails': 'count:21',
'chapters': 'count:20',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = yoda
'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend',
'only_matching': True,
}, {
# service_name = url
# dead link, service_name = url
'url': 'https://slideslive.com/38922070/learning-transferable-skills-1',
'only_matching': True,
}, {
# service_name = vimeo
# dead link, service_name = vimeo
'url': 'https://slideslive.com/38921896/retrospectives-a-venue-for-selfreflection-in-ml-research-3',
'only_matching': True,
}]
_WEBPAGE_TESTS = [{
# only XML slides info
'url': 'https://iclr.cc/virtual_2020/poster_Hklr204Fvr.html',
'info_dict': {
'id': '38925850',
'ext': 'mp4',
'title': 'Towards a Deep Network Architecture for Structured Smoothness',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:8',
'timestamp': 1629671508,
'upload_date': '20210822',
'chapters': 'count:7',
},
'params': {
'skip_download': 'm3u8',
},
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
# Reference: https://slideslive.com/embed_presentation.js
for embed_id in re.findall(r'(?s)new\s+SlidesLiveEmbed\s*\([^)]+\bpresentationId:\s*["\'](\d+)["\']', webpage):
url_parsed = urllib.parse.urlparse(url)
origin = f'{url_parsed.scheme}://{url_parsed.netloc}'
yield update_url_query(
f'https://slideslive.com/embed/presentation/{embed_id}', {
'embed_parent_url': url,
'embed_container_origin': origin,
})
def _download_embed_webpage_handle(self, video_id, headers):
return self._download_webpage_handle(
f'https://slideslive.com/embed/presentation/{video_id}', video_id,
headers=headers, query=traverse_obj(headers, {
'embed_parent_url': 'Referer',
'embed_container_origin': 'Origin',
}))
def _extract_custom_m3u8_info(self, m3u8_data):
m3u8_dict = {}
@@ -108,6 +349,8 @@ class SlidesLiveIE(InfoExtractor):
'VOD-VIDEO-ID': 'service_id',
'VOD-VIDEO-SERVERS': 'video_servers',
'VOD-SUBTITLES': 'subtitles',
'VOD-SLIDES-JSON-URL': 'slides_json_url',
'VOD-SLIDES-XML-URL': 'slides_xml_url',
}
for line in m3u8_data.splitlines():
@@ -126,9 +369,33 @@ class SlidesLiveIE(InfoExtractor):
return m3u8_dict
def _extract_formats(self, cdn_hostname, path, video_id):
formats = []
formats.extend(self._extract_m3u8_formats(
f'https://{cdn_hostname}/{path}/master.m3u8',
video_id, 'mp4', m3u8_id='hls', fatal=False, live=True))
formats.extend(self._extract_mpd_formats(
f'https://{cdn_hostname}/{path}/master.mpd',
video_id, mpd_id='dash', fatal=False))
return formats
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
webpage, urlh = self._download_embed_webpage_handle(
video_id, headers=traverse_obj(parse_qs(url), {
'Referer': ('embed_parent_url', -1),
'Origin': ('embed_container_origin', -1)}))
redirect_url = urlh.geturl()
if 'domain_not_allowed' in redirect_url:
domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False)
if not domain:
raise ExtractorError(
'This is an embed-only presentation. Try passing --referer', expected=True)
webpage, _ = self._download_embed_webpage_handle(video_id, headers={
'Referer': f'https://{domain}/',
'Origin': f'https://{domain}',
})
player_token = self._search_regex(r'data-player-token="([^"]+)"', webpage, 'player token')
player_data = self._download_webpage(
f'https://ben.slideslive.com/player/{video_id}', video_id,
@@ -139,6 +406,50 @@ class SlidesLiveIE(InfoExtractor):
assert service_name in ('url', 'yoda', 'vimeo', 'youtube')
service_id = player_info['service_id']
slides_info_url = None
slides, slides_info = [], []
if player_info.get('slides_json_url'):
slides_info_url = player_info['slides_json_url']
slides = traverse_obj(self._download_json(
slides_info_url, video_id, fatal=False,
note='Downloading slides JSON', errnote=False), 'slides', expected_type=list) or []
for slide_id, slide in enumerate(slides, start=1):
slides_info.append((
slide_id, traverse_obj(slide, ('image', 'name')),
int_or_none(slide.get('time'), scale=1000)))
if not slides and player_info.get('slides_xml_url'):
slides_info_url = player_info['slides_xml_url']
slides = self._download_xml(
slides_info_url, video_id, fatal=False,
note='Downloading slides XML', errnote='Failed to download slides info')
for slide_id, slide in enumerate(slides.findall('./slide'), start=1):
slides_info.append((
slide_id, xpath_text(slide, './slideName', 'name'),
int_or_none(xpath_text(slide, './timeSec', 'time'))))
slides_version = int(self._search_regex(
r'https?://slides\.slideslive\.com/\d+/v(\d+)/\w+\.(?:json|xml)',
slides_info_url, 'slides version', default=0))
if slides_version < 4:
slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s.jpg'
else:
slide_url_template = 'https://slides.slideslive.com/%s/slides/original/%s.png'
chapters, thumbnails = [], []
if url_or_none(player_info.get('thumbnail')):
thumbnails.append({'id': 'cover', 'url': player_info['thumbnail']})
for slide_id, slide_path, start_time in slides_info:
if slide_path:
thumbnails.append({
'id': f'{slide_id:03d}',
'url': slide_url_template % (video_id, slide_path),
})
chapters.append({
'title': f'Slide {slide_id:03d}',
'start_time': start_time,
})
subtitles = {}
for sub in traverse_obj(player_info, ('subtitles', ...), expected_type=dict):
webvtt_url = url_or_none(sub.get('webvtt_url'))
@@ -154,25 +465,15 @@ class SlidesLiveIE(InfoExtractor):
'title': player_info.get('title') or self._html_search_meta('title', webpage, default=''),
'timestamp': unified_timestamp(player_info.get('timestamp')),
'is_live': player_info.get('playlist_type') != 'vod',
'thumbnail': url_or_none(player_info.get('thumbnail')),
'thumbnails': thumbnails,
'chapters': chapters,
'subtitles': subtitles,
}
if service_name in ('url', 'yoda'):
if service_name == 'url':
info['url'] = service_id
else:
cdn_hostname = player_info['video_servers'][0]
formats = []
formats.extend(self._extract_m3u8_formats(
f'https://{cdn_hostname}/{service_id}/master.m3u8',
video_id, 'mp4', m3u8_id='hls', fatal=False, live=True))
formats.extend(self._extract_mpd_formats(
f'https://{cdn_hostname}/{service_id}/master.mpd',
video_id, mpd_id='dash', fatal=False))
info.update({
'formats': formats,
})
if service_name == 'url':
info['url'] = service_id
elif service_name == 'yoda':
info['formats'] = self._extract_formats(player_info['video_servers'][0], service_id, video_id)
else:
info.update({
'_type': 'url_transparent',
@@ -185,4 +486,37 @@ class SlidesLiveIE(InfoExtractor):
f'https://player.vimeo.com/video/{service_id}',
{'http_headers': {'Referer': url}})
return info
video_slides = traverse_obj(slides, (..., 'video', 'id'))
if not video_slides:
return info
def entries():
yield info
service_data = self._download_json(
f'https://ben.slideslive.com/player/{video_id}/slides_video_service_data',
video_id, fatal=False, query={
'player_token': player_token,
'videos': ','.join(video_slides),
}, note='Downloading video slides info', errnote='Failed to download video slides info') or {}
for slide_id, slide in enumerate(slides, 1):
if not traverse_obj(slide, ('video', 'service')) == 'yoda':
continue
video_path = traverse_obj(slide, ('video', 'id'))
cdn_hostname = traverse_obj(service_data, (
video_path, 'video_servers', ...), get_all=False)
if not cdn_hostname or not video_path:
continue
formats = self._extract_formats(cdn_hostname, video_path, video_id)
if not formats:
continue
yield {
'id': f'{video_id}-{slide_id:03d}',
'title': f'{info["title"]} - Slide {slide_id:03d}',
'timestamp': info['timestamp'],
'duration': int_or_none(traverse_obj(slide, ('video', 'duration_ms')), scale=1000),
'formats': formats,
}
return self.playlist_result(entries(), f'{video_id}-playlist', info['title'])
yt_dlp/extractor/soundcloud.py
+21
View File
@@ -782,6 +782,27 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
'%s (%s)' % (user['username'], resource.capitalize()))
class SoundcloudUserPermalinkIE(SoundcloudPagedPlaylistBaseIE):
_VALID_URL = r'https?://api\.soundcloud\.com/users/(?P<id>\d+)'
IE_NAME = 'soundcloud:user:permalink'
_TESTS = [{
'url': 'https://api.soundcloud.com/users/30909869',
'info_dict': {
'id': '30909869',
'title': 'neilcic',
},
'playlist_mincount': 23,
}]
def _real_extract(self, url):
user_id = self._match_id(url)
user = self._download_json(
self._resolv_url(url), user_id, 'Downloading user info', headers=self._HEADERS)
return self._extract_playlist(
f'{self._API_V2_BASE}stream/users/{user["id"]}', str(user['id']), user.get('username'))
class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
_VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
IE_NAME = 'soundcloud:trackstation'
yt_dlp/extractor/spankbang.py
+3 -4
View File
@@ -177,7 +177,6 @@ class SpankBangPlaylistIE(InfoExtractor):
def _real_extract(self, url):
mobj = self._match_valid_url(url)
playlist_id = mobj.group('id')
display_id = mobj.group('display_id')
webpage = self._download_webpage(
url, playlist_id, headers={'Cookie': 'country=US; mobile=on'})
@@ -186,11 +185,11 @@ class SpankBangPlaylistIE(InfoExtractor):
urljoin(url, mobj.group('path')),
ie=SpankBangIE.ie_key(), video_id=mobj.group('id'))
for mobj in re.finditer(
r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/%s(?:(?!\1).)*)\1'
% re.escape(display_id), webpage)]
r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/[^"\'](?:(?!\1).)*)\1',
webpage)]
title = self._html_search_regex(
r'<h1>([^<]+)\s+playlist\s*<', webpage, 'playlist title',
r'<em>([^<]+)</em>\s+playlist\s*<', webpage, 'playlist title',
fatal=False)
return self.playlist_result(entries, playlist_id, title)
yt_dlp/extractor/stv.py
+2
View File
@@ -73,6 +73,8 @@ class STVPlayerIE(InfoExtractor):
})
programme = result.get('programme') or {}
if programme.get('drmEnabled'):
self.report_drm(video_id)
return {
'_type': 'url_transparent',
yt_dlp/extractor/tencent.py
+2 -2
View File
@@ -32,7 +32,7 @@ class TencentBaseIE(InfoExtractor):
padding_mode='whitespace').hex().upper()
def _get_video_api_response(self, video_url, video_id, series_id, subtitle_format, video_format, video_quality):
guid = ''.join([random.choice(string.digits + string.ascii_lowercase) for _ in range(16)])
guid = ''.join(random.choices(string.digits + string.ascii_lowercase, k=16))
ckey = self._get_ckey(video_id, video_url, guid)
query = {
'vid': video_id,
@@ -55,7 +55,7 @@ class TencentBaseIE(InfoExtractor):
'platform': self._PLATFORM,
# For VQQ
'guid': guid,
'flowid': ''.join(random.choice(string.digits + string.ascii_lowercase) for _ in range(32)),
'flowid': ''.join(random.choices(string.digits + string.ascii_lowercase, k=32)),
}
return self._search_json(r'QZOutputJson=', self._download_webpage(
yt_dlp/extractor/testurl.py
+6 -5
View File
@@ -23,11 +23,12 @@ class TestURLIE(InfoExtractor):
if len(matching_extractors) == 0:
raise ExtractorError(f'No extractors matching {extractor_id!r} found', expected=True)
elif len(matching_extractors) > 1:
try: # Check for exact match
extractor = next(
ie for ie in matching_extractors
if ie.IE_NAME.lower() == extractor_id.lower())
except StopIteration:
extractor = next(( # Check for exact match
ie for ie in matching_extractors if ie.IE_NAME.lower() == extractor_id.lower()
), None) or next(( # Check for exact match without plugin suffix
ie for ie in matching_extractors if ie.IE_NAME.split('+')[0].lower() == extractor_id.lower()
), None)
if not extractor:
raise ExtractorError(
'Found multiple matching extractors: %s' % ' '.join(ie.IE_NAME for ie in matching_extractors),
expected=True)
yt_dlp/extractor/thisvid.py
+226
View File
@@ -0,0 +1,226 @@
import itertools
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
clean_html,
get_element_by_class,
int_or_none,
url_or_none,
urljoin,
)
class ThisVidIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+)'
_TESTS = [{
'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
'md5': '839becb572995687e11a69dc4358a386',
'info_dict': {
'id': '3533241',
'ext': 'mp4',
'title': 'Sitting on ball tight jeans',
'description': 'md5:372353bb995883d1b65fddf507489acd',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
'uploader_id': '150629',
'uploader': 'jeanslevisjeans',
'display_id': 'sitting-on-ball-tight-jeans',
'age_limit': 18,
}
}, {
'url': 'https://thisvid.com/embed/3533241/',
'md5': '839becb572995687e11a69dc4358a386',
'info_dict': {
'id': '3533241',
'ext': 'mp4',
'title': 'Sitting on ball tight jeans',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
'uploader_id': '150629',
'uploader': 'jeanslevisjeans',
'display_id': 'sitting-on-ball-tight-jeans',
'age_limit': 18,
}
}]
def _real_extract(self, url):
main_id, type_ = re.match(self._VALID_URL, url).group('id', 'type')
webpage = self._download_webpage(url, main_id)
title = self._html_search_regex(
r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>',
webpage, 'title')
if type_ == 'embed':
# look for more metadata
video_alt_url = url_or_none(self._search_regex(
rf'''video_alt_url\s*:\s+'({self._VALID_URL}/)',''',
webpage, 'video_alt_url', default=None))
if video_alt_url and video_alt_url != url:
webpage = self._download_webpage(
video_alt_url, main_id,
note='Redirecting embed to main page', fatal=False) or webpage
video_holder = get_element_by_class('video-holder', webpage) or ''
if '>This video is a private video' in video_holder:
self.raise_login_required(
(clean_html(video_holder) or 'Private video').partition('\n')[0])
uploader = self._html_search_regex(
r'''(?s)<span\b[^>]*>Added by:\s*</span><a\b[^>]+\bclass\s*=\s*["']author\b[^>]+\bhref\s*=\s*["']https://thisvid\.com/members/([0-9]+/.{3,}?)\s*</a>''',
webpage, 'uploader', default='')
uploader = re.split(r'''/["'][^>]*>\s*''', uploader)
if len(uploader) == 2:
# id must be non-empty, uploader could be ''
uploader_id, uploader = uploader
uploader = uploader or None
else:
uploader_id = uploader = None
return self.url_result(
url, ie='Generic', url_transparent=True,
title=title,
age_limit=18,
uploader=uploader,
uploader_id=uploader_id)
class ThisVidPlaylistBaseIE(InfoExtractor):
_PLAYLIST_URL_RE = None
@classmethod
def _find_urls(cls, html):
for m in re.finditer(rf'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>{cls._PLAYLIST_URL_RE}\b)[^>]+>''', html):
yield m.group('url')
def _generate_playlist_entries(self, url, playlist_id, html=None):
page_url = url
for page in itertools.count(1):
if not html:
html = self._download_webpage(
page_url, playlist_id, note=f'Downloading page {page}',
fatal=False) or ''
yield from self._find_urls(html)
next_page = get_element_by_class('pagination-next', html) or ''
if next_page:
# member list page
next_page = urljoin(url, self._search_regex(
r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?!#)(?:(?!\1).)+)''',
next_page, 'next page link', group='url', default=None))
# in case a member page should have pagination-next with empty link, not just `else:`
if next_page is None:
# playlist page
parsed_url = urllib.parse.urlparse(page_url)
base_path, _, num = parsed_url.path.rpartition('/')
num = int_or_none(num)
if num is None:
base_path, num = parsed_url.path.rstrip('/'), 1
parsed_url = parsed_url._replace(path=f'{base_path}/{num + 1}')
next_page = urllib.parse.urlunparse(parsed_url)
if page_url == next_page:
next_page = None
if not next_page:
return
page_url, html = next_page, None
def _make_playlist_result(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
title = re.split(
r'(?i)\s*\|\s*ThisVid\.com\s*$',
self._og_search_title(webpage, default=None)
or self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title', webpage, 'title', fatal=False) or '', 1)[0] or None
return self.playlist_from_matches(
self._generate_playlist_entries(url, playlist_id, webpage),
playlist_id=playlist_id, playlist_title=title, ie=ThisVidIE)
class ThisVidMemberIE(ThisVidPlaylistBaseIE):
_VALID_URL = r'https?://thisvid\.com/members/(?P<id>\d+)'
_TESTS = [{
'url': 'https://thisvid.com/members/2140501/',
'info_dict': {
'id': '2140501',
'title': 'Rafflesia\'s Profile',
},
'playlist_mincount': 16,
}, {
'url': 'https://thisvid.com/members/2140501/favourite_videos/',
'info_dict': {
'id': '2140501',
'title': 'Rafflesia\'s Favourite Videos',
},
'playlist_mincount': 15,
}, {
'url': 'https://thisvid.com/members/636468/public_videos/',
'info_dict': {
'id': '636468',
'title': 'Happymouth\'s Public Videos',
},
'playlist_mincount': 196,
}]
_PLAYLIST_URL_RE = ThisVidIE._VALID_URL
def _real_extract(self, url):
return self._make_playlist_result(url)
class ThisVidPlaylistIE(ThisVidPlaylistBaseIE):
_VALID_URL = r'https?://thisvid\.com/playlist/(?P<id>\d+)/video/(?P<video_id>[A-Za-z0-9-]+)'
_TESTS = [{
'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
'info_dict': {
'id': '6615',
'title': 'Underwear Stuff',
},
'playlist_mincount': 200,
}, {
'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
'info_dict': {
'id': '1072387',
'ext': 'mp4',
'title': 'Big Italian Booty 28',
'description': 'md5:1bccf7b13765e18fb27bf764dba7ede2',
'uploader_id': '367912',
'uploader': 'Jcmusclefun',
'age_limit': 18,
'display_id': 'big-italian-booty-28',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+1072387/preview\.jpg',
},
'params': {
'noplaylist': True,
},
}]
_PLAYLIST_URL_RE = _VALID_URL
def _generate_playlist_entries(self, url, playlist_id, html=None):
for wrapped_url in super()._generate_playlist_entries(url, playlist_id, html):
video_id = re.match(self._VALID_URL, wrapped_url).group('video_id')
yield urljoin(url, f'/videos/{video_id}/')
def _real_extract(self, url):
playlist_id, video_id = self._match_valid_url(url).group('id', 'video_id')
if not self._yes_playlist(playlist_id, video_id):
redirect_url = urljoin(url, f'/videos/{video_id}/')
return self.url_result(redirect_url, ThisVidIE)
result = self._make_playlist_result(url)
# Fix duplicated title (`the title - the title` => `the title`)
title = result['title']
t_len = len(title)
if t_len > 5 and t_len % 2 != 0:
t_len = t_len // 2
if title[t_len] == '-':
first, second = map(str.strip, (title[:t_len], title[t_len + 1:]))
if first and first == second:
result['title'] = first
return result
yt_dlp/extractor/tiktok.py
+45 -5
View File
@@ -11,6 +11,7 @@ from ..utils import (
HEADRequest,
LazyList,
UnsupportedError,
UserNotLive,
get_element_by_id,
get_first,
int_or_none,
@@ -49,7 +50,7 @@ class TikTokBaseIE(InfoExtractor):
def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True,
note='Downloading API JSON', errnote='Unable to download API page'):
self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160)))
self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choices('0123456789abcdef', k=160)))
webpage_cookies = self._get_cookies(self._WEBPAGE_HOST)
if webpage_cookies.get('sid_tt'):
self._set_cookie(self._API_HOSTNAME, 'sid_tt', webpage_cookies['sid_tt'].value)
@@ -68,8 +69,8 @@ class TikTokBaseIE(InfoExtractor):
'build_number': app_version,
'manifest_version_code': manifest_app_version,
'update_version_code': manifest_app_version,
'openudid': ''.join(random.choice('0123456789abcdef') for _ in range(16)),
'uuid': ''.join([random.choice(string.digits) for _ in range(16)]),
'openudid': ''.join(random.choices('0123456789abcdef', k=16)),
'uuid': ''.join(random.choices(string.digits, k=16)),
'_rticket': int(time.time() * 1000),
'ts': int(time.time()),
'device_brand': 'Google',
@@ -638,7 +639,7 @@ class TikTokUserIE(TikTokBaseIE):
'max_cursor': 0,
'min_cursor': 0,
'retry_type': 'no_retry',
'device_id': ''.join(random.choice(string.digits) for _ in range(19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
'device_id': ''.join(random.choices(string.digits, k=19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
}
for page in itertools.count(1):
@@ -686,7 +687,7 @@ class TikTokBaseListIE(TikTokBaseIE): # XXX: Conventionally, base classes shoul
'cursor': 0,
'count': 20,
'type': 5,
'device_id': ''.join(random.choice(string.digits) for i in range(19))
'device_id': ''.join(random.choices(string.digits, k=19))
}
for page in itertools.count(1):
@@ -980,3 +981,42 @@ class TikTokVMIE(InfoExtractor):
if self.suitable(new_url): # Prevent infinite loop in case redirect fails
raise UnsupportedError(new_url)
return self.url_result(new_url)
class TikTokLiveIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/live'
IE_NAME = 'tiktok:live'
_TESTS = [{
'url': 'https://www.tiktok.com/@iris04201/live',
'only_matching': True,
}]
def _real_extract(self, url):
uploader = self._match_id(url)
webpage = self._download_webpage(url, uploader, headers={'User-Agent': 'User-Agent:Mozilla/5.0'})
room_id = self._html_search_regex(r'snssdk\d*://live\?room_id=(\d+)', webpage, 'room ID', default=None)
if not room_id:
raise UserNotLive(video_id=uploader)
live_info = traverse_obj(self._download_json(
'https://www.tiktok.com/api/live/detail/', room_id, query={
'aid': '1988',
'roomID': room_id,
}), 'LiveRoomInfo', expected_type=dict, default={})
if 'status' not in live_info:
raise ExtractorError('Unexpected response from TikTok API')
# status = 2 if live else 4
if not int_or_none(live_info['status']) == 2:
raise UserNotLive(video_id=uploader)
return {
'id': room_id,
'title': live_info.get('title') or self._html_search_meta(['og:title', 'twitter:title'], webpage, default=''),
'uploader': uploader,
'uploader_id': traverse_obj(live_info, ('ownerInfo', 'id')),
'creator': traverse_obj(live_info, ('ownerInfo', 'nickname')),
'concurrent_view_count': traverse_obj(live_info, ('liveRoomStats', 'userCount'), expected_type=int),
'formats': self._extract_m3u8_formats(live_info['liveUrl'], room_id, 'mp4', live=True),
'is_live': True,
}
yt_dlp/extractor/trtcocuk.py
+48
View File
@@ -0,0 +1,48 @@
from .common import InfoExtractor
from ..utils import ExtractorError, int_or_none, parse_iso8601, traverse_obj
class TrtCocukVideoIE(InfoExtractor):
_VALID_URL = r'https?://www\.trtcocuk\.net\.tr/video/(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://www.trtcocuk.net.tr/video/kaptan-pengu-ve-arkadaslari-1',
'info_dict': {
'id': '3789738',
'ext': 'mp4',
'season_number': 1,
'series': '"Kaptan Pengu ve Arkadaşları"',
'season': 'Season 1',
'title': 'Kaptan Pengu ve Arkadaşları 1 Bölüm İzle TRT Çocuk',
'release_date': '20201209',
'release_timestamp': 1607513774,
}
}, {
'url': 'https://www.trtcocuk.net.tr/video/sef-rokanin-lezzet-dunyasi-17',
'info_dict': {
'id': '10260842',
'ext': 'mp4',
'series': '"Şef Roka\'nın Lezzet Dünyası"',
'title': 'Şef Roka\'nın Lezzet Dünyası 17 Bölüm İzle TRT Çocuk',
}
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
nuxtjs_data = self._search_nuxt_data(webpage, display_id)['data']
try:
video_url = self._parse_json(nuxtjs_data['video'], display_id)
except ExtractorError:
video_url = nuxtjs_data['video']
formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, display_id)
return {
'id': str(nuxtjs_data['id']),
'formats': formats,
'subtitles': subtitles,
'season_number': int_or_none(nuxtjs_data.get('season')),
'release_timestamp': parse_iso8601(nuxtjs_data.get('publishedDate')),
'series': traverse_obj(nuxtjs_data, ('show', 0, 'title')),
'title': self._html_extract_title(webpage) # TODO: get better title
}
yt_dlp/extractor/twitcasting.py
+22 -12
View File
@@ -38,7 +38,7 @@ class TwitCastingIE(InfoExtractor):
'description': 'Twitter Oficial da cantora brasileira Ivete Sangalo.',
'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20110822',
'timestamp': 1314010824,
'timestamp': 1313978424,
'duration': 32,
'view_count': int,
},
@@ -52,10 +52,10 @@ class TwitCastingIE(InfoExtractor):
'ext': 'mp4',
'title': 'Live playing something #3689740',
'uploader_id': 'mttbernardini',
'description': 'Salve, io sono Matto (ma con la e). Questa è la mia presentazione, in quanto sono letteralmente matto (nel senso di strano), con qualcosa in più.',
'description': 'md5:1dc7efa2f1ab932fcd119265cebeec69',
'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20120212',
'timestamp': 1329028024,
'upload_date': '20120211',
'timestamp': 1328995624,
'duration': 681,
'view_count': int,
},
@@ -64,15 +64,22 @@ class TwitCastingIE(InfoExtractor):
'videopassword': 'abc',
},
}, {
'note': 'archive is split in 2 parts',
'url': 'https://twitcasting.tv/loft_heaven/movie/685979292',
'info_dict': {
'id': '685979292',
'ext': 'mp4',
'title': '南波一海のhear_here “ナタリー望月哲さんに聞く編集と「渋谷系狂騒曲」”',
'duration': 6964.599334,
'title': '【無料配信】南波一海のhear/here “ナタリー望月哲さんに聞く編集と「渋谷系狂騒曲」”',
'uploader_id': 'loft_heaven',
'description': 'md5:3a0c7b53019df987ce545c935538bacf',
'upload_date': '20210604',
'timestamp': 1622802114,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 6964,
'view_count': int,
},
'params': {
'skip_download': True,
},
'playlist_mincount': 2,
}]
def _parse_data_movie_playlist(self, dmp, video_id):
@@ -88,15 +95,18 @@ class TwitCastingIE(InfoExtractor):
def _real_extract(self, url):
uploader_id, video_id = self._match_valid_url(url).groups()
webpage, urlh = self._download_webpage_handle(url, video_id)
video_password = self.get_param('videopassword')
request_data = None
if video_password:
request_data = urlencode_postdata({
'password': video_password,
**self._hidden_inputs(webpage),
}, encoding='utf-8')
webpage, urlh = self._download_webpage_handle(
url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'})
webpage, urlh = self._download_webpage_handle(
url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'},
note='Trying video password')
if urlh.geturl() != url and request_data:
webpage = self._download_webpage(
urlh.geturl(), video_id, data=request_data,
@@ -122,7 +132,7 @@ class TwitCastingIE(InfoExtractor):
duration = (try_get(video_js_data, lambda x: sum(float_or_none(y.get('duration')) for y in x) / 1000)
or parse_duration(clean_html(get_element_by_class('tw-player-duration-time', webpage))))
view_count = str_to_int(self._search_regex(
(r'Total\s*:\s*([\d,]+)\s*Views', r'総視聴者\s*:\s*([\d,]+)\s*</'), webpage, 'views', None))
(r'Total\s*:\s*Views\s*([\d,]+)', r'総視聴者\s*:\s*([\d,]+)\s*</'), webpage, 'views', None))
timestamp = unified_timestamp(self._search_regex(
r'data-toggle="true"[^>]+datetime="([^"]+)"',
webpage, 'datetime', None))
yt_dlp/extractor/udemy.py
+15 -5
View File
@@ -11,8 +11,10 @@ from ..utils import (
int_or_none,
js_to_json,
sanitized_Request,
smuggle_url,
try_get,
unescapeHTML,
unsmuggle_url,
url_or_none,
urlencode_postdata,
)
@@ -106,7 +108,7 @@ class UdemyIE(InfoExtractor):
% (course_id, lecture_id),
lecture_id, 'Downloading lecture JSON', query={
'fields[lecture]': 'title,description,view_html,asset',
'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data',
'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data,course_is_drmed',
})
def _handle_error(self, response):
@@ -199,16 +201,19 @@ class UdemyIE(InfoExtractor):
def _real_extract(self, url):
lecture_id = self._match_id(url)
course_id = unsmuggle_url(url, {})[1].get('course_id')
webpage = self._download_webpage(url, lecture_id)
course_id, _ = self._extract_course_info(webpage, lecture_id)
webpage = None
if not course_id:
webpage = self._download_webpage(url, lecture_id)
course_id, _ = self._extract_course_info(webpage, lecture_id)
try:
lecture = self._download_lecture(course_id, lecture_id)
except ExtractorError as e:
# Error could possibly mean we are not enrolled in the course
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
webpage = webpage or self._download_webpage(url, lecture_id)
self._enroll_course(url, webpage, course_id)
lecture = self._download_lecture(course_id, lecture_id)
else:
@@ -391,6 +396,9 @@ class UdemyIE(InfoExtractor):
if f.get('url'):
formats.append(f)
if not formats and asset.get('course_is_drmed'):
self.report_drm(video_id)
return {
'id': video_id,
'title': title,
@@ -449,7 +457,9 @@ class UdemyCourseIE(UdemyIE): # XXX: Do not subclass from concrete IE
if lecture_id:
entry = {
'_type': 'url_transparent',
'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
'url': smuggle_url(
f'https://www.udemy.com/{course_path}/learn/v4/t/lecture/{entry["id"]}',
{'course_id': course_id}),
'title': entry.get('title'),
'ie_key': UdemyIE.ie_key(),
}
yt_dlp/extractor/urplay.py
+26 -2
View File
@@ -14,12 +14,13 @@ class URPlayIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
'md5': 'ff5b0c89928f8083c74bbd5099c9292d',
'md5': '5ba36643c77cc3d34ffeadad89937d1e',
'info_dict': {
'id': '203704',
'ext': 'mp4',
'title': 'UR Samtiden - Livet, universum och rymdens märkliga musik : Om vetenskap, kritiskt tänkande och motstånd',
'description': 'md5:5344508a52aa78c1ced6c1b8b9e44e9a',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1513292400,
'upload_date': '20171214',
'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
@@ -29,6 +30,24 @@ class URPlayIE(InfoExtractor):
'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
'age_limit': 15,
},
}, {
'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
'info_dict': {
'id': '222967',
'ext': 'mp4',
'title': 'En förälders dagbok : Mitt barn skadar sig själv',
'description': 'md5:9f771eef03a732a213b367b52fe826ca',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1629676800,
'upload_date': '20210823',
'series': 'En förälders dagbok',
'duration': 1740,
'age_limit': 15,
'episode_number': 3,
'categories': 'count:2',
'tags': 'count:7',
'episode': 'Mitt barn skadar sig själv',
},
}, {
'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
'info_dict': {
@@ -36,12 +55,17 @@ class URPlayIE(InfoExtractor):
'ext': 'mp4',
'title': 'Tripp, Trapp, Träd : Sovkudde',
'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1440086400,
'upload_date': '20150820',
'series': 'Tripp, Trapp, Träd',
'duration': 865,
'age_limit': 1,
'episode_number': 1,
'categories': [],
'tags': ['Sova'],
'episode': 'Sovkudde',
'season': 'Säsong 1',
},
}, {
'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
@@ -69,7 +93,7 @@ class URPlayIE(InfoExtractor):
urplayer_streams = urplayer_data.get('streamingInfo', {})
for k, v in urplayer_streams.get('raw', {}).items():
if not (k in ('sd', 'hd') and isinstance(v, dict)):
if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)):
continue
file_http = v.get('location')
if file_http:
yt_dlp/extractor/videa.py
+1 -1
View File
@@ -119,7 +119,7 @@ class VideaIE(InfoExtractor):
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
query = parse_qs(player_url)
random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
random_seed = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
query['_s'] = random_seed
query['_t'] = result[:16]
yt_dlp/extractor/videoken.py
+336
View File
@@ -0,0 +1,336 @@
import base64
import functools
import math
import re
import time
import urllib.parse
from .common import InfoExtractor
from .slideslive import SlidesLiveIE
from ..utils import (
ExtractorError,
InAdvancePagedList,
int_or_none,
traverse_obj,
update_url_query,
url_or_none,
)
class VideoKenBaseIE(InfoExtractor):
_ORGANIZATIONS = {
'videos.icts.res.in': 'icts',
'videos.cncf.io': 'cncf',
'videos.neurips.cc': 'neurips',
}
_BASE_URL_RE = rf'https?://(?P<host>{"|".join(map(re.escape, _ORGANIZATIONS))})/'
_PAGE_SIZE = 12
def _get_org_id_and_api_key(self, org, video_id):
details = self._download_json(
f'https://analytics.videoken.com/api/videolake/{org}/details', video_id,
note='Downloading organization ID and API key', headers={
'Accept': 'application/json',
})
return details['id'], details['apikey']
def _create_slideslive_url(self, video_url, video_id, referer):
if not video_url and not video_id:
return
elif not video_url or 'embed/sign-in' in video_url:
video_url = f'https://slideslive.com/embed/{video_id.lstrip("slideslive-")}'
if url_or_none(referer):
return update_url_query(video_url, {
'embed_parent_url': referer,
'embed_container_origin': f'https://{urllib.parse.urlparse(referer).netloc}',
})
return video_url
def _extract_videos(self, videos, url):
for video in traverse_obj(videos, (('videos', 'results'), ...)):
video_id = traverse_obj(video, 'youtube_id', 'videoid')
if not video_id:
continue
ie_key = None
if traverse_obj(video, 'type', 'source') == 'youtube':
video_url = video_id
ie_key = 'Youtube'
else:
video_url = traverse_obj(video, 'embed_url', 'embeddableurl')
if urllib.parse.urlparse(video_url).netloc == 'slideslive.com':
ie_key = SlidesLiveIE
video_url = self._create_slideslive_url(video_url, video_id, url)
if not video_url:
continue
yield self.url_result(video_url, ie_key, video_id)
class VideoKenIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'(?:(?:topic|category)/[^/#?]+/)?video/(?P<id>[\w-]+)'
_TESTS = [{
# neurips -> videoken -> slideslive
'url': 'https://videos.neurips.cc/video/slideslive-38922815',
'info_dict': {
'id': '38922815',
'ext': 'mp4',
'title': 'Efficient Processing of Deep Neural Network: from Algorithms to Hardware Architectures',
'timestamp': 1630939331,
'upload_date': '20210906',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:330',
'chapters': 'count:329',
},
'params': {
'skip_download': 'm3u8',
},
'expected_warnings': ['Failed to download VideoKen API JSON'],
}, {
# neurips -> videoken -> slideslive -> youtube
'url': 'https://videos.neurips.cc/topic/machine%20learning/video/slideslive-38923348',
'info_dict': {
'id': '2Xa_dt78rJE',
'ext': 'mp4',
'display_id': '38923348',
'title': 'Machine Education',
'description': 'Watch full version of this video at https://slideslive.com/38923348.',
'channel': 'SlidesLive Videos - G2',
'channel_id': 'UCOExahQQ588Da8Nft_Ltb9w',
'channel_url': 'https://www.youtube.com/channel/UCOExahQQ588Da8Nft_Ltb9w',
'uploader': 'SlidesLive Videos - G2',
'uploader_id': 'UCOExahQQ588Da8Nft_Ltb9w',
'uploader_url': 'http://www.youtube.com/channel/UCOExahQQ588Da8Nft_Ltb9w',
'duration': 2504,
'timestamp': 1618922125,
'upload_date': '20200131',
'age_limit': 0,
'channel_follower_count': int,
'view_count': int,
'availability': 'unlisted',
'live_status': 'not_live',
'playable_in_embed': True,
'categories': ['People & Blogs'],
'tags': [],
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:78',
'chapters': 'count:77',
},
'params': {
'skip_download': 'm3u8',
},
'expected_warnings': ['Failed to download VideoKen API JSON'],
}, {
# icts -> videoken -> youtube
'url': 'https://videos.icts.res.in/topic/random%20variable/video/zysIsojYdvc',
'info_dict': {
'id': 'zysIsojYdvc',
'ext': 'mp4',
'title': 'Small-worlds, complex networks and random graphs (Lecture 3) by Remco van der Hofstad',
'description': 'md5:87433069d79719eeadc1962cc2ace00b',
'channel': 'International Centre for Theoretical Sciences',
'channel_id': 'UCO3xnVTHzB7l-nc8mABUJIQ',
'channel_url': 'https://www.youtube.com/channel/UCO3xnVTHzB7l-nc8mABUJIQ',
'uploader': 'International Centre for Theoretical Sciences',
'uploader_id': 'ICTStalks',
'uploader_url': 'http://www.youtube.com/user/ICTStalks',
'duration': 3372,
'upload_date': '20191004',
'age_limit': 0,
'live_status': 'not_live',
'availability': 'public',
'playable_in_embed': True,
'channel_follower_count': int,
'like_count': int,
'view_count': int,
'categories': ['Science & Technology'],
'tags': [],
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:42',
'chapters': 'count:20',
},
'params': {
'skip_download': 'm3u8',
},
}, {
'url': 'https://videos.cncf.io/category/478/video/IL4nxbmUIX8',
'only_matching': True,
}, {
'url': 'https://videos.cncf.io/topic/kubernetes/video/YAM2d7yTrrI',
'only_matching': True,
}, {
'url': 'https://videos.icts.res.in/video/d7HuP_abpKU',
'only_matching': True,
}]
def _real_extract(self, url):
hostname, video_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], video_id)
details = self._download_json(
'https://analytics.videoken.com/api/videoinfo_private', video_id, query={
'videoid': video_id,
'org_id': org_id,
}, headers={'Accept': 'application/json'}, note='Downloading VideoKen API JSON',
errnote='Failed to download VideoKen API JSON', fatal=False)
if details:
return next(self._extract_videos({'videos': [details]}, url))
# fallback for API error 400 response
elif video_id.startswith('slideslive-'):
return self.url_result(
self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id)
elif re.match(r'^[\w-]{11}$', video_id):
self.url_result(video_id, 'Youtube', video_id)
else:
raise ExtractorError('Unable to extract without VideoKen API response')
class VideoKenPlayerIE(VideoKenBaseIE):
_VALID_URL = r'https?://player\.videoken\.com/embed/slideslive-(?P<id>\d+)'
_TESTS = [{
'url': 'https://player.videoken.com/embed/slideslive-38968434',
'info_dict': {
'id': '38968434',
'ext': 'mp4',
'title': 'Deep Learning with Label Differential Privacy',
'timestamp': 1643377020,
'upload_date': '20220128',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:30',
'chapters': 'count:29',
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
return self.url_result(
self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id)
class VideoKenPlaylistIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'(?:category/\d+/)?playlist/(?P<id>\d+)'
_TESTS = [{
'url': 'https://videos.icts.res.in/category/1822/playlist/381',
'playlist_mincount': 117,
'info_dict': {
'id': '381',
'title': 'Cosmology - The Next Decade',
},
}]
def _real_extract(self, url):
hostname, playlist_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], playlist_id)
videos = self._download_json(
f'https://analytics.videoken.com/api/{org_id}/playlistitems/{playlist_id}/',
playlist_id, headers={'Accept': 'application/json'}, note='Downloading API JSON')
return self.playlist_result(self._extract_videos(videos, url), playlist_id, videos.get('title'))
class VideoKenCategoryIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'category/(?P<id>\d+)/?(?:$|[?#])'
_TESTS = [{
'url': 'https://videos.icts.res.in/category/1822/',
'playlist_mincount': 500,
'info_dict': {
'id': '1822',
'title': 'Programs',
},
}, {
'url': 'https://videos.neurips.cc/category/350/',
'playlist_mincount': 34,
'info_dict': {
'id': '350',
'title': 'NeurIPS 2018',
},
}, {
'url': 'https://videos.cncf.io/category/479/',
'playlist_mincount': 328,
'info_dict': {
'id': '479',
'title': 'KubeCon + CloudNativeCon Europe\'19',
},
}]
def _get_category_page(self, category_id, org_id, page=1, note=None):
return self._download_json(
f'https://analytics.videoken.com/api/videolake/{org_id}/category_videos', category_id,
fatal=False, note=note if note else f'Downloading category page {page}',
query={
'category_id': category_id,
'page_number': page,
'length': self._PAGE_SIZE,
}, headers={'Accept': 'application/json'}) or {}
def _entries(self, category_id, org_id, url, page):
videos = self._get_category_page(category_id, org_id, page + 1)
yield from self._extract_videos(videos, url)
def _real_extract(self, url):
hostname, category_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], category_id)
category_info = self._get_category_page(category_id, org_id, note='Downloading category info')
category = category_info['category_name']
total_pages = math.ceil(int(category_info['recordsTotal']) / self._PAGE_SIZE)
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, category_id, org_id, url),
total_pages, self._PAGE_SIZE), category_id, category)
class VideoKenTopicIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'topic/(?P<id>[^/#?]+)/?(?:$|[?#])'
_TESTS = [{
'url': 'https://videos.neurips.cc/topic/machine%20learning/',
'playlist_mincount': 500,
'info_dict': {
'id': 'machine_learning',
'title': 'machine learning',
},
}, {
'url': 'https://videos.icts.res.in/topic/gravitational%20waves/',
'playlist_mincount': 77,
'info_dict': {
'id': 'gravitational_waves',
'title': 'gravitational waves'
},
}, {
'url': 'https://videos.cncf.io/topic/prometheus/',
'playlist_mincount': 134,
'info_dict': {
'id': 'prometheus',
'title': 'prometheus',
},
}]
def _get_topic_page(self, topic, org_id, search_id, api_key, page=1, note=None):
return self._download_json(
'https://es.videoken.com/api/v1.0/get_results', topic, fatal=False, query={
'orgid': org_id,
'size': self._PAGE_SIZE,
'query': topic,
'page': page,
'sort': 'upload_desc',
'filter': 'all',
'token': api_key,
'is_topic': 'true',
'category': '',
'searchid': search_id,
}, headers={'Accept': 'application/json'},
note=note if note else f'Downloading topic page {page}') or {}
def _entries(self, topic, org_id, search_id, api_key, url, page):
videos = self._get_topic_page(topic, org_id, search_id, api_key, page + 1)
yield from self._extract_videos(videos, url)
def _real_extract(self, url):
hostname, topic_id = self._match_valid_url(url).group('host', 'id')
topic = urllib.parse.unquote(topic_id)
topic_id = topic.replace(' ', '_')
org_id, api_key = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], topic)
search_id = base64.b64encode(f':{topic}:{int(time.time())}:transient'.encode()).decode()
total_pages = int_or_none(self._get_topic_page(
topic, org_id, search_id, api_key, note='Downloading topic info')['total_no_of_pages'])
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, topic, org_id, search_id, api_key, url),
total_pages, self._PAGE_SIZE), topic_id, topic)
yt_dlp/extractor/viu.py
+1 -1
View File
@@ -251,7 +251,7 @@ class ViuOTTIE(InfoExtractor):
return self._user_token
def _get_token(self, country_code, video_id):
rand = ''.join(random.choice('0123456789') for _ in range(10))
rand = ''.join(random.choices('0123456789', k=10))
return self._download_json(
f'https://api-gateway-global.viu.com/api/auth/token?v={rand}000', video_id,
headers={'Content-Type': 'application/json'}, note='Getting bearer token',
yt_dlp/extractor/volejtv.py
+40
View File
@@ -0,0 +1,40 @@
from .common import InfoExtractor
class VolejTVIE(InfoExtractor):
_VALID_URL = r'https?://volej\.tv/video/(?P<id>\d+)'
_TESTS = [{
'url': 'https://volej.tv/video/725742/',
'info_dict': {
'id': '725742',
'ext': 'mp4',
'description': 'Zápas VK Královo Pole vs VK Prostějov 10.12.2022 v 19:00 na Volej.TV',
'thumbnail': 'https://volej.tv/images/og/16/17186/og.png',
'title': 'VK Královo Pole vs VK Prostějov',
}
}, {
'url': 'https://volej.tv/video/725605/',
'info_dict': {
'id': '725605',
'ext': 'mp4',
'thumbnail': 'https://volej.tv/images/og/15/17185/og.png',
'title': 'VK Lvi Praha vs VK Euro Sitex Příbram',
'description': 'Zápas VK Lvi Praha vs VK Euro Sitex Příbram 11.12.2022 v 19:00 na Volej.TV',
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
json_data = self._search_json(
r'<\s*!\[CDATA[^=]+=', webpage, 'CDATA', video_id)
formats, subtitle = self._extract_m3u8_formats_and_subtitles(
json_data['urls']['hls'], video_id)
return {
'id': video_id,
'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage),
'description': self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage),
'formats': formats,
'subtitles': subtitle,
}
yt_dlp/extractor/vrv.py
+1 -1
View File
@@ -30,7 +30,7 @@ class VRVBaseIE(InfoExtractor):
base_url = self._API_DOMAIN + '/core/' + path
query = [
('oauth_consumer_key', self._API_PARAMS['oAuthKey']),
('oauth_nonce', ''.join([random.choice(string.ascii_letters) for _ in range(32)])),
('oauth_nonce', ''.join(random.choices(string.ascii_letters, k=32))),
('oauth_signature_method', 'HMAC-SHA1'),
('oauth_timestamp', int(time.time())),
]
yt_dlp/extractor/wistia.py
+27 -14
View File
@@ -6,12 +6,15 @@ from base64 import b64decode
from .common import InfoExtractor
from ..utils import (
ExtractorError,
HEADRequest,
determine_ext,
float_or_none,
int_or_none,
parse_qs,
traverse_obj,
try_get,
update_url_query,
urlhandle_detect_ext,
)
@@ -34,6 +37,16 @@ class WistiaBaseIE(InfoExtractor):
return embed_config
def _get_real_ext(self, url):
ext = determine_ext(url, default_ext='bin')
if ext == 'bin':
urlh = self._request_webpage(
HEADRequest(url), None, note='Checking media extension',
errnote='HEAD request returned error', fatal=False)
if urlh:
ext = urlhandle_detect_ext(urlh, default='bin')
return 'mp4' if ext == 'mov' else ext
def _extract_media(self, embed_config):
data = embed_config['media']
video_id = data['hashedId']
@@ -51,13 +64,13 @@ class WistiaBaseIE(InfoExtractor):
continue
elif atype in ('still', 'still_image'):
thumbnails.append({
'url': aurl,
'url': aurl.replace('.bin', f'.{self._get_real_ext(aurl)}'),
'width': int_or_none(a.get('width')),
'height': int_or_none(a.get('height')),
'filesize': int_or_none(a.get('size')),
})
else:
aext = a.get('ext')
aext = a.get('ext') or self._get_real_ext(aurl)
display_name = a.get('display_name')
format_id = atype
if atype and atype.endswith('_video') and display_name:
@@ -169,26 +182,26 @@ class WistiaIE(WistiaBaseIE):
'md5': '10c1ce9c4dde638202513ed17a3767bd',
'info_dict': {
'id': 'a6ndpko1wg',
'ext': 'bin',
'ext': 'mp4',
'title': 'Episode 2: Boxed Water\'s retention is thirsty',
'upload_date': '20210324',
'description': 'md5:da5994c2c2d254833b412469d9666b7a',
'duration': 966.0,
'timestamp': 1616614369,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/53dc60239348dc9b9fba3755173ea4c2.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/53dc60239348dc9b9fba3755173ea4c2.png',
}
}, {
'url': 'wistia:5vd7p4bct5',
'md5': 'b9676d24bf30945d97060638fbfe77f0',
'info_dict': {
'id': '5vd7p4bct5',
'ext': 'bin',
'ext': 'mp4',
'title': 'md5:eaa9f64c4efd7b5f098b9b6118597679',
'description': 'md5:a9bea0315f0616aa5df2dc413ddcdd0f',
'upload_date': '20220915',
'timestamp': 1663258727,
'duration': 623.019,
'thumbnail': r're:https?://embed(?:-ssl)?.wistia.com/.+\.(?:jpg|bin)$',
'thumbnail': r're:https?://embed(?:-ssl)?.wistia.com/.+\.jpg$',
},
}, {
'url': 'wistia:sh7fpupwlt',
@@ -208,25 +221,25 @@ class WistiaIE(WistiaBaseIE):
'url': 'https://www.weidert.com/blog/wistia-channels-video-marketing-tool',
'info_dict': {
'id': 'cqwukac3z1',
'ext': 'bin',
'ext': 'mp4',
'title': 'How Wistia Channels Can Help Capture Inbound Value From Your Video Content',
'duration': 158.125,
'timestamp': 1618974400,
'description': 'md5:27abc99a758573560be72600ef95cece',
'upload_date': '20210421',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/6c551820ae950cdee2306d6cbe9ef742.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/6c551820ae950cdee2306d6cbe9ef742.jpg',
}
}, {
'url': 'https://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
'md5': 'b9676d24bf30945d97060638fbfe77f0',
'info_dict': {
'id': '5vd7p4bct5',
'ext': 'bin',
'ext': 'mp4',
'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
'upload_date': '20220915',
'timestamp': 1663258727,
'duration': 623.019,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/83e6ec693e2c05a0ce65809cbaead86a.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/83e6ec693e2c05a0ce65809cbaead86a.jpg',
'description': 'a Paywall Videos video',
},
}]
@@ -302,9 +315,9 @@ class WistiaChannelIE(WistiaBaseIE):
'url': 'https://fast.wistia.net/embed/channel/3802iirk0l?wchannelid=3802iirk0l&wmediaid=sp5dqjzw3n',
'info_dict': {
'id': 'sp5dqjzw3n',
'ext': 'bin',
'ext': 'mp4',
'title': 'The Roof S2: The Modern CRO',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/dadfa9233eaa505d5e0c85c23ff70741.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/dadfa9233eaa505d5e0c85c23ff70741.png',
'duration': 86.487,
'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season.\n',
'timestamp': 1619790290,
@@ -334,12 +347,12 @@ class WistiaChannelIE(WistiaBaseIE):
'info_dict': {
'id': 'pz0m0l0if3',
'title': 'A Framework for Improving Product Team Performance',
'ext': 'bin',
'ext': 'mp4',
'timestamp': 1653935275,
'upload_date': '20220530',
'description': 'Learn how to help your company improve and achieve your product related goals.',
'duration': 1854.39,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/12fd19e56413d9d6f04e2185c16a6f8854e25226.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/12fd19e56413d9d6f04e2185c16a6f8854e25226.png',
},
'params': {'noplaylist': True, 'skip_download': True},
}]
yt_dlp/extractor/xanimu.py
+51
View File
@@ -0,0 +1,51 @@
import re
from .common import InfoExtractor
from ..utils import int_or_none
class XanimuIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?xanimu\.com/(?P<id>[^/]+)/?'
_TESTS = [{
'url': 'https://xanimu.com/51944-the-princess-the-frog-hentai/',
'md5': '899b88091d753d92dad4cb63bbf357a7',
'info_dict': {
'id': '51944-the-princess-the-frog-hentai',
'ext': 'mp4',
'title': 'The Princess + The Frog Hentai',
'thumbnail': 'https://xanimu.com/storage/2020/09/the-princess-and-the-frog-hentai.jpg',
'description': r're:^Enjoy The Princess \+ The Frog Hentai',
'duration': 207.0,
'age_limit': 18
}
}, {
'url': 'https://xanimu.com/huge-expansion/',
'only_matching': True
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
formats = []
for format in ['videoHigh', 'videoLow']:
format_url = self._search_json(r'var\s+%s\s*=' % re.escape(format), webpage, format,
video_id, default=None, contains_pattern=r'[\'"]([^\'"]+)[\'"]')
if format_url:
formats.append({
'url': format_url,
'format_id': format,
'quality': -2 if format.endswith('Low') else None,
})
return {
'id': video_id,
'formats': formats,
'title': self._search_regex(r'[\'"]headline[\'"]:\s*[\'"]([^"]+)[\'"]', webpage,
'title', default=None) or self._html_extract_title(webpage),
'thumbnail': self._html_search_meta('thumbnailUrl', webpage, default=None),
'description': self._html_search_meta('description', webpage, default=None),
'duration': int_or_none(self._search_regex(r'duration:\s*[\'"]([^\'"]+?)[\'"]',
webpage, 'duration', fatal=False)),
'age_limit': 18
}
yt_dlp/extractor/yandexvideo.py
+2 -2
View File
@@ -270,9 +270,9 @@ class ZenYandexIE(InfoExtractor):
for s_url in stream_urls:
ext = determine_ext(s_url)
if ext == 'mpd':
formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
formats.extend(self._extract_mpd_formats(s_url, video_id, mpd_id='dash'))
elif ext == 'm3u8':
formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
formats.extend(self._extract_m3u8_formats(s_url, video_id, 'mp4'))
return {
'id': video_id,
'title': video_json.get('title') or self._og_search_title(webpage),
yt_dlp/extractor/youku.py
+23 -13
View File
@@ -96,31 +96,41 @@ class YoukuIE(InfoExtractor):
'thumbnail': r're:^https?://.*',
'uploader': '明月庄主moon',
'uploader_id': '38465621',
'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UMTUzODYyNDg0',
'tags': list,
},
}, {
'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
'url': 'https://v.youku.com/v_show/id_XNTA2NTA0MjA1Mg==.html',
'info_dict': {
'id': 'XMjIyNzAzMTQ4NA',
'id': 'XNTA2NTA0MjA1Mg',
'ext': 'mp4',
'title': '卡马乔国足开大脚长传冲吊集锦',
'duration': 289,
'title': 'Minecraft我的世界：建造超大巨型航空飞机，菜鸟vs高手vs黑客',
'duration': 542.13,
'thumbnail': r're:^https?://.*',
'uploader': '阿卜杜拉之星',
'uploader_id': '2382249',
'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==',
'uploader': '波哥游戏解说',
'uploader_id': '156688084',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UNjI2NzUyMzM2',
'tags': list,
},
}, {
'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
'only_matching': True,
'url': 'https://v.youku.com/v_show/id_XNTE1MzczOTg4MA==.html',
'info_dict': {
'id': 'XNTE1MzczOTg4MA',
'ext': 'mp4',
'title': '国产超A特工片',
'duration': 362.97,
'thumbnail': r're:^https?://.*',
'uploader': '陈晓娟说历史',
'uploader_id': '1640913339',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UNjU2MzY1MzM1Ng==',
'tags': list,
},
}]
@staticmethod
def get_ysuid():
return '%d%s' % (int(time.time()), ''.join([
random.choice(string.ascii_letters) for i in range(3)]))
return '%d%s' % (int(time.time()), ''.join(
random.choices(string.ascii_letters, k=3)))
def get_format_name(self, fm):
_dict = {
@@ -151,7 +161,7 @@ class YoukuIE(InfoExtractor):
# request basic data
basic_data_params = {
'vid': video_id,
'ccode': '0532',
'ccode': '0524',
'client_ip': '192.168.1.1',
'utid': cna,
'client_ts': time.time() / 1000,
yt_dlp/extractor/youtube.py
+47 -16
View File
@@ -292,7 +292,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors"""
_RESERVED_NAMES = (
r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
r'browse|oembed|get_video_info|iframe_api|s/player|source|'
r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
@@ -2544,6 +2544,35 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'tags': [],
},
'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
}, {
'note': 'Audio formats with Dynamic Range Compression',
'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
'info_dict': {
'id': 'Tq92D6wQ1mg',
'ext': 'weba',
'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
'channel_follower_count': int,
'description': 'md5:17eccca93a786d51bc67646756894066',
'upload_date': '20191228',
'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
'playable_in_embed': True,
'like_count': int,
'categories': ['Entertainment'],
'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
'age_limit': 18,
'channel': 'Projekt Melody',
'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
'view_count': int,
'availability': 'needs_auth',
'comment_count': int,
'live_status': 'not_live',
'uploader': 'Projekt Melody',
'duration': 106,
},
'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
}
]
@@ -2621,18 +2650,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
"""
@returns (manifest_url, manifest_stream_number, is_live) or None
"""
with lock:
refetch_manifest(format_id, delay)
for retry in self.RetryManager(fatal=False):
with lock:
refetch_manifest(format_id, delay)
f = next((f for f in formats if f['format_id'] == format_id), None)
if not f:
if not is_live:
self.to_screen(f'{video_id}: Video is no longer live')
else:
self.report_warning(
f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
return None
return f['manifest_url'], f['manifest_stream_number'], is_live
f = next((f for f in formats if f['format_id'] == format_id), None)
if not f:
if not is_live:
retry.error = f'{video_id}: Video is no longer live'
else:
retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
continue
return f['manifest_url'], f['manifest_stream_number'], is_live
return None
for f in formats:
f['is_live'] = is_live
@@ -3553,7 +3583,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
itag = str_or_none(fmt.get('itag'))
audio_track = fmt.get('audioTrack') or {}
stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
if stream_id in stream_ids:
continue
@@ -3634,11 +3664,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
dct = {
'asr': int_or_none(fmt.get('audioSampleRate')),
'filesize': int_or_none(fmt.get('contentLength')),
'format_id': itag,
'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
'format_note': join_nonempty(
'%s%s' % (audio_track.get('displayName') or '',
' (default)' if language_preference > 0 else ''),
fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
'DRC' if fmt.get('isDrc') else None,
try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
@@ -3647,13 +3678,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'fps': int_or_none(fmt.get('fps')) or None,
'audio_channels': fmt.get('audioChannels'),
'height': height,
'quality': q(quality),
'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
'has_drm': bool(fmt.get('drmFamilies')),
'tbr': tbr,
'url': fmt_url,
'width': int_or_none(fmt.get('width')),
'language': join_nonempty(audio_track.get('id', '').split('.')[0],
'desc' if language_preference < -1 else ''),
'desc' if language_preference < -1 else '') or None,
'language_preference': language_preference,
# Strictly de-prioritize damaged and 3gp formats
'preference': -10 if is_damaged else -2 if itag == '17' else None,