[ie/cu.ntv.co.jp] Fix extractor (#13302)

Closes #10976
Authored by: nullpos, doe1080

Co-authored-by: doe1080 <98906116+doe1080@users.noreply.github.com>

yt_dlp/extractor/dreisat.py

@@ -64,7 +64,7 @@ class DreiSatIE(ZDFBaseIE):
             'title': 'dein buch  - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
             'description': 'md5:bae51bfc22f15563ce3acbf97d2e8844',
             'duration': 5399.0,
-            'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1743329640903',
+            'thumbnail': 'https://www.3sat.de/assets/buchmesse-kerkeling-100~original?cb=1747256996338',
             'chapters': 'count:24',
             'episode': 'dein buch  - Das Beste von der Leipziger Buchmesse 2025 - Teil 1',
             'episode_id': 'POS_1ef236cc-b390-401e-acd0-4fb4b04315fb',

yt_dlp/extractor/nobelprize.py

@@ -1,59 +1,57 @@
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
-    get_element_by_attribute,
+    UnsupportedError,
+    clean_html,
     int_or_none,
-    js_to_json,
-    mimetype2ext,
-    update_url_query,
+    parse_duration,
+    parse_qs,
+    str_or_none,
+    update_url,
 )
+from ..utils.traversal import find_element, traverse_obj
 
 
 class NobelPrizeIE(InfoExtractor):
-    _WORKING = False
-    _VALID_URL = r'https?://(?:www\.)?nobelprize\.org/mediaplayer.*?\bid=(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.nobelprize.org/mediaplayer/?id=2636',
-        'md5': '04c81e5714bb36cc4e2232fee1d8157f',
+    _VALID_URL = r'https?://(?:(?:mediaplayer|www)\.)?nobelprize\.org/mediaplayer/'
+    _TESTS = [{
+        'url': 'https://www.nobelprize.org/mediaplayer/?id=2636',
         'info_dict': {
             'id': '2636',
             'ext': 'mp4',
             'title': 'Announcement of the 2016 Nobel Prize in Physics',
-            'description': 'md5:05beba57f4f5a4bbd4cf2ef28fcff739',
+            'description': 'md5:1a2d8a6ca80c88fb3b9a326e0b0e8e43',
+            'duration': 1560.0,
+            'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
+            'timestamp': 1504883793,
+            'upload_date': '20170908',
         },
-    }
+    }, {
+        'url': 'https://mediaplayer.nobelprize.org/mediaplayer/?qid=12693',
+        'info_dict': {
+            'id': '12693',
+            'ext': 'mp4',
+            'title': 'Nobel Lecture by Peter Higgs',
+            'description': 'md5:9b12e275dbe3a8138484e70e00673a05',
+            'duration': 1800.0,
+            'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
+            'timestamp': 1504883793,
+            'upload_date': '20170908',
+        },
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        media = self._parse_json(self._search_regex(
-            r'(?s)var\s*config\s*=\s*({.+?});', webpage,
-            'config'), video_id, js_to_json)['media']
-        title = media['title']
-
-        formats = []
-        for source in media.get('source', []):
-            source_src = source.get('src')
-            if not source_src:
-                continue
-            ext = mimetype2ext(source.get('type')) or determine_ext(source_src)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    source_src, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    update_url_query(source_src, {'hdcore': '3.7.0'}),
-                    video_id, f4m_id='hds', fatal=False))
-            else:
-                formats.append({
-                    'url': source_src,
-                })
+        video_id = traverse_obj(parse_qs(url), (
+            ('id', 'qid'), -1, {int_or_none}, {str_or_none}, any))
+        if not video_id:
+            raise UnsupportedError(url)
+        webpage = self._download_webpage(
+            update_url(url, netloc='mediaplayer.nobelprize.org'), video_id)
 
         return {
+            **self._search_json_ld(webpage, video_id),
             'id': video_id,
-            'title': title,
-            'description': get_element_by_attribute('itemprop', 'description', webpage),
-            'duration': int_or_none(media.get('duration')),
-            'formats': formats,
+            'title': self._html_search_meta('caption', webpage),
+            'description': traverse_obj(webpage, (
+                {find_element(tag='span', attr='itemprop', value='description')}, {clean_html})),
+            'duration': parse_duration(self._html_search_meta('duration', webpage)),
         }

yt_dlp/extractor/ntvcojp.py

@@ -1,55 +1,82 @@
-from .common import InfoExtractor
+from .streaks import StreaksBaseIE
 from ..utils import (
-    ExtractorError,
-    smuggle_url,
-    traverse_obj,
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    url_or_none,
 )
+from ..utils.traversal import require, traverse_obj
 
 
-class NTVCoJpCUIE(InfoExtractor):
+class NTVCoJpCUIE(StreaksBaseIE):
     IE_NAME = 'cu.ntv.co.jp'
-    IE_DESC = 'Nippon Television Network'
-    _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P<id>[^/?&#]+)'
-    _TEST = {
-        'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/',
+    IE_DESC = '日テレ無料TADA!'
+    _VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program-list|search)(?P<id>[\w-]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://cu.ntv.co.jp/gaki_20250525/',
         'info_dict': {
-            'id': '5978891207001',
+            'id': 'gaki_20250525',
             'ext': 'mp4',
-            'title': '桜エビと炒り卵がポイント！ 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸',
-            'upload_date': '20181213',
-            'description': 'md5:1985b51a9abc285df0104d982a325f2a',
-            'uploader_id': '3855502814001',
-            'timestamp': 1544669941,
+            'title': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
+            'cast': 'count:2',
+            'description': 'md5:1e1db556224d627d4d2f74370c650927',
+            'display_id': 'ref:gaki_20250525',
+            'duration': 1450,
+            'episode': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
+            'episode_id': '000000010172808',
+            'episode_number': 255,
+            'genres': ['variety'],
+            'live_status': 'not_live',
+            'modified_date': '20250525',
+            'modified_timestamp': 1748145537,
+            'release_date': '20250525',
+            'release_timestamp': 1748145539,
+            'series': 'ダウンタウンのガキの使いやあらへんで！',
+            'series_id': 'gaki',
+            'thumbnail': r're:https?://.+\.jpg',
+            'timestamp': 1748145197,
+            'upload_date': '20250525',
+            'uploader': '日本テレビ放送網',
+            'uploader_id': '0x7FE2',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        player_config = self._search_nuxt_data(webpage, display_id)
-        video_id = traverse_obj(player_config, ('movie', 'video_id'))
-        if not video_id:
-            raise ExtractorError('Failed to extract video ID for Brightcove')
-        account_id = traverse_obj(player_config, ('player', 'account')) or '3855502814001'
-        title = traverse_obj(player_config, ('movie', 'name'))
-        if not title:
-            og_title = self._og_search_title(webpage, fatal=False) or traverse_obj(player_config, ('player', 'title'))
-            if og_title:
-                title = og_title.split('(', 1)[0].strip()
-        description = (traverse_obj(player_config, ('movie', 'description'))
-                       or self._html_search_meta(['description', 'og:description'], webpage))
+
+        info = self._search_json(
+            r'window\.app\s*=', webpage, 'video info',
+            display_id)['falcorCache']['catalog']['episode'][display_id]['value']
+        media_id = traverse_obj(info, (
+            'streaks_data', 'mediaid', {str_or_none}, {require('Streaks media ID')}))
+        non_phonetic = (lambda _, v: v['is_phonetic'] is False, 'value', {str})
+
         return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}),
-            'ie_key': 'BrightcoveNew',
+            **self._extract_from_streaks_api('ntv-tada', media_id, headers={
+                'X-Streaks-Api-Key': 'df497719056b44059a0483b8faad1f4a',
+            }),
+            **traverse_obj(info, {
+                'id': ('content_id', {str_or_none}),
+                'title': ('title', *non_phonetic, any),
+                'age_limit': ('is_adult_only_content', {lambda x: 18 if x else None}),
+                'cast': ('credit', ..., 'name', *non_phonetic),
+                'genres': ('genre', ..., {str}),
+                'release_timestamp': ('pub_date', {parse_iso8601}),
+                'tags': ('tags', ..., {str}),
+                'thumbnail': ('artwork', ..., 'url', any, {url_or_none}),
+            }),
+            **traverse_obj(info, ('tv_episode_info', {
+                'duration': ('duration', {int_or_none}),
+                'episode_number': ('episode_number', {int}),
+                'series': ('parent_show_title', *non_phonetic, any),
+                'series_id': ('show_content_id', {str}),
+            })),
+            **traverse_obj(info, ('custom_data', {
+                'description': ('program_detail', {str}),
+                'episode': ('episode_title', {str}),
+                'episode_id': ('episode_id', {str_or_none}),
+                'uploader': ('network_name', {str}),
+                'uploader_id': ('network_id', {str}),
+            })),
         }

yt_dlp/extractor/srmediathek.py

@@ -1,57 +1,102 @@
 from .ard import ARDMediathekBaseIE
 from ..utils import (
     ExtractorError,
-    get_element_by_attribute,
+    clean_html,
+    extract_attributes,
+    parse_duration,
+    parse_qs,
+    unified_strdate,
+)
+from ..utils.traversal import (
+    find_element,
+    require,
+    traverse_obj,
 )
 
 
 class SRMediathekIE(ARDMediathekBaseIE):
-    _WORKING = False
     IE_NAME = 'sr:mediathek'
     IE_DESC = 'Saarländischer Rundfunk'
-    _VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
+    _CLS_COMMON = 'teaser__image__caption__text teaser__image__caption__text--'
+    _VALID_URL = r'https?://(?:www\.)?sr-mediathek\.de/index\.php\?.*?&id=(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
+        'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=141317',
         'info_dict': {
-            'id': '28455',
+            'id': '141317',
             'ext': 'mp4',
-            'title': 'sportarena (26.10.2014)',
-            'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-        'skip': 'no longer available',
-    }, {
-        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
-        'info_dict': {
-            'id': '37682',
-            'ext': 'mp4',
-            'title': 'Love, Cakes and Rock\'n\'Roll',
-            'description': 'md5:18bf9763631c7d326c22603681e1123d',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+            'title': 'Kärnten, da will ich hin!',
+            'channel': 'SR Fernsehen',
+            'description': 'md5:7732e71e803379a499732864a572a456',
+            'duration': 1788.0,
+            'release_date': '20250525',
+            'series': 'da will ich hin!',
+            'series_id': 'DWIH',
+            'thumbnail': r're:https?://.+\.jpg',
         },
     }, {
-        'url': 'http://sr-mediathek.de/index.php?seite=7&id=7480',
-        'only_matching': True,
+        'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=153853',
+        'info_dict': {
+            'id': '153853',
+            'ext': 'mp3',
+            'title': 'Kappes, Klöße, Kokosmilch: Bruschetta mit Nduja',
+            'channel': 'SR 3',
+            'description': 'md5:3935798de3562b10c4070b408a15e225',
+            'duration': 139.0,
+            'release_date': '20250523',
+            'series': 'Kappes, Klöße, Kokosmilch',
+            'series_id': 'SR3_KKK_A',
+            'thumbnail': r're:https?://.+\.jpg',
+        },
+    }, {
+        'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=31406&pnr=&tbl=pf',
+        'info_dict': {
+            'id': '31406',
+            'ext': 'mp3',
+            'title': 'Das Leben schwer nehmen, ist einfach zu anstrengend',
+            'channel': 'SR 1',
+            'description': 'md5:3e03fd556af831ad984d0add7175fb0c',
+            'duration': 1769.0,
+            'release_date': '20230717',
+            'series': 'Abendrot',
+            'series_id': 'SR1_AB_P',
+            'thumbnail': r're:https?://.+\.jpg',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
 
-        if '>Der gew&uuml;nschte Beitrag ist leider nicht mehr verf&uuml;gbar.<' in webpage:
+        if description == 'Der gewünschte Beitrag ist leider nicht mehr vorhanden.':
             raise ExtractorError(f'Video {video_id} is no longer available', expected=True)
 
-        media_collection_url = self._search_regex(
-            r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
-        info = self._extract_media_info(media_collection_url, webpage, video_id)
-        info.update({
+        player_url = traverse_obj(webpage, (
+            {find_element(tag='div', id=f'player{video_id}', html=True)},
+            {extract_attributes}, 'data-mediacollection-ardplayer',
+            {self._proto_relative_url}, {require('player URL')}))
+        article = traverse_obj(webpage, (
+            {find_element(cls='article__content')},
+            {find_element(tag='p')}, {clean_html}))
+
+        return {
+            **self._extract_media_info(player_url, webpage, video_id),
             'id': video_id,
-            'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
-            'description': self._og_search_description(webpage),
+            'title': traverse_obj(webpage, (
+                {find_element(cls='ardplayer-title')}, {clean_html})),
+            'channel': traverse_obj(webpage, (
+                {find_element(cls=f'{self._CLS_COMMON}subheadline')},
+                {lambda x: x.split('|')[0]}, {clean_html})),
+            'description': description,
+            'duration': parse_duration(self._search_regex(
+                r'(\d{2}:\d{2}:\d{2})', article, 'duration')),
+            'release_date': unified_strdate(self._search_regex(
+                r'(\d{2}\.\d{2}\.\d{4})', article, 'release_date')),
+            'series': traverse_obj(webpage, (
+                {find_element(cls=f'{self._CLS_COMMON}headline')}, {clean_html})),
+            'series_id': traverse_obj(webpage, (
+                {find_element(cls='teaser__link', html=True)},
+                {extract_attributes}, 'href', {parse_qs}, 'sen', ..., {str}, any)),
             'thumbnail': self._og_search_thumbnail(webpage),
-        })
-        return info
+        }

yt_dlp/extractor/zdf.py

@@ -6,6 +6,7 @@ import time
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    ISO639Utils,
     determine_ext,
     filter_dict,
     float_or_none,
@@ -118,10 +119,7 @@ class ZDFBaseIE(InfoExtractor):
                         if ext == 'm3u8':
                             fmts = self._extract_m3u8_formats(
                                 format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        elif ext == 'mpd':
-                            fmts = self._extract_mpd_formats(
-                                format_url, video_id, mpd_id='dash', fatal=False)
-                        else:
+                        elif ext in ('mp4', 'webm'):
                             height = int_or_none(quality.get('highestVerticalResolution'))
                             width = round(aspect_ratio * height) if aspect_ratio and height else None
                             fmts = [{
@@ -132,16 +130,31 @@ class ZDFBaseIE(InfoExtractor):
                                 'format_id': join_nonempty('http', stream.get('type')),
                                 'tbr': int_or_none(self._search_regex(r'_(\d+)k_', format_url, 'tbr', default=None)),
                             }]
+                        else:
+                            self.report_warning(f'Skipping unsupported extension "{ext}"', video_id=video_id)
+                            fmts = []
+
                         f_class = variant.get('class')
                         for f in fmts:
+                            f_lang = ISO639Utils.short2long(
+                                (f.get('language') or variant.get('language') or '').lower())
+                            is_audio_only = f.get('vcodec') == 'none'
                             formats.append({
                                 **f,
-                                'format_id': join_nonempty(f.get('format_id'), is_dgs and 'dgs'),
+                                'format_id': join_nonempty(f['format_id'], is_dgs and 'dgs'),
                                 'format_note': join_nonempty(
-                                    f_class, is_dgs and 'German Sign Language', f.get('format_note'), delim=', '),
-                                'language': variant.get('language') or f.get('language'),
+                                    not is_audio_only and f_class,
+                                    is_dgs and 'German Sign Language',
+                                    f.get('format_note'), delim=', '),
                                 'preference': -2 if is_dgs else -1,
-                                'language_preference': 10 if f_class == 'main' else -10 if f_class == 'ad' else -1,
+                                'language': f_lang,
+                                'language_preference': (
+                                    -10 if ((is_audio_only and f.get('format_note') == 'Audiodeskription')
+                                            or (not is_audio_only and f_class == 'ad'))
+                                    else 10 if f_lang == 'deu' and f_class == 'main'
+                                    else 5 if f_lang == 'deu'
+                                    else 1 if f_class == 'main'
+                                    else -1),
                             })
 
         return {
@@ -333,12 +346,13 @@ class ZDFIE(ZDFBaseIE):
             'title': 'Dobrindt schließt Steuererhöhungen aus',
             'description': 'md5:9a117646d7b8df6bc902eb543a9c9023',
             'duration': 325,
-            'thumbnail': 'https://www.zdf.de/assets/dobrindt-csu-berlin-direkt-100~1920x1080?cb=1743357653736',
+            'thumbnail': 'https://www.zdfheute.de/assets/dobrindt-csu-berlin-direkt-100~1920x1080?cb=1743357653736',
             'timestamp': 1743374520,
             'upload_date': '20250330',
             '_old_archive_ids': ['zdf 250330_clip_2_bdi'],
         },
     }, {
+        # FUNK video (hosted on a different CDN, has atypical PTMD and HLS files)
         'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
         'md5': '57af4423db0455a3975d2dc4578536bc',
         'info_dict': {
@@ -651,6 +665,7 @@ class ZDFChannelIE(ZDFBaseIE):
             'description': 'md5:6edad39189abf8431795d3d6d7f986b3',
         },
         'playlist_count': 242,
+        'skip': 'Video count changes daily, needs support for playlist_maxcount',
     }]
 
     _PAGE_SIZE = 24