[update] Detect GitHub error page responses (#17555)

Closes #17550
Authored by: bashonly

README.md

@@ -1978,6 +1978,9 @@ The following extractors use this feature:
 * `client`: Client to extract video data from. The currently available clients are `android` and `web`. Only one client can be used. The `web` client is used by default, and it only works with account cookies or login credentials. The `android` client only works with previously cached OAuth tokens
 * `original_format_policy`: Policy for when to try extracting original formats. One of `always`, `never`, or `auto`. The default `auto` policy tries to avoid exceeding the web client's API rate-limit by only making an extra request when Vimeo publicizes the video's downloadability
 
+#### adn
+* `profile_id`: The numeric ID of the premium account profile that will be used to download videos (default: `1`)
+
 #### zan
 * `split_angles`: Split multi-angle streams into separate angle formats. Forces re-encoding of the video stream during download, and requires ffmpeg. Either `true` or `false` (default)
 

devscripts/prepare_manpage.py

@@ -66,8 +66,7 @@ def convert_code_blocks(readme):
 
 def move_sections(readme):
     MOVE_TAG_TEMPLATE = '<!-- MANPAGE: MOVE "%s" SECTION HERE -->'
-    sections = re.findall(r'(?m)^%s$' % (
-        re.escape(MOVE_TAG_TEMPLATE).replace(r'\%', '%') % '(.+)'), readme)
+    sections = re.findall(r'(?m)^%s$' % (re.escape(MOVE_TAG_TEMPLATE) % '(.+)'), readme)
 
     for section_name in sections:
         move_tag = MOVE_TAG_TEMPLATE % section_name

test/helper.py

@@ -3,7 +3,6 @@ import hashlib
 import json
 import os.path
 import re
-import ssl
 import sys
 import types
 
@@ -318,36 +317,6 @@ def expect_info_dict(self, got_dict, expected_dict):
             'Missing keys in test definition: {}'.format(', '.join(sorted(missing_keys))))
 
 
-def assertRegexpMatches(self, text, regexp, msg=None):
-    if hasattr(self, 'assertRegexp'):
-        return self.assertRegexp(text, regexp, msg)
-    else:
-        m = re.match(regexp, text)
-        if not m:
-            note = f'Regexp didn\'t match: {regexp!r} not found'
-            if len(text) < 1000:
-                note += f' in {text!r}'
-            if msg is None:
-                msg = note
-            else:
-                msg = note + ', ' + msg
-            self.assertTrue(m, msg)
-
-
-def assertGreaterEqual(self, got, expected, msg=None):
-    if not (got >= expected):
-        if msg is None:
-            msg = f'{got!r} not greater than or equal to {expected!r}'
-        self.assertTrue(got >= expected, msg)
-
-
-def assertLessEqual(self, got, expected, msg=None):
-    if not (got <= expected):
-        if msg is None:
-            msg = f'{got!r} not less than or equal to {expected!r}'
-        self.assertTrue(got <= expected, msg)
-
-
 def assertEqual(self, got, expected, msg=None):
     if got != expected:
         if msg is None:
@@ -366,12 +335,7 @@ def expect_warnings(ydl, warnings_re):
 
 
 def http_server_port(httpd):
-    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
-        # In Jython SSLSocket is not a subclass of socket.socket
-        sock = httpd.socket.sock
-    else:
-        sock = httpd.socket
-    return sock.getsockname()[1]
+    return httpd.server_address[1]
 
 
 def verify_address_availability(address):

test/test_YoutubeDL.py

@@ -15,7 +15,7 @@ import contextlib
 import copy
 import json
 
-from test.helper import FakeYDL, assertRegexpMatches, try_rm
+from test.helper import FakeYDL, try_rm
 from yt_dlp import YoutubeDL
 from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.postprocessor.common import PostProcessor
@@ -860,10 +860,10 @@ class TestYoutubeDL(unittest.TestCase):
     def test_format_note(self):
         ydl = YoutubeDL()
         self.assertEqual(ydl._format_note({}), '')
-        assertRegexpMatches(self, ydl._format_note({
+        self.assertRegex(ydl._format_note({
             'vbr': 10,
         }), r'^\s*10k$')
-        assertRegexpMatches(self, ydl._format_note({
+        self.assertRegex(ydl._format_note({
             'fps': 30,
         }), r'^30fps$')
 

test/test_download.py

@@ -13,8 +13,6 @@ import hashlib
 import json
 
 from test.helper import (
-    assertGreaterEqual,
-    assertLessEqual,
     expect_info_dict,
     expect_warnings,
     get_params,
@@ -201,8 +199,8 @@ def generator(test_case, tname):
             num_entries = len(res_dict.get('entries', []))
             if 'playlist_mincount' in test_case:
                 mincount = test_case['playlist_mincount']
-                assertGreaterEqual(
-                    self, num_entries, mincount,
+                self.assertGreaterEqual(
+                    num_entries, mincount,
                     f'Expected at least {mincount} entries in playlist {test_url}, but got only {num_entries}')
             if 'playlist_count' in test_case:
                 count = test_case['playlist_count']
@@ -212,8 +210,8 @@ def generator(test_case, tname):
                     f'Expected exactly {count} entries in playlist {test_url}, but got {got}')
             if 'playlist_maxcount' in test_case:
                 maxcount = test_case['playlist_maxcount']
-                assertLessEqual(
-                    self, num_entries, maxcount,
+                self.assertLessEqual(
+                    num_entries, maxcount,
                     f'Expected at most {maxcount} entries in playlist {test_url}, but got more')
             if 'playlist_duration_sum' in test_case:
                 got_duration = sum(e['duration'] for e in res_dict['entries'])
@@ -241,8 +239,8 @@ def generator(test_case, tname):
                         if params.get('test'):
                             expected_minsize = max(expected_minsize, 10000)
                         got_fsize = os.path.getsize(tc_filename)
-                        assertGreaterEqual(
-                            self, got_fsize, expected_minsize,
+                        self.assertGreaterEqual(
+                            got_fsize, expected_minsize,
                             f'Expected {tc_filename} to be at least {format_bytes(expected_minsize)}, '
                             f'but it\'s only {format_bytes(got_fsize)} ')
                     if 'md5' in tc:

test/test_networking.py

@@ -338,7 +338,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
         https_server_thread.start()
 
         with handler(verify=False) as rh:
-            with pytest.raises(SSLError, match=r'(?i)ssl(?:v3|/tls).alert.handshake.failure') as exc_info:
+            with pytest.raises(SSLError, match=r'(?i)(?:sslv3|tls).alert.handshake.failure') as exc_info:
                 validate_and_send(rh, Request(f'https://127.0.0.1:{https_port}/headers'))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 
@@ -984,28 +984,15 @@ class TestUrllibRequestHandler(TestRequestHandlerBase):
             ):
                 validate_and_send(rh, Request(f'https://127.0.0.1:{self.https_port}/headers'))
 
-    @pytest.mark.parametrize('req,match,version_check', [
+    @pytest.mark.parametrize('req,match', [
         # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
-        # bpo-39603: Check implemented in 3.7.9+, 3.8.5+
-        (
-            Request('http://127.0.0.1', method='GET\n'),
-            'method can\'t contain control characters',
-            lambda v: v < (3, 7, 9) or (3, 8, 0) <= v < (3, 8, 5),
-        ),
+        (Request('http://127.0.0.1', method='GET\n'), 'method can\'t contain control characters'),
         # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1265
-        # bpo-38576: Check implemented in 3.7.8+, 3.8.3+
-        (
-            Request('http://127.0.0. 1', method='GET'),
-            'URL can\'t contain control characters',
-            lambda v: v < (3, 7, 8) or (3, 8, 0) <= v < (3, 8, 3),
-        ),
+        (Request('http://127.0.0. 1', method='GET'), 'URL can\'t contain control characters'),
         # https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1288C31-L1288C50
-        (Request('http://127.0.0.1', headers={'foo\n': 'bar'}), 'Invalid header name', None),
+        (Request('http://127.0.0.1', headers={'foo\n': 'bar'}), 'Invalid header name'),
     ])
-    def test_httplib_validation_errors(self, handler, req, match, version_check):
-        if version_check and version_check(sys.version_info):
-            pytest.skip(f'Python {sys.version} version does not have the required validation for this test.')
-
+    def test_httplib_validation_errors(self, handler, req, match):
         with handler() as rh:
             with pytest.raises(RequestError, match=match) as exc_info:
                 validate_and_send(rh, req)

test/test_utils.py

@@ -1347,15 +1347,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
         self.assertEqual(extract_attributes('<e x="Fáilte 世界 \U0001f600">'), {'x': 'Fáilte 世界 \U0001f600'})
         self.assertEqual(extract_attributes('<e x="décompose&#769;">'), {'x': 'décompose\u0301'})
-        # "Narrow" Python builds don't support unicode code points outside BMP.
-        try:
-            chr(0x10000)
-            supports_outside_bmp = True
-        except ValueError:
-            supports_outside_bmp = False
-        if supports_outside_bmp:
         self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})
-        # Malformed HTML should not break attributes extraction on older Python
+        # Malformed HTML should not break attribute extraction
         self.assertEqual(extract_attributes('<mal"formed/>'), {})
 
     def test_clean_html(self):

test/test_websockets.py

@@ -186,7 +186,7 @@ class TestWebsSocketRequestHandlerConformance:
 
     def test_ssl_error(self, handler):
         with handler(verify=False) as rh:
-            with pytest.raises(SSLError, match=r'ssl(?:v3|/tls) alert handshake failure') as exc_info:
+            with pytest.raises(SSLError, match=r'(?:sslv3|tls) alert handshake failure') as exc_info:
                 ws_validate_and_send(rh, Request(self.bad_wss_host))
             assert not issubclass(exc_info.type, CertificateVerifyError)
 

yt_dlp/YoutubeDL.py

@@ -2398,8 +2398,7 @@ class YoutubeDL:
             selectors = []
             current_selector = None
             for type_, string_, start, _, _ in tokens:
-                # ENCODING is only defined in Python 3.x
-                if type_ == getattr(tokenize, 'ENCODING', None):
+                if type_ == tokenize.ENCODING:
                     continue
                 elif type_ in [tokenize.NAME, tokenize.NUMBER]:
                     current_selector = FormatSelector(SINGLE, string_, [])

yt_dlp/aes.py

@@ -293,7 +293,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     - Mode of operation is 'counter'
 
     @param {str} data                    Base64 encoded string
-    @param {str,unicode} password        Password (will be encoded with utf-8)
+    @param {str} password                Password (will be encoded with UTF-8)
     @param {int} key_size_bytes          Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
     @returns {str}                       Decrypted data
     """

yt_dlp/compat/__init__.py

@@ -8,9 +8,8 @@ passthrough_module(__name__, '._deprecated')
 del passthrough_module
 
 
-# HTMLParseError has been deprecated in Python 3.3 and removed in
-# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
-# and uniform cross-version exception handling
+# HTMLParseError was deprecated in Python 3.3 and removed in Python 3.5.
+# Keep a replacement for API compatibility and uniform exception handling.
 class compat_HTMLParseError(ValueError):
     pass
 

yt_dlp/downloader/ism.py

@@ -154,7 +154,7 @@ def write_piff_header(stream, params):
         sample_entry_payload += u16.pack(0x18)  # depth
         sample_entry_payload += s16.pack(-1)  # pre defined
 
-        codec_private_data = binascii.unhexlify(params['codec_private_data'].encode())
+        codec_private_data = binascii.unhexlify(params['codec_private_data'])
         if fourcc in ('H264', 'AVC1'):
             sps, pps = codec_private_data.split(u32.pack(1))[1:]
             avcc_payload = u8.pack(1)  # configuration version

yt_dlp/extractor/_extractors.py

@@ -1221,6 +1221,7 @@ from .nhk import (
 from .nhl import NHLIE
 from .nick import NickIE
 from .niconico import (
+    NiconicoChannelIE,
     NiconicoHistoryIE,
     NiconicoIE,
     NiconicoLiveIE,

yt_dlp/extractor/adn.py

@@ -50,14 +50,13 @@ class ADNIE(ADNBaseIE):
     _VALID_URL = r'https?://(?:www\.)?animationdigitalnetwork\.com/(?:(?P<lang>de)/)?video/[^/?#]+/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://animationdigitalnetwork.com/video/558-fruits-basket/9841-episode-1-a-ce-soir',
-        'md5': '1c9ef066ceb302c86f80c2b371615261',
+        'md5': '3999b7b235ffb3591a385d1913cd3cd1',
         'info_dict': {
             'id': '9841',
             'ext': 'mp4',
             'title': 'Fruits Basket - Episode 1',
-            'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
             'series': 'Fruits Basket',
-            'duration': 1437,
+            'duration': 1436,
             'release_date': '20190405',
             'comment_count': int,
             'average_rating': float,
@@ -174,6 +173,8 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
     def _real_extract(self, url):
         lang, video_id = self._match_valid_url(url).group('lang', 'id')
         self._HEADERS['X-Target-Distribution'] = lang or 'fr'
+        if 'Authorization' in self._HEADERS:
+            self._HEADERS['X-Profile-ID'] = self._configuration_arg('profile_id', ['1'])[0]
         video_base_url = self._PLAYER_BASE_URL + f'video/{video_id}/'
         player = self._download_json(
             video_base_url + 'configuration', video_id,

yt_dlp/extractor/bandcamp.py

@@ -25,7 +25,35 @@ from ..utils import (
 from ..utils.traversal import find_element, find_elements, traverse_obj
 
 
-class BandcampIE(InfoExtractor):
+class BandcampBaseIE(InfoExtractor):
+    # Initially try without impersonation, retry with impersonation
+    def _download_webpage(self, *args, **kwargs):
+        impersonate = kwargs.pop('impersonate', None) or True
+        kwargs.pop('require_impersonation', None)
+        webpage = super()._download_webpage(*args, **kwargs)
+        if webpage:
+            if self._html_extract_title(webpage) == 'Client Challenge':
+                self.write_debug('Got client challenge webpage response')
+            else:
+                return webpage
+
+        res = self._download_webpage_handle(*args, impersonate=impersonate, require_impersonation=True, **kwargs)
+        if res is False:
+            return False
+
+        webpage, urlh = res
+        if self._html_extract_title(webpage) == 'Client Challenge':
+            raise ExtractorError(f'Got client challenge webpage response with {urlh.extensions.get("impersonate")}')
+
+        return webpage
+
+    def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True):
+        return self._parse_json(self._html_search_regex(
+            rf'data-{attr}=(["\'])({{.+?}})\1', webpage,
+            attr + ' data', group=2), video_id, fatal=fatal)
+
+
+class BandcampIE(BandcampBaseIE):
     _VALID_URL = r'https?://(?P<uploader>[^/]+)\.bandcamp\.com/track/(?P<id>[^/?#&]+)'
     _EMBED_REGEX = [r'<meta property="og:url"[^>]*?content="(?P<url>.*?bandcamp\.com.*?)"']
     _TESTS = [{
@@ -149,14 +177,9 @@ class BandcampIE(InfoExtractor):
         'skip': 'embed detection is broken',
     }]
 
-    def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True):
-        return self._parse_json(self._html_search_regex(
-            rf'data-{attr}=(["\'])({{.+?}})\1', webpage,
-            attr + ' data', group=2), video_id, fatal=fatal)
-
     def _real_extract(self, url):
         title, uploader = self._match_valid_url(url).group('id', 'uploader')
-        webpage = self._download_webpage(url, title, impersonate=True)
+        webpage = self._download_webpage(url, title)
         tralbum = self._extract_data_attr(webpage, title)
         thumbnail = self._og_search_thumbnail(webpage)
 
@@ -202,7 +225,7 @@ class BandcampIE(InfoExtractor):
             track_id = str(tralbum['id'])
 
             download_webpage = self._download_webpage(
-                download_link, track_id, 'Downloading free downloads page', impersonate=True)
+                download_link, track_id, 'Downloading free downloads page')
 
             blob = self._extract_data_attr(download_webpage, track_id, 'blob')
 
@@ -284,7 +307,7 @@ class BandcampIE(InfoExtractor):
         }
 
 
-class BandcampAlbumIE(BandcampIE):  # XXX: Do not subclass from concrete IE
+class BandcampAlbumIE(BandcampBaseIE):
     IE_NAME = 'Bandcamp:album'
     _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com/album/(?P<id>[^/?#&]+)'
 
@@ -389,7 +412,7 @@ class BandcampAlbumIE(BandcampIE):  # XXX: Do not subclass from concrete IE
     def _real_extract(self, url):
         uploader_id, album_id = self._match_valid_url(url).groups()
         playlist_id = album_id or uploader_id
-        webpage = self._download_webpage(url, playlist_id, impersonate=True)
+        webpage = self._download_webpage(url, playlist_id)
         tralbum = self._extract_data_attr(webpage, playlist_id)
         track_info = tralbum.get('trackinfo')
         if not track_info:
@@ -414,7 +437,7 @@ class BandcampAlbumIE(BandcampIE):  # XXX: Do not subclass from concrete IE
         }
 
 
-class BandcampWeeklyIE(BandcampIE):  # XXX: Do not subclass from concrete IE
+class BandcampWeeklyIE(BandcampBaseIE):
     IE_NAME = 'Bandcamp:weekly'
     _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/radio/?\?(?:[^#]+&)?show=(?P<id>\d+)'
     _TESTS = [{
@@ -478,7 +501,7 @@ class BandcampWeeklyIE(BandcampIE):  # XXX: Do not subclass from concrete IE
         }
 
 
-class BandcampUserIE(InfoExtractor):
+class BandcampUserIE(BandcampBaseIE):
     IE_NAME = 'Bandcamp:user'
     _VALID_URL = r'https?://(?!www\.)(?P<id>[^.]+)\.bandcamp\.com(?:/music)?/?(?:[#?]|$)'
 

yt_dlp/extractor/common.py

@@ -432,29 +432,28 @@ class InfoExtractor:
 
     chapter:        Name or title of the chapter the video belongs to.
     chapter_number: Number of the chapter the video belongs to, as an integer.
-    chapter_id:     Id of the chapter the video belongs to, as a unicode string.
+    chapter_id:     Id of the chapter the video belongs to.
 
     The following fields should only be used when the video is an episode of some
     series, programme or podcast:
 
     series:         Title of the series or programme the video episode belongs to.
-    series_id:      Id of the series or programme the video episode belongs to, as a unicode string.
+    series_id:      Id of the series or programme the video episode belongs to.
     season:         Title of the season the video episode belongs to.
     season_number:  Number of the season the video episode belongs to, as an integer.
-    season_id:      Id of the season the video episode belongs to, as a unicode string.
+    season_id:      Id of the season the video episode belongs to.
     episode:        Title of the video episode. Unlike mandatory video title field,
                     this field should denote the exact title of the video episode
                     without any kind of decoration.
     episode_number: Number of the video episode within a season, as an integer.
-    episode_id:     Id of the video episode, as a unicode string.
+    episode_id:     Id of the video episode.
 
     The following fields should only be used when the media is a track or a part of
     a music album:
 
     track:          Title of the track.
     track_number:   Number of the track within an album or a disc, as an integer.
-    track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
-                    as a unicode string.
+    track_id:       Id of the track (useful for custom indexing, e.g. 6.iii).
     artists:        List of artists of the track.
     composers:      List of composers of the piece.
     genres:         List of genres of the track.
@@ -487,7 +486,7 @@ class InfoExtractor:
     creator:        Use "creators" instead.
                     The creator of the video.
 
-    Unless mentioned otherwise, the fields should be Unicode strings.
+    Unless mentioned otherwise, the fields should be strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.
 

yt_dlp/extractor/globalplayer.py

@@ -1,14 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    join_nonempty,
-    parse_duration,
-    str_or_none,
-    traverse_obj,
-    unified_strdate,
-    unified_timestamp,
-    urlhandle_detect_ext,
-)
+from ..utils import url_or_none
+from ..utils.traversal import require, traverse_obj
 
 
 class GlobalPlayerBaseIE(InfoExtractor):
@@ -16,29 +8,11 @@ class GlobalPlayerBaseIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         return self._search_nextjs_data(webpage, video_id)['props']['pageProps']
 
-    def _request_ext(self, url, video_id):
-        return urlhandle_detect_ext(self._request_webpage(  # Server rejects HEAD requests
-            url, video_id, note='Determining source extension'))
-
-    def _extract_audio(self, episode, series):
-        return {
-            'vcodec': 'none',
-            **traverse_obj(series, {
-                'series': 'title',
-                'series_id': 'id',
-                'thumbnail': 'imageUrl',
-                'uploader': 'itunesAuthor',  # podcasts only
-            }),
-            **traverse_obj(episode, {
-                'id': 'id',
-                'description': ('description', {clean_html}),
-                'duration': ('duration', {parse_duration}),
-                'thumbnail': 'imageUrl',
-                'url': 'streamUrl',
-                'timestamp': (('pubDate', 'startDate'), {unified_timestamp}),
-                'title': 'title',
-            }, get_all=False),
-        }
+    @staticmethod
+    def _get_playback_url(data):
+        return traverse_obj(data, (
+            'playback', lambda _, v: v['canUse'] == 'true',
+            'url', {url_or_none}, any, {require('playback URL')}))
 
 
 class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
@@ -48,11 +22,10 @@ class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
         'info_dict': {
             'id': '2mx1E',
             'ext': 'aac',
-            'display_id': 'smoothchill-uk',
-            'title': 're:^Smooth Chill.+$',
-            'thumbnail': 'https://herald.musicradio.com/media/f296ade8-50c9-4f60-911f-924e96873620.png',
-            'description': 'Music To Chill To',
             'live_status': 'is_live',
+            'thumbnail': 'md5:d5040f26c7c4061014a44866129b900e',
+            'description': 'md5:6e183929da9001778895f32ae85124bc',
+            'title': 're:^Smooth Chill.+$',
         },
     }, {
         # national station
@@ -60,11 +33,10 @@ class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
         'info_dict': {
             'id': '2mwx4',
             'ext': 'aac',
-            'description': 'turn up the feel good!',
-            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
             'live_status': 'is_live',
+            'description': 'md5:492d07dfea8addadd15650ef40c10d02',
+            'thumbnail': 'md5:6f13378a53ce55bcf57365a654e1b490',
             'title': 're:^Heart UK.+$',
-            'display_id': 'heart-uk',
         },
     }, {
         # regional variation
@@ -72,110 +44,131 @@ class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
         'info_dict': {
             'id': 'AMqg',
             'ext': 'aac',
-            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
-            'title': 're:^Heart London.+$',
             'live_status': 'is_live',
-            'display_id': 'heart-london',
-            'description': 'turn up the feel good!',
+            'description': 'md5:492d07dfea8addadd15650ef40c10d02',
+            'thumbnail': 'md5:6f13378a53ce55bcf57365a654e1b490',
+            'title': 're:^Heart London.+$',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        station = self._get_page_props(url, video_id)['station']
-        stream_url = station['streamUrl']
+        meta = self._get_page_props(url, video_id)['station']
+        station_id = meta['id']
+
+        data = self._download_json(f'https://bff-web-guacamole.musicradio.com/playables/{station_id}', video_id)
 
         return {
-            'id': station['id'],
-            'display_id': join_nonempty('brandSlug', 'slug', from_dict=station) or station.get('legacyStationPrefix'),
-            'url': stream_url,
-            'ext': self._request_ext(stream_url, video_id),
+            'id': station_id,
+            'url': self._get_playback_url(data),
+            'ext': 'aac',
             'vcodec': 'none',
             'is_live': True,
-            **traverse_obj(station, {
-                'title': (('name', 'brandName'), {str_or_none}),
-                'description': 'tagline',
-                'thumbnail': 'brandLogo',
-            }, get_all=False),
+            **traverse_obj(meta, {
+                'thumbnail': ('brandLogo', {url_or_none}),
+                'description': ('tagline', {str}),
+                'title': ('name', {str}),
+            }),
         }
 
 
 class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
     _VALID_URL = r'https?://www\.globalplayer\.com/playlists/(?P<id>\w+)'
     _TESTS = [{
-        # "live playlist"
+        # live playlist
         'url': 'https://www.globalplayer.com/playlists/8bLk/',
         'info_dict': {
             'id': '8bLk',
             'ext': 'aac',
             'live_status': 'is_live',
-            'description': 'md5:e10f5e10b01a7f2c14ba815509fbb38d',
-            'thumbnail': 'https://images.globalplayer.com/images/551379?width=450&signature=oMLPZIoi5_dBSHnTMREW0Xg76mA=',
+            'thumbnail': 'md5:391a13cc087b42f626e9e65bbeaf0a11',
+            'description': 'md5:f015f2f6c6f6a807669ebcc9a0ca147c',
             'title': 're:^Classic FM Hall of Fame.+$',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        station = self._get_page_props(url, video_id)['playlistData']
-        stream_url = station['streamUrl']
+        meta = self._get_page_props(url, video_id)['playlistData']
 
         return {
-            'id': video_id,
-            'url': stream_url,
-            'ext': self._request_ext(stream_url, video_id),
+            'url': meta['streamUrl'],
+            'ext': 'aac',
             'vcodec': 'none',
+            'id': video_id,
             'is_live': True,
-            **traverse_obj(station, {
-                'title': 'title',
-                'description': 'description',
-                'thumbnail': 'image',
+            **traverse_obj(meta, {
+                'thumbnail': ('image', {url_or_none}),
+                'description': ('description', {str}),
+                'title': ('title', {str}),
             }),
         }
 
 
 class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
-    _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)/|catchup/\w+/\w+/)(?P<id>\w+)/?(?:$|[?#])'
+    _VALID_URL = r'https?://www\.globalplayer\.com/(?P<path>(?P<podcast>podcasts)/|catchup/\w+/\w+/)(?P<id>\w+)/?(?:$|[?#])'
     _TESTS = [{
         # podcast
         'url': 'https://www.globalplayer.com/podcasts/42KuaM/',
-        'playlist_mincount': 5,
+        'playlist_mincount': 2,
         'info_dict': {
             'id': '42KuaM',
-            'title': 'Filthy Ritual',
             'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
-            'categories': ['Society & Culture', 'True Crime'],
-            'uploader': 'Global',
-            'description': 'md5:da5b918eac9ae319454a10a563afacf9',
+            'description': 'md5:17b7b9e3c76b2f4d9e31ccc4f0b66e32',
+            'title': 'Filthy Ritual',
         },
     }, {
         # radio catchup
         'url': 'https://www.globalplayer.com/catchup/lbc/uk/46vyD7z/',
-        'playlist_mincount': 3,
+        'playlist_mincount': 2,
         'info_dict': {
             'id': '46vyD7z',
-            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'thumbnail': 'md5:664ad62a8fb920a2b8e264ed780eee3d',
+            'description': 'md5:53b6fa5ef71a3cff6628551bcc416384',
             'title': 'Nick Ferrari',
-            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
         },
     }]
 
     def _real_extract(self, url):
-        video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
+        video_id, path, podcast = self._match_valid_url(url).group('id', 'path', 'podcast')
         props = self._get_page_props(url, video_id)
-        series = props['podcastInfo'] if podcast else props['catchupInfo']
+        if podcast:
+            meta = props['podcastInfo']['metadata']
+            blocks = props['podcastInfo']['blocks'][1]['items']
+        else:
+            catchup = props['catchupShow'] if 'catchupShow' in props else props['catchupInfo']
+            meta = catchup['metadata']
+            blocks = catchup['blocks'][1]['items']
+
+        def _entries():
+            for block in blocks:
+                entry_id = block['id']
+                data = self._download_json(
+                    f'https://bff-web-guacamole.musicradio.com/playables/{entry_id}',
+                    video_id, f'Downloading metadata JSON for {entry_id}')
+
+                yield {
+                    'id': entry_id,
+                    'url': self._get_playback_url(data),
+                    'vcodec': 'none',
+                    'extractor': GlobalPlayerAudioEpisodeIE.IE_NAME,
+                    'extractor_key': GlobalPlayerAudioEpisodeIE.ie_key(),
+                    'webpage_url': f'https://www.globalplayer.com/{path}episodes/{entry_id}',
+                    **traverse_obj(block, {
+                        'thumbnail': ('image', 'url', {url_or_none}),
+                        'description': ('description', {str}),
+                        'title': ('title', {str}),
+                    }),
+                }
 
         return {
             '_type': 'playlist',
             'id': video_id,
-            'entries': [self._extract_audio(ep, series) for ep in traverse_obj(
-                        series, ('episodes', lambda _, v: v['id'] and v['streamUrl']))],
-            'categories': traverse_obj(series, ('categories', ..., 'name')) or None,
-            **traverse_obj(series, {
-                'description': 'description',
-                'thumbnail': 'imageUrl',
-                'title': 'title',
-                'uploader': 'itunesAuthor',  # podcasts only
+            'entries': _entries(),
+            **traverse_obj(meta, {
+                'thumbnail': ('image', 'url', {url_or_none}),
+                'description': ('description', {str}),
+                'title': ('title', {str}),
             }),
         }
 
@@ -184,44 +177,42 @@ class GlobalPlayerAudioEpisodeIE(GlobalPlayerBaseIE):
     _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)|catchup/\w+/\w+)/episodes/(?P<id>\w+)/?(?:$|[?#])'
     _TESTS = [{
         # podcast
-        'url': 'https://www.globalplayer.com/podcasts/episodes/7DrfNnE/',
+        'url': 'https://www.globalplayer.com/podcasts/episodes/7DrorSc/',
         'info_dict': {
-            'id': '7DrfNnE',
+            'id': '7DrorSc',
             'ext': 'mp3',
-            'title': 'Filthy Ritual - Trailer',
-            'description': 'md5:1f1562fd0f01b4773b590984f94223e0',
             'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
-            'duration': 225.0,
-            'timestamp': 1681254900,
-            'series': 'Filthy Ritual',
-            'series_id': '42KuaM',
-            'upload_date': '20230411',
-            'uploader': 'Global',
+            'description': 'md5:372e5aa2b531f9eba863dfc67d007c1c',
+            'title': 'Filthy Ritual - Trailer',
         },
     }, {
-        # radio catchup
-        'url': 'https://www.globalplayer.com/catchup/lbc/uk/episodes/2zGq26Vcv1fCWhddC4JAwETXWe/',
+        # radio catchup - test urls are removed after 7 days
+        'url': 'https://www.globalplayer.com/catchup/lbc/uk/episodes/2zGmrV6DnvogKkNCXkwkQ8HQTA/',
         'info_dict': {
-            'id': '2zGq26Vcv1fCWhddC4JAwETXWe',
+            'id': '2zGmrV6DnvogKkNCXkwkQ8HQTA',
             'ext': 'm4a',
-            'timestamp': 1682056800,
-            'series': 'Nick Ferrari',
-            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
-            'upload_date': '20230421',
-            'series_id': '46vyD7z',
-            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'thumbnail': 'md5:664ad62a8fb920a2b8e264ed780eee3d',
+            'description': 'md5:53b6fa5ef71a3cff6628551bcc416384',
             'title': 'Nick Ferrari',
-            'duration': 10800.0,
         },
     }]
 
     def _real_extract(self, url):
         video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
         props = self._get_page_props(url, video_id)
-        episode = props['podcastEpisode'] if podcast else props['catchupEpisode']
+        meta = props['podcastEpisode']['metadata'] if podcast else props['catchupEpisode']['metadata']
+        data = self._download_json(f'https://bff-web-guacamole.musicradio.com/playables/{video_id}', video_id)
 
-        return self._extract_audio(
-            episode, traverse_obj(episode, 'podcast', 'show', expected_type=dict) or {})
+        return {
+            'id': video_id,
+            'url': self._get_playback_url(data),
+            'vcodec': 'none',
+            **traverse_obj(meta, {
+                'thumbnail': ('image', 'url', {url_or_none}),
+                'description': ('description', {str}),
+                'title': ('title', {str}),
+            }),
+        }
 
 
 class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
@@ -231,9 +222,8 @@ class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
         'info_dict': {
             'id': '2JsSZ7Gm2uP',
             'ext': 'mp4',
-            'description': 'md5:6a9f063c67c42f218e42eee7d0298bfd',
             'thumbnail': 'md5:d4498af48e15aae4839ce77b97d39550',
-            'upload_date': '20230420',
+            'description': 'md5:6a9f063c67c42f218e42eee7d0298bfd',
             'title': 'Treble Malakai Bayoh sings a sublime Handel aria at Classic FM Live',
         },
     }]
@@ -245,10 +235,9 @@ class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
         return {
             'id': video_id,
             **traverse_obj(meta, {
-                'url': 'url',
-                'thumbnail': ('image', 'url'),
-                'title': 'title',
-                'upload_date': ('publish_date', {unified_strdate}),
-                'description': 'description',
+                'url': ('url', {url_or_none}),
+                'thumbnail': ('image', 'url', {url_or_none}),
+                'description': ('description', {str}),
+                'title': ('title', {str}),
             }),
         }

yt_dlp/extractor/go.py

@@ -116,41 +116,41 @@ class GoIE(AdobePassIE):
         'params': {'skip_download': 'm3u8'},
         'skip': 'This video requires AdobePass MSO credentials',
     }, {
-        'url': 'https://www.freeform.com/episode/bda0eaf7-761a-4838-aa44-96f794000844/playlist/PL553044961',
+        'url': 'https://www.freeform.com/episode/235128d8-2609-4df4-9874-0b0b687fe9f9/playlist/PL5539647334',
         'info_dict': {
-            'id': 'VDKA39007340',
+            'id': 'VDKA39623200',
             'ext': 'mp4',
-            'title': 'Angel\'s Landing',
-            'description': 'md5:91bf084e785c968fab16734df7313446',
+            'title': 'New House / New Rules',
+            'description': 'md5:6f38b4b1649dc9f3a9e7acf911a70056',
             'age_limit': 14,
-            'duration': 2523,
+            'duration': 2733,
             'thumbnail': r're:https?://.+/.+\.jpg',
-            'series': 'How I Escaped My Cult',
-            'season': 'Season 1',
-            'season_number': 1,
-            'episode': 'Episode 2',
-            'episode_number': 2,
-            'timestamp': 1740038400.0,
-            'upload_date': '20250220',
+            'series': 'Project Runway',
+            'season': 'Season 21',
+            'season_number': 21,
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'timestamp': 1754020800,
+            'upload_date': '20250801',
         },
         'params': {'skip_download': 'm3u8'},
     }, {
-        'url': 'https://www.nationalgeographic.com/tv/episode/ca694661-1186-41ae-8089-82f64d69b16d/playlist/PL554408064',
+        'url': 'https://www.nationalgeographic.com/tv/episode/df0e5bd8-f9bb-4c92-9f94-74dec4dad8a7/playlist/PL553044961',
         'info_dict': {
-            'id': 'VDKA39492078',
+            'id': 'VDKA35475602',
             'ext': 'mp4',
-            'title': 'Heart of the Emperors',
-            'description': 'md5:4fc50a2878f030bb3a7eac9124dca677',
+            'title': 'The Pol Shebang',
+            'description': 'md5:ccea1210c5ebcdc5c1a435f01bdb8b82',
             'age_limit': 0,
-            'duration': 2775,
+            'duration': 1323,
             'thumbnail': r're:https?://.+/.+\.jpg',
-            'series': 'Secrets of the Penguins',
+            'series': 'The Incredible Pol Farm',
             'season': 'Season 1',
             'season_number': 1,
-            'episode': 'Episode 1',
-            'episode_number': 1,
-            'timestamp': 1745204400.0,
-            'upload_date': '20250421',
+            'episode': 'Episode 14',
+            'episode_number': 14,
+            'timestamp': 1704614400,
+            'upload_date': '20240107',
         },
         'params': {'skip_download': 'm3u8'},
     }, {
@@ -190,7 +190,10 @@ class GoIE(AdobePassIE):
 
         site_info = self._SITE_INFO[site]
         brand = site_info['brand']
-        video_data = self._extract_videos(brand, video_id)[0]
+        videos = self._extract_videos(brand, video_id)
+        if not videos:
+            self.report_drm(video_id)
+        video_data = videos[0]
         video_id = video_data['id']
         title = video_data['title']
 

yt_dlp/extractor/itv.py

@@ -114,7 +114,7 @@ class ITVIE(InfoExtractor):
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_subs, featureset_subs = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
+             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
              if try_get(featureset, lambda x: x[2]) == 'outband-webvtt'),
             (None, None))
 
@@ -143,7 +143,7 @@ class ITVIE(InfoExtractor):
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_video, featureset_video = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
+             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
              if set(try_get(featureset, lambda x: x[:2]) or []) == {'aes', 'hls'}),
             (None, None))
         if not platform_tag_video or not featureset_video:

yt_dlp/extractor/niconico.py

@@ -3,6 +3,7 @@ import functools
 import itertools
 import json
 import re
+import urllib.parse
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..networking.exceptions import HTTPError
@@ -14,6 +15,7 @@ from ..utils import (
     extract_attributes,
     float_or_none,
     int_or_none,
+    join_nonempty,
     parse_bitrate,
     parse_iso8601,
     parse_qs,
@@ -31,8 +33,10 @@ from ..utils import (
 )
 from ..utils.traversal import (
     find_element,
+    find_elements,
     require,
     traverse_obj,
+    trim_str,
 )
 
 
@@ -1080,3 +1084,98 @@ class NiconicoLiveIE(NiconicoBaseIE):
             'thumbnails': thumbnails,
             'formats': formats,
         }
+
+
+class NiconicoChannelIE(NiconicoBaseIE):
+    IE_NAME = 'niconico:channel'
+
+    _PAGE_SIZE = 20
+    _SEARCH_PAGE_SIZE = 32
+    _VALID_URL = [
+        r'https?://ch\.nicovideo\.jp/(?P<id>[\w-]+)/(?P<type>video)/?(?P<slug>continuation|member|pay|so\d{8})?(?:[/?#]|$)',
+        r'https?://ch\.nicovideo\.jp/(?P<type>search)/(?P<id>[^/?#]+)',
+    ]
+    _TESTS = [{
+        'url': 'https://ch.nicovideo.jp/higurashianime/video',
+        'info_dict': {
+            'id': 'higurashianime',
+            'title': '「ひぐらしのなく頃に」オフィシャルチャンネル',
+        },
+        'playlist_mincount': 54,
+    }, {
+        'url': 'https://ch.nicovideo.jp/amiami-ssr/video?page=2',
+        'info_dict': {
+            'id': 'amiami-ssr',
+            'title': 'あみあみSSRチャンネル',
+        },
+        'playlist_count': 20,
+    }, {
+        'url': 'https://ch.nicovideo.jp/yukarisama/video/pay',
+        'info_dict': {
+            'id': 'yukarisama',
+            'title': '縁結びのゆかり様',
+        },
+        'playlist_mincount': 16,
+    }, {
+        'url': 'https://ch.nicovideo.jp/mokou1/video/member',
+        'info_dict': {
+            'id': 'mokou1',
+            'title': 'もこう。',
+        },
+        'playlist_mincount': 49,
+    }, {
+        'url': 'https://ch.nicovideo.jp/amiami-ch/video/continuation',
+        'info_dict': {
+            'id': 'amiami-ch',
+            'title': 'あみあみチャンネル',
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://ch.nicovideo.jp/search/%E3%81%AF%E3%81%AA%E3%81%BE%E3%81%8D%E3%81%93%E3%82%82%E3%81%A1%E3%81%83?channel_id=ch2585696&type=video',
+        'info_dict': {
+            'id': 'secondshot',
+            'title': 'セカンドショットちゃんねる - はなまきこもちぃ',
+        },
+        'playlist_mincount': 115,
+    }, {
+        'url': 'https://ch.nicovideo.jp/amiami-ch/video/so44060088',
+        'only_matching': True,
+    }]
+
+    def _fetch_page(self, url, playlist_id, page):
+        page += 1
+        webpage = self._download_webpage(
+            url, playlist_id, f'Downloading page {page}', query={'page': page})
+        for url in traverse_obj(webpage, (
+            {find_elements(cls='watchLink', html=True)},
+            ..., {extract_attributes}, 'href', {url_or_none},
+        )):
+            yield self.url_result(url, NiconicoIE)
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        display_id = urllib.parse.unquote(mobj.group('id'))
+        playlist_type = mobj.group('type')
+
+        if playlist_type == 'search':
+            keyword = display_id
+            page_size = self._SEARCH_PAGE_SIZE
+        else:
+            if (slug := mobj.group('slug')) and slug.startswith('so'):
+                return self.url_result(
+                    f'{self._BASE_URL}/watch/{slug}', NiconicoIE)
+            keyword = None
+            page_size = self._PAGE_SIZE
+
+        webpage = self._download_webpage(url, display_id)
+        channel_name = traverse_obj(webpage, (
+            {find_element(cls='channel_name')}, {find_element(tag='a', html=True)},
+            {extract_attributes}, 'href', {str}, {trim_str(start='/')}, filter))
+        site_name = self._og_search_property('site_name', webpage, default=None)
+
+        fetch_page = functools.partial(self._fetch_page, url, display_id)
+        page = traverse_obj(parse_qs(url), ('page', -1, {int_or_none}))
+        entries = fetch_page(page - 1) if page else OnDemandPagedList(fetch_page, page_size)
+
+        return self.playlist_result(
+            entries, channel_name, join_nonempty(site_name, keyword, delim=' - '))

yt_dlp/extractor/showroomlive.py

@@ -17,7 +17,11 @@ from ..utils.traversal import (
 )
 
 
-class ShowRoomLiveIE(InfoExtractor):
+class ShowRoomBaseIE(InfoExtractor):
+    _API_BASE = 'https://www.showroom-live.com/api'
+
+
+class ShowRoomLiveIE(ShowRoomBaseIE):
     IE_NAME = 'showroom:live'
     IE_DESC = 'SHOWROOM'
 
@@ -29,19 +33,11 @@ class ShowRoomLiveIE(InfoExtractor):
 
     def _real_extract(self, url):
         broadcaster_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, broadcaster_id, headers={'Accept-Language': 'ja'})
-        nuxt_data = self._search_nuxt_json(webpage, broadcaster_id)['data']
-
-        cookies = self._get_cookies(url)
-        sr_id = traverse_obj(cookies, ('sr_id', 'value', {str}, filter))
-        if not sr_id:
-            self.raise_login_required()
-
-        room_profile = traverse_obj(nuxt_data, (
-            f'roomProfile-{broadcaster_id}-{sr_id}', {dict}))
-        start_timestamp = traverse_obj(room_profile, ('current_live_started_at', {int_or_none}))
-        is_live = traverse_obj(room_profile, ('is_onlive', {bool}))
+        room_status = self._download_json(
+            f'{self._API_BASE}/room/status', broadcaster_id,
+            query={'room_url_key': broadcaster_id})
+        start_timestamp = traverse_obj(room_status, ('started_at', {int_or_none}))
+        is_live = traverse_obj(room_status, ('is_live', {bool}))
 
         if not is_live:
             if start_timestamp:
@@ -59,12 +55,15 @@ class ShowRoomLiveIE(InfoExtractor):
                 }
             raise UserNotLive(video_id=broadcaster_id)
 
-        room_id = traverse_obj(room_profile, ('room_id', {str_or_none}))
+        room_id = traverse_obj(room_status, ('room_id', {str_or_none}))
+        room_profile = self._download_json(
+            f'{self._API_BASE}/room/profile',
+            broadcaster_id, query={'room_id': room_id})
         room_name = traverse_obj(room_profile, (
             ('room_name', 'main_name'), {clean_html}, filter, any))
 
         streaming_url_list = self._download_json(
-            'https://www.showroom-live.com/api/live/streaming_url',
+            f'{self._API_BASE}/live/streaming_url',
             broadcaster_id, query={'room_id': room_id})
         m3u8_url = traverse_obj(streaming_url_list, (
             'streaming_url_list', lambda _, v: v['type'] == 'hls_all',
@@ -84,13 +83,13 @@ class ShowRoomLiveIE(InfoExtractor):
                 'description': ('description', {clean_html}, filter),
                 'genres': ('genre_name', {clean_html}, filter, all, filter),
                 'tags': ('live_tags', ..., 'name', {clean_html}, filter, all, filter),
-                'thumbnail': ('image_square', {url_or_none}),
+                'thumbnail': (('image_square', 'image'), {url_or_none}, any),
                 'view_count': ('view_num', {int_or_none}),
             }),
         }
 
 
-class ShowRoomVodIE(InfoExtractor):
+class ShowRoomVodIE(ShowRoomBaseIE):
     IE_NAME = 'showroom:vod'
 
     _VALID_URL = r'https?://(?:www\.)?showroom-live\.com/episode/watch\?(?:[^#]+&)?id=(?P<id>\w+)'
@@ -111,7 +110,7 @@ class ShowRoomVodIE(InfoExtractor):
             {extract_attributes}, 'data-episode', {json.loads}, {dict}))
 
         streaming_url_list = self._download_json(
-            'https://www.showroom-live.com/api/episode/streaming_url',
+            f'{self._API_BASE}/episode/streaming_url',
             episode_id, query={'episode_id': episode_id})
         m3u8_url = traverse_obj(streaming_url_list, (
             'streaming_url_list', 'hls_all',

yt_dlp/extractor/ted.py

@@ -46,7 +46,7 @@ class TedTalkIE(TedBaseIE):
         webpage = self._download_webpage(url, display_id)
         talk_info = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['videoData']
         video_id = talk_info['id']
-        player_data = self._parse_json(talk_info.get('playerData'), video_id)
+        player_data = talk_info.get('videoPlayerData') or {}
 
         http_url = None
         formats, subtitles = [], {}
@@ -193,8 +193,8 @@ class TedPlaylistIE(TedBaseIE):
         'url': 'https://www.ted.com/playlists/171/the_most_popular_talks_of_all',
         'info_dict': {
             'id': '171',
-            'title': 'The most popular talks of all time',
-            'description': 'md5:d2f22831dc86c7040e733a3cb3993d78',
+            'title': 'The most popular TED Talks of all time',
+            'description': 'md5:5346ef094754d2edd7e1a4cd3a166168',
         },
         'playlist_mincount': 25,
     }]

yt_dlp/extractor/tubitv.py

@@ -1,23 +1,17 @@
-import re
-
 from .common import InfoExtractor
-from ..networking import Request
 from ..utils import (
     ExtractorError,
     int_or_none,
     js_to_json,
     strip_or_none,
-    traverse_obj,
     url_or_none,
-    urlencode_postdata,
 )
+from ..utils.traversal import require, traverse_obj
 
 
 class TubiTvIE(InfoExtractor):
     IE_NAME = 'tubitv'
     _VALID_URL = r'https?://(?:www\.)?tubitv\.com/(?:[a-z]{2}-[a-z]{2}/)?(?P<type>video|movies|tv-shows)/(?P<id>\d+)'
-    _LOGIN_URL = 'http://tubitv.com/login'
-    _NETRC_MACHINE = 'tubitv'
     _TESTS = [{
         'url': 'https://tubitv.com/movies/100004539/the-39-steps',
         'info_dict': {
@@ -27,7 +21,7 @@ class TubiTvIE(InfoExtractor):
             'description': 'md5:bb2f2dd337f0dc58c06cb509943f54c8',
             'uploader_id': 'abc2558d54505d4f0f32be94f2e7108c',
             'release_year': 1935,
-            'thumbnail': r're:^https?://.+\.(jpe?g|png)$',
+            'thumbnail': r're:^https?://canvas-lb\.tubitv\.com/.+',
             'duration': 5187,
         },
         'params': {'skip_download': 'm3u8'},
@@ -44,7 +38,7 @@ class TubiTvIE(InfoExtractor):
             'season_number': 1,
             'uploader_id': '2a9273e728c510d22aa5c57d0646810b',
             'release_year': 2011,
-            'thumbnail': r're:^https?://.+\.(jpe?g|png)$',
+            'thumbnail': r're:^https?://canvas-lb\.tubitv\.com/.+',
             'duration': 1376,
         },
         'params': {'skip_download': 'm3u8'},
@@ -82,24 +76,11 @@ class TubiTvIE(InfoExtractor):
     _UNPLAYABLE_FORMATS = ('hlsv6_widevine', 'hlsv6_widevine_nonclearlead', 'hlsv6_playready_psshv0',
                            'hlsv6_fairplay', 'dash_widevine', 'dash_widevine_nonclearlead')
 
-    def _perform_login(self, username, password):
-        self.report_login()
-        form_data = {
-            'username': username,
-            'password': password,
-        }
-        payload = urlencode_postdata(form_data)
-        request = Request(self._LOGIN_URL, payload)
-        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
-        login_page = self._download_webpage(
-            request, None, False, 'Wrong login info')
-        if not re.search(r'id="tubi-logout"', login_page):
-            raise ExtractorError(
-                'Login failed (invalid username/password)', expected=True)
-
     def _real_extract(self, url):
         video_id, video_type = self._match_valid_url(url).group('id', 'type')
-        webpage = self._download_webpage(f'https://tubitv.com/{video_type}/{video_id}/', video_id)
+        webpage = self._download_webpage(
+            f'https://tubitv.com/{video_type}/{video_id}/', video_id,
+            headers=self.geo_verification_headers())
         video_data = self._search_json(
             r'window\.__data\s*=', webpage, 'data', video_id,
             transform_source=js_to_json)['video']['byId'][video_id]
@@ -113,7 +94,11 @@ class TubiTvIE(InfoExtractor):
             if resource_type == 'dash':
                 formats.extend(self._extract_mpd_formats(manifest_url, video_id, mpd_id=resource_type, fatal=False))
             elif resource_type in ('hlsv3', 'hlsv6'):
-                formats.extend(self._extract_m3u8_formats(manifest_url, video_id, 'mp4', m3u8_id=resource_type, fatal=False))
+                fmts = self._extract_m3u8_formats(manifest_url, video_id, 'mp4', m3u8_id=resource_type, fatal=False)
+                for fmt in fmts:
+                    if 'Audio Description' in fmt.get('format_note', ''):
+                        fmt['language_preference'] = -10
+                formats.extend(fmts)
             elif resource_type in self._UNPLAYABLE_FORMATS:
                 drm_formats = True
             else:
@@ -162,31 +147,34 @@ class TubiTvShowIE(InfoExtractor):
             'id': 'the-joy-of-painting-with-bob-ross',
         },
     }, {
-        'url': 'https://tubitv.com/series/2311/the-saddle-club/season-1',
+        'url': 'https://tubitv.com/series/300000435/the-saddle-club/season-1',
         'playlist_count': 26,
         'info_dict': {
             'id': 'the-saddle-club-season-1',
         },
     }, {
-        'url': 'https://tubitv.com/series/2311/the-saddle-club/season-3',
-        'playlist_count': 19,
+        'url': 'https://tubitv.com/series/300000435/the-saddle-club/season-2',
+        'playlist_count': 26,
         'info_dict': {
-            'id': 'the-saddle-club-season-3',
+            'id': 'the-saddle-club-season-2',
         },
     }, {
-        'url': 'https://tubitv.com/series/2311/the-saddle-club/',
-        'playlist_mincount': 71,
+        'url': 'https://tubitv.com/series/300000435/the-saddle-club/',
+        'playlist_mincount': 52,
         'info_dict': {
             'id': 'the-saddle-club',
         },
     }]
 
     def _entries(self, show_url, playlist_id, selected_season):
-        webpage = self._download_webpage(show_url, playlist_id)
+        webpage = self._download_webpage(show_url, playlist_id, headers=self.geo_verification_headers())
 
-        data = self._search_json(
-            r'window\.__REACT_QUERY_STATE__\s*=', webpage, 'data', playlist_id,
-            transform_source=js_to_json)['queries'][0]['state']['data']
+        react_query_state = self._search_json(
+            r'window\.__REACT_QUERY_STATE__\s*=', webpage,
+            'react query state', playlist_id, transform_source=js_to_json)
+        data = traverse_obj(react_query_state, (
+            'queries', lambda _, v: v['state']['data']['seasons'][0],
+            'state', 'data', any, {require('season data')}))
 
         # v['number'] is already a decimal string, but stringify to protect against API changes
         path = [lambda _, v: str(v['number']) == selected_season] if selected_season else [..., {dict}]

yt_dlp/extractor/twitch.py

@@ -44,7 +44,7 @@ class TwitchBaseIE(InfoExtractor):
         'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a',
         'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f',
         'ClipsCards__User': '1cd671bfa12cec480499c087319f26d21925e9695d1f80225aae6a4354f23088',
-        'ShareClipRenderStatus': '0a02bb974443b576f5579aab0fef1d4b7f44e58a8a256f0c5adfead0db70640f',
+        'ShareClipRenderStatus': '2db6a3b20eabf510bd3cf465ae2408834b59eb6b8af89ca73ab1486cacecfb63',
         'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077',
         'StreamMetadata': 'ad022ca32220d5523d03a23cbcb5beaa1e0999889c1f8f78f9f2520dafb5cae6',
         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',

yt_dlp/extractor/youtube/pot/_director.py

@@ -76,7 +76,7 @@ class YoutubeIEContentProviderLogger(IEContentProviderLogger):
         if self.log_level <= self.LogLevel.ERROR:
             self.__ie._downloader.report_error(
                 self._format_msg(message), is_error=False,
-                tb=''.join(traceback.format_exception(None, cause, cause.__traceback__)) if cause else None)
+                tb=''.join(traceback.format_exception(cause)) if cause else None)
 
 
 class PoTokenCache:

yt_dlp/networking/_helper.py

@@ -108,8 +108,6 @@ def make_ssl_context(
     context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
     context.check_hostname = verify
     context.verify_mode = ssl.CERT_REQUIRED if verify else ssl.CERT_NONE
-    # OpenSSL 1.1.1+ Python 3.8+ keylog file
-    if hasattr(context, 'keylog_filename'):
     context.keylog_filename = os.environ.get('SSLKEYLOGFILE') or None
 
     # Some servers may reject requests if ALPN extension is not sent. See:

yt_dlp/networking/_requests.py

@@ -123,7 +123,7 @@ class RequestsResponseAdapter(Response):
         # Work around issue with `.read(amt)` then `.read()`
         # See: https://github.com/urllib3/urllib3/issues/3636
         if amt is None:
-            # compat: py3.9: Python 3.9 preallocates the whole read buffer, read in chunks
+            # Read in chunks to avoid preallocating a large buffer
             read_chunk = functools.partial(self.fp.read, 1 << 20, decode_content=True)
             return b''.join(iter(read_chunk, b''))
         # Interact with urllib3 response directly.

yt_dlp/networking/_urllib.py

@@ -296,13 +296,9 @@ class UrllibResponseAdapter(Response):
     """
 
     def __init__(self, res: http.client.HTTPResponse | urllib.response.addinfourl):
-        # addinfourl: In Python 3.9+, .status was introduced and .getcode() was deprecated [1]
-        # HTTPResponse: .getcode() was deprecated, .status always existed [2]
-        # 1. https://docs.python.org/3/library/urllib.request.html#urllib.response.addinfourl.getcode
-        # 2. https://docs.python.org/3.10/library/http.client.html#http.client.HTTPResponse.status
         super().__init__(
             fp=res, headers=res.headers, url=res.url,
-            status=getattr(res, 'status', None) or res.getcode(), reason=getattr(res, 'reason', None))
+            status=res.status, reason=getattr(res, 'reason', None))
 
     def read(self, amt=None):
         if self.closed:

yt_dlp/update.py

@@ -227,6 +227,10 @@ def _make_label(origin, tag, version=None):
     return f'{origin}@{tag}'
 
 
+class _GitHubError(Exception):
+    pass
+
+
 @dataclass
 class UpdateInfo:
     """
@@ -319,7 +323,14 @@ class Updater:
         path = 'latest/download' if tag == 'latest' else f'download/{tag}'
         url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
         self.ydl.write_debug(f'Downloading {name} from {url}')
-        return self.ydl.urlopen(url).read()
+        response = self.ydl.urlopen(url)
+        # GitHub may send an error webpage response with HTTP status 200
+        # See https://github.com/yt-dlp/yt-dlp/issues/17550
+        prefix = response.read(512)
+        if b'<!DOCTYPE html>' in prefix:
+            raise _GitHubError('got error webpage instead of release asset')
+
+        return prefix + response.read()
 
     def _call_api(self, tag):
         tag = f'tags/{tag}' if tag != 'latest' else tag
@@ -358,7 +369,7 @@ class Updater:
         for tag in source_tags:
             try:
                 return self._download_asset('_update_spec', tag=tag).decode()
-            except network_exceptions as error:
+            except (_GitHubError, *network_exceptions) as error:
                 if isinstance(error, HTTPError) and error.status == 404:
                     continue
                 self._report_network_error(f'fetch update spec: {error}')
@@ -462,7 +473,7 @@ class Updater:
         if not is_non_updateable():
             try:
                 hashes = self._download_asset('SHA2-256SUMS', result_tag)
-            except network_exceptions as error:
+            except (_GitHubError, *network_exceptions) as error:
                 if not isinstance(error, HTTPError) or error.status != 404:
                     self._report_network_error(f'fetch checksums: {error}')
                     return None
@@ -525,7 +536,7 @@ class Updater:
 
         try:
             newcontent = self._download_asset(update_info.binary_name, update_info.tag)
-        except network_exceptions as e:
+        except (_GitHubError, *network_exceptions) as e:
             if isinstance(e, HTTPError) and e.status == 404:
                 return self._report_error(
                     f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
@@ -609,7 +620,7 @@ class Updater:
         self.ydl._download_retcode = 100
 
     def _report_permission_error(self, file):
-        self._report_error(f'Unable to write to {file}; try running as administrator', True)
+        self._report_error(f'Insufficient permissions to write to {file}', True)
 
     def _report_network_error(self, action, delim=';', tag=None):
         if not tag:

yt_dlp/utils/_utils.py

@@ -238,11 +238,9 @@ def find_xpath_attr(node, xpath, key, val=None):
     expr = xpath + (f'[@{key}]' if val is None else f"[@{key}='{val}']")
     return node.find(expr)
 
-# On python2.6 the xml.etree.ElementTree.Element methods don't support
-# the namespace parameter
-
 
 def xpath_with_ns(path, ns_map):
+    """Expand namespace-prefixed names to Clark notation."""
     components = [c.split(':') for c in path.split('/')]
     replaced = []
     for c in components:
@@ -876,7 +874,7 @@ class Popen(subprocess.Popen):
 
         self.__text_mode = kwargs.get('encoding') or kwargs.get('errors') or text or kwargs.get('universal_newlines')
         if text is True:
-            kwargs['universal_newlines'] = True  # For 3.6 compatibility
+            kwargs['text'] = True
             kwargs.setdefault('encoding', 'utf-8')
             kwargs.setdefault('errors', 'replace')
 
@@ -1012,7 +1010,7 @@ class ExtractorError(YoutubeDLError):
     def format_traceback(self):
         return join_nonempty(
             self.traceback and ''.join(traceback.format_tb(self.traceback)),
-            self.cause and ''.join(traceback.format_exception(None, self.cause, self.cause.__traceback__)[1:]),
+            self.cause and ''.join(traceback.format_exception(self.cause)[1:]),
             delim='\n') or None
 
     def __setattr__(self, name, value):
@@ -1960,11 +1958,7 @@ def setproctitle(title):
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:
         return
-    except TypeError:
-        # LoadLibrary in Windows Python 2.7.13 only expects
-        # a bytestring, but since unicode_literals turns
-        # every string into a unicode string, it fails.
-        return
+
     title_bytes = title.encode()
     buf = ctypes.create_string_buffer(len(title_bytes))
     buf.value = title_bytes
@@ -2655,11 +2649,10 @@ def multipart_encode(data, boundary=None):
     Encode a dict to RFC 7578-compliant form-data
 
     data:
-        A dict where keys and values can be either Unicode or bytes-like
-        objects.
+        A dict where keys and values can be either str or bytes-like objects.
     boundary:
-        If specified a Unicode object, it's used as the boundary. Otherwise
-        a random boundary is generated.
+        An ASCII string to use as the boundary. If omitted, a random boundary
+        is generated.
 
     Reference: https://tools.ietf.org/html/rfc7578
     """
@@ -3422,7 +3415,7 @@ def ass_subtitles_timecode(seconds):
 def dfxp2srt(dfxp_data):
     """
     @param dfxp_data A bytes-like object containing DFXP data
-    @returns A unicode object containing converted SRT data
+    @returns A string containing the converted SRT data
     """
     LEGACY_NAMESPACES = (
         (b'http://www.w3.org/ns/ttml', [