Merge remote-tracking branch 'origin' into yt-live-from-start-range

yt_dlp/YoutubeDL.py

@@ -13,6 +13,7 @@ import os
 import random
 import re
 import shutil
+import string
 import subprocess
 import sys
 import tempfile
@@ -20,10 +21,9 @@ import time
 import tokenize
 import traceback
 import unicodedata
-import urllib.request
-from string import Formatter, ascii_letters
 
 from .cache import Cache
+from .compat import urllib  # isort: split
 from .compat import compat_os_name, compat_shlex_quote
 from .cookies import load_cookies
 from .downloader import (
@@ -129,7 +129,6 @@ from .utils import (
     parse_filesize,
     preferredencoding,
     prepend_extension,
-    register_socks_protocols,
     remove_terminal_sequences,
     render_table,
     replace_extension,
@@ -195,6 +194,7 @@ class YoutubeDL:
     ap_username:       Multiple-system operator account username.
     ap_password:       Multiple-system operator account password.
     usenetrc:          Use netrc for authentication instead.
+    netrc_location:    Location of the netrc file. Defaults to ~/.netrc.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.
     no_warnings:       Do not print out anything for warnings.
@@ -285,7 +285,7 @@ class YoutubeDL:
                        subtitles. The language can be prefixed with a "-" to
                        exclude it from the requested languages, e.g. ['all', '-live_chat']
     keepvideo:         Keep the video file after post-processing
-    daterange:         A DateRange object, download only if the upload_date is in the range.
+    daterange:         A utils.DateRange object, download only if the upload_date is in the range.
     skip_download:     Skip the actual download of the video file
     cachedir:          Location of the cache files in the filesystem.
                        False to disable filesystem cache.
@@ -334,13 +334,13 @@ class YoutubeDL:
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.
     extract_flat:      Whether to resolve and process url_results further
-                       * False:     Always process (default)
+                       * False:     Always process. Default for API
                        * True:      Never process
                        * 'in_playlist': Do not process inside playlist/multi_video
                        * 'discard': Always process, but don't return the result
                                     from inside playlist/multi_video
                        * 'discard_in_playlist': Same as "discard", but only for
-                                    playlists (not multi_video)
+                                    playlists (not multi_video). Default for CLI
     wait_for_video:    If given, wait for scheduled streams to become available.
                        The value should be a tuple containing the range
                        (min_secs, max_secs) to wait between retries
@@ -420,7 +420,12 @@ class YoutubeDL:
                        - Raise utils.DownloadCancelled(msg) to abort remaining
                          downloads when a video is rejected.
                        match_filter_func in utils.py is one example for this.
-    no_color:          Do not emit color codes in output.
+    color:             A Dictionary with output stream names as keys
+                       and their respective color policy as values.
+                       Can also just be a single color policy,
+                       in which case it applies to all outputs.
+                       Valid stream names are 'stdout' and 'stderr'.
+                       Valid color policies are one of 'always', 'auto', 'no_color' or 'never'.
     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
                        HTTP header
     geo_bypass_country:
@@ -477,7 +482,7 @@ class YoutubeDL:
                        can also be used
 
     The following options are used by the extractors:
-    extractor_retries: Number of times to retry for known errors
+    extractor_retries: Number of times to retry for known errors (default: 3)
     dynamic_mpd:       Whether to process dynamic DASH manifests (default: True)
     hls_split_discontinuity: Split HLS playlists to different formats at
                        discontinuities such as ad breaks (default: False)
@@ -542,6 +547,7 @@ class YoutubeDL:
                        data will be downloaded and processed by extractor.
                        You can reduce network I/O by disabling it if you don't
                        care about HLS. (only for youtube)
+    no_color:          Same as `color='no_color'`
     """
 
     _NUMERIC_FIELDS = {
@@ -608,9 +614,24 @@ class YoutubeDL:
         except Exception as e:
             self.write_debug(f'Failed to enable VT mode: {e}')
 
+        if self.params.get('no_color'):
+            if self.params.get('color') is not None:
+                self.report_warning('Overwriting params from "color" with "no_color"')
+            self.params['color'] = 'no_color'
+
+        term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
+
+        def process_color_policy(stream):
+            stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream]
+            policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
+            if policy in ('auto', None):
+                return term_allow_color and supports_terminal_sequences(stream)
+            assert policy in ('always', 'never', 'no_color')
+            return {'always': True, 'never': False}.get(policy, policy)
+
         self._allow_colors = Namespace(**{
-            type_: not self.params.get('no_color') and supports_terminal_sequences(stream)
-            for type_, stream in self._out_files.items_ if type_ != 'console'
+            name: process_color_policy(stream)
+            for name, stream in self._out_files.items_ if name != 'console'
         })
 
         # The code is left like this to be reused for future deprecations
@@ -743,7 +764,6 @@ class YoutubeDL:
                 when=when)
 
         self._setup_opener()
-        register_socks_protocols()
 
         def preload_download_archive(fn):
             """Preload the archive, if any is specified"""
@@ -980,7 +1000,7 @@ class YoutubeDL:
             text = text.encode(encoding, 'ignore').decode(encoding)
             if fallback is not None and text != original_text:
                 text = fallback
-        return format_text(text, f) if allow_colors else text if fallback is None else fallback
+        return format_text(text, f) if allow_colors is True else text if fallback is None else fallback
 
     def _format_out(self, *args, **kwargs):
         return self._format_text(self._out_files.out, self._allow_colors.out, *args, **kwargs)
@@ -1083,7 +1103,7 @@ class YoutubeDL:
         # correspondingly that is not what we want since we need to keep
         # '%%' intact for template dict substitution step. Working around
         # with boundary-alike separator hack.
-        sep = ''.join(random.choices(ascii_letters, k=32))
+        sep = ''.join(random.choices(string.ascii_letters, k=32))
         outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$')
 
         # outtmpl should be expand_path'ed before template dict substitution
@@ -1242,7 +1262,7 @@ class YoutubeDL:
                 return list(obj)
             return repr(obj)
 
-        class _ReplacementFormatter(Formatter):
+        class _ReplacementFormatter(string.Formatter):
             def get_field(self, field_name, args, kwargs):
                 if field_name.isdigit():
                     return args[0], -1
@@ -2072,86 +2092,86 @@ class YoutubeDL:
 
         def _parse_filter(tokens):
             filter_parts = []
-            for type, string, start, _, _ in tokens:
-                if type == tokenize.OP and string == ']':
+            for type, string_, start, _, _ in tokens:
+                if type == tokenize.OP and string_ == ']':
                     return ''.join(filter_parts)
                 else:
-                    filter_parts.append(string)
+                    filter_parts.append(string_)
 
         def _remove_unused_ops(tokens):
             # Remove operators that we don't use and join them with the surrounding strings.
             # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
             ALLOWED_OPS = ('/', '+', ',', '(', ')')
             last_string, last_start, last_end, last_line = None, None, None, None
-            for type, string, start, end, line in tokens:
-                if type == tokenize.OP and string == '[':
+            for type, string_, start, end, line in tokens:
+                if type == tokenize.OP and string_ == '[':
                     if last_string:
                         yield tokenize.NAME, last_string, last_start, last_end, last_line
                         last_string = None
-                    yield type, string, start, end, line
+                    yield type, string_, start, end, line
                     # everything inside brackets will be handled by _parse_filter
-                    for type, string, start, end, line in tokens:
-                        yield type, string, start, end, line
-                        if type == tokenize.OP and string == ']':
+                    for type, string_, start, end, line in tokens:
+                        yield type, string_, start, end, line
+                        if type == tokenize.OP and string_ == ']':
                             break
-                elif type == tokenize.OP and string in ALLOWED_OPS:
+                elif type == tokenize.OP and string_ in ALLOWED_OPS:
                     if last_string:
                         yield tokenize.NAME, last_string, last_start, last_end, last_line
                         last_string = None
-                    yield type, string, start, end, line
+                    yield type, string_, start, end, line
                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
                     if not last_string:
-                        last_string = string
+                        last_string = string_
                         last_start = start
                         last_end = end
                     else:
-                        last_string += string
+                        last_string += string_
             if last_string:
                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 
         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
-            for type, string, start, _, _ in tokens:
+            for type, string_, start, _, _ in tokens:
                 # ENCODING is only defined in python 3.x
                 if type == getattr(tokenize, 'ENCODING', None):
                     continue
                 elif type in [tokenize.NAME, tokenize.NUMBER]:
-                    current_selector = FormatSelector(SINGLE, string, [])
+                    current_selector = FormatSelector(SINGLE, string_, [])
                 elif type == tokenize.OP:
-                    if string == ')':
+                    if string_ == ')':
                         if not inside_group:
                             # ')' will be handled by the parentheses group
                             tokens.restore_last_token()
                         break
-                    elif inside_merge and string in ['/', ',']:
+                    elif inside_merge and string_ in ['/', ',']:
                         tokens.restore_last_token()
                         break
-                    elif inside_choice and string == ',':
+                    elif inside_choice and string_ == ',':
                         tokens.restore_last_token()
                         break
-                    elif string == ',':
+                    elif string_ == ',':
                         if not current_selector:
                             raise syntax_error('"," must follow a format selector', start)
                         selectors.append(current_selector)
                         current_selector = None
-                    elif string == '/':
+                    elif string_ == '/':
                         if not current_selector:
                             raise syntax_error('"/" must follow a format selector', start)
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
-                    elif string == '[':
+                    elif string_ == '[':
                         if not current_selector:
                             current_selector = FormatSelector(SINGLE, 'best', [])
                         format_filter = _parse_filter(tokens)
                         current_selector.filters.append(format_filter)
-                    elif string == '(':
+                    elif string_ == '(':
                         if current_selector:
                             raise syntax_error('Unexpected "("', start)
                         group = _parse_format_selection(tokens, inside_group=True)
                         current_selector = FormatSelector(GROUP, group, [])
-                    elif string == '+':
+                    elif string_ == '+':
                         if not current_selector:
                             raise syntax_error('Unexpected "+"', start)
                         selector_1 = current_selector
@@ -2160,7 +2180,7 @@ class YoutubeDL:
                             raise syntax_error('Expected a selector', start)
                         current_selector = FormatSelector(MERGE, (selector_1, selector_2), [])
                     else:
-                        raise syntax_error(f'Operator not recognized: "{string}"', start)
+                        raise syntax_error(f'Operator not recognized: "{string_}"', start)
                 elif type == tokenize.ENDMARKER:
                     break
             if current_selector:
@@ -2386,8 +2406,10 @@ class YoutubeDL:
 
     def _calc_headers(self, info_dict):
         res = merge_headers(self.params['http_headers'], info_dict.get('http_headers') or {})
-
-        cookies = self._calc_cookies(info_dict['url'])
+        if 'Youtubedl-No-Compression' in res:  # deprecated
+            res.pop('Youtubedl-No-Compression', None)
+            res['Accept-Encoding'] = 'identity'
+        cookies = self.cookiejar.get_cookie_header(info_dict['url'])
         if cookies:
             res['Cookie'] = cookies
 
@@ -2399,9 +2421,8 @@ class YoutubeDL:
         return res
 
     def _calc_cookies(self, url):
-        pr = sanitized_Request(url)
-        self.cookiejar.add_cookie_header(pr)
-        return pr.get_header('Cookie')
+        self.deprecation_warning('"YoutubeDL._calc_cookies" is deprecated and may be removed in a future version')
+        return self.cookiejar.get_cookie_header(url)
 
     def _sort_thumbnails(self, thumbnails):
         thumbnails.sort(key=lambda t: (
@@ -2728,21 +2749,22 @@ class YoutubeDL:
             return info_dict
 
         format_selector = self.format_selector
-        if format_selector is None:
-            req_format = self._default_format_spec(info_dict, download=download)
-            self.write_debug('Default format spec: %s' % req_format)
-            format_selector = self.build_format_selector(req_format)
-
         while True:
             if interactive_format_selection:
-                req_format = input(
-                    self._format_screen('\nEnter format selector: ', self.Styles.EMPHASIS))
+                req_format = input(self._format_screen('\nEnter format selector ', self.Styles.EMPHASIS)
+                                   + '(Press ENTER for default, or Ctrl+C to quit)'
+                                   + self._format_screen(': ', self.Styles.EMPHASIS))
                 try:
-                    format_selector = self.build_format_selector(req_format)
+                    format_selector = self.build_format_selector(req_format) if req_format else None
                 except SyntaxError as err:
                     self.report_error(err, tb=False, is_error=False)
                     continue
 
+            if format_selector is None:
+                req_format = self._default_format_spec(info_dict, download=download)
+                self.write_debug(f'Default format spec: {req_format}')
+                format_selector = self.build_format_selector(req_format)
+
             formats_to_download = list(format_selector({
                 'formats': formats,
                 'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats),
@@ -2902,7 +2924,7 @@ class YoutubeDL:
 
             fmt = '%({})s'
             if tmpl.startswith('{'):
-                tmpl = f'.{tmpl}'
+                tmpl, fmt = f'.{tmpl}', '%({})j'
             if tmpl.endswith('='):
                 tmpl, fmt = tmpl[:-1], '{0} = %({0})#j'
             return '\n'.join(map(fmt.format, [tmpl] if mobj.group('dict') else tmpl.split(',')))
@@ -2941,7 +2963,8 @@ class YoutubeDL:
         print_field('url', 'urls')
         print_field('thumbnail', optional=True)
         print_field('description', optional=True)
-        print_field('filename', optional=True)
+        if filename:
+            print_field('filename')
         if self.params.get('forceduration') and info_copy.get('duration') is not None:
             self.to_stdout(formatSeconds(info_copy['duration']))
         print_field('format')
@@ -3422,8 +3445,8 @@ class YoutubeDL:
         if remove_private_keys:
             reject = lambda k, v: v is None or k.startswith('__') or k in {
                 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
-                'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber',
-                '_format_sort_fields',
+                'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
+                'playlist_autonumber', '_format_sort_fields',
             }
         else:
             reject = lambda k, v: False
@@ -3492,7 +3515,7 @@ class YoutubeDL:
                 *files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)')
         return infodict
 
-    def run_all_pps(self, key, info, *, additional_pps=None, fatal=True):
+    def run_all_pps(self, key, info, *, additional_pps=None):
         if key != 'video':
             self._forceprint(key, info)
         for pp in (additional_pps or []) + self._pps[key]:
@@ -3771,9 +3794,14 @@ class YoutubeDL:
 
         def get_encoding(stream):
             ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__))
+            additional_info = []
+            if os.environ.get('TERM', '').lower() == 'dumb':
+                additional_info.append('dumb')
             if not supports_terminal_sequences(stream):
                 from .utils import WINDOWS_VT_MODE  # Must be imported locally
-                ret += ' (No VT)' if WINDOWS_VT_MODE is False else ' (No ANSI)'
+                additional_info.append('No VT' if WINDOWS_VT_MODE is False else 'No ANSI')
+            if additional_info:
+                ret = f'{ret} ({",".join(additional_info)})'
             return ret
 
         encoding_str = 'Encodings: locale %s, fs %s, pref %s, %s' % (
@@ -3998,7 +4026,7 @@ class YoutubeDL:
             # that way it will silently go on when used with unsupporting IE
             return ret
         elif not subtitles:
-            self.to_screen('[info] There\'s no subtitles for the requested languages')
+            self.to_screen('[info] There are no subtitles for the requested languages')
             return ret
         sub_filename_base = self.prepare_filename(info_dict, 'subtitle')
         if not sub_filename_base:
@@ -4052,7 +4080,7 @@ class YoutubeDL:
         if write_all or self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails') or []
             if not thumbnails:
-                self.to_screen(f'[info] There\'s no {label} thumbnails to download')
+                self.to_screen(f'[info] There are no {label} thumbnails to download')
                 return ret
         multiple = write_all and len(thumbnails) > 1
 

yt_dlp/__init__.py

@@ -14,6 +14,7 @@ import os
 import re
 import sys
 import time
+import traceback
 
 from .compat import compat_shlex_quote
 from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
@@ -451,6 +452,10 @@ def validate_options(opts):
         elif ed and proto == 'default':
             default_downloader = ed.get_basename()
 
+    for policy in opts.color.values():
+        if policy not in ('always', 'auto', 'no_color', 'never'):
+            raise ValueError(f'"{policy}" is not a valid color policy')
+
     warnings, deprecation_warnings = [], []
 
     # Common mistake: -f best
@@ -909,7 +914,7 @@ def parse_options(argv=None):
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': opts.match_filter,
-        'no_color': opts.no_color,
+        'color': opts.color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'hls_use_mpegts': opts.hls_use_mpegts,
@@ -953,14 +958,18 @@ def _real_main(argv=None):
         if opts.rm_cachedir:
             ydl.cache.remove()
 
-        updater = Updater(ydl, opts.update_self if isinstance(opts.update_self, str) else None)
-        if opts.update_self and updater.update() and actual_use:
-            if updater.cmd:
-                return updater.restart()
-            # This code is reachable only for zip variant in py < 3.10
-            # It makes sense to exit here, but the old behavior is to continue
-            ydl.report_warning('Restart yt-dlp to use the updated version')
-            # return 100, 'ERROR: The program must exit for the update to complete'
+        try:
+            updater = Updater(ydl, opts.update_self)
+            if opts.update_self and updater.update() and actual_use:
+                if updater.cmd:
+                    return updater.restart()
+                # This code is reachable only for zip variant in py < 3.10
+                # It makes sense to exit here, but the old behavior is to continue
+                ydl.report_warning('Restart yt-dlp to use the updated version')
+                # return 100, 'ERROR: The program must exit for the update to complete'
+        except Exception:
+            traceback.print_exc()
+            ydl._download_retcode = 100
 
         if not actual_use:
             if pre_process:

yt_dlp/compat/urllib/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa: F405
+from urllib import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib')
+del passthrough_module

yt_dlp/compat/urllib/request.py

@@ -0,0 +1,40 @@
+# flake8: noqa: F405
+from urllib.request import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib.request')
+del passthrough_module
+
+
+from .. import compat_os_name
+
+if compat_os_name == 'nt':
+    # On older python versions, proxies are extracted from Windows registry erroneously. [1]
+    # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
+    # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
+    # it to http on these older python versions to avoid issues
+    # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported.
+    # 1: https://github.com/python/cpython/issues/86793
+    # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698
+    import sys
+    from urllib.request import getproxies_environment, getproxies_registry
+
+    def getproxies_registry_patched():
+        proxies = getproxies_registry()
+        if (
+            sys.version_info >= (3, 10, 5)  # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
+            or (3, 9, 13) <= sys.version_info < (3, 10)  # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
+        ):
+            return proxies
+
+        for scheme in ('https', 'ftp'):
+            if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
+                proxies[scheme] = 'http' + proxies[scheme][len(scheme):]
+
+        return proxies
+
+    def getproxies():
+        return getproxies_environment() or getproxies_registry_patched()
+
+del compat_os_name

yt_dlp/cookies.py

@@ -1,7 +1,9 @@
 import base64
+import collections
 import contextlib
 import http.cookiejar
 import http.cookies
+import io
 import json
 import os
 import re
@@ -11,6 +13,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import urllib.request
 from datetime import datetime, timedelta, timezone
 from enum import Enum, auto
 from hashlib import pbkdf2_hmac
@@ -29,11 +32,14 @@ from .dependencies import (
 from .minicurses import MultilinePrinter, QuietMultilinePrinter
 from .utils import (
     Popen,
-    YoutubeDLCookieJar,
     error_to_str,
+    escape_url,
     expand_path,
     is_path_like,
+    sanitize_url,
+    str_or_none,
     try_call,
+    write_string,
 )
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
@@ -347,7 +353,9 @@ class ChromeCookieDecryptor:
         Linux:
         - cookies are either v10 or v11
             - v10: AES-CBC encrypted with a fixed key
+                - also attempts empty password if decryption fails
             - v11: AES-CBC encrypted with an OS protected key (keyring)
+                - also attempts empty password if decryption fails
             - v11 keys can be stored in various places depending on the activate desktop environment [2]
 
         Mac:
@@ -362,7 +370,7 @@ class ChromeCookieDecryptor:
 
     Sources:
     - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
-    - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
+    - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_linux.cc
         - KeyStorageLinux::CreateService
     """
 
@@ -384,6 +392,7 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
     def __init__(self, browser_keyring_name, logger, *, keyring=None):
         self._logger = logger
         self._v10_key = self.derive_key(b'peanuts')
+        self._empty_key = self.derive_key(b'')
         self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
         self._browser_keyring_name = browser_keyring_name
         self._keyring = keyring
@@ -396,25 +405,36 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
     @staticmethod
     def derive_key(password):
         # values from
-        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_linux.cc
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 
     def decrypt(self, encrypted_value):
+        """
+
+        following the same approach as the fix in [1]: if cookies fail to decrypt then attempt to decrypt
+        with an empty password. The failure detection is not the same as what chromium uses so the
+        results won't be perfect
+
+        References:
+            - [1] https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/
+                - a bugfix to try an empty password as a fallback
+        """
         version = encrypted_value[:3]
         ciphertext = encrypted_value[3:]
 
         if version == b'v10':
             self._cookie_counts['v10'] += 1
-            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key, self._empty_key), self._logger)
 
         elif version == b'v11':
             self._cookie_counts['v11'] += 1
             if self._v11_key is None:
                 self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
                 return None
-            return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v11_key, self._empty_key), self._logger)
 
         else:
+            self._logger.warning(f'unknown cookie version: "{version}"', only_once=True)
             self._cookie_counts['other'] += 1
             return None
 
@@ -429,7 +449,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
     @staticmethod
     def derive_key(password):
         # values from
-        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 
     def decrypt(self, encrypted_value):
@@ -442,12 +462,12 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
 
-            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+            return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key,), self._logger)
 
         else:
             self._cookie_counts['other'] += 1
             # other prefixes are considered 'old data' which were stored as plaintext
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_mac.mm
             return encrypted_value
 
 
@@ -467,7 +487,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
 
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
             #   kNonceLength
             nonce_length = 96 // 8
             # boringssl
@@ -484,23 +504,27 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
         else:
             self._cookie_counts['other'] += 1
             # any other prefix means the data is DPAPI encrypted
-            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode()
 
 
 def _extract_safari_cookies(profile, logger):
-    if profile is not None:
-        logger.error('safari does not support profiles')
     if sys.platform != 'darwin':
         raise ValueError(f'unsupported platform: {sys.platform}')
 
-    cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
-
-    if not os.path.isfile(cookies_path):
-        logger.debug('Trying secondary cookie location')
-        cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
+    if profile:
+        cookies_path = os.path.expanduser(profile)
         if not os.path.isfile(cookies_path):
-            raise FileNotFoundError('could not find safari cookies database')
+            raise FileNotFoundError('custom safari cookies database not found')
+
+    else:
+        cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
+
+        if not os.path.isfile(cookies_path):
+            logger.debug('Trying secondary cookie location')
+            cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
+            if not os.path.isfile(cookies_path):
+                raise FileNotFoundError('could not find safari cookies database')
 
     with open(cookies_path, 'rb') as f:
         cookies_data = f.read()
@@ -663,27 +687,35 @@ class _LinuxDesktopEnvironment(Enum):
     """
     OTHER = auto()
     CINNAMON = auto()
+    DEEPIN = auto()
     GNOME = auto()
-    KDE = auto()
+    KDE3 = auto()
+    KDE4 = auto()
+    KDE5 = auto()
+    KDE6 = auto()
     PANTHEON = auto()
+    UKUI = auto()
     UNITY = auto()
     XFCE = auto()
+    LXQT = auto()
 
 
 class _LinuxKeyring(Enum):
     """
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.h
     SelectedLinuxBackend
     """
-    KWALLET = auto()
-    GNOMEKEYRING = auto()
-    BASICTEXT = auto()
+    KWALLET4 = auto()  # this value is just called KWALLET in the chromium source but it is for KDE4 only
+    KWALLET5 = auto()
+    KWALLET6 = auto()
+    GNOME_KEYRING = auto()
+    BASIC_TEXT = auto()
 
 
 SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
 
 
-def _get_linux_desktop_environment(env):
+def _get_linux_desktop_environment(env, logger):
     """
     https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
     GetDesktopEnvironment
@@ -698,51 +730,97 @@ def _get_linux_desktop_environment(env):
                 return _LinuxDesktopEnvironment.GNOME
             else:
                 return _LinuxDesktopEnvironment.UNITY
+        elif xdg_current_desktop == 'Deepin':
+            return _LinuxDesktopEnvironment.DEEPIN
         elif xdg_current_desktop == 'GNOME':
             return _LinuxDesktopEnvironment.GNOME
         elif xdg_current_desktop == 'X-Cinnamon':
             return _LinuxDesktopEnvironment.CINNAMON
         elif xdg_current_desktop == 'KDE':
-            return _LinuxDesktopEnvironment.KDE
+            kde_version = env.get('KDE_SESSION_VERSION', None)
+            if kde_version == '5':
+                return _LinuxDesktopEnvironment.KDE5
+            elif kde_version == '6':
+                return _LinuxDesktopEnvironment.KDE6
+            elif kde_version == '4':
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                logger.info(f'unknown KDE version: "{kde_version}". Assuming KDE4')
+                return _LinuxDesktopEnvironment.KDE4
         elif xdg_current_desktop == 'Pantheon':
             return _LinuxDesktopEnvironment.PANTHEON
         elif xdg_current_desktop == 'XFCE':
             return _LinuxDesktopEnvironment.XFCE
+        elif xdg_current_desktop == 'UKUI':
+            return _LinuxDesktopEnvironment.UKUI
+        elif xdg_current_desktop == 'LXQt':
+            return _LinuxDesktopEnvironment.LXQT
+        else:
+            logger.info(f'XDG_CURRENT_DESKTOP is set to an unknown value: "{xdg_current_desktop}"')
+
     elif desktop_session is not None:
-        if desktop_session in ('mate', 'gnome'):
+        if desktop_session == 'deepin':
+            return _LinuxDesktopEnvironment.DEEPIN
+        elif desktop_session in ('mate', 'gnome'):
             return _LinuxDesktopEnvironment.GNOME
-        elif 'kde' in desktop_session:
-            return _LinuxDesktopEnvironment.KDE
-        elif 'xfce' in desktop_session:
+        elif desktop_session in ('kde4', 'kde-plasma'):
+            return _LinuxDesktopEnvironment.KDE4
+        elif desktop_session == 'kde':
+            if 'KDE_SESSION_VERSION' in env:
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                return _LinuxDesktopEnvironment.KDE3
+        elif 'xfce' in desktop_session or desktop_session == 'xubuntu':
             return _LinuxDesktopEnvironment.XFCE
+        elif desktop_session == 'ukui':
+            return _LinuxDesktopEnvironment.UKUI
+        else:
+            logger.info(f'DESKTOP_SESSION is set to an unknown value: "{desktop_session}"')
+
     else:
         if 'GNOME_DESKTOP_SESSION_ID' in env:
             return _LinuxDesktopEnvironment.GNOME
         elif 'KDE_FULL_SESSION' in env:
-            return _LinuxDesktopEnvironment.KDE
+            if 'KDE_SESSION_VERSION' in env:
+                return _LinuxDesktopEnvironment.KDE4
+            else:
+                return _LinuxDesktopEnvironment.KDE3
     return _LinuxDesktopEnvironment.OTHER
 
 
 def _choose_linux_keyring(logger):
     """
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
-    SelectBackend
+    SelectBackend in [1]
+
+    There is currently support for forcing chromium to use BASIC_TEXT by creating a file called
+    `Disable Local Encryption` [1] in the user data dir. The function to write this file (`WriteBackendUse()` [1])
+    does not appear to be called anywhere other than in tests, so the user would have to create this file manually
+    and so would be aware enough to tell yt-dlp to use the BASIC_TEXT keyring.
+
+    References:
+        - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/key_storage_util_linux.cc
     """
-    desktop_environment = _get_linux_desktop_environment(os.environ)
+    desktop_environment = _get_linux_desktop_environment(os.environ, logger)
     logger.debug(f'detected desktop environment: {desktop_environment.name}')
-    if desktop_environment == _LinuxDesktopEnvironment.KDE:
-        linux_keyring = _LinuxKeyring.KWALLET
-    elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
-        linux_keyring = _LinuxKeyring.BASICTEXT
+    if desktop_environment == _LinuxDesktopEnvironment.KDE4:
+        linux_keyring = _LinuxKeyring.KWALLET4
+    elif desktop_environment == _LinuxDesktopEnvironment.KDE5:
+        linux_keyring = _LinuxKeyring.KWALLET5
+    elif desktop_environment == _LinuxDesktopEnvironment.KDE6:
+        linux_keyring = _LinuxKeyring.KWALLET6
+    elif desktop_environment in (
+        _LinuxDesktopEnvironment.KDE3, _LinuxDesktopEnvironment.LXQT, _LinuxDesktopEnvironment.OTHER
+    ):
+        linux_keyring = _LinuxKeyring.BASIC_TEXT
     else:
-        linux_keyring = _LinuxKeyring.GNOMEKEYRING
+        linux_keyring = _LinuxKeyring.GNOME_KEYRING
     return linux_keyring
 
 
-def _get_kwallet_network_wallet(logger):
+def _get_kwallet_network_wallet(keyring, logger):
     """ The name of the wallet used to store network passwords.
 
-    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/kwallet_dbus.cc
     KWalletDBus::NetworkWallet
     which does a dbus call to the following function:
     https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
@@ -750,10 +828,22 @@ def _get_kwallet_network_wallet(logger):
     """
     default_wallet = 'kdewallet'
     try:
+        if keyring == _LinuxKeyring.KWALLET4:
+            service_name = 'org.kde.kwalletd'
+            wallet_path = '/modules/kwalletd'
+        elif keyring == _LinuxKeyring.KWALLET5:
+            service_name = 'org.kde.kwalletd5'
+            wallet_path = '/modules/kwalletd5'
+        elif keyring == _LinuxKeyring.KWALLET6:
+            service_name = 'org.kde.kwalletd6'
+            wallet_path = '/modules/kwalletd6'
+        else:
+            raise ValueError(keyring)
+
         stdout, _, returncode = Popen.run([
             'dbus-send', '--session', '--print-reply=literal',
-            '--dest=org.kde.kwalletd5',
-            '/modules/kwalletd5',
+            f'--dest={service_name}',
+            wallet_path,
             'org.kde.KWallet.networkWallet'
         ], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
 
@@ -768,8 +858,8 @@ def _get_kwallet_network_wallet(logger):
         return default_wallet
 
 
-def _get_kwallet_password(browser_keyring_name, logger):
-    logger.debug('using kwallet-query to obtain password from kwallet')
+def _get_kwallet_password(browser_keyring_name, keyring, logger):
+    logger.debug(f'using kwallet-query to obtain password from {keyring.name}')
 
     if shutil.which('kwallet-query') is None:
         logger.error('kwallet-query command not found. KWallet and kwallet-query '
@@ -777,7 +867,7 @@ def _get_kwallet_password(browser_keyring_name, logger):
                      'included in the kwallet package for your distribution')
         return b''
 
-    network_wallet = _get_kwallet_network_wallet(logger)
+    network_wallet = _get_kwallet_network_wallet(keyring, logger)
 
     try:
         stdout, _, returncode = Popen.run([
@@ -799,8 +889,9 @@ def _get_kwallet_password(browser_keyring_name, logger):
                 # checks hasEntry. To verify this:
                 # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
                 # while starting chrome.
-                # this may be a bug as the intended behaviour is to generate a random password and store
-                # it, but that doesn't matter here.
+                # this was identified as a bug later and fixed in
+                # https://chromium.googlesource.com/chromium/src/+/bbd54702284caca1f92d656fdcadf2ccca6f4165%5E%21/#F0
+                # https://chromium.googlesource.com/chromium/src/+/5463af3c39d7f5b6d11db7fbd51e38cc1974d764
                 return b''
             else:
                 logger.debug('password found')
@@ -838,11 +929,11 @@ def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
     keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
     logger.debug(f'Chosen keyring: {keyring.name}')
 
-    if keyring == _LinuxKeyring.KWALLET:
-        return _get_kwallet_password(browser_keyring_name, logger)
-    elif keyring == _LinuxKeyring.GNOMEKEYRING:
+    if keyring in (_LinuxKeyring.KWALLET4, _LinuxKeyring.KWALLET5, _LinuxKeyring.KWALLET6):
+        return _get_kwallet_password(browser_keyring_name, keyring, logger)
+    elif keyring == _LinuxKeyring.GNOME_KEYRING:
         return _get_gnome_keyring_password(browser_keyring_name, logger)
-    elif keyring == _LinuxKeyring.BASICTEXT:
+    elif keyring == _LinuxKeyring.BASIC_TEXT:
         # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
         return None
     assert False, f'Unknown keyring {keyring}'
@@ -867,6 +958,10 @@ def _get_mac_keyring_password(browser_keyring_name, logger):
 
 
 def _get_windows_v10_key(browser_root, logger):
+    """
+    References:
+        - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/sync/os_crypt_win.cc
+    """
     path = _find_most_recently_used_file(browser_root, 'Local State', logger)
     if path is None:
         logger.error('could not find local state file')
@@ -875,11 +970,13 @@ def _get_windows_v10_key(browser_root, logger):
     with open(path, encoding='utf8') as f:
         data = json.load(f)
     try:
+        # kOsCryptEncryptedKeyPrefName in [1]
         base64_key = data['os_crypt']['encrypted_key']
     except KeyError:
         logger.error('no encrypted key in Local State')
         return None
     encrypted_key = base64.b64decode(base64_key)
+    # kDPAPIKeyPrefix in [1]
     prefix = b'DPAPI'
     if not encrypted_key.startswith(prefix):
         logger.error('invalid key')
@@ -891,13 +988,15 @@ def pbkdf2_sha1(password, salt, iterations, key_length):
     return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
 
 
-def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
-    plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
-    try:
-        return plaintext.decode()
-    except UnicodeDecodeError:
-        logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
-        return None
+def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16):
+    for key in keys:
+        plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
+        try:
+            return plaintext.decode()
+        except UnicodeDecodeError:
+            pass
+    logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
+    return None
 
 
 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
@@ -1091,3 +1190,139 @@ class LenientSimpleCookie(http.cookies.SimpleCookie):
 
             else:
                 morsel = None
+
+
+class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
+    """
+    See [1] for cookie file format.
+
+    1. https://curl.haxx.se/docs/http-cookies.html
+    """
+    _HTTPONLY_PREFIX = '#HttpOnly_'
+    _ENTRY_LEN = 7
+    _HEADER = '''# Netscape HTTP Cookie File
+# This file is generated by yt-dlp.  Do not edit.
+
+'''
+    _CookieFileEntry = collections.namedtuple(
+        'CookieFileEntry',
+        ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
+
+    def __init__(self, filename=None, *args, **kwargs):
+        super().__init__(None, *args, **kwargs)
+        if is_path_like(filename):
+            filename = os.fspath(filename)
+        self.filename = filename
+
+    @staticmethod
+    def _true_or_false(cndn):
+        return 'TRUE' if cndn else 'FALSE'
+
+    @contextlib.contextmanager
+    def open(self, file, *, write=False):
+        if is_path_like(file):
+            with open(file, 'w' if write else 'r', encoding='utf-8') as f:
+                yield f
+        else:
+            if write:
+                file.truncate(0)
+            yield file
+
+    def _really_save(self, f, ignore_discard=False, ignore_expires=False):
+        now = time.time()
+        for cookie in self:
+            if (not ignore_discard and cookie.discard
+                    or not ignore_expires and cookie.is_expired(now)):
+                continue
+            name, value = cookie.name, cookie.value
+            if value is None:
+                # cookies.txt regards 'Set-Cookie: foo' as a cookie
+                # with no name, whereas http.cookiejar regards it as a
+                # cookie with no value.
+                name, value = '', name
+            f.write('%s\n' % '\t'.join((
+                cookie.domain,
+                self._true_or_false(cookie.domain.startswith('.')),
+                cookie.path,
+                self._true_or_false(cookie.secure),
+                str_or_none(cookie.expires, default=''),
+                name, value
+            )))
+
+    def save(self, filename=None, *args, **kwargs):
+        """
+        Save cookies to a file.
+        Code is taken from CPython 3.6
+        https://github.com/python/cpython/blob/8d999cbf4adea053be6dbb612b9844635c4dfb8e/Lib/http/cookiejar.py#L2091-L2117 """
+
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
+
+        # Store session cookies with `expires` set to 0 instead of an empty string
+        for cookie in self:
+            if cookie.expires is None:
+                cookie.expires = 0
+
+        with self.open(filename, write=True) as f:
+            f.write(self._HEADER)
+            self._really_save(f, *args, **kwargs)
+
+    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+        """Load cookies from a file."""
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
+
+        def prepare_line(line):
+            if line.startswith(self._HTTPONLY_PREFIX):
+                line = line[len(self._HTTPONLY_PREFIX):]
+            # comments and empty lines are fine
+            if line.startswith('#') or not line.strip():
+                return line
+            cookie_list = line.split('\t')
+            if len(cookie_list) != self._ENTRY_LEN:
+                raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
+            cookie = self._CookieFileEntry(*cookie_list)
+            if cookie.expires_at and not cookie.expires_at.isdigit():
+                raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
+            return line
+
+        cf = io.StringIO()
+        with self.open(filename) as f:
+            for line in f:
+                try:
+                    cf.write(prepare_line(line))
+                except http.cookiejar.LoadError as e:
+                    if f'{line.strip()} '[0] in '[{"':
+                        raise http.cookiejar.LoadError(
+                            'Cookies file must be Netscape formatted, not JSON. See  '
+                            'https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp')
+                    write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
+                    continue
+        cf.seek(0)
+        self._really_load(cf, filename, ignore_discard, ignore_expires)
+        # Session cookies are denoted by either `expires` field set to
+        # an empty string or 0. MozillaCookieJar only recognizes the former
+        # (see [1]). So we need force the latter to be recognized as session
+        # cookies on our own.
+        # Session cookies may be important for cookies-based authentication,
+        # e.g. usually, when user does not check 'Remember me' check box while
+        # logging in on a site, some important cookies are stored as session
+        # cookies so that not recognizing them will result in failed login.
+        # 1. https://bugs.python.org/issue17164
+        for cookie in self:
+            # Treat `expires=0` cookies as session cookies
+            if cookie.expires == 0:
+                cookie.expires = None
+                cookie.discard = True
+
+    def get_cookie_header(self, url):
+        """Generate a Cookie HTTP header for a given url"""
+        cookie_req = urllib.request.Request(escape_url(sanitize_url(url)))
+        self.add_cookie_header(cookie_req)
+        return cookie_req.get_header('Cookie')

yt_dlp/downloader/__init__.py

@@ -30,7 +30,7 @@ from .hls import HlsFD
 from .http import HttpFD
 from .ism import IsmFD
 from .mhtml import MhtmlFD
-from .niconico import NiconicoDmcFD
+from .niconico import NiconicoDmcFD, NiconicoLiveFD
 from .rtmp import RtmpFD
 from .rtsp import RtspFD
 from .websocket import WebSocketFragmentFD
@@ -50,6 +50,7 @@ PROTOCOL_MAP = {
     'ism': IsmFD,
     'mhtml': MhtmlFD,
     'niconico_dmc': NiconicoDmcFD,
+    'niconico_live': NiconicoLiveFD,
     'fc2_live': FC2LiveFD,
     'websocket_frag': WebSocketFragmentFD,
     'youtube_live_chat': YoutubeLiveChatFD,

yt_dlp/downloader/common.py

@@ -51,8 +51,9 @@ class FileDownloader:
     ratelimit:          Download speed limit, in bytes/sec.
     continuedl:         Attempt to continue downloads if possible
     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
-    retries:            Number of times to retry for HTTP error 5xx
-    file_access_retries:   Number of times to retry on file access error
+    retries:            Number of times to retry for expected network errors.
+                        Default is 0 for API, but 10 for CLI
+    file_access_retries:   Number of times to retry on file access error (default: 3)
     buffersize:         Size of download buffer in bytes.
     noresizebuffer:     Do not automatically resize the download buffer.
     continuedl:         Try to continue downloads if possible.
@@ -138,17 +139,21 @@ class FileDownloader:
     def format_percent(percent):
         return '  N/A%' if percent is None else f'{percent:>5.1f}%'
 
-    @staticmethod
-    def calc_eta(start, now, total, current):
+    @classmethod
+    def calc_eta(cls, start_or_rate, now_or_remaining, total=NO_DEFAULT, current=NO_DEFAULT):
+        if total is NO_DEFAULT:
+            rate, remaining = start_or_rate, now_or_remaining
+            if None in (rate, remaining):
+                return None
+            return int(float(remaining) / rate)
+
+        start, now = start_or_rate, now_or_remaining
         if total is None:
             return None
         if now is None:
             now = time.time()
-        dif = now - start
-        if current == 0 or dif < 0.001:  # One millisecond
-            return None
-        rate = float(current) / dif
-        return int((float(total) - float(current)) / rate)
+        rate = cls.calc_speed(start, now, current)
+        return rate and int((float(total) - float(current)) / rate)
 
     @staticmethod
     def calc_speed(start, now, bytes):
@@ -165,6 +170,12 @@ class FileDownloader:
     def format_retries(retries):
         return 'inf' if retries == float('inf') else int(retries)
 
+    @staticmethod
+    def filesize_or_none(unencoded_filename):
+        if os.path.isfile(unencoded_filename):
+            return os.path.getsize(unencoded_filename)
+        return 0
+
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
@@ -225,7 +236,7 @@ class FileDownloader:
                 sleep_func=fd.params.get('retry_sleep_functions', {}).get('file_access'))
 
         def wrapper(self, func, *args, **kwargs):
-            for retry in RetryManager(self.params.get('file_access_retries'), error_callback, fd=self):
+            for retry in RetryManager(self.params.get('file_access_retries', 3), error_callback, fd=self):
                 try:
                     return func(self, *args, **kwargs)
                 except OSError as err:
@@ -285,7 +296,8 @@ class FileDownloader:
             self._multiline = BreaklineStatusPrinter(self.ydl._out_files.out, lines)
         else:
             self._multiline = MultilinePrinter(self.ydl._out_files.out, lines, not self.params.get('quiet'))
-        self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
+        self._multiline.allow_colors = self.ydl._allow_colors.out and self.ydl._allow_colors.out != 'no_color'
+        self._multiline._HAVE_FULLCAP = self.ydl._allow_colors.out
 
     def _finish_multiline_status(self):
         self._multiline.end()

yt_dlp/downloader/external.py

@@ -23,7 +23,6 @@ from ..utils import (
     encodeArgument,
     encodeFilename,
     find_available_port,
-    handle_youtubedl_headers,
     remove_end,
     sanitized_Request,
     traverse_obj,
@@ -529,10 +528,9 @@ class FFmpegFD(ExternalFD):
         selected_formats = info_dict.get('requested_formats') or [info_dict]
         for i, fmt in enumerate(selected_formats):
             if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
-                headers_dict = handle_youtubedl_headers(fmt['http_headers'])
                 # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
                 # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
-                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in headers_dict.items())])
+                args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in fmt['http_headers'].items())])
 
             if start_time:
                 args += ['-ss', str(start_time)]

yt_dlp/downloader/fragment.py

@@ -34,8 +34,8 @@ class FragmentFD(FileDownloader):
 
     Available options:
 
-    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH
-                        and hlsnative only)
+    fragment_retries:   Number of times to retry a fragment for HTTP error
+                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
     skip_unavailable_fragments:
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
@@ -121,6 +121,11 @@ class FragmentFD(FileDownloader):
             'request_data': request_data,
             'ctx_id': ctx.get('ctx_id'),
         }
+        frag_resume_len = 0
+        if ctx['dl'].params.get('continuedl', True):
+            frag_resume_len = self.filesize_or_none(self.temp_name(fragment_filename))
+        fragment_info_dict['frag_resume_len'] = ctx['frag_resume_len'] = frag_resume_len
+
         success, _ = ctx['dl'].download(fragment_filename, fragment_info_dict)
         if not success:
             return False
@@ -155,9 +160,7 @@ class FragmentFD(FileDownloader):
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
-        if 'live' not in ctx:
-            ctx['live'] = False
-        if not ctx['live']:
+        if not ctx.setdefault('live', False):
             total_frags_str = '%d' % ctx['total_frags']
             ad_frags = ctx.get('ad_frags', 0)
             if ad_frags:
@@ -173,12 +176,11 @@ class FragmentFD(FileDownloader):
         })
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
-        resume_len = 0
 
         # Establish possible resume length
-        if os.path.isfile(encodeFilename(tmpfilename)):
+        resume_len = self.filesize_or_none(tmpfilename)
+        if resume_len > 0:
             open_mode = 'ab'
-            resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
         # Should be initialized before ytdl file check
         ctx.update({
@@ -187,7 +189,9 @@ class FragmentFD(FileDownloader):
         })
 
         if self.__do_ytdl_file(ctx):
-            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename'])))
+            continuedl = self.params.get('continuedl', True)
+            if continuedl and ytdl_file_exists:
                 self._read_ytdl_file(ctx)
                 is_corrupt = ctx.get('ytdl_corrupt') is True
                 is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
@@ -201,7 +205,12 @@ class FragmentFD(FileDownloader):
                     if 'ytdl_corrupt' in ctx:
                         del ctx['ytdl_corrupt']
                     self._write_ytdl_file(ctx)
+
             else:
+                if not continuedl:
+                    if ytdl_file_exists:
+                        self._read_ytdl_file(ctx)
+                    ctx['fragment_index'] = resume_len = 0
                 self._write_ytdl_file(ctx)
                 assert ctx['fragment_index'] == 0
 
@@ -274,12 +283,10 @@ class FragmentFD(FileDownloader):
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                if not ctx['live']:
-                    state['eta'] = self.calc_eta(
-                        start, time_now, estimated_size - resume_len,
-                        state['downloaded_bytes'] - resume_len)
                 ctx['speed'] = state['speed'] = self.calc_speed(
-                    ctx['fragment_started'], time_now, frag_downloaded_bytes)
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0))
+                if not ctx['live']:
+                    state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state, info_dict)
 
@@ -297,7 +304,7 @@ class FragmentFD(FileDownloader):
 
         to_file = ctx['tmpfilename'] != '-'
         if to_file:
-            downloaded_bytes = os.path.getsize(encodeFilename(ctx['tmpfilename']))
+            downloaded_bytes = self.filesize_or_none(ctx['tmpfilename'])
         else:
             downloaded_bytes = ctx['complete_frags_downloaded_bytes']
 

yt_dlp/downloader/http.py

@@ -45,8 +45,8 @@ class HttpFD(FileDownloader):
         ctx.tmpfilename = self.temp_name(filename)
         ctx.stream = None
 
-        # Do not include the Accept-Encoding header
-        headers = {'Youtubedl-no-compression': 'True'}
+        # Disable compression
+        headers = {'Accept-Encoding': 'identity'}
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
@@ -150,7 +150,8 @@ class HttpFD(FileDownloader):
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload
-                    self.report_unable_to_resume()
+                    elif range_start > 0:
+                        self.report_unable_to_resume()
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
                 ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))

yt_dlp/downloader/niconico.py

@@ -1,8 +1,17 @@
+import json
 import threading
+import time
 
 from . import get_suitable_downloader
 from .common import FileDownloader
-from ..utils import sanitized_Request
+from .external import FFmpegFD
+from ..utils import (
+    DownloadError,
+    str_or_none,
+    sanitized_Request,
+    WebSocketsWrapper,
+    try_get,
+)
 
 
 class NiconicoDmcFD(FileDownloader):
@@ -50,3 +59,93 @@ class NiconicoDmcFD(FileDownloader):
                     timer[0].cancel()
                     download_complete = True
         return success
+
+
+class NiconicoLiveFD(FileDownloader):
+    """ Downloads niconico live without being stopped """
+
+    def real_download(self, filename, info_dict):
+        video_id = info_dict['video_id']
+        ws_url = info_dict['url']
+        ws_extractor = info_dict['ws']
+        ws_origin_host = info_dict['origin']
+        cookies = info_dict.get('cookies')
+        live_quality = info_dict.get('live_quality', 'high')
+        live_latency = info_dict.get('live_latency', 'high')
+        dl = FFmpegFD(self.ydl, self.params or {})
+
+        new_info_dict = info_dict.copy()
+        new_info_dict.update({
+            'protocol': 'm3u8',
+        })
+
+        def communicate_ws(reconnect):
+            if reconnect:
+                ws = WebSocketsWrapper(ws_url, {
+                    'Cookies': str_or_none(cookies) or '',
+                    'Origin': f'https://{ws_origin_host}',
+                    'Accept': '*/*',
+                    'User-Agent': self.params['http_headers']['User-Agent'],
+                })
+                if self.ydl.params.get('verbose', False):
+                    self.to_screen('[debug] Sending startWatching request')
+                ws.send(json.dumps({
+                    'type': 'startWatching',
+                    'data': {
+                        'stream': {
+                            'quality': live_quality,
+                            'protocol': 'hls+fmp4',
+                            'latency': live_latency,
+                            'chasePlay': False
+                        },
+                        'room': {
+                            'protocol': 'webSocket',
+                            'commentable': True
+                        },
+                        'reconnect': True,
+                    }
+                }))
+            else:
+                ws = ws_extractor
+            with ws:
+                while True:
+                    recv = ws.recv()
+                    if not recv:
+                        continue
+                    data = json.loads(recv)
+                    if not data or not isinstance(data, dict):
+                        continue
+                    if data.get('type') == 'ping':
+                        # pong back
+                        ws.send(r'{"type":"pong"}')
+                        ws.send(r'{"type":"keepSeat"}')
+                    elif data.get('type') == 'disconnect':
+                        self.write_debug(data)
+                        return True
+                    elif data.get('type') == 'error':
+                        self.write_debug(data)
+                        message = try_get(data, lambda x: x['body']['code'], str) or recv
+                        return DownloadError(message)
+                    elif self.ydl.params.get('verbose', False):
+                        if len(recv) > 100:
+                            recv = recv[:100] + '...'
+                        self.to_screen('[debug] Server said: %s' % recv)
+
+        def ws_main():
+            reconnect = False
+            while True:
+                try:
+                    ret = communicate_ws(reconnect)
+                    if ret is True:
+                        return
+                except BaseException as e:
+                    self.to_screen('[%s] %s: Connection error occured, reconnecting after 10 seconds: %s' % ('niconico:live', video_id, str_or_none(e)))
+                    time.sleep(10)
+                    continue
+                finally:
+                    reconnect = True
+
+        thread = threading.Thread(target=ws_main, daemon=True)
+        thread.start()
+
+        return dl.download(filename, new_info_dict)

yt_dlp/extractor/_extractors.py

@@ -204,7 +204,11 @@ from .bfmtv import (
     BFMTVLiveIE,
     BFMTVArticleIE,
 )
-from .bibeltv import BibelTVIE
+from .bibeltv import (
+    BibelTVLiveIE,
+    BibelTVSeriesIE,
+    BibelTVVideoIE,
+)
 from .bigflix import BigflixIE
 from .bigo import BigoIE
 from .bild import BildIE
@@ -247,7 +251,6 @@ from .bongacams import BongaCamsIE
 from .bostonglobe import BostonGlobeIE
 from .box import BoxIE
 from .boxcast import BoxCastVideoIE
-from .booyah import BooyahClipsIE
 from .bpb import BpbIE
 from .br import (
     BRIE,
@@ -281,6 +284,10 @@ from .camdemy import (
     CamdemyIE,
     CamdemyFolderIE
 )
+from .camfm import (
+    CamFMEpisodeIE,
+    CamFMShowIE
+)
 from .cammodels import CamModelsIE
 from .camsoda import CamsodaIE
 from .camtasia import CamtasiaEmbedIE
@@ -288,12 +295,6 @@ from .camwithher import CamWithHerIE
 from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
-from .canvas import (
-    CanvasIE,
-    CanvasEenIE,
-    VrtNUIE,
-    DagelijkseKostIE,
-)
 from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,
@@ -310,14 +311,14 @@ from .cbs import (
     CBSIE,
     ParamountPressExpressIE,
 )
-from .cbslocal import (
-    CBSLocalIE,
-    CBSLocalArticleIE,
-)
 from .cbsinteractive import CBSInteractiveIE
 from .cbsnews import (
     CBSNewsEmbedIE,
     CBSNewsIE,
+    CBSLocalIE,
+    CBSLocalArticleIE,
+    CBSLocalLiveIE,
+    CBSNewsLiveIE,
     CBSNewsLiveVideoIE,
 )
 from .cbssports import (
@@ -404,9 +405,12 @@ from .crowdbunker import (
     CrowdBunkerIE,
     CrowdBunkerChannelIE,
 )
+from .crtvg import CrtvgIE
 from .crunchyroll import (
     CrunchyrollBetaIE,
     CrunchyrollBetaShowIE,
+    CrunchyrollMusicIE,
+    CrunchyrollArtistIE,
 )
 from .cspan import CSpanIE, CSpanCongressIE
 from .ctsnews import CtsNewsIE
@@ -423,6 +427,10 @@ from .cybrary import (
     CybraryIE,
     CybraryCourseIE
 )
+from .dacast import (
+    DacastVODIE,
+    DacastPlaylistIE,
+)
 from .daftsex import DaftsexIE
 from .dailymail import DailyMailIE
 from .dailymotion import (
@@ -536,6 +544,7 @@ from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
+from .elevensports import ElevenSportsIE
 from .ellentube import (
     EllenTubeIE,
     EllenTubeVideoIE,
@@ -784,6 +793,7 @@ from .ichinanalive import (
     IchinanaLiveIE,
     IchinanaLiveClipIE,
 )
+from .idolplus import IdolPlusIE
 from .ign import (
     IGNIE,
     IGNVideoIE,
@@ -868,6 +878,7 @@ from .japandiet import (
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .joj import JojIE
+from .jstream import JStreamIE
 from .jwplatform import JWPlatformIE
 from .kakao import KakaoIE
 from .kaltura import KalturaIE
@@ -877,7 +888,6 @@ from .karaoketv import KaraoketvIE
 from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .kelbyone import KelbyOneIE
-from .ketnet import KetnetIE
 from .khanacademy import (
     KhanAcademyIE,
     KhanAcademyUnitIE,
@@ -1147,6 +1157,7 @@ from .myvi import (
 )
 from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
+from .mzaalo import MzaaloIE
 from .n1 import (
     N1InfoAssetIE,
     N1InfoIIE,
@@ -1195,6 +1206,7 @@ from .nebula import (
     NebulaSubscriptionsIE,
     NebulaChannelIE,
 )
+from .nekohacker import NekoHackerIE
 from .nerdcubed import NerdCubedFeedIE
 from .netzkino import NetzkinoIE
 from .neteasemusic import (
@@ -1264,6 +1276,7 @@ from .niconico import (
     NicovideoSearchIE,
     NicovideoSearchURLIE,
     NicovideoTagURLIE,
+    NiconicoLiveIE,
 )
 from .ninecninemedia import (
     NineCNineMediaIE,
@@ -1373,6 +1386,7 @@ from .orf import (
     ORFIPTVIE,
 )
 from .outsidetv import OutsideTVIE
+from .owncloud import OwnCloudIE
 from .packtpub import (
     PacktPubIE,
     PacktPubCourseIE,
@@ -1474,7 +1488,6 @@ from .polskieradio import (
     PolskieRadioPlayerIE,
     PolskieRadioPodcastIE,
     PolskieRadioPodcastListIE,
-    PolskieRadioRadioKierowcowIE,
 )
 from .popcorntimes import PopcorntimesIE
 from .popcorntv import PopcornTVIE
@@ -1544,6 +1557,8 @@ from .radlive import (
     RadLiveSeasonIE,
 )
 from .rai import (
+    RaiIE,
+    RaiCulturaIE,
     RaiPlayIE,
     RaiPlayLiveIE,
     RaiPlayPlaylistIE,
@@ -1552,7 +1567,6 @@ from .rai import (
     RaiPlaySoundPlaylistIE,
     RaiNewsIE,
     RaiSudtirolIE,
-    RaiIE,
 )
 from .raywenderlich import (
     RayWenderlichIE,
@@ -1574,6 +1588,7 @@ from .rcti import (
     RCTIPlusTVIE,
 )
 from .rds import RDSIE
+from .recurbate import RecurbateIE
 from .redbee import ParliamentLiveUKIE, RTBFIE
 from .redbulltv import (
     RedBullTVIE,
@@ -2080,7 +2095,6 @@ from .tvp import (
 )
 from .tvplay import (
     TVPlayIE,
-    ViafreeIE,
     TVPlayHomeIE,
 )
 from .tvplayer import TVPlayerIE
@@ -2264,7 +2278,12 @@ from .voxmedia import (
     VoxMediaVolumeIE,
     VoxMediaIE,
 )
-from .vrt import VRTIE
+from .vrt import (
+    VRTIE,
+    VrtNUIE,
+    KetnetIE,
+    DagelijkseKostIE,
+)
 from .vrak import VrakIE
 from .vrv import (
     VRVIE,
@@ -2315,7 +2334,16 @@ from .weibo import (
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .weverse import (
+    WeverseIE,
+    WeverseMediaIE,
+    WeverseMomentIE,
+    WeverseLiveTabIE,
+    WeverseMediaTabIE,
+    WeverseLiveIE,
+)
 from .wevidi import WeVidiIE
+from .weyyak import WeyyakIE
 from .whyp import WhypIE
 from .wikimedia import WikimediaIE
 from .willow import WillowIE
@@ -2344,6 +2372,12 @@ from .wsj import (
     WSJArticleIE,
 )
 from .wwe import WWEIE
+from .wykop import (
+    WykopDigIE,
+    WykopDigCommentIE,
+    WykopPostIE,
+    WykopPostCommentIE,
+)
 from .xanimu import XanimuIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
@@ -2463,6 +2497,7 @@ from .zingmp3 import (
     ZingMp3WeekChartIE,
     ZingMp3ChartMusicVideoIE,
     ZingMp3UserIE,
+    ZingMp3HubIE,
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE

yt_dlp/extractor/aenetworks.py

@@ -3,6 +3,8 @@ from ..utils import (
     ExtractorError,
     GeoRestrictedError,
     int_or_none,
+    remove_start,
+    traverse_obj,
     update_url_query,
     urlencode_postdata,
 )
@@ -72,7 +74,14 @@ class AENetworksBaseIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
         requestor_id, brand = self._DOMAIN_MAP[domain]
         result = self._download_json(
             'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
-            filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
+            filter_value, query={'filter[%s]' % filter_key: filter_value})
+        result = traverse_obj(
+            result, ('results',
+                     lambda k, v: k == 0 and v[filter_key] == filter_value),
+            get_all=False)
+        if not result:
+            raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
+                                 video_id=remove_start(filter_value, '/'))
         title = result['title']
         video_id = result['id']
         media_url = result['publicUrl']
@@ -123,7 +132,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
-        'skip': 'This video is only available for users of participating TV providers.',
+        'skip': 'Geo-restricted - This content is not available in your location.'
     }, {
         'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
         'info_dict': {
@@ -140,6 +149,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'This video is only available for users of participating TV providers.',
     }, {
         'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
         'only_matching': True
@@ -303,6 +313,7 @@ class HistoryTopicIE(AENetworksBaseIE):
 class HistoryPlayerIE(AENetworksBaseIE):
     IE_NAME = 'history:player'
     _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
+    _TESTS = []
 
     def _real_extract(self, url):
         domain, video_id = self._match_valid_url(url).groups()

yt_dlp/extractor/anvato.py

@@ -336,7 +336,7 @@ class AnvatoIE(InfoExtractor):
             elif media_format == 'm3u8-variant' or ext == 'm3u8':
                 # For some videos the initial m3u8 URL returns JSON instead
                 manifest_json = self._download_json(
-                    video_url, video_id, note='Downloading manifest JSON', errnote=False)
+                    video_url, video_id, note='Downloading manifest JSON', fatal=False)
                 if manifest_json:
                     video_url = manifest_json.get('master_m3u8')
                     if not video_url:
@@ -392,14 +392,6 @@ class AnvatoIE(InfoExtractor):
                 url = smuggle_url(url, {'token': anvplayer_data['token']})
             yield cls.url_result(url, AnvatoIE, video_id)
 
-    def _extract_anvato_videos(self, webpage, video_id):
-        anvplayer_data = self._parse_json(
-            self._html_search_regex(
-                self._ANVP_RE, webpage, 'Anvato player data', group='anvp'),
-            video_id)
-        return self._get_anvato_videos(
-            anvplayer_data['accessKey'], anvplayer_data['video'], 'default')  # cbslocal token = 'default'
-
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         self._initialize_geo_bypass({

yt_dlp/extractor/ard.py

@@ -13,6 +13,7 @@ from ..utils import (
     try_get,
     unified_strdate,
     unified_timestamp,
+    update_url,
     update_url_query,
     url_or_none,
     xpath_text,
@@ -408,6 +409,23 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
 
     _TESTS = [{
+        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
+        'md5': '3fd5fead7a370a819341129c8d713136',
+        'info_dict': {
+            'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
+            'id': '12172961',
+            'title': 'Wolfsland - Die traurigen Schwestern',
+            'description': r're:^Als der Polizeiobermeister Raaben',
+            'duration': 5241,
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
+            'timestamp': 1670710500,
+            'upload_date': '20221210',
+            'ext': 'mp4',
+            'age_limit': 12,
+            'episode': 'Wolfsland - Die traurigen Schwestern',
+            'series': 'Filme im MDR'
+        },
+    }, {
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
         'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
         'info_dict': {
@@ -424,7 +442,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'skip': 'Error',
     }, {
         'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
-        'md5': 'f1837e563323b8a642a8ddeff0131f51',
+        'md5': '1e73ded21cb79bac065117e80c81dc88',
         'info_dict': {
             'id': '10049223',
             'ext': 'mp4',
@@ -432,13 +450,11 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'timestamp': 1636398000,
             'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
             'upload_date': '20211108',
-        },
-    }, {
-        'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
-        'playlist_count': 6,
-        'info_dict': {
-            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
-            'title': 'beforeigners/beforeigners/staffel-1',
+            'display_id': 'tagesschau-oder-tagesschau-20-00-uhr/das-erste',
+            'duration': 915,
+            'episode': 'tagesschau, 20:00 Uhr',
+            'series': 'tagesschau',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -602,6 +618,9 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     show {
       title
     }
+    image {
+      src
+    }
     synopsis
     title
     tracking {
@@ -640,6 +659,15 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'description': description,
             'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
             'series': try_get(player_page, lambda x: x['show']['title']),
+            'thumbnail': (media_collection.get('_previewImage')
+                          or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
+                          or self.get_thumbnail_from_html(display_id, url)),
         })
         info.update(self._ARD_extract_episode_info(info['title']))
         return info
+
+    def get_thumbnail_from_html(self, display_id, url):
+        webpage = self._download_webpage(url, display_id, fatal=False) or ''
+        return (
+            self._og_search_thumbnail(webpage, default=None)
+            or self._html_search_meta('thumbnailUrl', webpage, default=None))

yt_dlp/extractor/bibeltv.py

@@ -1,27 +1,197 @@
+from functools import partial
+
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    determine_ext,
+    format_field,
+    int_or_none,
+    js_to_json,
+    orderedSet,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
 
 
-class BibelTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?:crn/)?(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://www.bibeltv.de/mediathek/videos/329703-sprachkurs-in-malaiisch',
-        'md5': '252f908192d611de038b8504b08bf97f',
-        'info_dict': {
-            'id': 'ref:329703',
-            'ext': 'mp4',
-            'title': 'Sprachkurs in Malaiisch',
-            'description': 'md5:3e9f197d29ee164714e67351cf737dfe',
-            'timestamp': 1608316701,
-            'uploader_id': '5840105145001',
-            'upload_date': '20201218',
+class BibelTVBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['AT', 'CH', 'DE']
+    _GEO_BYPASS = False
+
+    API_URL = 'https://www.bibeltv.de/mediathek/api'
+    AUTH_TOKEN = 'j88bRXY8DsEqJ9xmTdWhrByVi5Hm'
+
+    def _extract_formats_and_subtitles(self, data, crn_id, *, is_live=False):
+        formats = []
+        subtitles = {}
+        for media_url in traverse_obj(data, (..., 'src', {url_or_none})):
+            media_ext = determine_ext(media_url)
+            if media_ext == 'm3u8':
+                m3u8_formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
+                    media_url, crn_id, live=is_live)
+                formats.extend(m3u8_formats)
+                subtitles.update(m3u8_subs)
+            elif media_ext == 'mpd':
+                mpd_formats, mpd_subs = self._extract_mpd_formats_and_subtitles(media_url, crn_id)
+                formats.extend(mpd_formats)
+                subtitles.update(mpd_subs)
+            elif media_ext == 'mp4':
+                formats.append({'url': media_url})
+            else:
+                self.report_warning(f'Unknown format {media_ext!r}')
+
+        return formats, subtitles
+
+    @staticmethod
+    def _extract_base_info(data):
+        return {
+            'id': data['crn'],
+            **traverse_obj(data, {
+                'title': 'title',
+                'description': 'description',
+                'duration': ('duration', {partial(int_or_none, scale=1000)}),
+                'timestamp': ('schedulingStart', {parse_iso8601}),
+                'season_number': 'seasonNumber',
+                'episode_number': 'episodeNumber',
+                'view_count': 'viewCount',
+                'like_count': 'likeCount',
+            }),
+            'thumbnails': orderedSet(traverse_obj(data, ('images', ..., {
+                'url': ('url', {url_or_none}),
+            }))),
         }
-    }, {
-        'url': 'https://www.bibeltv.de/mediathek/videos/crn/326374',
-        'only_matching': True,
+
+    def _extract_url_info(self, data):
+        return {
+            '_type': 'url',
+            'url': format_field(data, 'slug', 'https://www.bibeltv.de/mediathek/videos/%s'),
+            **self._extract_base_info(data),
+        }
+
+    def _extract_video_info(self, data):
+        crn_id = data['crn']
+
+        if data.get('drm'):
+            self.report_drm(crn_id)
+
+        json_data = self._download_json(
+            format_field(data, 'id', f'{self.API_URL}/video/%s'), crn_id,
+            headers={'Authorization': self.AUTH_TOKEN}, fatal=False,
+            errnote='No formats available') or {}
+
+        formats, subtitles = self._extract_formats_and_subtitles(
+            traverse_obj(json_data, ('video', 'videoUrls', ...)), crn_id)
+
+        return {
+            '_type': 'video',
+            **self._extract_base_info(data),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class BibelTVVideoIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV single video'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?P<id>\d+)[\w-]+'
+    IE_NAME = 'bibeltv:video'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/mediathek/videos/344436-alte-wege',
+        'md5': 'ec1c07efe54353780512e8a4103b612e',
+        'info_dict': {
+            'id': '344436',
+            'ext': 'mp4',
+            'title': 'Alte Wege',
+            'description': 'md5:2f4eb7294c9797a47b8fd13cccca22e9',
+            'timestamp': 1677877071,
+            'duration': 150.0,
+            'upload_date': '20230303',
+            'thumbnail': r're:https://bibeltv\.imgix\.net/[\w-]+\.jpg',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'format': '6',
+        },
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5840105145001/default_default/index.html?videoId=ref:%s'
 
     def _real_extract(self, url):
         crn_id = self._match_id(url)
-        return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % crn_id, 'BrightcoveNew')
+        video_data = traverse_obj(
+            self._search_nextjs_data(self._download_webpage(url, crn_id), crn_id),
+            ('props', 'pageProps', 'videoPageData', 'videos', 0, {dict}))
+        if not video_data:
+            raise ExtractorError('Missing video data.')
+
+        return self._extract_video_info(video_data)
+
+
+class BibelTVSeriesIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV series playlist'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/serien/(?P<id>\d+)[\w-]+'
+    IE_NAME = 'bibeltv:series'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/mediathek/serien/333485-ein-wunder-fuer-jeden-tag',
+        'playlist_mincount': 400,
+        'info_dict': {
+            'id': '333485',
+            'title': 'Ein Wunder für jeden Tag',
+            'description': 'Tägliche Kurzandacht mit Déborah Rosenkranz.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        crn_id = self._match_id(url)
+        webpage = self._download_webpage(url, crn_id)
+        nextjs_data = self._search_nextjs_data(webpage, crn_id)
+        series_data = traverse_obj(nextjs_data, ('props', 'pageProps', 'seriePageData', {dict}))
+        if not series_data:
+            raise ExtractorError('Missing series data.')
+
+        return self.playlist_result(
+            traverse_obj(series_data, ('videos', ..., {dict}, {self._extract_url_info})),
+            crn_id, series_data.get('title'), clean_html(series_data.get('description')))
+
+
+class BibelTVLiveIE(BibelTVBaseIE):
+    IE_DESC = 'BibelTV live program'
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/livestreams/(?P<id>[\w-]+)'
+    IE_NAME = 'bibeltv:live'
+
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/livestreams/bibeltv/',
+        'info_dict': {
+            'id': 'bibeltv',
+            'ext': 'mp4',
+            'title': 're:Bibel TV',
+            'live_status': 'is_live',
+            'thumbnail': 'https://streampreview.bibeltv.de/bibeltv.webp',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.bibeltv.de/livestreams/impuls/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        stream_id = self._match_id(url)
+        webpage = self._download_webpage(url, stream_id)
+        stream_data = self._search_json(
+            r'\\"video\\":', webpage, 'bibeltvData', stream_id,
+            transform_source=lambda jstring: js_to_json(jstring.replace('\\"', '"')))
+
+        formats, subtitles = self._extract_formats_and_subtitles(
+            traverse_obj(stream_data, ('src', ...)), stream_id, is_live=True)
+
+        return {
+            'id': stream_id,
+            'title': stream_data.get('title'),
+            'thumbnail': stream_data.get('poster'),
+            'is_live': True,
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/bilibili.py

@@ -1,7 +1,9 @@
 import base64
 import functools
+import hashlib
 import itertools
 import math
+import time
 import urllib.error
 import urllib.parse
 
@@ -26,6 +28,7 @@ from ..utils import (
     srt_subtitles_timecode,
     str_or_none,
     traverse_obj,
+    try_call,
     unified_timestamp,
     unsmuggle_url,
     url_or_none,
@@ -514,19 +517,63 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
             'id': '3985676',
         },
         'playlist_mincount': 178,
+    }, {
+        'url': 'https://space.bilibili.com/313580179/video',
+        'info_dict': {
+            'id': '313580179',
+        },
+        'playlist_mincount': 92,
     }]
 
+    def _extract_signature(self, playlist_id):
+        session_data = self._download_json('https://api.bilibili.com/x/web-interface/nav', playlist_id, fatal=False)
+
+        key_from_url = lambda x: x[x.rfind('/') + 1:].split('.')[0]
+        img_key = traverse_obj(
+            session_data, ('data', 'wbi_img', 'img_url', {key_from_url})) or '34478ba821254d9d93542680e3b86100'
+        sub_key = traverse_obj(
+            session_data, ('data', 'wbi_img', 'sub_url', {key_from_url})) or '7e16a90d190a4355a78fd00b32a38de6'
+
+        session_key = img_key + sub_key
+
+        signature_values = []
+        for position in (
+            46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39,
+            12, 38, 41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63,
+            57, 62, 11, 36, 20, 34, 44, 52
+        ):
+            char_at_position = try_call(lambda: session_key[position])
+            if char_at_position:
+                signature_values.append(char_at_position)
+
+        return ''.join(signature_values)[:32]
+
     def _real_extract(self, url):
         playlist_id, is_video_url = self._match_valid_url(url).group('id', 'video')
         if not is_video_url:
             self.to_screen('A channel URL was given. Only the channel\'s videos will be downloaded. '
                            'To download audios, add a "/audio" to the URL')
 
+        signature = self._extract_signature(playlist_id)
+
         def fetch_page(page_idx):
+            query = {
+                'keyword': '',
+                'mid': playlist_id,
+                'order': 'pubdate',
+                'order_avoided': 'true',
+                'platform': 'web',
+                'pn': page_idx + 1,
+                'ps': 30,
+                'tid': 0,
+                'web_location': 1550101,
+                'wts': int(time.time()),
+            }
+            query['w_rid'] = hashlib.md5(f'{urllib.parse.urlencode(query)}{signature}'.encode()).hexdigest()
+
             try:
-                response = self._download_json('https://api.bilibili.com/x/space/arc/search',
-                                               playlist_id, note=f'Downloading page {page_idx}',
-                                               query={'mid': playlist_id, 'pn': page_idx + 1, 'jsonp': 'jsonp'})
+                response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
+                                               playlist_id, note=f'Downloading page {page_idx}', query=query)
             except ExtractorError as e:
                 if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
                     raise ExtractorError(
@@ -556,9 +603,9 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
 class BilibiliSpaceAudioIE(BilibiliSpaceBaseIE):
     _VALID_URL = r'https?://space\.bilibili\.com/(?P<id>\d+)/audio'
     _TESTS = [{
-        'url': 'https://space.bilibili.com/3985676/audio',
+        'url': 'https://space.bilibili.com/313580179/audio',
         'info_dict': {
-            'id': '3985676',
+            'id': '313580179',
         },
         'playlist_mincount': 1,
     }]

yt_dlp/extractor/booyah.py

@@ -1,86 +0,0 @@
-from .common import InfoExtractor
-from ..utils import int_or_none, str_or_none, traverse_obj
-
-
-class BooyahBaseIE(InfoExtractor):
-    _BOOYAH_SESSION_KEY = None
-
-    def _real_initialize(self):
-        BooyahBaseIE._BOOYAH_SESSION_KEY = self._request_webpage(
-            'https://booyah.live/api/v3/auths/sessions', None, data=b'').getheader('booyah-session-key')
-
-    def _get_comments(self, video_id):
-        comment_json = self._download_json(
-            f'https://booyah.live/api/v3/playbacks/{video_id}/comments/tops', video_id,
-            headers={'Booyah-Session-Key': self._BOOYAH_SESSION_KEY}, fatal=False) or {}
-
-        return [{
-            'id': comment.get('comment_id'),
-            'author': comment.get('from_nickname'),
-            'author_id': comment.get('from_uid'),
-            'author_thumbnail': comment.get('from_thumbnail'),
-            'text': comment.get('content'),
-            'timestamp': comment.get('create_time'),
-            'like_count': comment.get('like_cnt'),
-        } for comment in comment_json.get('comment_list') or ()]
-
-
-class BooyahClipsIE(BooyahBaseIE):
-    _VALID_URL = r'https?://booyah.live/clips/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://booyah.live/clips/13887261322952306617',
-        'info_dict': {
-            'id': '13887261322952306617',
-            'ext': 'mp4',
-            'view_count': int,
-            'duration': 30,
-            'channel_id': 90565760,
-            'like_count': int,
-            'title': 'Cayendo con estilo 😎',
-            'uploader': '♡LɪꜱGΛ​MER​',
-            'comment_count': int,
-            'uploader_id': '90565760',
-            'thumbnail': 'https://resmambet-a.akamaihd.net/mambet-storage/Clip/90565760/90565760-27204374-fba0-409d-9d7b-63a48b5c0e75.jpg',
-            'upload_date': '20220617',
-            'timestamp': 1655490556,
-            'modified_timestamp': 1655490556,
-            'modified_date': '20220617',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json(
-            f'https://booyah.live/api/v3/playbacks/{video_id}', video_id,
-            headers={'Booyah-Session-key': self._BOOYAH_SESSION_KEY})
-
-        formats = []
-        for video_data in json_data['playback']['endpoint_list']:
-            formats.extend(({
-                'url': video_data.get('stream_url'),
-                'ext': 'mp4',
-                'height': video_data.get('resolution'),
-            }, {
-                'url': video_data.get('download_url'),
-                'ext': 'mp4',
-                'format_note': 'Watermarked',
-                'height': video_data.get('resolution'),
-                'preference': -10,
-            }))
-
-        return {
-            'id': video_id,
-            'title': traverse_obj(json_data, ('playback', 'name')),
-            'thumbnail': traverse_obj(json_data, ('playback', 'thumbnail_url')),
-            'formats': formats,
-            'view_count': traverse_obj(json_data, ('playback', 'views')),
-            'like_count': traverse_obj(json_data, ('playback', 'likes')),
-            'duration': traverse_obj(json_data, ('playback', 'duration')),
-            'comment_count': traverse_obj(json_data, ('playback', 'comment_cnt')),
-            'channel_id': traverse_obj(json_data, ('playback', 'channel_id')),
-            'uploader': traverse_obj(json_data, ('user', 'nickname')),
-            'uploader_id': str_or_none(traverse_obj(json_data, ('user', 'uid'))),
-            'modified_timestamp': int_or_none(traverse_obj(json_data, ('playback', 'update_time_ms')), 1000),
-            'timestamp': int_or_none(traverse_obj(json_data, ('playback', 'create_time_ms')), 1000),
-            '__post_extractor': self.extract_comments(video_id, self._get_comments(video_id)),
-        }

yt_dlp/extractor/bravotv.py

@@ -1,5 +1,6 @@
 from .adobepass import AdobePassIE
 from ..utils import (
+    HEADRequest,
     extract_attributes,
     float_or_none,
     get_element_html_by_class,
@@ -153,8 +154,11 @@ class BravoTVIE(AdobePassIE):
         if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
             chapters = None
 
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            update_url_query(f'{tp_url}/stream.m3u8', query), video_id, 'mp4', m3u8_id='hls')
+        m3u8_url = self._request_webpage(HEADRequest(
+            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
+        if 'mpeg_cenc' in m3u8_url:
+            self.report_drm(video_id)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
 
         return {
             'id': video_id,

yt_dlp/extractor/camfm.py

@@ -0,0 +1,85 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    get_elements_by_class,
+    join_nonempty,
+    traverse_obj,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class CamFMShowIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
+    _TESTS = [{
+        'playlist_mincount': 5,
+        'url': 'https://camfm.co.uk/shows/soul-mining/',
+        'info_dict': {
+            'id': 'soul-mining',
+            'thumbnail': 'md5:6a873091f92c936f23bdcce80f75e66a',
+            'title': 'Soul Mining',
+            'description': 'Telling the stories of jazz, funk and soul from all corners of the world.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        page = self._download_webpage(url, show_id)
+
+        return {
+            '_type': 'playlist',
+            'id': show_id,
+            'entries': [self.url_result(urljoin('https://camfm.co.uk', i), CamFMEpisodeIE)
+                        for i in re.findall(r"javascript:popup\('(/player/[^']+)', 'listen'", page)],
+            'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
+                r'<img[^>]+class="thumb-expand"[^>]+src="([^"]+)"', page, 'thumbnail', fatal=False)),
+            'title': self._html_search_regex('<h1>([^<]+)</h1>', page, 'title', fatal=False),
+            'description': clean_html(get_element_by_class('small-12 medium-8 cell', page))
+        }
+
+
+class CamFMEpisodeIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/player/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://camfm.co.uk/player/43336',
+        'skip': 'Episode will expire - don\'t actually know when, but it will go eventually',
+        'info_dict': {
+            'id': '43336',
+            'title': 'AITAA: Am I the Agony Aunt? - 19:00 Tue 16/05/2023',
+            'ext': 'mp3',
+            'upload_date': '20230516',
+            'description': 'md5:f165144f94927c0f1bfa2ee6e6ab7bbf',
+            'timestamp': 1684263600,
+            'series': 'AITAA: Am I the Agony Aunt?',
+            'thumbnail': 'md5:5980a831360d0744c3764551be3d09c1',
+            'categories': ['Entertainment'],
+        }
+    }]
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        page = self._download_webpage(url, episode_id)
+        audios = self._parse_html5_media_entries('https://audio.camfm.co.uk', page, episode_id)
+
+        caption = get_element_by_class('caption', page)
+        series = clean_html(re.sub(r'<span[^<]+<[^<]+>', '', caption))
+
+        card_section = get_element_by_class('card-section', page)
+        date = self._html_search_regex('>Aired at ([^<]+)<', card_section, 'air date', fatal=False)
+
+        return {
+            'id': episode_id,
+            'title': join_nonempty(series, date, delim=' - '),
+            'formats': traverse_obj(audios, (..., 'formats', ...)),
+            'timestamp': unified_timestamp(date),  # XXX: Does not account for UK's daylight savings
+            'series': series,
+            'description': clean_html(re.sub(r'<b>[^<]+</b><br[^>]+/>', '', card_section)),
+            'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
+                r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)',
+                page, 'thumbnail', fatal=False)),
+            'categories': get_elements_by_class('label', caption),
+            'was_live': True,
+        }

yt_dlp/extractor/canvas.py

@@ -1,383 +0,0 @@
-import json
-
-
-from .common import InfoExtractor
-from .gigya import GigyaBaseIE
-from ..compat import compat_HTTPError
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    extract_attributes,
-    float_or_none,
-    get_element_by_class,
-    int_or_none,
-    merge_dicts,
-    str_or_none,
-    strip_or_none,
-    url_or_none,
-    urlencode_postdata
-)
-
-
-class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-        'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
-        'info_dict': {
-            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'ext': 'mp4',
-            'title': 'Nachtwacht: De Greystook',
-            'description': 'Nachtwacht: De Greystook',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1468.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-    _HLS_ENTRY_PROTOCOLS_MAP = {
-        'HLS': 'm3u8_native',
-        'HLS_AES': 'm3u8_native',
-    }
-    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        site_id, video_id = mobj.group('site_id'), mobj.group('id')
-
-        data = None
-        if site_id != 'vrtvideo':
-            # Old API endpoint, serves more formats but may fail for some videos
-            data = self._download_json(
-                'https://mediazone.vrt.be/api/v1/%s/assets/%s'
-                % (site_id, video_id), video_id, 'Downloading asset JSON',
-                'Unable to download asset JSON', fatal=False)
-
-        # New API endpoint
-        if not data:
-            vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
-                                             video_id, note='refreshtoken: Retrieve vrtnutoken',
-                                             errnote='refreshtoken failed')['vrtnutoken']
-            headers = self.geo_verification_headers()
-            headers.update({'Content-Type': 'application/json; charset=utf-8'})
-            vrtPlayerToken = self._download_json(
-                '%s/tokens' % self._REST_API_BASE, video_id,
-                'Downloading token', headers=headers, data=json.dumps({
-                    'identityToken': vrtnutoken
-                }).encode('utf-8'))['vrtPlayerToken']
-            data = self._download_json(
-                '%s/videos/%s' % (self._REST_API_BASE, video_id),
-                video_id, 'Downloading video JSON', query={
-                    'vrtPlayerToken': vrtPlayerToken,
-                    'client': 'null',
-                }, expected_status=400)
-            if 'title' not in data:
-                code = data.get('code')
-                if code == 'AUTHENTICATION_REQUIRED':
-                    self.raise_login_required()
-                elif code == 'INVALID_LOCATION':
-                    self.raise_geo_restricted(countries=['BE'])
-                raise ExtractorError(data.get('message') or code, expected=True)
-
-        # Note: The title may be an empty string
-        title = data['title'] or f'{site_id} {video_id}'
-        description = data.get('description')
-
-        formats = []
-        subtitles = {}
-        for target in data['targetUrls']:
-            format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
-            if not format_url or not format_type:
-                continue
-            format_type = format_type.upper()
-            if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
-                fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                    format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
-                    m3u8_id=format_type, fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            elif format_type == 'HDS':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_type, fatal=False))
-            elif format_type == 'MPEG_DASH':
-                fmts, subs = self._extract_mpd_formats_and_subtitles(
-                    format_url, video_id, mpd_id=format_type, fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            elif format_type == 'HSS':
-                fmts, subs = self._extract_ism_formats_and_subtitles(
-                    format_url, video_id, ism_id='mss', fatal=False)
-                formats.extend(fmts)
-                subtitles = self._merge_subtitles(subtitles, subs)
-            else:
-                formats.append({
-                    'format_id': format_type,
-                    'url': format_url,
-                })
-
-        subtitle_urls = data.get('subtitleUrls')
-        if isinstance(subtitle_urls, list):
-            for subtitle in subtitle_urls:
-                subtitle_url = subtitle.get('url')
-                if subtitle_url and subtitle.get('type') == 'CLOSED':
-                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
-
-        return {
-            'id': video_id,
-            'display_id': video_id,
-            'title': title,
-            'description': description,
-            'formats': formats,
-            'duration': float_or_none(data.get('duration'), 1000),
-            'thumbnail': data.get('posterImageUrl'),
-            'subtitles': subtitles,
-        }
-
-
-class CanvasEenIE(InfoExtractor):
-    IE_DESC = 'canvas.be and een.be'
-    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-        'md5': 'ed66976748d12350b118455979cca293',
-        'info_dict': {
-            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
-            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
-            'ext': 'flv',
-            'title': 'De afspraak veilt voor de Warmste Week',
-            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 49.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        # with subtitles
-        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
-        'info_dict': {
-            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
-            'display_id': 'pieter-0167',
-            'ext': 'mp4',
-            'title': 'Pieter 0167',
-            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 2553.08,
-            'subtitles': {
-                'nl': [{
-                    'ext': 'vtt',
-                }],
-            },
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'Pagina niet gevonden',
-    }, {
-        'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
-        'info_dict': {
-            'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
-            'display_id': 'emma-pakt-thilly-aan',
-            'ext': 'mp4',
-            'title': 'Emma pakt Thilly aan',
-            'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 118.24,
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
-        site_id, display_id = mobj.group('site_id'), mobj.group('id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = strip_or_none(self._search_regex(
-            r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
-            webpage, 'title', default=None) or self._og_search_title(
-            webpage, default=None))
-
-        video_id = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
-            group='id')
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': self._og_search_description(webpage),
-        }
-
-
-class VrtNUIE(GigyaBaseIE):
-    IE_DESC = 'VrtNU.be'
-    _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        # Available via old API endpoint
-        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
-        'info_dict': {
-            'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
-            'ext': 'mp4',
-            'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
-            'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
-            'duration': 1457.04,
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'series': 'Postbus X',
-            'season': 'Seizoen 1989',
-            'season_number': 1989,
-            'episode': 'De zwarte weduwe',
-            'episode_number': 1,
-            'timestamp': 1595822400,
-            'upload_date': '20200727',
-        },
-        'skip': 'This video is only available for registered users',
-        'expected_warnings': ['is not a supported codec'],
-    }, {
-        # Only available via new API endpoint
-        'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
-        'info_dict': {
-            'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
-            'ext': 'mp4',
-            'title': 'Aflevering 5',
-            'description': 'Wie valt door de mand tijdens een missie?',
-            'duration': 2967.06,
-            'season': 'Season 1',
-            'season_number': 1,
-            'episode_number': 5,
-        },
-        'skip': 'This video is only available for registered users',
-        'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
-    }]
-    _NETRC_MACHINE = 'vrtnu'
-    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
-    _CONTEXT_ID = 'R3595707040'
-
-    def _perform_login(self, username, password):
-        auth_info = self._gigya_login({
-            'APIKey': self._APIKEY,
-            'targetEnv': 'jssdk',
-            'loginID': username,
-            'password': password,
-            'authMode': 'cookie',
-        })
-
-        if auth_info.get('errorDetails'):
-            raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
-
-        # Sometimes authentication fails for no good reason, retry
-        login_attempt = 1
-        while login_attempt <= 3:
-            try:
-                self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
-                                      None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
-                                      query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
-
-                post_data = {
-                    'UID': auth_info['UID'],
-                    'UIDSignature': auth_info['UIDSignature'],
-                    'signatureTimestamp': auth_info['signatureTimestamp'],
-                    '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
-                }
-
-                self._request_webpage(
-                    'https://login.vrt.be/perform_login',
-                    None, note='Performing login', errnote='perform login failed',
-                    headers={}, query={
-                        'client_id': 'vrtnu-site'
-                    }, data=urlencode_postdata(post_data))
-
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                    login_attempt += 1
-                    self.report_warning('Authentication failed')
-                    self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
-                else:
-                    raise e
-            else:
-                break
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        attrs = extract_attributes(self._search_regex(
-            r'(<nui-media[^>]+>)', webpage, 'media element'))
-        video_id = attrs['videoid']
-        publication_id = attrs.get('publicationid')
-        if publication_id:
-            video_id = publication_id + '$' + video_id
-
-        page = (self._parse_json(self._search_regex(
-            r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
-            default='{}'), video_id, fatal=False) or {}).get('page') or {}
-
-        info = self._search_json_ld(webpage, display_id, default={})
-        return merge_dicts(info, {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'season_number': int_or_none(page.get('episode_season')),
-        })
-
-
-class DagelijkseKostIE(InfoExtractor):
-    IE_DESC = 'dagelijksekost.een.be'
-    _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
-        'md5': '30bfffc323009a3e5f689bef6efa2365',
-        'info_dict': {
-            'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
-            'display_id': 'hachis-parmentier-met-witloof',
-            'ext': 'mp4',
-            'title': 'Hachis parmentier met witloof',
-            'description': 'md5:9960478392d87f63567b5b117688cdc5',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 283.02,
-        },
-        'expected_warnings': ['is not a supported codec'],
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        title = strip_or_none(get_element_by_class(
-            'dish-metadata__title', webpage
-        ) or self._html_search_meta(
-            'twitter:title', webpage))
-
-        description = clean_html(get_element_by_class(
-            'dish-description', webpage)
-        ) or self._html_search_meta(
-            ('description', 'twitter:description', 'og:description'),
-            webpage)
-
-        video_id = self._html_search_regex(
-            r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
-            group='id')
-
-        return {
-            '_type': 'url_transparent',
-            'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
-            'ie_key': CanvasIE.ie_key(),
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-        }

yt_dlp/extractor/cbc.py

@@ -351,7 +351,9 @@ class CBCGemIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_info = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/assets/' + video_id, video_id)
+        video_info = self._download_json(
+            f'https://services.radio-canada.ca/ott/cbc-api/v2/assets/{video_id}',
+            video_id, expected_status=426)
 
         email, password = self._get_login_info()
         if email and password:
@@ -426,7 +428,7 @@ class CBCGemPlaylistIE(InfoExtractor):
         match = self._match_valid_url(url)
         season_id = match.group('id')
         show = match.group('show')
-        show_info = self._download_json(self._API_BASE + show, season_id)
+        show_info = self._download_json(self._API_BASE + show, season_id, expected_status=426)
         season = int(match.group('season'))
 
         season_info = next((s for s in show_info['seasons'] if s.get('season') == season), None)

yt_dlp/extractor/cbslocal.py

@@ -1,116 +0,0 @@
-from .anvato import AnvatoIE
-from .sendtonews import SendtoNewsIE
-from ..compat import compat_urlparse
-from ..utils import (
-    parse_iso8601,
-    unified_timestamp,
-)
-
-
-class CBSLocalIE(AnvatoIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL_BASE = r'https?://[a-z]+\.cbslocal\.com/'
-    _VALID_URL = _VALID_URL_BASE + r'video/(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
-        'info_dict': {
-            'id': '3580809',
-            'ext': 'mp4',
-            'title': 'A Very Blue Anniversary',
-            'description': 'CBS2’s Cindy Hsu has more.',
-            'thumbnail': 're:^https?://.*',
-            'timestamp': int,
-            'upload_date': r're:^\d{8}$',
-            'uploader': 'CBS',
-            'subtitles': {
-                'en': 'mincount:5',
-            },
-            'categories': [
-                'Stations\\Spoken Word\\WCBSTV',
-                'Syndication\\AOL',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\Yahoo',
-                'Content\\News',
-                'Content\\News\\Local News',
-            ],
-            'tags': ['CBS 2 News Weekends', 'Cindy Hsu', 'Blue Man Group'],
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        mcp_id = self._match_id(url)
-        return self.url_result(
-            'anvato:anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67:' + mcp_id, 'Anvato', mcp_id)
-
-
-class CBSLocalArticleIE(AnvatoIE):  # XXX: Do not subclass from concrete IE
-    _VALID_URL = CBSLocalIE._VALID_URL_BASE + r'\d+/\d+/\d+/(?P<id>[0-9a-z-]+)'
-
-    _TESTS = [{
-        # Anvato backend
-        'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
-        'md5': 'f0ee3081e3843f575fccef901199b212',
-        'info_dict': {
-            'id': '3401037',
-            'ext': 'mp4',
-            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
-            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
-            'thumbnail': 're:^https?://.*',
-            'timestamp': 1463440500,
-            'upload_date': '20160516',
-            'uploader': 'CBS',
-            'subtitles': {
-                'en': 'mincount:5',
-            },
-            'categories': [
-                'Stations\\Spoken Word\\KCBSTV',
-                'Syndication\\MSN',
-                'Syndication\\NDN',
-                'Syndication\\AOL',
-                'Syndication\\Yahoo',
-                'Syndication\\Tribune',
-                'Syndication\\Curb.tv',
-                'Content\\News'
-            ],
-            'tags': ['CBS 2 News Evening'],
-        },
-    }, {
-        # SendtoNews embed
-        'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
-        'info_dict': {
-            'id': 'GxfCe0Zo7D-175909-5588',
-        },
-        'playlist_count': 9,
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        sendtonews_url = SendtoNewsIE._extract_url(webpage)
-        if sendtonews_url:
-            return self.url_result(
-                compat_urlparse.urljoin(url, sendtonews_url),
-                ie=SendtoNewsIE.ie_key())
-
-        info_dict = self._extract_anvato_videos(webpage, display_id)
-
-        timestamp = unified_timestamp(self._html_search_regex(
-            r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
-            'released date', default=None)) or parse_iso8601(
-            self._html_search_meta('uploadDate', webpage))
-
-        info_dict.update({
-            'display_id': display_id,
-            'timestamp': timestamp,
-        })
-
-        return info_dict

yt_dlp/extractor/cbsnews.py

@@ -1,36 +1,153 @@
+import base64
 import re
+import urllib.error
+import urllib.parse
 import zlib
 
+from .anvato import AnvatoIE
 from .common import InfoExtractor
-from .cbs import CBSIE
-from ..compat import (
-    compat_b64decode,
-    compat_urllib_parse_unquote,
-)
+from .paramountplus import ParamountPlusIE
 from ..utils import (
+    ExtractorError,
+    HEADRequest,
+    UserNotLive,
+    determine_ext,
+    float_or_none,
+    format_field,
+    int_or_none,
+    make_archive_id,
+    mimetype2ext,
     parse_duration,
+    smuggle_url,
+    traverse_obj,
+    url_or_none,
 )
 
 
-class CBSNewsEmbedIE(CBSIE):  # XXX: Do not subclass from concrete IE
+class CBSNewsBaseIE(InfoExtractor):
+    _LOCALES = {
+        'atlanta': None,
+        'baltimore': 'BAL',
+        'boston': 'BOS',
+        'chicago': 'CHI',
+        'colorado': 'DEN',
+        'detroit': 'DET',
+        'losangeles': 'LA',
+        'miami': 'MIA',
+        'minnesota': 'MIN',
+        'newyork': 'NY',
+        'philadelphia': 'PHI',
+        'pittsburgh': 'PIT',
+        'sacramento': 'SAC',
+        'sanfrancisco': 'SF',
+        'texas': 'DAL',
+    }
+    _LOCALE_RE = '|'.join(map(re.escape, _LOCALES))
+    _ANVACK = '5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl'
+
+    def _get_item(self, webpage, display_id):
+        return traverse_obj(self._search_json(
+            r'CBSNEWS\.defaultPayload\s*=', webpage, 'payload', display_id,
+            default={}), ('items', 0, {dict})) or {}
+
+    def _get_video_url(self, item):
+        return traverse_obj(item, 'video', 'video2', expected_type=url_or_none)
+
+    def _extract_playlist(self, webpage, playlist_id):
+        entries = [self.url_result(embed_url, CBSNewsEmbedIE) for embed_url in re.findall(
+            r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage)]
+        if entries:
+            return self.playlist_result(
+                entries, playlist_id, self._html_search_meta(['og:title', 'twitter:title'], webpage),
+                self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
+
+    def _extract_video(self, item, video_url, video_id):
+        if mimetype2ext(item.get('format'), default=determine_ext(video_url)) == 'mp4':
+            formats = [{'url': video_url, 'ext': 'mp4'}]
+
+        else:
+            manifest = self._download_webpage(video_url, video_id, note='Downloading m3u8 information')
+
+            anvato_id = self._search_regex(r'anvato-(\d+)', manifest, 'Anvato ID', default=None)
+            # Prefer Anvato if available; cbsnews.com m3u8 formats are re-encoded from Anvato source
+            if anvato_id:
+                return self.url_result(
+                    smuggle_url(f'anvato:{self._ANVACK}:{anvato_id}', {'token': 'default'}),
+                    AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
+
+            formats, _ = self._parse_m3u8_formats_and_subtitles(
+                manifest, video_url, 'mp4', m3u8_id='hls', video_id=video_id)
+
+        def get_subtitles(subs_url):
+            return {
+                'en': [{
+                    'url': subs_url,
+                    'ext': 'dfxp',  # TTAF1
+                }],
+            } if url_or_none(subs_url) else None
+
+        episode_meta = traverse_obj(item, {
+            'season_number': ('season', {int_or_none}),
+            'episode_number': ('episode', {int_or_none}),
+        }) if item.get('isFullEpisode') else {}
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(item, {
+                'title': (None, ('fulltitle', 'title')),
+                'description': 'dek',
+                'timestamp': ('timestamp', {lambda x: float_or_none(x, 1000)}),
+                'duration': ('duration', {float_or_none}),
+                'subtitles': ('captions', {get_subtitles}),
+                'thumbnail': ('images', ('hd', 'sd'), {url_or_none}),
+                'is_live': ('type', {lambda x: x == 'live'}),
+            }, get_all=False),
+            **episode_meta,
+        }
+
+
+class CBSNewsEmbedIE(CBSNewsBaseIE):
     IE_NAME = 'cbsnews:embed'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/embed/video[^#]*#(?P<id>.+)'
     _TESTS = [{
         'url': 'https://www.cbsnews.com/embed/video/?v=1.c9b5b61492913d6660db0b2f03579ef25e86307a#1Vb7b9s2EP5XBAHbT6Gt98PAMKTJ0se6LVjWYWtdGBR1stlIpEBSTtwi%2F%2FvuJNkNhmHdGxgM2NL57vjd6zt%2B8PngdN%2Fyg79qeGvhzN%2FLGrS%2F%2BuBLB531V28%2B%2BO7Qg7%2Fy97r2z3xZ42NW8yLhDbA0S0KWlHnIijwKWJBHZZnHBa8Cgbpdf%2F89NM9Hi9fXifhpr8sr%2FlP848tn%2BTdXycX25zh4cdX%2FvHl6PmmPqnWQv9w8Ed%2B9GjYRim07bFEqdG%2BZVHuwTm65A7bVRrYtR5lAyMox7pigF6W4k%2By91mjspGsJ%2BwVae4%2BsvdnaO1p73HkXs%2FVisUDTGm7R8IcdnOROeq%2B19qT1amhA1VJtPenoTUgrtfKc9m7Rq8dP7nnjwOB7wg7ADdNt7VX64DWAWlKhPtmDEq22g4GF99x6Dk9E8OSsankHXqPNKDxC%2FdK7MLKTircTDgsI3mmj4OBdSq64dy7fd1x577RU1rt4cvMtOaulFYOd%2FLewRWvDO9lIgXFpZSnkZmjbv5SxKTPoQXClFbpsf%2Fhbbpzs0IB3vb8KkyzJQ%2BywOAgCrMpgRrz%2BKk4fvb7kFbR4XJCu0gAdtNO7woCwZTu%2BBUs9bam%2Fds71drVerpeisgrubLjAB4nnOSkWQnfr5W6o1ku5Xpr1MgrCbL0M0vUyDtfLLK15WiYp47xKWSLyjFVpwVmVJSLIoCjSOFkv3W7oKsVliwZJcB9nwXpZ5GEQQwY8jNKqKCBrgjTLeFxgdCIpazojDgnRtn43J6kG7nZ6cAbxh0EeFFk4%2B1u867cY5u4344n%2FxXjCqAjucdTHgLKojNKmSfO8KRsOFY%2FzKEYCKEJBzv90QA9nfm9gL%2BHulaFqUkz9ULUYxl62B3U%2FRVNLA8IhggaPycOoBuwOCESciDQVSSUgiOMsROB%2FhKfwCKOzEk%2B4k6rWd4uuT%2FwTDz7K7t3d3WLO8ISD95jSPQbayBacthbz86XVgxHwhex5zawzgDOmtp%2F3GPcXn0VXHdSS029%2Fj99UC%2FwJUvyKQ%2FzKyixIEVlYJOn4RxxuaH43Ty9fbJ5OObykHH435XAzJTHeOF4hhEUXD8URe%2FQ%2FBT%2BMpf8d5GN02Ox%2FfiGsl7TA7POu1xZ5%2BbTzcAVKMe48mqcC21hkacVEVScM26liVVBnrKkC4CLKyzAvHu0lhEaTKMFwI3a4SN9MsrfYzdBLq2vkwRD1gVviLT8kY9h2CHH6Y%2Bix6609weFtey4ESp60WtyeWMy%2BsmBuhsoKIyuoT%2Bq2R%2FrW5qi3g%2FvzS2j40DoixDP8%2BKP0yUdpXJ4l6Vla%2Bg9vce%2BC4yM5YlUcbA%2F0jLKdpmTwvsdN5z88nAIe08%2F0HgxeG1iv%2B6Hlhjh7uiW0SDzYNI92L401uha3JKYk268UVRzdOzNQvAaJqoXzAc80dAV440NZ1WVVAAMRYQ2KrGJFmDUsq8saWSnjvIj8t78y%2FRa3JRnbHVfyFpfwoDiGpPgjzekyUiKNlU3OMlwuLMmzgvEojllYVE2Z1HhImvsnk%2BuhusTEoB21PAtSFodeFK3iYhXEH9WOG2%2FkOE833sfeG%2Ff5cfHtEFNXgYes0%2FXj7aGivUgJ9XpusCtoNcNYVVnJVrrDo0OmJAutHCpuZul4W9lLcfy7BnuLPT02%2ByXsCTk%2B9zhzswIN04YueNSK%2BPtM0jS88QdLqSLJDTLsuGZJNolm2yO0PXh3UPnz9Ix5bfIAqxPjvETQsDCEiPG4QbqNyhBZISxybLnZYCrW5H3Axp690%2F0BJdXtDZ5ITuM4xj3f4oUHGzc5JeJmZKpp%2FjwKh4wMV%2FV1yx3emLoR0MwbG4K%2F%2BZgVep3PnzXGDHZ6a3i%2Fk%2BJrONDN13%2Bnq6tBTYk4o7cLGhBtqCC4KwacGHpEVuoH5JNro%2FE6JfE6d5RydbiR76k%2BW5wioDHBIjw1euhHjUGRB0y5A97KoaPx6MlL%2BwgboUVtUFRI%2FLemgTpdtF59ii7pab08kuPcfWzs0l%2FRI5takWnFpka0zOgWRtYcuf9aIxZMxlwr6IiGpsb6j2DQUXPl%2FimXI599Ev7fWjoPD78A',
-        'only_matching': True,
+        'info_dict': {
+            'id': '6ZP4cXvo9FaX3VLH7MF4CgY30JFpY_GA',
+            'ext': 'mp4',
+            'title': 'Cops investigate gorilla incident at Cincinnati Zoo',
+            'description': 'md5:fee7441ab8aaeb3c693482394738102b',
+            'duration': 350,
+            'timestamp': 1464719713,
+            'upload_date': '20160531',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _real_extract(self, url):
-        item = self._parse_json(zlib.decompress(compat_b64decode(
-            compat_urllib_parse_unquote(self._match_id(url))),
-            -zlib.MAX_WBITS).decode('utf-8'), None)['video']['items'][0]
-        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
+        item = traverse_obj(self._parse_json(zlib.decompress(base64.b64decode(
+            urllib.parse.unquote(self._match_id(url))),
+            -zlib.MAX_WBITS).decode(), None), ('video', 'items', 0, {dict})) or {}
+
+        video_id = item['mpxRefId']
+        video_url = self._get_video_url(item)
+        if not video_url:
+            # Old embeds redirect user to ParamountPlus but most links are 404
+            pplus_url = f'https://www.paramountplus.com/shows/video/{video_id}'
+            try:
+                self._request_webpage(HEADRequest(pplus_url), video_id)
+                return self.url_result(pplus_url, ParamountPlusIE)
+            except ExtractorError:
+                self.raise_no_formats('This video is no longer available', True, video_id)
+
+        return self._extract_video(item, video_url, video_id)
 
 
-class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
+class CBSNewsIE(CBSNewsBaseIE):
     IE_NAME = 'cbsnews'
     IE_DESC = 'CBS News'
-    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|video)/(?P<id>[\w-]+)'
 
     _TESTS = [
         {
@@ -47,10 +164,7 @@ class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
                 'timestamp': 1476046464,
                 'upload_date': '20161009',
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+            'skip': 'This video is no longer available',
         },
         {
             'url': 'https://www.cbsnews.com/video/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -61,48 +175,234 @@ class CBSNewsIE(CBSIE):  # XXX: Do not subclass from concrete IE
                 'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
                 'upload_date': '20140404',
                 'timestamp': 1396650660,
-                'uploader': 'CBSI-NEW',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 205,
                 'subtitles': {
                     'en': [{
-                        'ext': 'ttml',
+                        'ext': 'dfxp',
                     }],
                 },
             },
             'params': {
-                # m3u8 download
-                'skip_download': True,
+                'skip_download': 'm3u8',
             },
         },
         {
             # 48 hours
             'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
             'info_dict': {
+                'id': 'maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved',
                 'title': 'Cold as Ice',
                 'description': 'Can a childhood memory solve the 1957 murder of 7-year-old Maria Ridulph?',
             },
             'playlist_mincount': 7,
         },
+        {
+            'url': 'https://www.cbsnews.com/video/032823-cbs-evening-news/',
+            'info_dict': {
+                'id': '_2wuO7hD9LwtyM_TwSnVwnKp6kxlcXgE',
+                'ext': 'mp4',
+                'title': 'CBS Evening News, March 28, 2023',
+                'description': 'md5:db20615aae54adc1d55a1fd69dc75d13',
+                'duration': 1189,
+                'timestamp': 1680042600,
+                'upload_date': '20230328',
+                'season': 'Season 2023',
+                'season_number': 2023,
+                'episode': 'Episode 83',
+                'episode_number': 83,
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': 'm3u8',
+            },
+        },
     ]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
-        entries = []
-        for embed_url in re.findall(r'<iframe[^>]+data-src="(https?://(?:www\.)?cbsnews\.com/embed/video/[^#]*#[^"]+)"', webpage):
-            entries.append(self.url_result(embed_url, CBSNewsEmbedIE.ie_key()))
-        if entries:
-            return self.playlist_result(
-                entries, playlist_title=self._html_search_meta(['og:title', 'twitter:title'], webpage),
-                playlist_description=self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage))
+        playlist = self._extract_playlist(webpage, display_id)
+        if playlist:
+            return playlist
 
-        item = self._parse_json(self._html_search_regex(
-            r'CBSNEWS\.defaultPayload\s*=\s*({.+})',
-            webpage, 'video JSON info'), display_id)['items'][0]
-        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
+        item = self._get_item(webpage, display_id)
+        video_id = item.get('mpxRefId') or display_id
+        video_url = self._get_video_url(item)
+        if not video_url:
+            self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
+
+        return self._extract_video(item, video_url, video_id)
+
+
+class CBSLocalBaseIE(CBSNewsBaseIE):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        item = self._get_item(webpage, display_id)
+        video_id = item.get('mpxRefId') or display_id
+        anvato_id = None
+        video_url = self._get_video_url(item)
+
+        if not video_url:
+            anv_params = self._search_regex(
+                r'<iframe[^>]+\bdata-src="https?://w3\.mp\.lura\.live/player/prod/v3/anvload\.html\?key=([^"]+)"',
+                webpage, 'Anvato URL', default=None)
+
+            if not anv_params:
+                playlist = self._extract_playlist(webpage, display_id)
+                if playlist:
+                    return playlist
+                self.raise_no_formats('No video content was found', expected=True, video_id=video_id)
+
+            anv_data = self._parse_json(base64.urlsafe_b64decode(f'{anv_params}===').decode(), video_id)
+            anvato_id = anv_data['v']
+            return self.url_result(
+                smuggle_url(f'anvato:{anv_data.get("anvack") or self._ANVACK}:{anvato_id}', {
+                    'token': anv_data.get('token') or 'default',
+                }), AnvatoIE, url_transparent=True, _old_archive_ids=[make_archive_id(self, anvato_id)])
+
+        return self._extract_video(item, video_url, video_id)
+
+
+class CBSLocalIE(CBSLocalBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/(?:live/)?video/(?P<id>[\w-]+)'
+    _TESTS = [{
+        # Anvato video via defaultPayload JSON
+        'url': 'https://www.cbsnews.com/newyork/video/1st-cannabis-dispensary-opens-in-queens/',
+        'info_dict': {
+            'id': '6376747',
+            'ext': 'mp4',
+            'title': '1st cannabis dispensary opens in Queens',
+            'description': 'The dispensary is women-owned and located in Jamaica.',
+            'uploader': 'CBS',
+            'duration': 20,
+            'timestamp': 1680193657,
+            'upload_date': '20230330',
+            'categories': ['Stations\\Spoken Word\\WCBSTV', 'Content\\Google', 'Content\\News', 'Content\\News\\Local News'],
+            'tags': 'count:11',
+            'thumbnail': 're:^https?://.*',
+            '_old_archive_ids': ['cbslocal 6376747'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        # cbsnews.com video via defaultPayload JSON
+        'url': 'https://www.cbsnews.com/newyork/live/video/20230330171655-the-city-is-sounding-the-alarm-on-dangerous-social-media-challenges/',
+        'info_dict': {
+            'id': 'sJqfw7YvgSC6ant2zVmzt3y1jYKoL5J3',
+            'ext': 'mp4',
+            'title': 'the city is sounding the alarm on dangerous social media challenges',
+            'description': 'md5:8eccc9b1b73be5138a52e9c4350d2cd6',
+            'thumbnail': 'https://images-cbsn.cbsnews.com/prod/2023/03/30/story_22509622_1680196925.jpg',
+            'duration': 41.0,
+            'timestamp': 1680196615,
+            'upload_date': '20230330',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+
+class CBSLocalArticleIE(CBSLocalBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?:{CBSNewsBaseIE._LOCALE_RE})/news/(?P<id>[\w-]+)'
+    _TESTS = [{
+        # Anvato video via iframe embed
+        'url': 'https://www.cbsnews.com/newyork/news/mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service/',
+        'playlist_count': 2,
+        'info_dict': {
+            'id': 'mta-station-agents-leaving-their-booths-to-provide-more-direct-customer-service',
+            'title': 'MTA station agents begin leaving their booths to provide more direct customer service',
+            'description': 'The more than 2,200 agents will provide face-to-face customer service to passengers.',
+        },
+    }, {
+        'url': 'https://www.cbsnews.com/losangeles/news/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis/',
+        'md5': 'f0ee3081e3843f575fccef901199b212',
+        'info_dict': {
+            'id': '3401037',
+            'ext': 'mp4',
+            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
+            'thumbnail': 're:^https?://.*',
+            'timestamp': 1463440500,
+            'upload_date': '20160516',
+        },
+        'skip': 'Video has been removed',
+    }]
+
+
+class CBSNewsLiveBaseIE(CBSNewsBaseIE):
+    def _get_id(self, url):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _real_extract(self, url):
+        video_id = self._get_id(url)
+        if not video_id:
+            raise ExtractorError('Livestream is not available', expected=True)
+
+        data = traverse_obj(self._download_json(
+            'https://feeds-cbsn.cbsnews.com/2.0/rundown/', video_id, query={
+                'partner': 'cbsnsite',
+                'edition': video_id,
+                'type': 'live',
+            }), ('navigation', 'data', 0, {dict}))
+
+        video_url = traverse_obj(data, (('videoUrlDAI', ('videoUrl', 'base')), {url_or_none}), get_all=False)
+        if not video_url:
+            raise UserNotLive(video_id=video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': True,
+            **traverse_obj(data, {
+                'title': 'headline',
+                'description': 'rundown_slug',
+                'thumbnail': ('images', 'thumbnail_url_hd', {url_or_none}),
+            }),
+        }
+
+
+class CBSLocalLiveIE(CBSNewsLiveBaseIE):
+    _VALID_URL = rf'https?://(?:www\.)?cbsnews\.com/(?P<id>{CBSNewsBaseIE._LOCALE_RE})/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.cbsnews.com/losangeles/live/',
+        'info_dict': {
+            'id': 'CBSN-LA',
+            'ext': 'mp4',
+            'title': str,
+            'description': r're:KCBS/CBSN_LA.CRISPIN.\w+.RUNDOWN \w+ \w+',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'live_status': 'is_live',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _get_id(self, url):
+        return format_field(self._LOCALES, self._match_id(url), 'CBSN-%s')
+
+
+class CBSNewsLiveIE(CBSNewsLiveBaseIE):
+    IE_NAME = 'cbsnews:live'
+    IE_DESC = 'CBS News Livestream'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.cbsnews.com/live/',
+        'info_dict': {
+            'id': 'CBSN-US',
+            'ext': 'mp4',
+            'title': str,
+            'description': r're:\w+ \w+ CRISPIN RUNDOWN',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'live_status': 'is_live',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _get_id(self, url):
+        return 'CBSN-US'
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):
@@ -111,7 +411,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[^/?#]+)'
 
     # Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
         'info_dict': {
             'id': 'clinton-sanders-prepare-to-face-off-in-nh',
@@ -120,7 +420,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
             'duration': 334,
         },
         'skip': 'Video gone',
-    }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -131,13 +431,13 @@ class CBSNewsLiveVideoIE(InfoExtractor):
                 'dvr_slug': display_id,
             })
 
-        formats = self._extract_akamai_formats(video_info['url'], display_id)
-
         return {
             'id': display_id,
             'display_id': display_id,
-            'title': video_info['headline'],
-            'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
-            'duration': parse_duration(video_info.get('segmentDur')),
-            'formats': formats,
+            'formats': self._extract_akamai_formats(video_info['url'], display_id),
+            **traverse_obj(video_info, {
+                'title': 'headline',
+                'thumbnail': ('thumbnail_url_hd', {url_or_none}),
+                'duration': ('segmentDur', {parse_duration}),
+            }),
         }

yt_dlp/extractor/comedycentral.py

@@ -2,7 +2,7 @@ from .mtv import MTVServicesInfoExtractor
 
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist)/(?P<id>[0-9a-z]{6})'
+    _VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?|collection-playlist|movies)/(?P<id>[0-9a-z]{6})'
     _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
 
     _TESTS = [{
@@ -25,6 +25,9 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
     }, {
         'url': 'https://www.cc.com/collection-playlist/cosnej/stand-up-specials/t6vtjb',
         'only_matching': True,
+    }, {
+        'url': 'https://www.cc.com/movies/tkp406/a-cluesterfuenke-christmas',
+        'only_matching': True,
     }]
 
 

yt_dlp/extractor/common.py

@@ -314,6 +314,11 @@ class InfoExtractor:
                         * "author" - human-readable name of the comment author
                         * "author_id" - user ID of the comment author
                         * "author_thumbnail" - The thumbnail of the comment author
+                        * "author_url" - The url to the comment author's page
+                        * "author_is_verified" - Whether the author is verified
+                                                 on the platform
+                        * "author_is_uploader" - Whether the comment is made by
+                                                 the video uploader
                         * "id" - Comment ID
                         * "html" - Comment as HTML
                         * "text" - Plain text of the comment
@@ -325,8 +330,8 @@ class InfoExtractor:
                         * "dislike_count" - Number of negative ratings of the comment
                         * "is_favorited" - Whether the comment is marked as
                                            favorite by the video uploader
-                        * "author_is_uploader" - Whether the comment is made by
-                                                 the video uploader
+                        * "is_pinned" - Whether the comment is pinned to
+                                        the top of the comments
     age_limit:      Age restriction for the video, as an integer (years)
     webpage_url:    The URL to the video webpage, if given to yt-dlp it
                     should allow to get the same result again. (It will be set
@@ -350,6 +355,10 @@ class InfoExtractor:
                         * "start_time" - The start time of the chapter in seconds
                         * "end_time" - The end time of the chapter in seconds
                         * "title" (optional, string)
+    heatmap:        A list of dictionaries, with the following entries:
+                        * "start_time" - The start time of the data point in seconds
+                        * "end_time" - The end time of the data point in seconds
+                        * "value" - The normalized value of the data point (float between 0 and 1)
     playable_in_embed: Whether this video is allowed to play in embedded
                     players on other sites. Can be True (=always allowed),
                     False (=never allowed), None (=unknown), or a string
@@ -3455,7 +3464,7 @@ class InfoExtractor:
 
     def _get_cookies(self, url):
         """ Return a http.cookies.SimpleCookie with the cookies for the url """
-        return LenientSimpleCookie(self._downloader._calc_cookies(url))
+        return LenientSimpleCookie(self._downloader.cookiejar.get_cookie_header(url))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """

yt_dlp/extractor/crtvg.py

@@ -0,0 +1,34 @@
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class CrtvgIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?crtvg\.es/tvg/a-carta/[^/#?]+-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.crtvg.es/tvg/a-carta/os-caimans-do-tea-5839623',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': '5839623',
+            'title': 'Os caimáns do Tea',
+            'ext': 'mp4',
+            'description': 'md5:f71cfba21ae564f0a6f415b31de1f842',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._search_regex(r'var\s+url\s*=\s*["\']([^"\']+)', webpage, 'video url')
+        formats = self._extract_m3u8_formats(video_url + '/playlist.m3u8', video_id, fatal=False)
+        formats.extend(self._extract_mpd_formats(video_url + '/manifest.mpd', video_id, fatal=False))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': remove_end(self._html_search_meta(
+                ['og:title', 'twitter:title'], webpage, 'title', default=None), ' | CRTVG'),
+            'description': self._html_search_meta('description', webpage, 'description', default=None),
+            'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage, 'thumbnail', default=None),
+        }

yt_dlp/extractor/crunchyroll.py

@@ -1,28 +1,37 @@
 import base64
-import urllib.parse
+import urllib.error
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
     format_field,
+    int_or_none,
     join_nonempty,
+    parse_age_limit,
+    parse_count,
     parse_iso8601,
     qualities,
+    remove_start,
+    time_seconds,
     traverse_obj,
-    try_get,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
 class CrunchyrollBaseIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
+    _BASE_URL = 'https://www.crunchyroll.com'
     _API_BASE = 'https://api.crunchyroll.com'
     _NETRC_MACHINE = 'crunchyroll'
-    params = None
+    _AUTH_HEADERS = None
+    _API_ENDPOINT = None
+    _BASIC_AUTH = None
+    _QUERY = {}
 
     @property
     def is_logged_in(self):
-        return self._get_cookies(self._LOGIN_URL).get('etp_rt')
+        return self._get_cookies(self._BASE_URL).get('etp_rt')
 
     def _perform_login(self, username, password):
         if self.is_logged_in:
@@ -35,7 +44,7 @@ class CrunchyrollBaseIE(InfoExtractor):
                 'device_id': 'whatvalueshouldbeforweb',
                 'device_type': 'com.crunchyroll.static',
                 'access_token': 'giKq5eY27ny3cqz',
-                'referer': self._LOGIN_URL
+                'referer': f'{self._BASE_URL}/welcome/login'
             })
         if upsell_response['code'] != 'ok':
             raise ExtractorError('Could not get session id')
@@ -43,149 +52,89 @@ class CrunchyrollBaseIE(InfoExtractor):
 
         login_response = self._download_json(
             f'{self._API_BASE}/login.1.json', None, 'Logging in',
-            data=urllib.parse.urlencode({
+            data=urlencode_postdata({
                 'account': username,
                 'password': password,
                 'session_id': session_id
-            }).encode('ascii'))
+            }))
         if login_response['code'] != 'ok':
             raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
         if not self.is_logged_in:
             raise ExtractorError('Login succeeded but did not set etp_rt cookie')
 
-    def _get_embedded_json(self, webpage, display_id):
-        initial_state = self._parse_json(self._search_regex(
-            r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'), display_id)
-        app_config = self._parse_json(self._search_regex(
-            r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'), display_id)
-        return initial_state, app_config
+    def _update_query(self, lang):
+        if lang in CrunchyrollBaseIE._QUERY:
+            return
 
-    def _get_params(self, lang):
-        if not CrunchyrollBaseIE.params:
-            if self._get_cookies(f'https://www.crunchyroll.com/{lang}').get('etp_rt'):
-                grant_type, key = 'etp_rt_cookie', 'accountAuthClientId'
-            else:
-                grant_type, key = 'client_id', 'anonClientId'
+        webpage = self._download_webpage(
+            f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})')
 
-            initial_state, app_config = self._get_embedded_json(self._download_webpage(
-                f'https://www.crunchyroll.com/{lang}', None, note='Retrieving main page'), None)
-            api_domain = app_config['cxApiParams']['apiDomain'].replace('beta.crunchyroll.com', 'www.crunchyroll.com')
+        initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None)
+        CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, {
+            'locale': ('localization', 'locale'),
+        }) or None
 
-            auth_response = self._download_json(
-                f'{api_domain}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
-                headers={
-                    'Authorization': 'Basic ' + str(base64.b64encode(('%s:' % app_config['cxApiParams'][key]).encode('ascii')), 'ascii')
-                }, data=f'grant_type={grant_type}'.encode('ascii'))
-            policy_response = self._download_json(
-                f'{api_domain}/index/v2', None, note='Retrieving signed policy',
-                headers={
-                    'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
-                })
-            cms = policy_response.get('cms_web')
-            bucket = cms['bucket']
-            params = {
-                'Policy': cms['policy'],
-                'Signature': cms['signature'],
-                'Key-Pair-Id': cms['key_pair_id']
-            }
-            locale = traverse_obj(initial_state, ('localization', 'locale'))
-            if locale:
-                params['locale'] = locale
-            CrunchyrollBaseIE.params = (api_domain, bucket, params)
-        return CrunchyrollBaseIE.params
+        if CrunchyrollBaseIE._BASIC_AUTH:
+            return
 
+        app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None)
+        cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId']
+        self.write_debug(f'Using cxApiParam={cx_api_param}')
+        CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
 
-class CrunchyrollBetaIE(CrunchyrollBaseIE):
-    IE_NAME = 'crunchyroll'
-    _VALID_URL = r'''(?x)
-        https?://(?:beta|www)\.crunchyroll\.com/
-        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        watch/(?P<id>\w+)
-        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
-    _TESTS = [{
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
-        'info_dict': {
-            'id': 'GY2P1Q98Y',
-            'ext': 'mp4',
-            'duration': 1380.241,
-            'timestamp': 1459632600,
-            'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
-            'title': 'World Trigger Episode 73 – To the Future',
-            'upload_date': '20160402',
-            'series': 'World Trigger',
-            'series_id': 'GR757DMKY',
-            'season': 'World Trigger',
-            'season_id': 'GR9P39NJ6',
-            'season_number': 1,
-            'episode': 'To the Future',
-            'episode_number': 73,
-            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
-            'chapters': 'count:2',
-        },
-        'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
-    }, {
-        'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
-        'info_dict': {
-            'id': 'GYE5WKQGR',
-            'ext': 'mp4',
-            'duration': 366.459,
-            'timestamp': 1476788400,
-            'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
-            'title': 'SHELTER Episode  – Porter Robinson presents Shelter the Animation',
-            'upload_date': '20161018',
-            'series': 'SHELTER',
-            'series_id': 'GYGG09WWY',
-            'season': 'SHELTER',
-            'season_id': 'GR09MGK4R',
-            'season_number': 1,
-            'episode': 'Porter Robinson presents Shelter the Animation',
-            'episode_number': 0,
-            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg$',
-            'chapters': 'count:0',
-        },
-        'params': {'skip_download': True},
-        'skip': 'Video is Premium only',
-    }, {
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
-        'only_matching': True,
-    }, {
-        'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
-        'only_matching': True,
-    }]
+    def _update_auth(self):
+        if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
+            return
 
-    def _real_extract(self, url):
-        lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-        api_domain, bucket, params = self._get_params(lang)
+        assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand'
+        grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id'
+        auth_response = self._download_json(
+            f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
+            headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
 
-        episode_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
-            note='Retrieving episode metadata', query=params)
-        if episode_response.get('is_premium_only') and not bucket.endswith('crunchyroll'):
-            if self.is_logged_in:
-                raise ExtractorError('This video is for premium members only', expected=True)
-            else:
-                self.raise_login_required('This video is for premium members only')
+        CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
+        CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
 
-        stream_response = self._download_json(
-            f'{api_domain}{episode_response["__links__"]["streams"]["href"]}', display_id,
-            note='Retrieving stream info', query=params)
-        get_streams = lambda name: (traverse_obj(stream_response, name) or {}).items()
+    def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}):
+        self._update_query(lang)
+        self._update_auth()
 
-        requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
-        hardsub_preference = qualities(requested_hardsubs[::-1])
+        if not endpoint.startswith('/'):
+            endpoint = f'/{endpoint}'
+
+        return self._download_json(
+            f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}',
+            headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query})
+
+    def _call_api(self, path, internal_id, lang, note='api', query={}):
+        if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'):
+            path = f'/content/v2/{self._API_ENDPOINT}/{path}'
+
+        try:
+            result = self._call_base_api(
+                path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
+        except ExtractorError as error:
+            if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
+                return None
+            raise
+
+        if not result:
+            raise ExtractorError(f'Unexpected response when downloading {note} JSON')
+        return result
+
+    def _extract_formats(self, stream_response, display_id=None):
         requested_formats = self._configuration_arg('format') or ['adaptive_hls']
-
         available_formats = {}
-        for stream_type, streams in get_streams('streams'):
+        for stream_type, streams in traverse_obj(
+                stream_response, (('streams', ('data', 0)), {dict.items}, ...)):
             if stream_type not in requested_formats:
                 continue
-            for stream in streams.values():
-                if not stream.get('url'):
-                    continue
+            for stream in traverse_obj(streams, lambda _, v: v['url']):
                 hardsub_lang = stream.get('hardsub_locale') or ''
                 format_id = join_nonempty(stream_type, format_field(stream, 'hardsub_locale', 'hardsub-%s'))
                 available_formats[hardsub_lang] = (stream_type, format_id, hardsub_lang, stream['url'])
 
+        requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
         if '' in available_formats and 'all' not in requested_hardsubs:
             full_format_langs = set(requested_hardsubs)
             self.to_screen(
@@ -196,6 +145,8 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
         else:
             full_format_langs = set(map(str.lower, available_formats))
 
+        audio_locale = traverse_obj(stream_response, ((None, 'meta'), 'audio_locale'), get_all=False)
+        hardsub_preference = qualities(requested_hardsubs[::-1])
         formats = []
         for stream_type, format_id, hardsub_lang, stream_url in available_formats.values():
             if stream_type.endswith('hls'):
@@ -214,63 +165,292 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
                 continue
             for f in adaptive_formats:
                 if f.get('acodec') != 'none':
-                    f['language'] = stream_response.get('audio_locale')
+                    f['language'] = audio_locale
                 f['quality'] = hardsub_preference(hardsub_lang.lower())
             formats.extend(adaptive_formats)
 
-        chapters = None
+        return formats
+
+    def _extract_subtitles(self, data):
+        subtitles = {}
+
+        for locale, subtitle in traverse_obj(data, ((None, 'meta'), 'subtitles', {dict.items}, ...)):
+            subtitles[locale] = [traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})]
+
+        return subtitles
+
+
+class CrunchyrollCmsBaseIE(CrunchyrollBaseIE):
+    _API_ENDPOINT = 'cms'
+    _CMS_EXPIRY = None
+
+    def _call_cms_api_signed(self, path, internal_id, lang, note='api'):
+        if not CrunchyrollCmsBaseIE._CMS_EXPIRY or CrunchyrollCmsBaseIE._CMS_EXPIRY <= time_seconds():
+            response = self._call_base_api('index/v2', None, lang, 'Retrieving signed policy')['cms_web']
+            CrunchyrollCmsBaseIE._CMS_QUERY = {
+                'Policy': response['policy'],
+                'Signature': response['signature'],
+                'Key-Pair-Id': response['key_pair_id'],
+            }
+            CrunchyrollCmsBaseIE._CMS_BUCKET = response['bucket']
+            CrunchyrollCmsBaseIE._CMS_EXPIRY = parse_iso8601(response['expires']) - 10
+
+        if not path.startswith('/cms/v2'):
+            path = f'/cms/v2{CrunchyrollCmsBaseIE._CMS_BUCKET}/{path}'
+
+        return self._call_base_api(
+            path, internal_id, lang, f'Downloading {note} JSON (signed cms)', query=CrunchyrollCmsBaseIE._CMS_QUERY)
+
+
+class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
+    IE_NAME = 'crunchyroll'
+    _VALID_URL = r'''(?x)
+        https?://(?:beta\.|www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        watch/(?!concert|musicvideo)(?P<id>\w+)'''
+    _TESTS = [{
+        # Premium only
+        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y/to-the-future',
+        'info_dict': {
+            'id': 'GY2P1Q98Y',
+            'ext': 'mp4',
+            'duration': 1380.241,
+            'timestamp': 1459632600,
+            'description': 'md5:a022fbec4fbb023d43631032c91ed64b',
+            'title': 'World Trigger Episode 73 – To the Future',
+            'upload_date': '20160402',
+            'series': 'World Trigger',
+            'series_id': 'GR757DMKY',
+            'season': 'World Trigger',
+            'season_id': 'GR9P39NJ6',
+            'season_number': 1,
+            'episode': 'To the Future',
+            'episode_number': 73,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'chapters': 'count:2',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': 'm3u8', 'format': 'all[format_id~=hardsub]'},
+    }, {
+        # Premium only
+        'url': 'https://www.crunchyroll.com/watch/GYE5WKQGR',
+        'info_dict': {
+            'id': 'GYE5WKQGR',
+            'ext': 'mp4',
+            'duration': 366.459,
+            'timestamp': 1476788400,
+            'description': 'md5:74b67283ffddd75f6e224ca7dc031e76',
+            'title': 'SHELTER – Porter Robinson presents Shelter the Animation',
+            'upload_date': '20161018',
+            'series': 'SHELTER',
+            'series_id': 'GYGG09WWY',
+            'season': 'SHELTER',
+            'season_id': 'GR09MGK4R',
+            'season_number': 1,
+            'episode': 'Porter Robinson presents Shelter the Animation',
+            'episode_number': 0,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': True},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GJWU2VKK3/cherry-blossom-meeting-and-a-coming-blizzard',
+        'info_dict': {
+            'id': 'GJWU2VKK3',
+            'ext': 'mp4',
+            'duration': 1420.054,
+            'description': 'md5:2d1c67c0ec6ae514d9c30b0b99a625cd',
+            'title': 'The Ice Guy and His Cool Female Colleague Episode 1 – Cherry Blossom Meeting and a Coming Blizzard',
+            'series': 'The Ice Guy and His Cool Female Colleague',
+            'series_id': 'GW4HM75NP',
+            'season': 'The Ice Guy and His Cool Female Colleague',
+            'season_id': 'GY9PC21VE',
+            'season_number': 1,
+            'episode': 'Cherry Blossom Meeting and a Coming Blizzard',
+            'episode_number': 1,
+            'chapters': 'count:2',
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'timestamp': 1672839000,
+            'upload_date': '20230104',
+            'age_limit': 14,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GM8F313NQ',
+        'info_dict': {
+            'id': 'GM8F313NQ',
+            'ext': 'mp4',
+            'title': 'Garakowa -Restore the World-',
+            'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
+            'duration': 3996.104,
+            'age_limit': 13,
+            'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/G62PEZ2E6',
+        'info_dict': {
+            'id': 'G62PEZ2E6',
+            'description': 'md5:8d2f8b6b9dd77d87810882e7d2ee5608',
+            'age_limit': 13,
+            'duration': 65.138,
+            'title': 'Garakowa -Restore the World-',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
+        'only_matching': True,
+    }, {
+        'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
+        'only_matching': True,
+    }]
+    # We want to support lazy playlist filtering and movie listings cannot be inside a playlist
+    _RETURN_TYPE = 'video'
+
+    def _real_extract(self, url):
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
+
+        # We need to use unsigned API call to allow ratings query string
+        response = traverse_obj(self._call_api(
+            f'objects/{internal_id}', internal_id, lang, 'object info', {'ratings': 'true'}), ('data', 0, {dict}))
+        if not response:
+            raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
+
+        object_type = response.get('type')
+        if object_type == 'episode':
+            result = self._transform_episode_response(response)
+
+        elif object_type == 'movie':
+            result = self._transform_movie_response(response)
+
+        elif object_type == 'movie_listing':
+            first_movie_id = traverse_obj(response, ('movie_listing_metadata', 'first_movie_id'))
+            if not self._yes_playlist(internal_id, first_movie_id):
+                return self.url_result(f'{self._BASE_URL}/{lang}watch/{first_movie_id}', CrunchyrollBetaIE, first_movie_id)
+
+            def entries():
+                movies = self._call_api(f'movie_listings/{internal_id}/movies', internal_id, lang, 'movie list')
+                for movie_response in traverse_obj(movies, ('data', ...)):
+                    yield self.url_result(
+                        f'{self._BASE_URL}/{lang}watch/{movie_response["id"]}',
+                        CrunchyrollBetaIE, **self._transform_movie_response(movie_response))
+
+            return self.playlist_result(entries(), **self._transform_movie_response(response))
+
+        else:
+            raise ExtractorError(f'Unknown object type {object_type}')
+
+        # There might be multiple audio languages for one object (`<object>_metadata.versions`),
+        # so we need to get the id from `streams_link` instead or we dont know which language to choose
+        streams_link = response.get('streams_link')
+        if not streams_link and traverse_obj(response, (f'{object_type}_metadata', 'is_premium_only')):
+            message = f'This {object_type} is for premium members only'
+            if self.is_logged_in:
+                raise ExtractorError(message, expected=True)
+            self.raise_login_required(message)
+
+        # We need go from unsigned to signed api to avoid getting soft banned
+        stream_response = self._call_cms_api_signed(remove_start(
+            streams_link, '/content/v2/cms/'), internal_id, lang, 'stream info')
+        result['formats'] = self._extract_formats(stream_response, internal_id)
+        result['subtitles'] = self._extract_subtitles(stream_response)
+
         # if no intro chapter is available, a 403 without usable data is returned
-        intro_chapter = self._download_json(f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
-                                            display_id, fatal=False, errnote=False)
+        intro_chapter = self._download_json(
+            f'https://static.crunchyroll.com/datalab-intro-v2/{internal_id}.json',
+            internal_id, note='Downloading chapter info', fatal=False, errnote=False)
         if isinstance(intro_chapter, dict):
-            chapters = [{
+            result['chapters'] = [{
                 'title': 'Intro',
                 'start_time': float_or_none(intro_chapter.get('startTime')),
-                'end_time': float_or_none(intro_chapter.get('endTime'))
+                'end_time': float_or_none(intro_chapter.get('endTime')),
             }]
 
+        def calculate_count(item):
+            return parse_count(''.join((item['displayed'], item.get('unit') or '')))
+
+        result.update(traverse_obj(response, ('rating', {
+            'like_count': ('up', {calculate_count}),
+            'dislike_count': ('down', {calculate_count}),
+        })))
+
+        return result
+
+    @staticmethod
+    def _transform_episode_response(data):
+        metadata = traverse_obj(data, (('episode_metadata', None), {dict}), get_all=False) or {}
         return {
-            'id': internal_id,
-            'title': '%s Episode %s – %s' % (
-                episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
-            'description': try_get(episode_response, lambda x: x['description'].replace(r'\r\n', '\n')),
-            'duration': float_or_none(episode_response.get('duration_ms'), 1000),
-            'timestamp': parse_iso8601(episode_response.get('upload_date')),
-            'series': episode_response.get('series_title'),
-            'series_id': episode_response.get('series_id'),
-            'season': episode_response.get('season_title'),
-            'season_id': episode_response.get('season_id'),
-            'season_number': episode_response.get('season_number'),
-            'episode': episode_response.get('title'),
-            'episode_number': episode_response.get('sequence_number'),
-            'formats': formats,
-            'thumbnails': [{
-                'url': thumb.get('source'),
-                'width': thumb.get('width'),
-                'height': thumb.get('height'),
-            } for thumb in traverse_obj(episode_response, ('images', 'thumbnail', ..., ...)) or []],
-            'subtitles': {
-                lang: [{
-                    'url': subtitle_data.get('url'),
-                    'ext': subtitle_data.get('format')
-                }] for lang, subtitle_data in get_streams('subtitles')
-            },
-            'chapters': chapters
+            'id': data['id'],
+            'title': ' \u2013 '.join((
+                ('%s%s' % (
+                    format_field(metadata, 'season_title'),
+                    format_field(metadata, 'episode', ' Episode %s'))),
+                format_field(data, 'title'))),
+            **traverse_obj(data, {
+                'episode': ('title', {str}),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', 'thumbnail', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+            }),
+            **traverse_obj(metadata, {
+                'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
+                'timestamp': ('upload_date', {parse_iso8601}),
+                'series': ('series_title', {str}),
+                'series_id': ('series_id', {str}),
+                'season': ('season_title', {str}),
+                'season_id': ('season_id', {str}),
+                'season_number': ('season_number', ({int}, {float_or_none})),
+                'episode_number': ('sequence_number', ({int}, {float_or_none})),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+                'language': ('audio_locale', {str}),
+            }, get_all=False),
+        }
+
+    @staticmethod
+    def _transform_movie_response(data):
+        metadata = traverse_obj(data, (('movie_metadata', 'movie_listing_metadata', None), {dict}), get_all=False) or {}
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', 'thumbnail', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+            }),
+            **traverse_obj(metadata, {
+                'duration': ('duration_ms', {lambda x: float_or_none(x, 1000)}),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+            }),
         }
 
 
-class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
+class CrunchyrollBetaShowIE(CrunchyrollCmsBaseIE):
     IE_NAME = 'crunchyroll:playlist'
     _VALID_URL = r'''(?x)
-        https?://(?:beta|www)\.crunchyroll\.com/
+        https?://(?:beta\.|www\.)?crunchyroll\.com/
         (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        series/(?P<id>\w+)
-        (?:/(?P<display_id>[\w-]+))?/?(?:[?#]|$)'''
+        series/(?P<id>\w+)'''
     _TESTS = [{
         'url': 'https://www.crunchyroll.com/series/GY19NQ2QR/Girl-Friend-BETA',
         'info_dict': {
             'id': 'GY19NQ2QR',
             'title': 'Girl Friend BETA',
+            'description': 'md5:99c1b22ee30a74b536a8277ced8eb750',
+            # XXX: `thumbnail` does not get set from `thumbnails` in playlist
+            #  'thumbnail': r're:^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'age_limit': 14,
         },
         'playlist_mincount': 10,
     }, {
@@ -279,41 +459,163 @@ class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
     }]
 
     def _real_extract(self, url):
-        lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'id', 'display_id')
-        api_domain, bucket, params = self._get_params(lang)
-
-        series_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/series/{internal_id}', display_id,
-            note='Retrieving series metadata', query=params)
-
-        seasons_response = self._download_json(
-            f'{api_domain}/cms/v2{bucket}/seasons?series_id={internal_id}', display_id,
-            note='Retrieving season list', query=params)
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
 
         def entries():
-            for season in seasons_response['items']:
-                episodes_response = self._download_json(
-                    f'{api_domain}/cms/v2{bucket}/episodes?season_id={season["id"]}', display_id,
-                    note=f'Retrieving episode list for {season.get("slug_title")}', query=params)
-                for episode in episodes_response['items']:
-                    episode_id = episode['id']
-                    episode_display_id = episode['slug_title']
-                    yield {
-                        '_type': 'url',
-                        'url': f'https://www.crunchyroll.com/{lang}watch/{episode_id}/{episode_display_id}',
-                        'ie_key': CrunchyrollBetaIE.ie_key(),
-                        'id': episode_id,
-                        'title': '%s Episode %s – %s' % (episode.get('season_title'), episode.get('episode'), episode.get('title')),
-                        'description': try_get(episode, lambda x: x['description'].replace(r'\r\n', '\n')),
-                        'duration': float_or_none(episode.get('duration_ms'), 1000),
-                        'series': episode.get('series_title'),
-                        'series_id': episode.get('series_id'),
-                        'season': episode.get('season_title'),
-                        'season_id': episode.get('season_id'),
-                        'season_number': episode.get('season_number'),
-                        'episode': episode.get('title'),
-                        'episode_number': episode.get('sequence_number'),
-                        'language': episode.get('audio_locale'),
-                    }
+            seasons_response = self._call_cms_api_signed(f'seasons?series_id={internal_id}', internal_id, lang, 'seasons')
+            for season in traverse_obj(seasons_response, ('items', ..., {dict})):
+                episodes_response = self._call_cms_api_signed(
+                    f'episodes?season_id={season["id"]}', season["id"], lang, 'episode list')
+                for episode_response in traverse_obj(episodes_response, ('items', ..., {dict})):
+                    yield self.url_result(
+                        f'{self._BASE_URL}/{lang}watch/{episode_response["id"]}',
+                        CrunchyrollBetaIE, **CrunchyrollBetaIE._transform_episode_response(episode_response))
 
-        return self.playlist_result(entries(), internal_id, series_response.get('title'))
+        return self.playlist_result(
+            entries(), internal_id,
+            **traverse_obj(self._call_api(f'series/{internal_id}', internal_id, lang, 'series'), ('data', 0, {
+                'title': ('title', {str}),
+                'description': ('description', {lambda x: x.replace(r'\r\n', '\n')}),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+                'thumbnails': ('images', ..., ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                })
+            })))
+
+
+class CrunchyrollMusicIE(CrunchyrollBaseIE):
+    IE_NAME = 'crunchyroll:music'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})'''
+    _TESTS = [{
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MV88BB7F2C',
+            'display_id': 'crossing-field',
+            'title': 'Crossing Field',
+            'track': 'Crossing Field',
+            'artist': 'LiSA',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'genre': ['Anime'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MC2E2AC135',
+            'display_id': 'live-is-smile-always-364joker-at-yokohama-arena',
+            'title': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
+            'track': 'LiVE is Smile Always-364+JOKER- at YOKOHAMA ARENA',
+            'artist': 'LiSA',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'description': 'md5:747444e7e6300907b7a43f0a0503072e',
+            'genre': ['J-Pop'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena',
+        'only_matching': True,
+    }]
+    _API_ENDPOINT = 'music'
+
+    def _real_extract(self, url):
+        lang, internal_id, object_type = self._match_valid_url(url).group('lang', 'id', 'type')
+        path, name = {
+            'concert': ('concerts', 'concert info'),
+            'musicvideo': ('music_videos', 'music video info'),
+        }[object_type]
+        response = traverse_obj(self._call_api(f'{path}/{internal_id}', internal_id, lang, name), ('data', 0, {dict}))
+        if not response:
+            raise ExtractorError(f'No video with id {internal_id} could be found (possibly region locked?)', expected=True)
+
+        streams_link = response.get('streams_link')
+        if not streams_link and response.get('isPremiumOnly'):
+            message = f'This {response.get("type") or "media"} is for premium members only'
+            if self.is_logged_in:
+                raise ExtractorError(message, expected=True)
+            self.raise_login_required(message)
+
+        result = self._transform_music_response(response)
+        stream_response = self._call_api(streams_link, internal_id, lang, 'stream info')
+        result['formats'] = self._extract_formats(stream_response, internal_id)
+
+        return result
+
+    @staticmethod
+    def _transform_music_response(data):
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'display_id': 'slug',
+                'title': 'title',
+                'track': 'title',
+                'artist': ('artist', 'name'),
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n') or None}),
+                'thumbnails': ('images', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+                'genre': ('genres', ..., 'displayValue'),
+                'age_limit': ('maturity_ratings', -1, {parse_age_limit}),
+            }),
+        }
+
+
+class CrunchyrollArtistIE(CrunchyrollBaseIE):
+    IE_NAME = 'crunchyroll:artist'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?crunchyroll\.com/
+        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        artist/(?P<id>\w{10})'''
+    _TESTS = [{
+        'url': 'https://www.crunchyroll.com/artist/MA179CB50D',
+        'info_dict': {
+            'id': 'MA179CB50D',
+            'title': 'LiSA',
+            'genre': ['J-Pop', 'Anime', 'Rock'],
+            'description': 'md5:16d87de61a55c3f7d6c454b73285938e',
+        },
+        'playlist_mincount': 83,
+    }, {
+        'url': 'https://www.crunchyroll.com/artist/MA179CB50D/lisa',
+        'only_matching': True,
+    }]
+    _API_ENDPOINT = 'music'
+
+    def _real_extract(self, url):
+        lang, internal_id = self._match_valid_url(url).group('lang', 'id')
+        response = traverse_obj(self._call_api(
+            f'artists/{internal_id}', internal_id, lang, 'artist info'), ('data', 0))
+
+        def entries():
+            for attribute, path in [('concerts', 'concert'), ('videos', 'musicvideo')]:
+                for internal_id in traverse_obj(response, (attribute, ...)):
+                    yield self.url_result(f'{self._BASE_URL}/watch/{path}/{internal_id}', CrunchyrollMusicIE, internal_id)
+
+        return self.playlist_result(entries(), **self._transform_artist_response(response))
+
+    @staticmethod
+    def _transform_artist_response(data):
+        return {
+            'id': data['id'],
+            **traverse_obj(data, {
+                'title': 'name',
+                'description': ('description', {str}, {lambda x: x.replace(r'\r\n', '\n')}),
+                'thumbnails': ('images', ..., ..., {
+                    'url': ('source', {url_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                }),
+                'genre': ('genres', ..., 'displayValue'),
+            }),
+        }

yt_dlp/extractor/dacast.py

@@ -0,0 +1,158 @@
+import hashlib
+import re
+import time
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    classproperty,
+    float_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class DacastBaseIE(InfoExtractor):
+    _URL_TYPE = None
+
+    @classproperty
+    def _VALID_URL(cls):
+        return fr'https?://iframe\.dacast\.com/{cls._URL_TYPE}/(?P<user_id>[\w-]+)/(?P<id>[\w-]+)'
+
+    @classproperty
+    def _EMBED_REGEX(cls):
+        return [rf'<iframe[^>]+\bsrc=["\'](?P<url>{cls._VALID_URL})']
+
+    _API_INFO_URL = 'https://playback.dacast.com/content/info'
+
+    @classmethod
+    def _get_url_from_id(cls, content_id):
+        user_id, media_id = content_id.split(f'-{cls._URL_TYPE}-')
+        return f'https://iframe.dacast.com/{cls._URL_TYPE}/{user_id}/{media_id}'
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for content_id in re.findall(
+                rf'<script[^>]+\bsrc=["\']https://player\.dacast\.com/js/player\.js\?contentId=([\w-]+-{cls._URL_TYPE}-[\w-]+)["\']', webpage):
+            yield cls._get_url_from_id(content_id)
+
+
+class DacastVODIE(DacastBaseIE):
+    _URL_TYPE = 'vod'
+    _TESTS = [{
+        'url': 'https://iframe.dacast.com/vod/acae82153ef4d7a7344ae4eaa86af534/1c6143e3-5a06-371d-8695-19b96ea49090',
+        'info_dict': {
+            'id': '1c6143e3-5a06-371d-8695-19b96ea49090',
+            'ext': 'mp4',
+            'uploader_id': 'acae82153ef4d7a7344ae4eaa86af534',
+            'title': '2_4||Adnexal mass characterisation: O-RADS US and MRI||N. Bharwani, London/UK',
+            'thumbnail': 'https://universe-files.dacast.com/26137208-5858-65c1-5e9a-9d6b6bd2b6c2',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.dacast.com/support/knowledgebase/how-can-i-embed-a-video-on-my-website/',
+        'info_dict': {
+            'id': 'b6674869-f08a-23c5-1d7b-81f5309e1a90',
+            'ext': 'mp4',
+            'title': '4-HowToEmbedVideo.mp4',
+            'uploader_id': '3b67c4a9-3886-4eb1-d0eb-39b23b14bef3',
+            'thumbnail': 'https://universe-files.dacast.com/d26ab48f-a52a-8783-c42e-a90290ba06b6.png',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://gist.githubusercontent.com/bashonly/4ad249ef2910346fbdf3809b220f11ee/raw/87349778d4af1a80b1fcc3beb9c88108de5858f5/dacast_embeds.html',
+        'info_dict': {
+            'id': 'e7df418e-a83b-7a7f-7b5e-1a667981e8fa',
+            'ext': 'mp4',
+            'title': 'Evening Service 2-5-23',
+            'uploader_id': '943bb1ab3c03695ba85330d92d6d226e',
+            'thumbnail': 'https://universe-files.dacast.com/337472b3-e92c-2ea4-7eb7-5700da477f67',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
+        query = {'contentId': f'{user_id}-vod-{video_id}', 'provider': 'universe'}
+        info = self._download_json(self._API_INFO_URL, video_id, query=query, fatal=False)
+        access = self._download_json(
+            'https://playback.dacast.com/content/access', video_id,
+            note='Downloading access JSON', query=query, expected_status=403)
+
+        error = access.get('error')
+        if error in ('Broadcaster has been blocked', 'Content is offline'):
+            raise ExtractorError(error, expected=True)
+        elif error:
+            raise ExtractorError(f'Dacast API says "{error}"')
+
+        hls_url = access['hls']
+        hls_aes = {}
+
+        if 'DRM_EXT' in hls_url:
+            self.report_drm(video_id)
+        elif '/uspaes/' in hls_url:
+            # From https://player.dacast.com/js/player.js
+            ts = int(time.time())
+            signature = hashlib.sha1(
+                f'{10413792000 - ts}{ts}YfaKtquEEpDeusCKbvYszIEZnWmBcSvw').digest().hex()
+            hls_aes['uri'] = f'https://keys.dacast.com/uspaes/{video_id}.key?s={signature}&ts={ts}'
+
+        for retry in self.RetryManager():
+            try:
+                formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
+            except ExtractorError as e:
+                # CDN will randomly respond with 403
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    retry.error = e
+                    continue
+                raise
+
+        return {
+            'id': video_id,
+            'uploader_id': user_id,
+            'formats': formats,
+            'hls_aes': hls_aes or None,
+            **traverse_obj(info, ('contentInfo', {
+                'title': 'title',
+                'duration': ('duration', {float_or_none}),
+                'thumbnail': ('thumbnailUrl', {url_or_none}),
+            })),
+        }
+
+
+class DacastPlaylistIE(DacastBaseIE):
+    _URL_TYPE = 'playlist'
+    _TESTS = [{
+        'url': 'https://iframe.dacast.com/playlist/943bb1ab3c03695ba85330d92d6d226e/b632eb053cac17a9c9a02bcfc827f2d8',
+        'playlist_mincount': 28,
+        'info_dict': {
+            'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
+            'title': 'Archive Sermons',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://gist.githubusercontent.com/bashonly/7efb606f49f3c6e07ea0327de5a661d1/raw/05a16eac830245ea301fb0a585023bec71e6093c/dacast_playlist_embed.html',
+        'playlist_mincount': 28,
+        'info_dict': {
+            'id': 'b632eb053cac17a9c9a02bcfc827f2d8',
+            'title': 'Archive Sermons',
+        },
+    }]
+
+    def _real_extract(self, url):
+        user_id, playlist_id = self._match_valid_url(url).group('user_id', 'id')
+        info = self._download_json(
+            self._API_INFO_URL, playlist_id, note='Downloading playlist JSON', query={
+                'contentId': f'{user_id}-playlist-{playlist_id}',
+                'provider': 'universe',
+            })['contentInfo']
+
+        def entries(info):
+            for video in traverse_obj(info, ('features', 'playlist', 'contents', lambda _, v: v['id'])):
+                yield self.url_result(
+                    DacastVODIE._get_url_from_id(video['id']), DacastVODIE, video['id'], video.get('title'))
+
+        return self.playlist_result(entries(info), playlist_id, info.get('title'))

yt_dlp/extractor/daftsex.py

@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from ..compat import compat_b64decode
 from ..utils import (
+    ExtractorError,
     int_or_none,
     js_to_json,
     parse_count,
@@ -12,21 +13,24 @@ from ..utils import (
 
 
 class DaftsexIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?daftsex\.com/watch/(?P<id>-?\d+_\d+)'
+    _VALID_URL = r'https?://(?:www\.)?daft\.sex/watch/(?P<id>-?\d+_\d+)'
     _TESTS = [{
-        'url': 'https://daftsex.com/watch/-35370899_456246186',
-        'md5': 'd95135e6cea2d905bea20dbe82cda64a',
+        'url': 'https://daft.sex/watch/-35370899_456246186',
+        'md5': '64c04ef7b4c7b04b308f3b0c78efe7cd',
         'info_dict': {
             'id': '-35370899_456246186',
             'ext': 'mp4',
             'title': 'just relaxing',
-            'description': 'just relaxing - Watch video Watch video in high quality',
+            'description': 'just relaxing – Watch video Watch video in high quality',
             'upload_date': '20201113',
             'timestamp': 1605261911,
-            'thumbnail': r're:https://[^/]+/impf/-43BuMDIawmBGr3GLcZ93CYwWf2PBv_tVWoS1A/dnu41DnARU4\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=6af2c26ff4a45e55334189301c867384&type=video_thumb',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'age_limit': 18,
+            'duration': 15.0,
+            'view_count': int
         },
     }, {
-        'url': 'https://daftsex.com/watch/-156601359_456242791',
+        'url': 'https://daft.sex/watch/-156601359_456242791',
         'info_dict': {
             'id': '-156601359_456242791',
             'ext': 'mp4',
@@ -36,6 +40,7 @@ class DaftsexIE(InfoExtractor):
             'timestamp': 1600250735,
             'thumbnail': 'https://psv153-1.crazycloud.ru/videos/-156601359/456242791/thumb.jpg?extra=i3D32KaBbBFf9TqDRMAVmQ',
         },
+        'skip': 'deleted / private'
     }]
 
     def _real_extract(self, url):
@@ -60,7 +65,7 @@ class DaftsexIE(InfoExtractor):
             webpage, 'player color', fatal=False) or ''
 
         embed_page = self._download_webpage(
-            'https://daxab.com/player/%s?color=%s' % (player_hash, player_color),
+            'https://dxb.to/player/%s?color=%s' % (player_hash, player_color),
             video_id, headers={'Referer': url})
         video_params = self._parse_json(
             self._search_regex(
@@ -94,15 +99,19 @@ class DaftsexIE(InfoExtractor):
                 'age_limit': 18,
             }
 
-        item = self._download_json(
+        items = self._download_json(
             f'{server_domain}/method/video.get/{video_id}', video_id,
             headers={'Referer': url}, query={
                 'token': video_params['video']['access_token'],
                 'videos': video_id,
                 'ckey': video_params['c_key'],
                 'credentials': video_params['video']['credentials'],
-            })['response']['items'][0]
+            })['response']['items']
 
+        if not items:
+            raise ExtractorError('Video is not available', video_id=video_id, expected=True)
+
+        item = items[0]
         formats = []
         for f_id, f_url in item.get('files', {}).items():
             if f_id == 'external':

yt_dlp/extractor/digitalconcerthall.py

@@ -11,7 +11,7 @@ from ..utils import (
 
 class DigitalConcertHallIE(InfoExtractor):
     IE_DESC = 'DigitalConcertHall extractor'
-    _VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/concert/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/(?P<type>film|concert)/(?P<id>[0-9]+)'
     _OAUTH_URL = 'https://api.digitalconcerthall.com/v2/oauth2/token'
     _ACCESS_TOKEN = None
     _NETRC_MACHINE = 'digitalconcerthall'
@@ -40,6 +40,19 @@ class DigitalConcertHallIE(InfoExtractor):
         },
         'params': {'skip_download': 'm3u8'},
         'playlist_count': 3,
+    }, {
+        'url': 'https://www.digitalconcerthall.com/en/film/388',
+        'info_dict': {
+            'id': '388',
+            'ext': 'mp4',
+            'title': 'The Berliner Philharmoniker and Frank Peter Zimmermann',
+            'description': 'md5:cfe25a7044fa4be13743e5089b5b5eb2',
+            'thumbnail': r're:^https?://images.digitalconcerthall.com/cms/thumbnails.*\.jpg$',
+            'upload_date': '20220714',
+            'timestamp': 1657785600,
+            'album_artist': 'Frank Peter Zimmermann / Benedikt von Bernstorff / Jakob von Bernstorff',
+        },
+        'params': {'skip_download': 'm3u8'},
     }]
 
     def _perform_login(self, username, password):
@@ -75,7 +88,7 @@ class DigitalConcertHallIE(InfoExtractor):
         if not self._ACCESS_TOKEN:
             self.raise_login_required(method='password')
 
-    def _entries(self, items, language, **kwargs):
+    def _entries(self, items, language, type_, **kwargs):
         for item in items:
             video_id = item['id']
             stream_info = self._download_json(
@@ -103,11 +116,11 @@ class DigitalConcertHallIE(InfoExtractor):
                     'start_time': chapter.get('time'),
                     'end_time': try_get(chapter, lambda x: x['time'] + x['duration']),
                     'title': chapter.get('text'),
-                } for chapter in item['cuepoints']] if item.get('cuepoints') else None,
+                } for chapter in item['cuepoints']] if item.get('cuepoints') and type_ == 'concert' else None,
             }
 
     def _real_extract(self, url):
-        language, video_id = self._match_valid_url(url).group('language', 'id')
+        language, type_, video_id = self._match_valid_url(url).group('language', 'type', 'id')
         if not language:
             language = 'en'
 
@@ -120,18 +133,18 @@ class DigitalConcertHallIE(InfoExtractor):
         }]
 
         vid_info = self._download_json(
-            f'https://api.digitalconcerthall.com/v2/concert/{video_id}', video_id, headers={
+            f'https://api.digitalconcerthall.com/v2/{type_}/{video_id}', video_id, headers={
                 'Accept': 'application/json',
                 'Accept-Language': language
             })
         album_artist = ' / '.join(traverse_obj(vid_info, ('_links', 'artist', ..., 'name')) or '')
+        videos = [vid_info] if type_ == 'film' else traverse_obj(vid_info, ('_embedded', ..., ...))
 
         return {
             '_type': 'playlist',
             'id': video_id,
             'title': vid_info.get('title'),
-            'entries': self._entries(traverse_obj(vid_info, ('_embedded', ..., ...)), language,
-                                     thumbnails=thumbnails, album_artist=album_artist),
+            'entries': self._entries(videos, language, thumbnails=thumbnails, album_artist=album_artist, type_=type_),
             'thumbnails': thumbnails,
             'album_artist': album_artist,
         }

yt_dlp/extractor/elevensports.py

@@ -0,0 +1,59 @@
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class ElevenSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?elevensports\.com/view/event/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://elevensports.com/view/event/clf46yr3kenn80jgrqsjmwefk',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': 'clf46yr3kenn80jgrqsjmwefk',
+            'title': 'Cleveland SC vs Lionsbridge FC',
+            'ext': 'mp4',
+            'description': 'md5:03b5238d6549f4ea1fddadf69b5e0b58',
+            'upload_date': '20230323',
+            'timestamp': 1679612400,
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        'url': 'https://elevensports.com/view/event/clhpyd53b06160jez74qhgkmf',
+        'md5': 'c0958d9ff90e4503a75544358758921d',
+        'info_dict': {
+            'id': 'clhpyd53b06160jez74qhgkmf',
+            'title': 'AJNLF vs ARRAF',
+            'ext': 'mp4',
+            'description': 'md5:c8c5e75c78f37c6d15cd6c475e43a8c1',
+            'upload_date': '20230521',
+            'timestamp': 1684684800,
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        event_id = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['event']['mclsEventId']
+        event_data = self._download_json(
+            f'https://mcls-api.mycujoo.tv/bff/events/v1beta1/{event_id}', video_id,
+            headers={'Authorization': 'Bearer FBVKACGN37JQC5SFA0OVK8KKSIOP153G'})
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            event_data['streams'][0]['full_url'], video_id, 'mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(event_data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'timestamp': ('start_time', {parse_iso8601}),
+                'thumbnail': ('thumbnail_url', {url_or_none}),
+            }),
+        }

yt_dlp/extractor/europa.py

@@ -6,6 +6,7 @@ from ..utils import (
     parse_iso8601,
     parse_qs,
     qualities,
+    traverse_obj,
     unified_strdate,
     xpath_text
 )
@@ -92,42 +93,17 @@ class EuropaIE(InfoExtractor):
 
 class EuroParlWebstreamIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:multimedia|webstreaming)\.europarl\.europa\.eu/[^/#?]+/
-        (?:embed/embed\.html\?event=|(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
+        https?://multimedia\.europarl\.europa\.eu/[^/#?]+/
+        (?:(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
     '''
     _TESTS = [{
         'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
         'info_dict': {
-            'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
-            'ext': 'mp4',
-            'release_timestamp': 1663137900,
-            'title': 'Plenary session',
-            'release_date': '20220914',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/eu-cop27-un-climate-change-conference-in-sharm-el-sheikh-egypt-ep-delegation-meets-with-ngo-represen_20221114-1600-SPECIAL-OTHER',
-        'info_dict': {
-            'id': 'a8428de8-b9cd-6a2e-11e4-3805d9c9ff5c',
-            'ext': 'mp4',
-            'release_timestamp': 1668434400,
-            'release_date': '20221114',
-            'title': 'md5:d3550280c33cc70e0678652e3d52c028',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        # embed webpage
-        'url': 'https://webstreaming.europarl.europa.eu/ep/embed/embed.html?event=20220914-0900-PLENARY&language=en&autoplay=true&logo=true',
-        'info_dict': {
-            'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
+            'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
             'ext': 'mp4',
             'title': 'Plenary session',
+            'release_timestamp': 1663139069,
             'release_date': '20220914',
-            'release_timestamp': 1663137900,
         },
         'params': {
             'skip_download': True,
@@ -144,30 +120,54 @@ class EuroParlWebstreamIE(InfoExtractor):
             'live_status': 'is_live',
         },
         'skip': 'not live anymore'
+    }, {
+        'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
+        'info_dict': {
+            'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
+            'ext': 'mp4',
+            'release_date': '20230301',
+            'title': 'Committee on Culture and Education',
+            'release_timestamp': 1677666641,
+        }
+    }, {
+        # live stream
+        'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
+        'info_dict': {
+            'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
+            'ext': 'mp4',
+            'release_date': '20230524',
+            'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
+            'release_timestamp': 1684911541,
+            'live_status': 'is_live',
+        },
+        'skip': 'Not live anymore'
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
 
         json_info = self._download_json(
-            'https://vis-api.vuplay.co.uk/event/external', display_id,
+            'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
             query={
-                'player_key': 'europarl|718f822c-a48c-4841-9947-c9cb9bb1743c',
-                'external_id': display_id,
+                'api-version': 1.0,
+                'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
+                'externalReference': display_id
             })
 
-        formats, subtitles = self._extract_mpd_formats_and_subtitles(json_info['streaming_url'], display_id)
-        fmts, subs = self._extract_m3u8_formats_and_subtitles(
-            json_info['streaming_url'].replace('.mpd', '.m3u8'), display_id)
-
-        formats.extend(fmts)
-        self._merge_subtitles(subs, target=subtitles)
+        formats, subtitles = [], {}
+        for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
+            fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
+            formats.extend(fmt)
+            self._merge_subtitles(subs, target=subtitles)
 
         return {
             'id': json_info['id'],
-            'title': json_info.get('title'),
+            'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
             'formats': formats,
             'subtitles': subtitles,
-            'release_timestamp': parse_iso8601(json_info.get('published_start')),
-            'is_live': 'LIVE' in json_info.get('state', '')
+            'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
+            'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live'
         }

yt_dlp/extractor/eurosport.py

@@ -3,7 +3,7 @@ from ..utils import traverse_obj
 
 
 class EurosportIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.eurosport\.com/\w+/[\w-]+/\d+/[\w-]+_(?P<id>vid\d+)'
+    _VALID_URL = r'https?://www\.eurosport\.com/\w+/(?:[\w-]+/[\d-]+/)?[\w-]+_(?P<id>vid\d+)'
     _TESTS = [{
         'url': 'https://www.eurosport.com/tennis/roland-garros/2022/highlights-rafael-nadal-brushes-aside-caper-ruud-to-win-record-extending-14th-french-open-title_vid1694147/video.shtml',
         'info_dict': {
@@ -44,6 +44,32 @@ class EurosportIE(InfoExtractor):
             'description': 'md5:32bbe3a773ac132c57fb1e8cca4b7c71',
             'upload_date': '20220727',
         }
+    }, {
+        'url': 'https://www.eurosport.com/football/champions-league/2022-2023/pep-guardiola-emotionally-destroyed-after-manchester-city-win-over-bayern-munich-in-champions-league_vid1896254/video.shtml',
+        'info_dict': {
+            'id': '3096477',
+            'ext': 'mp4',
+            'title': 'md5:82edc17370124c7a19b3cf518517583b',
+            'duration': 84.0,
+            'description': 'md5:b3f44ef7f5b5b95b24a273b163083feb',
+            'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/04/12/3682873-74947393-2560-1440.jpg',
+            'timestamp': 1681292028,
+            'upload_date': '20230412',
+            'display_id': 'vid1896254',
+        }
+    }, {
+        'url': 'https://www.eurosport.com/football/last-year-s-semi-final-pain-was-still-there-pep-guardiola-after-man-city-reach-cl-final_vid1914115/video.shtml',
+        'info_dict': {
+            'id': '3149108',
+            'ext': 'mp4',
+            'title': '\'Last year\'s semi-final pain was still there\' - Pep Guardiola after Man City reach CL final',
+            'description': 'md5:89ef142fe0170a66abab77fac2955d8e',
+            'display_id': 'vid1914115',
+            'timestamp': 1684403618,
+            'thumbnail': 'https://imgresizer.eurosport.com/unsafe/1280x960/smart/filters:format(jpeg)/origin-imgresizer.eurosport.com/2023/05/18/3707254-75435008-2560-1440.jpg',
+            'duration': 105.0,
+            'upload_date': '20230518',
+        }
     }]
 
     _TOKEN = None

yt_dlp/extractor/facebook.py

@@ -390,7 +390,10 @@ class FacebookIE(InfoExtractor):
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
             title = get_first(media, ('title', 'text'))
             description = get_first(media, ('creation_story', 'comet_sections', 'message', 'story', 'message', 'text'))
-            uploader_data = get_first(media, 'owner') or get_first(post, ('node', 'actors', ...)) or {}
+            uploader_data = (
+                get_first(media, ('owner', {dict}))
+                or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name']))
+                or get_first(post, ('node', 'actors', ..., {dict})) or {})
 
             page_title = title or self._html_search_regex((
                 r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
@@ -415,16 +418,17 @@ class FacebookIE(InfoExtractor):
             # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
             if thumbnail and not re.search(r'\.(?:jpg|png)', thumbnail):
                 thumbnail = None
-            view_count = parse_count(self._search_regex(
-                r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
-                default=None))
             info_dict = {
                 'description': description,
                 'uploader': uploader,
                 'uploader_id': uploader_data.get('id'),
                 'timestamp': timestamp,
                 'thumbnail': thumbnail,
-                'view_count': view_count,
+                'view_count': parse_count(self._search_regex(
+                    (r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)',),
+                    webpage, 'view count', default=None)),
+                'concurrent_view_count': get_first(post, (
+                    ('video', (..., ..., 'attachments', ..., 'media')), 'liveViewerCount', {int_or_none})),
             }
 
             info_json_ld = self._search_json_ld(webpage, video_id, default={})

yt_dlp/extractor/idolplus.py

@@ -0,0 +1,115 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj, try_call, url_or_none
+
+
+class IdolPlusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?idolplus\.com/z[us]/(?:concert/|contents/?\?(?:[^#]+&)?albumId=)(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://idolplus.com/zs/contents?albumId=M012077298PPV00',
+        'md5': '2ace3f4661c943a2f7e79f0b88cea1e7',
+        'info_dict': {
+            'id': 'M012077298PPV00',
+            'ext': 'mp4',
+            'title': '[MultiCam] Aegyo on Top of Aegyo (IZ*ONE EATING TRIP)',
+            'release_date': '20200707',
+            'formats': 'count:65',
+        },
+        'params': {'format': '532-KIM_MINJU'},
+    }, {
+        'url': 'https://idolplus.com/zs/contents?albumId=M01232H058PPV00&catId=E9TX5',
+        'info_dict': {
+            'id': 'M01232H058PPV00',
+            'ext': 'mp4',
+            'title': 'YENA (CIRCLE CHART MUSIC AWARDS 2022 RED CARPET)',
+            'release_date': '20230218',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        # live stream
+        'url': 'https://idolplus.com/zu/contents?albumId=M012323174PPV00',
+        'info_dict': {
+            'id': 'M012323174PPV00',
+            'ext': 'mp4',
+            'title': 'Hanteo Music Awards 2022 DAY2',
+            'release_date': '20230211',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://idolplus.com/zs/concert/M012323039PPV00',
+        'info_dict': {
+            'id': 'M012323039PPV00',
+            'ext': 'mp4',
+            'title': 'CIRCLE CHART MUSIC AWARDS 2022',
+            'release_date': '20230218',
+            'formats': 'count:5',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data_list = traverse_obj(self._download_json(
+            'https://idolplus.com/api/zs/viewdata/ruleset/build', video_id,
+            headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
+                'rulesetId': 'contents',
+                'albumId': video_id,
+                'distribute': 'PRD',
+                'loggedIn': 'false',
+                'region': 'zs',
+                'countryGroup': '00010',
+                'lang': 'en',
+                'saId': '999999999998',
+            }), ('data', 'viewData', ...))
+
+        player_data = {}
+        while data_list:
+            player_data = data_list.pop()
+            if traverse_obj(player_data, 'type') == 'player':
+                break
+            elif traverse_obj(player_data, ('dataList', ...)):
+                data_list += player_data['dataList']
+
+        formats = self._extract_m3u8_formats(traverse_obj(player_data, (
+            'vodPlayerList', 'vodProfile', 0, 'vodServer', 0, 'video_url', {url_or_none})), video_id)
+
+        subtitles = {}
+        for caption in traverse_obj(player_data, ('vodPlayerList', 'caption')) or []:
+            subtitles.setdefault(caption.get('lang') or 'und', []).append({
+                'url': caption.get('smi_url'),
+                'ext': 'vtt',
+            })
+
+        # Add member multicams as alternative formats
+        if (traverse_obj(player_data, ('detail', 'has_cuesheet')) == 'Y'
+                and traverse_obj(player_data, ('detail', 'is_omni_member')) == 'Y'):
+            cuesheet = traverse_obj(self._download_json(
+                'https://idolplus.com/gapi/contents/v1.0/content/cuesheet', video_id,
+                'Downloading JSON metadata for member multicams',
+                headers={'App_type': 'web', 'Country_Code': 'KR'}, query={
+                    'ALBUM_ID': video_id,
+                    'COUNTRY_GRP': '00010',
+                    'LANG': 'en',
+                    'SA_ID': '999999999998',
+                    'COUNTRY_CODE': 'KR',
+                }), ('data', 'cuesheet_item', 0))
+
+            for member in traverse_obj(cuesheet, ('members', ...)):
+                index = try_call(lambda: int(member['omni_view_index']) - 1)
+                member_video_url = traverse_obj(cuesheet, ('omni_view', index, 'cdn_url', 0, 'url', {url_or_none}))
+                if not member_video_url:
+                    continue
+                member_formats = self._extract_m3u8_formats(
+                    member_video_url, video_id, note=f'Downloading m3u8 for multicam {member["name"]}')
+                for mf in member_formats:
+                    mf['format_id'] = f'{mf["format_id"]}-{member["name"].replace(" ", "_")}'
+                formats.extend(member_formats)
+
+        return {
+            'id': video_id,
+            'title': traverse_obj(player_data, ('detail', 'albumName')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'release_date': traverse_obj(player_data, ('detail', 'broadcastDate')),
+        }

yt_dlp/extractor/iwara.py

@@ -1,6 +1,7 @@
 import functools
 import urllib.parse
 import hashlib
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,7 +15,49 @@ from ..utils import (
 )
 
 
-class IwaraIE(InfoExtractor):
+# https://github.com/yt-dlp/yt-dlp/issues/6671
+class IwaraBaseIE(InfoExtractor):
+    _USERTOKEN = None
+    _MEDIATOKEN = None
+    _NETRC_MACHINE = 'iwara'
+
+    def _get_user_token(self, invalidate=False):
+        if not invalidate and self._USERTOKEN:
+            return self._USERTOKEN
+
+        username, password = self._get_login_info()
+        IwaraBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
+        if not IwaraBaseIE._USERTOKEN or invalidate:
+            IwaraBaseIE._USERTOKEN = self._download_json(
+                'https://api.iwara.tv/user/login', None, note='Logging in',
+                data=json.dumps({
+                    'email': username,
+                    'password': password
+                }).encode('utf-8'),
+                headers={
+                    'Content-Type': 'application/json'
+                })['token']
+
+            self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
+
+        return self._USERTOKEN
+
+    def _get_media_token(self, invalidate=False):
+        if not invalidate and self._MEDIATOKEN:
+            return self._MEDIATOKEN
+
+        IwaraBaseIE._MEDIATOKEN = self._download_json(
+            'https://api.iwara.tv/user/token', None, note='Fetching media token',
+            data=b'',  # Need to have some data here, even if it's empty
+            headers={
+                'Authorization': f'Bearer {self._get_user_token()}',
+                'Content-Type': 'application/json'
+            })['accessToken']
+
+        return self._MEDIATOKEN
+
+
+class IwaraIE(IwaraBaseIE):
     IE_NAME = 'iwara'
     _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos?/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
@@ -56,6 +99,26 @@ class IwaraIE(InfoExtractor):
             'timestamp': 1678732213,
             'modified_timestamp': 1679110271,
         },
+    }, {
+        'url': 'https://iwara.tv/video/blggmfno8ghl725bg',
+        'info_dict': {
+            'id': 'blggmfno8ghl725bg',
+            'ext': 'mp4',
+            'age_limit': 18,
+            'title': 'お外でおしっこしちゃう猫耳ロリメイド',
+            'description': 'md5:0342ba9bf6db09edbbb28729657c3611',
+            'uploader': 'Fe_Kurosabi',
+            'uploader_id': 'fekurosabi',
+            'tags': [
+                'pee'
+            ],
+            'like_count': 192,
+            'view_count': 12119,
+            'comment_count': 0,
+            'timestamp': 1598880567,
+            'modified_timestamp': 1598908995,
+            'availability': 'needs_auth',
+        },
     }]
 
     def _extract_formats(self, video_id, fileurl):
@@ -79,12 +142,18 @@ class IwaraIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True)
+        username, password = self._get_login_info()
+        headers = {
+            'Authorization': f'Bearer {self._get_media_token()}',
+        } if username and password else None
+        video_data = self._download_json(f'https://api.iwara.tv/video/{video_id}', video_id, expected_status=lambda x: True, headers=headers)
         errmsg = video_data.get('message')
         # at this point we can actually get uploaded user info, but do we need it?
         if errmsg == 'errors.privateVideo':
             self.raise_login_required('Private video. Login if you have permissions to watch')
-        elif errmsg:
+        elif errmsg == 'errors.notFound' and not username:
+            self.raise_login_required('Video may need login to view')
+        elif errmsg:  # None if success
             raise ExtractorError(f'Iwara says: {errmsg}')
 
         if not video_data.get('fileUrl'):
@@ -112,8 +181,17 @@ class IwaraIE(InfoExtractor):
             'formats': list(self._extract_formats(video_id, video_data.get('fileUrl'))),
         }
 
+    def _perform_login(self, username, password):
+        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
+            self.write_debug('Skipping logging in')
+            return
 
-class IwaraUserIE(InfoExtractor):
+        IwaraBaseIE._USERTOKEN = self._get_user_token(True)
+        self._get_media_token(True)
+        self.cache.store(self._NETRC_MACHINE, username, IwaraBaseIE._USERTOKEN)
+
+
+class IwaraUserIE(IwaraBaseIE):
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/profile/(?P<id>[^/?#&]+)'
     IE_NAME = 'iwara:user'
     _PER_PAGE = 32
@@ -165,7 +243,7 @@ class IwaraUserIE(InfoExtractor):
             playlist_id, traverse_obj(user_info, ('user', 'name')))
 
 
-class IwaraPlaylistIE(InfoExtractor):
+class IwaraPlaylistIE(IwaraBaseIE):
     # the ID is an UUID but I don't think it's necessary to write concrete regex
     _VALID_URL = r'https?://(?:www\.)?iwara\.tv/playlist/(?P<id>[0-9a-f-]+)'
     IE_NAME = 'iwara:playlist'

yt_dlp/extractor/jstream.py

@@ -0,0 +1,73 @@
+import base64
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    js_to_json,
+    remove_start,
+)
+
+
+class JStreamIE(InfoExtractor):
+    # group "id" only exists for compliance, not directly used in requests
+    # also all components are mandatory
+    _VALID_URL = r'jstream:(?P<host>www\d+):(?P<id>(?P<publisher>[a-z0-9]+):(?P<mid>\d+))'
+
+    _TESTS = [{
+        'url': 'jstream:www50:eqd638pvwx:752',
+        'info_dict': {
+            'id': 'eqd638pvwx:752',
+            'ext': 'mp4',
+            'title': '阪神淡路大震災 激震の記録2020年版　解説動画',
+            'duration': 672,
+            'thumbnail': r're:https?://eqd638pvwx\.eq\.webcdn\.stream\.ne\.jp/.+\.jpg',
+        },
+    }]
+
+    def _parse_jsonp(self, callback, string, video_id):
+        return self._search_json(rf'\s*{re.escape(callback)}\s*\(', string, callback, video_id)
+
+    def _find_formats(self, video_id, movie_list_hls, host, publisher, subtitles):
+        for value in movie_list_hls:
+            text = value.get('text') or ''
+            if not text.startswith('auto'):
+                continue
+            m3u8_id = remove_start(remove_start(text, 'auto'), '_') or None
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/{value.get("url")}', video_id, 'mp4', m3u8_id=m3u8_id)
+            self._merge_subtitles(subs, target=subtitles)
+            yield from fmts
+
+    def _real_extract(self, url):
+        host, publisher, mid, video_id = self._match_valid_url(url).group('host', 'publisher', 'mid', 'id')
+        video_info_jsonp = self._download_webpage(
+            f'https://{publisher}.eq.webcdn.stream.ne.jp/{host}/{publisher}/jmc_pub/eq_meta/v1/{mid}.jsonp',
+            video_id, 'Requesting video info')
+        video_info = self._parse_jsonp('metaDataResult', video_info_jsonp, video_id)['movie']
+        subtitles = {}
+        formats = list(self._find_formats(video_id, video_info.get('movie_list_hls'), host, publisher, subtitles))
+        self._remove_duplicate_formats(formats)
+        return {
+            'id': video_id,
+            'title': video_info.get('title'),
+            'duration': float_or_none(video_info.get('duration')),
+            'thumbnail': video_info.get('thumbnail_url'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        # check for eligiblity of webpage
+        # https://support.eq.stream.co.jp/hc/ja/articles/115008388147-%E3%83%97%E3%83%AC%E3%82%A4%E3%83%A4%E3%83%BCAPI%E3%81%AE%E3%82%B5%E3%83%B3%E3%83%97%E3%83%AB%E3%82%B3%E3%83%BC%E3%83%89
+        script_tag = re.search(r'<script\s*[^>]+?src="https://ssl-cache\.stream\.ne\.jp/(?P<host>www\d+)/(?P<publisher>[a-z0-9]+)/[^"]+?/if\.js"', webpage)
+        if not script_tag:
+            return
+        host, publisher = script_tag.groups()
+        for m in re.finditer(r'(?s)PlayerFactoryIF\.create\(\s*({[^\}]+?})\s*\)\s*;', webpage):
+            # TODO: using json.loads here as InfoExtractor._parse_json is not classmethod
+            info = json.loads(js_to_json(m.group(1)))
+            mid = base64.b64decode(info.get('m')).decode()
+            yield f'jstream:{host}:{publisher}:{mid}'

yt_dlp/extractor/ketnet.py

@@ -1,70 +0,0 @@
-from .canvas import CanvasIE
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class KetnetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://www.ketnet.be/kijken/n/nachtwacht/3/nachtwacht-s3a1-de-greystook',
-        'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
-        'info_dict': {
-            'id': 'pbs-pub-aef8b526-115e-4006-aa24-e59ff6c6ef6f$vid-ddb815bf-c8e7-467b-8879-6bad7a32cebd',
-            'ext': 'mp4',
-            'title': 'Nachtwacht - Reeks 3: Aflevering 1',
-            'description': 'De Nachtwacht krijgt te maken met een parasiet',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1468.02,
-            'timestamp': 1609225200,
-            'upload_date': '20201229',
-            'series': 'Nachtwacht',
-            'season': 'Reeks 3',
-            'episode': 'De Greystook',
-            'episode_number': 1,
-        },
-        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
-    }, {
-        'url': 'https://www.ketnet.be/themas/karrewiet/jaaroverzicht-20200/karrewiet-het-jaar-van-black-mamba',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        video = self._download_json(
-            'https://senior-bff.ketnet.be/graphql', display_id, query={
-                'query': '''{
-  video(id: "content/ketnet/nl/%s.model.json") {
-    description
-    episodeNr
-    imageUrl
-    mediaReference
-    programTitle
-    publicationDate
-    seasonTitle
-    subtitleVideodetail
-    titleVideodetail
-  }
-}''' % display_id,
-            })['data']['video']
-
-        mz_id = compat_urllib_parse_unquote(video['mediaReference'])
-
-        return {
-            '_type': 'url_transparent',
-            'id': mz_id,
-            'title': video['titleVideodetail'],
-            'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/' + mz_id,
-            'thumbnail': video.get('imageUrl'),
-            'description': video.get('description'),
-            'timestamp': parse_iso8601(video.get('publicationDate')),
-            'series': video.get('programTitle'),
-            'season': video.get('seasonTitle'),
-            'episode': video.get('subtitleVideodetail'),
-            'episode_number': int_or_none(video.get('episodeNr')),
-            'ie_key': CanvasIE.ie_key(),
-        }

yt_dlp/extractor/litv.py

@@ -4,8 +4,8 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    traverse_obj,
     smuggle_url,
+    traverse_obj,
     unsmuggle_url,
 )
 
@@ -113,7 +113,7 @@ class LiTVIE(InfoExtractor):
             entry_protocol='m3u8_native', m3u8_id='hls')
         for a_format in formats:
             # LiTV HLS segments doesn't like compressions
-            a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True
+            a_format.setdefault('http_headers', {})['Accept-Encoding'] = 'identity'
 
         title = program_info['title'] + program_info.get('secondaryMark', '')
         description = program_info.get('description')

yt_dlp/extractor/livestream.py

@@ -1,33 +1,36 @@
-import re
 import itertools
+import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
+from ..compat import compat_str, compat_urlparse
 from ..utils import (
-    find_xpath_attr,
-    xpath_attr,
-    xpath_with_ns,
-    xpath_text,
-    orderedSet,
-    update_url_query,
-    int_or_none,
-    float_or_none,
-    parse_iso8601,
     determine_ext,
+    find_xpath_attr,
+    float_or_none,
+    int_or_none,
+    orderedSet,
+    parse_iso8601,
+    traverse_obj,
+    update_url_query,
+    xpath_attr,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
+    _VALID_URL = r'''(?x)
+        https?://(?:new\.)?livestream\.com/
+        (?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))
+        (?:/events/(?P<event_id>\d+)|/(?P<event_name>[^/]+))?
+        (?:/videos/(?P<id>\d+))?
+    '''
     _EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"']
 
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
-        'md5': '53274c76ba7754fb0e8d072716f2292b',
+        'md5': '7876c5f5dc3e711b6b73acce4aac1527',
         'info_dict': {
             'id': '4719370',
             'ext': 'mp4',
@@ -37,22 +40,37 @@ class LivestreamIE(InfoExtractor):
             'duration': 5968.0,
             'like_count': int,
             'view_count': int,
+            'comment_count': int,
             'thumbnail': r're:^http://.*\.jpg$'
         }
     }, {
-        'url': 'http://new.livestream.com/tedx/cityenglish',
+        'url': 'https://livestream.com/coheedandcambria/websterhall',
         'info_dict': {
-            'title': 'TEDCity2.0 (English)',
-            'id': '2245590',
+            'id': '1585861',
+            'title': 'Live From Webster Hall'
+        },
+        'playlist_mincount': 1,
+    }, {
+        'url': 'https://livestream.com/dayananda/events/7954027',
+        'info_dict': {
+            'title': 'Live from Mevo',
+            'id': '7954027',
         },
         'playlist_mincount': 4,
     }, {
-        'url': 'http://new.livestream.com/chess24/tatasteelchess',
+        'url': 'https://livestream.com/accounts/82',
         'info_dict': {
-            'title': 'Tata Steel Chess',
-            'id': '3705884',
-        },
-        'playlist_mincount': 60,
+            'id': '253978',
+            'view_count': int,
+            'title': 'trsr',
+            'comment_count': int,
+            'like_count': int,
+            'upload_date': '20120306',
+            'timestamp': 1331042383,
+            'thumbnail': 'http://img.new.livestream.com/videos/0000000000000372/cacbeed6-fb68-4b5e-ad9c-e148124e68a9_640x427.jpg',
+            'duration': 15.332,
+            'ext': 'mp4'
+        }
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
@@ -179,7 +197,7 @@ class LivestreamIE(InfoExtractor):
             'is_live': is_live,
         }
 
-    def _extract_event(self, event_data):
+    def _generate_event_playlist(self, event_data):
         event_id = compat_str(event_data['id'])
         account_id = compat_str(event_data['owner_account_id'])
         feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
@@ -189,7 +207,6 @@ class LivestreamIE(InfoExtractor):
             return self._extract_stream_info(stream_info)
 
         last_video = None
-        entries = []
         for i in itertools.count(1):
             if last_video is None:
                 info_url = feed_root_url
@@ -197,31 +214,38 @@ class LivestreamIE(InfoExtractor):
                 info_url = '{root}?&id={id}&newer=-1&type=video'.format(
                     root=feed_root_url, id=last_video)
             videos_info = self._download_json(
-                info_url, event_id, 'Downloading page {0}'.format(i))['data']
+                info_url, event_id, f'Downloading page {i}')['data']
             videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
             if not videos_info:
                 break
             for v in videos_info:
                 v_id = compat_str(v['id'])
-                entries.append(self.url_result(
-                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
-                    'Livestream', v_id, v.get('caption')))
+                yield self.url_result(
+                    f'http://livestream.com/accounts/{account_id}/events/{event_id}/videos/{v_id}',
+                    LivestreamIE, v_id, v.get('caption'))
             last_video = videos_info[-1]['id']
-        return self.playlist_result(entries, event_id, event_data['full_name'])
 
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
         event = mobj.group('event_id') or mobj.group('event_name')
         account = mobj.group('account_id') or mobj.group('account_name')
-        api_url = self._API_URL_TEMPLATE % (account, event)
+        api_url = f'http://livestream.com/api/accounts/{account}'
+
         if video_id:
             video_data = self._download_json(
-                api_url + '/videos/%s' % video_id, video_id)
+                f'{api_url}/events/{event}/videos/{video_id}', video_id)
             return self._extract_video_info(video_data)
-        else:
-            event_data = self._download_json(api_url, video_id)
-            return self._extract_event(event_data)
+        elif event:
+            event_data = self._download_json(f'{api_url}/events/{event}', None)
+            return self.playlist_result(
+                self._generate_event_playlist(event_data), str(event_data['id']), event_data['full_name'])
+
+        account_data = self._download_json(api_url, None)
+        items = traverse_obj(account_data, (('upcoming_events', 'past_events'), 'data', ...))
+        return self.playlist_result(
+            itertools.chain.from_iterable(map(self._generate_event_playlist, items)),
+            account_data.get('id'), account_data.get('full_name'))
 
 
 # The original version of Livestream uses a different system

yt_dlp/extractor/mzaalo.py

@@ -0,0 +1,92 @@
+from .common import InfoExtractor
+from ..utils import (
+    parse_age_limit,
+    parse_duration,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class MzaaloIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mzaalo\.com/play/(?P<type>movie|original|clip)/(?P<id>[a-fA-F0-9-]+)/[\w-]+'
+    _TESTS = [{
+        # Movies
+        'url': 'https://www.mzaalo.com/play/movie/c0958d9f-f90e-4503-a755-44358758921d/Jamun',
+        'info_dict': {
+            'id': 'c0958d9f-f90e-4503-a755-44358758921d',
+            'title': 'Jamun',
+            'ext': 'mp4',
+            'description': 'md5:24fe9ebb9bbe5b36f7b54b90ab1e2f31',
+            'thumbnails': 'count:15',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 5527.0,
+            'language': 'hin',
+            'categories': ['Drama'],
+            'age_limit': 13,
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        # Shows
+        'url': 'https://www.mzaalo.com/play/original/93d42b2b-f373-4c2d-bca4-997412cb069d/Modi-Season-2-CM-TO-PM/Episode-1:Decision,-Not-Promises',
+        'info_dict': {
+            'id': '93d42b2b-f373-4c2d-bca4-997412cb069d',
+            'title': 'Episode 1:Decision, Not Promises',
+            'ext': 'mp4',
+            'description': 'md5:16f76058432a54774fbb2561a1955652',
+            'thumbnails': 'count:22',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2040.0,
+            'language': 'hin',
+            'categories': ['Drama'],
+            'age_limit': 13,
+        },
+        'params': {'skip_download': 'm3u8'}
+    }, {
+        # Streams/Clips
+        'url': 'https://www.mzaalo.com/play/clip/83cdbcb5-400a-42f1-a1d2-459053cfbda5/Manto-Ki-Kahaaniya',
+        'info_dict': {
+            'id': '83cdbcb5-400a-42f1-a1d2-459053cfbda5',
+            'title': 'Manto Ki Kahaaniya',
+            'ext': 'mp4',
+            'description': 'md5:c3c5f1d05f0fd1bfcb05b673d1cc9f2f',
+            'thumbnails': 'count:3',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1937.0,
+            'language': 'hin',
+        },
+        'params': {'skip_download': 'm3u8'}
+    }]
+
+    def _real_extract(self, url):
+        video_id, type_ = self._match_valid_url(url).group('id', 'type')
+        path = (f'partner/streamurl?&assetId={video_id}&getClipDetails=YES' if type_ == 'clip'
+                else f'api/v2/player/details?assetType={type_.upper()}&assetId={video_id}')
+        data = self._download_json(
+            f'https://production.mzaalo.com/platform/{path}', video_id, headers={
+                'Ocp-Apim-Subscription-Key': '1d0caac2702049b89a305929fdf4cbae',
+            })['data']
+
+        formats = self._extract_m3u8_formats(data['streamURL'], video_id)
+
+        subtitles = {}
+        for subs_lang, subs_url in traverse_obj(data, ('subtitles', {dict.items}, ...)):
+            if url_or_none(subs_url):
+                subtitles[subs_lang] = [{'url': subs_url, 'ext': 'vtt'}]
+
+        lang = traverse_obj(data, ('language', {str.lower}))
+        for f in formats:
+            f['language'] = lang
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'duration': ('duration', {parse_duration}),
+                'age_limit': ('maturity_rating', {parse_age_limit}),
+                'thumbnails': ('images', ..., {'url': {url_or_none}}),
+                'categories': ('genre', ..., {str}),
+            }),
+        }

yt_dlp/extractor/naver.py

@@ -21,7 +21,7 @@ from ..utils import (
 class NaverBaseIE(InfoExtractor):
     _CAPTION_EXT_RE = r'\.(?:ttml|vtt)'
 
-    @staticmethod  # NB: Used in VLiveWebArchiveIE
+    @staticmethod  # NB: Used in VLiveWebArchiveIE, WeverseIE
     def process_subtitles(vod_data, process_url):
         ret = {'subtitles': {}, 'automatic_captions': {}}
         for caption in traverse_obj(vod_data, ('captions', 'list', ...)):

yt_dlp/extractor/nekohacker.py

@@ -0,0 +1,217 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    extract_attributes,
+    get_element_by_class,
+    get_element_text_and_html_by_tag,
+    parse_duration,
+    traverse_obj,
+    try_call,
+    url_or_none,
+)
+
+
+class NekoHackerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
+    _TESTS = [{
+        'url': 'https://nekohacker.com/nekoverse/',
+        'info_dict': {
+            'id': 'nekoverse',
+            'title': 'Nekoverse',
+        },
+        'playlist': [
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
+                'md5': '44223701ebedba0467ebda4cc07fb3aa',
+                'info_dict': {
+                    'id': '1712',
+                    'ext': 'mp3',
+                    'title': 'Spaceship',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Spaceship',
+                    'track_number': 1,
+                    'duration': 195.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/02-City-Runner.mp3',
+                'md5': '8f853c71719389d32bbbd3f1a87b3f08',
+                'info_dict': {
+                    'id': '1713',
+                    'ext': 'mp3',
+                    'title': 'City Runner',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'City Runner',
+                    'track_number': 2,
+                    'duration': 148.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/03-Nature-Talk.mp3',
+                'md5': '5a8a8ae852720cee4c0ac95c7d1a7450',
+                'info_dict': {
+                    'id': '1714',
+                    'ext': 'mp3',
+                    'title': 'Nature Talk',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Nature Talk',
+                    'track_number': 3,
+                    'duration': 174.0
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/04-Crystal-World.mp3',
+                'md5': 'd8e59a48061764e50d92386a294abd50',
+                'info_dict': {
+                    'id': '1715',
+                    'ext': 'mp3',
+                    'title': 'Crystal World',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20221101',
+                    'album': 'Nekoverse',
+                    'artist': 'Neko Hacker',
+                    'track': 'Crystal World',
+                    'track_number': 4,
+                    'duration': 199.0
+                }
+            }
+        ]
+    }, {
+        'url': 'https://nekohacker.com/susume/',
+        'info_dict': {
+            'id': 'susume',
+            'title': '進め！むじなカンパニー',
+        },
+        'playlist': [
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-feat.-六科なじむ-CV_-日高里菜-割戶真友-CV_-金元寿子-軽井沢ユキ-CV_-上坂すみれ-出稼ぎガルシア-CV_-金子彩花-.mp3',
+                'md5': 'fb13f008aa81f26ba48f91fd2d6186ce',
+                'info_dict': {
+                    'id': '711',
+                    'ext': 'mp3',
+                    'title': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0',
+                    'track_number': 1,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-feat.-六科なじむ-CV_-日高里菜-.mp3',
+                'md5': '028803f70241df512b7764e73396fdd1',
+                'info_dict': {
+                    'id': '709',
+                    'ext': 'mp3',
+                    'title': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )',
+                    'track_number': 2,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/進め！むじなカンパニー-instrumental.mp3',
+                'md5': 'adde9e9a16e1da5e602b579c247d0fb9',
+                'info_dict': {
+                    'id': '710',
+                    'ext': 'mp3',
+                    'title': '進め！むじなカンパニー (instrumental)',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': '進め！むじなカンパニー (instrumental)',
+                    'track_number': 3,
+                    'duration': None
+                }
+            },
+            {
+                'url': 'https://nekohacker.com/wp-content/uploads/2021/01/むじな-de-なじむ-instrumental.mp3',
+                'md5': 'ebb0443039cf5f9ff7fd557ed9b23599',
+                'info_dict': {
+                    'id': '712',
+                    'ext': 'mp3',
+                    'title': 'むじな de なじむ (instrumental)',
+                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2021/01/OP表-1024x1024.png',
+                    'vcodec': 'none',
+                    'acodec': 'mp3',
+                    'release_date': '20210115',
+                    'album': '進め！むじなカンパニー',
+                    'artist': 'Neko Hacker',
+                    'track': 'むじな de なじむ (instrumental)',
+                    'track_number': 4,
+                    'duration': None
+                }
+            }
+        ]
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+        playlist = get_element_by_class('playlist', webpage)
+
+        if not playlist:
+            iframe = try_call(lambda: get_element_text_and_html_by_tag('iframe', webpage)[1]) or ''
+            iframe_src = url_or_none(extract_attributes(iframe).get('src'))
+            if not iframe_src:
+                raise ExtractorError('No playlist or embed found in webpage')
+            elif re.match(r'https?://(?:\w+\.)?spotify\.com/', iframe_src):
+                raise ExtractorError('Spotify embeds are not supported', expected=True)
+            return self.url_result(url, 'Generic')
+
+        entries = []
+        for track_number, track in enumerate(re.findall(r'(<li[^>]+data-audiopath[^>]+>)', playlist), 1):
+            entry = traverse_obj(extract_attributes(track), {
+                'url': ('data-audiopath', {url_or_none}),
+                'ext': ('data-audiopath', {determine_ext}),
+                'id': 'data-trackid',
+                'title': 'data-tracktitle',
+                'track': 'data-tracktitle',
+                'album': 'data-albumtitle',
+                'duration': ('data-tracktime', {parse_duration}),
+                'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
+                'thumbnail': ('data-albumart', {url_or_none}),
+            })
+            entries.append({
+                **entry,
+                'track_number': track_number,
+                'artist': 'Neko Hacker',
+                'vcodec': 'none',
+                'acodec': 'mp3' if entry['ext'] == 'mp3' else None,
+            })
+
+        return self.playlist_result(entries, playlist_id, traverse_obj(entries, (0, 'album')))

yt_dlp/extractor/nhk.py

@@ -67,7 +67,7 @@ class NhkBaseIE(InfoExtractor):
             info.update({
                 '_type': 'url_transparent',
                 'ie_key': 'Piksel',
-                'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + vod_id,
+                'url': 'https://movie-s.nhk.or.jp/v/refid/nhkworld/prefid/' + vod_id,
                 'id': vod_id,
             })
         else:
@@ -94,6 +94,19 @@ class NhkVodIE(NhkBaseIE):
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2061601/',
+        'info_dict': {
+            'id': 'yd8322ch',
+            'ext': 'mp4',
+            'description': 'md5:109c8b05d67a62d0592f2b445d2cd898',
+            'title': 'GRAND SUMO Highlights - [Recap] May Tournament Day 1 (Opening Day)',
+            'upload_date': '20230514',
+            'timestamp': 1684083791,
+            'series': 'GRAND SUMO Highlights',
+            'episode': '[Recap] May Tournament Day 1 (Opening Day)',
+            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1684084443/4028649.jpg?w=1920&h=1080',
+        },
+    }, {
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
         'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
@@ -104,6 +117,9 @@ class NhkVodIE(NhkBaseIE):
             'description': 'md5:5aee4a9f9d81c26281862382103b0ea5',
             'timestamp': 1565965194,
             'upload_date': '20190816',
+            'thumbnail': 'https://mz-edge.stream.co.jp/thumbs/aid/t1567086278/3715195.jpg?w=1920&h=1080',
+            'series': 'Dining with the Chef',
+            'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU',
         },
     }, {
         # audio clip
@@ -114,10 +130,7 @@ class NhkVodIE(NhkBaseIE):
             'title': "Japan's Top Inventions - Miniature Video Cameras",
             'description': 'md5:07ea722bdbbb4936fdd360b6a480c25b',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/',
         'only_matching': True,
@@ -133,7 +146,6 @@ class NhkVodIE(NhkBaseIE):
     }, {
         # video, alphabetic character in ID #29670
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
-        'only_matching': True,
         'info_dict': {
             'id': 'qfjay6cg',
             'ext': 'mp4',
@@ -142,7 +154,8 @@ class NhkVodIE(NhkBaseIE):
             'thumbnail': r're:^https?:/(/[a-z0-9.-]+)+\.jpg\?w=1920&h=1080$',
             'upload_date': '20210615',
             'timestamp': 1623722008,
-        }
+        },
+        'skip': '404 Not Found',
     }]
 
     def _real_extract(self, url):
@@ -153,12 +166,19 @@ class NhkVodProgramIE(NhkBaseIE):
     _VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
     _TESTS = [{
         # video program episodes
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/sumo',
+        'info_dict': {
+            'id': 'sumo',
+            'title': 'GRAND SUMO Highlights',
+        },
+        'playlist_mincount': 12,
+    }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
         'info_dict': {
             'id': 'japanrailway',
             'title': 'Japan Railway Journal',
         },
-        'playlist_mincount': 1,
+        'playlist_mincount': 12,
     }, {
         # video program clips
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway/?type=clip',

yt_dlp/extractor/niconico.py

@@ -5,13 +5,17 @@ import json
 import re
 import time
 
+from urllib.parse import urlparse
+
 from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
     compat_HTTPError,
 )
+from ..dependencies import websockets
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
+    WebSocketsWrapper,
     bug_reports_message,
     clean_html,
     float_or_none,
@@ -895,3 +899,162 @@ class NiconicoUserIE(InfoExtractor):
     def _real_extract(self, url):
         list_id = self._match_id(url)
         return self.playlist_result(self._entries(list_id), list_id, ie=NiconicoIE.ie_key())
+
+
+class NiconicoLiveIE(InfoExtractor):
+    IE_NAME = 'niconico:live'
+    IE_DESC = 'ニコニコ生放送'
+    _VALID_URL = r'https?://(?:sp\.)?live2?\.nicovideo\.jp/(?:watch|gate)/(?P<id>lv\d+)'
+    _TESTS = [{
+        'note': 'this test case includes invisible characters for title, pasting them as-is',
+        'url': 'https://live.nicovideo.jp/watch/lv339533123',
+        'info_dict': {
+            'id': 'lv339533123',
+            'title': '激辛ペヤング食べます‪( ;ᯅ; )‬（歌枠オーディション参加中）',
+            'view_count': 1526,
+            'comment_count': 1772,
+            'description': '初めましてもかって言います❕\nのんびり自由に適当に暮らしてます',
+            'uploader': 'もか',
+            'channel': 'ゲストさんのコミュニティ',
+            'channel_id': 'co5776900',
+            'channel_url': 'https://com.nicovideo.jp/community/co5776900',
+            'timestamp': 1670677328,
+            'is_live': True,
+        },
+        'skip': 'livestream',
+    }, {
+        'url': 'https://live2.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }, {
+        'url': 'https://sp.live.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }, {
+        'url': 'https://sp.live2.nicovideo.jp/watch/lv339533123',
+        'only_matching': True,
+    }]
+
+    _KNOWN_LATENCY = ('high', 'low')
+
+    def _real_extract(self, url):
+        if not websockets:
+            raise ExtractorError('websockets library is not available. Please install it.', expected=True)
+        video_id = self._match_id(url)
+        webpage, urlh = self._download_webpage_handle(f'https://live.nicovideo.jp/watch/{video_id}', video_id)
+
+        embedded_data = self._parse_json(unescapeHTML(self._search_regex(
+            r'<script\s+id="embedded-data"\s*data-props="(.+?)"', webpage, 'embedded data')), video_id)
+
+        ws_url = traverse_obj(embedded_data, ('site', 'relive', 'webSocketUrl'))
+        if not ws_url:
+            raise ExtractorError('The live hasn\'t started yet or already ended.', expected=True)
+        ws_url = update_url_query(ws_url, {
+            'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
+        })
+
+        hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
+        cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
+        latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
+        if latency not in self._KNOWN_LATENCY:
+            latency = 'high'
+
+        ws = WebSocketsWrapper(ws_url, {
+            'Cookies': str_or_none(cookies) or '',
+            'Origin': f'https://{hostname}',
+            'Accept': '*/*',
+            'User-Agent': self.get_param('http_headers')['User-Agent'],
+        })
+
+        self.write_debug('[debug] Sending HLS server request')
+        ws.send(json.dumps({
+            'type': 'startWatching',
+            'data': {
+                'stream': {
+                    'quality': 'abr',
+                    'protocol': 'hls+fmp4',
+                    'latency': latency,
+                    'chasePlay': False
+                },
+                'room': {
+                    'protocol': 'webSocket',
+                    'commentable': True
+                },
+                'reconnect': False,
+            }
+        }))
+
+        while True:
+            recv = ws.recv()
+            if not recv:
+                continue
+            data = json.loads(recv)
+            if not isinstance(data, dict):
+                continue
+            if data.get('type') == 'stream':
+                m3u8_url = data['data']['uri']
+                qualities = data['data']['availableQualities']
+                break
+            elif data.get('type') == 'disconnect':
+                self.write_debug(recv)
+                raise ExtractorError('Disconnected at middle of extraction')
+            elif data.get('type') == 'error':
+                self.write_debug(recv)
+                message = traverse_obj(data, ('body', 'code')) or recv
+                raise ExtractorError(message)
+            elif self.get_param('verbose', False):
+                if len(recv) > 100:
+                    recv = recv[:100] + '...'
+                self.write_debug('Server said: %s' % recv)
+
+        title = traverse_obj(embedded_data, ('program', 'title')) or self._html_search_meta(
+            ('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
+
+        raw_thumbs = traverse_obj(embedded_data, ('program', 'thumbnail')) or {}
+        thumbnails = []
+        for name, value in raw_thumbs.items():
+            if not isinstance(value, dict):
+                thumbnails.append({
+                    'id': name,
+                    'url': value,
+                    **parse_resolution(value, lenient=True),
+                })
+                continue
+
+            for k, img_url in value.items():
+                res = parse_resolution(k, lenient=True) or parse_resolution(img_url, lenient=True)
+                width, height = res.get('width'), res.get('height')
+
+                thumbnails.append({
+                    'id': f'{name}_{width}x{height}',
+                    'url': img_url,
+                    **res,
+                })
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', live=True)
+        for fmt, q in zip(formats, reversed(qualities[1:])):
+            fmt.update({
+                'format_id': q,
+                'protocol': 'niconico_live',
+                'ws': ws,
+                'video_id': video_id,
+                'cookies': cookies,
+                'live_latency': latency,
+                'origin': hostname,
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            **traverse_obj(embedded_data, {
+                'view_count': ('program', 'statistics', 'watchCount'),
+                'comment_count': ('program', 'statistics', 'commentCount'),
+                'uploader': ('program', 'supplier', 'name'),
+                'channel': ('socialGroup', 'name'),
+                'channel_id': ('socialGroup', 'id'),
+                'channel_url': ('socialGroup', 'socialGroupPageUrl'),
+            }),
+            'description': clean_html(traverse_obj(embedded_data, ('program', 'description'))),
+            'timestamp': int_or_none(traverse_obj(embedded_data, ('program', 'openTime'))),
+            'is_live': True,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

yt_dlp/extractor/owncloud.py

@@ -0,0 +1,80 @@
+import re
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    url_or_none,
+    urlencode_postdata,
+)
+
+
+class OwnCloudIE(InfoExtractor):
+    _INSTANCES_RE = '|'.join((
+        r'(?:[^\.]+\.)?sciebo\.de',
+        r'cloud\.uni-koblenz-landau\.de',
+    ))
+    _VALID_URL = rf'https?://(?:{_INSTANCES_RE})/s/(?P<id>[\w.-]+)'
+
+    _TESTS = [
+        {
+            'url': 'https://ruhr-uni-bochum.sciebo.de/s/wWhqZzh9jTumVFN',
+            'info_dict': {
+                'id': 'wWhqZzh9jTumVFN',
+                'ext': 'mp4',
+                'title': 'CmvpJST.mp4',
+            },
+        },
+        {
+            'url': 'https://ruhr-uni-bochum.sciebo.de/s/WNDuFu0XuFtmm3f',
+            'info_dict': {
+                'id': 'WNDuFu0XuFtmm3f',
+                'ext': 'mp4',
+                'title': 'CmvpJST.mp4',
+            },
+            'params': {
+                'videopassword': '12345',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+
+        if re.search(r'<label[^>]+for="password"', webpage):
+            webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
+
+        hidden_inputs = self._hidden_inputs(webpage)
+        title = hidden_inputs.get('filename')
+        parsed_url = urllib.parse.urlparse(url)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url_or_none(hidden_inputs.get('downloadURL')) or parsed_url._replace(
+                path=urllib.parse.urljoin(parsed_url.path, 'download')).geturl(),
+            'ext': determine_ext(title),
+        }
+
+    def _verify_video_password(self, webpage, url, video_id):
+        password = self.get_param('videopassword')
+        if password is None:
+            raise ExtractorError(
+                'This video is protected by a password, use the --video-password option',
+                expected=True)
+
+        validation_response = self._download_webpage(
+            url, video_id, 'Validating Password', 'Wrong password?',
+            data=urlencode_postdata({
+                'requesttoken': self._hidden_inputs(webpage)['requesttoken'],
+                'password': password,
+            }))
+
+        if re.search(r'<label[^>]+for="password"', validation_response):
+            warning = self._search_regex(
+                r'<div[^>]+class="warning">([^<]*)</div>', validation_response,
+                'warning', default='The password is wrong')
+            raise ExtractorError(f'Opening the video failed, {self.IE_NAME} said: {warning!r}', expected=True)
+        return validation_response

yt_dlp/extractor/piksel.py

@@ -7,8 +7,10 @@ from ..utils import (
     int_or_none,
     join_nonempty,
     parse_iso8601,
+    traverse_obj,
     try_get,
     unescapeHTML,
+    urljoin,
 )
 
 
@@ -63,11 +65,11 @@ class PikselIE(InfoExtractor):
         }
     ]
 
-    def _call_api(self, app_token, resource, display_id, query, fatal=True):
-        response = (self._download_json(
-            'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
-            display_id, query=query, fatal=fatal) or {}).get('response')
-        failure = try_get(response, lambda x: x['failure']['reason'])
+    def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.com', fatal=True):
+        url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5')
+        response = traverse_obj(
+            self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {}
+        failure = traverse_obj(response, ('failure', 'reason')) if response else 'Empty response from API'
         if failure:
             if fatal:
                 raise ExtractorError(failure, expected=True)
@@ -83,7 +85,7 @@ class PikselIE(InfoExtractor):
         ], webpage, 'app token')
         query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
         program = self._call_api(
-            app_token, 'program', display_id, query)['WsProgramResponse']['program']
+            app_token, 'program', display_id, query, url)['WsProgramResponse']['program']
         video_id = program['uuid']
         video_data = program['asset']
         title = video_data['title']
@@ -129,7 +131,7 @@ class PikselIE(InfoExtractor):
                 process_asset_files(try_get(self._call_api(
                     app_token, 'asset_file', display_id, {
                         'assetid': asset_id,
-                    }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
+                    }, url, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
 
         m3u8_url = dict_get(video_data, [
             'm3u8iPadURL',

yt_dlp/extractor/playsuisse.py

@@ -5,10 +5,16 @@ from ..utils import int_or_none, traverse_obj
 
 
 class PlaySuisseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?playsuisse\.ch/(?:watch|detail)/(?:[^#]*[?&]episodeId=)?(?P<id>[0-9]+)'
     _TESTS = [
         {
+            # Old URL
             'url': 'https://www.playsuisse.ch/watch/763211/0',
+            'only_matching': True,
+        },
+        {
+            # episode in a series
+            'url': 'https://www.playsuisse.ch/watch/763182?episodeId=763211',
             'md5': '82df2a470b2dfa60c2d33772a8a60cf8',
             'info_dict': {
                 'id': '763211',
@@ -21,11 +27,11 @@ class PlaySuisseIE(InfoExtractor):
                 'season_number': 1,
                 'episode': 'Knochen',
                 'episode_number': 1,
-                'thumbnail': 'md5:9260abe0c0ec9b69914d0a10d54c5878'
+                'thumbnail': 're:https://playsuisse-img.akamaized.net/',
             }
-        },
-        {
-            'url': 'https://www.playsuisse.ch/watch/808675/0',
+        }, {
+            # film
+            'url': 'https://www.playsuisse.ch/watch/808675',
             'md5': '818b94c1d2d7c4beef953f12cb8f3e75',
             'info_dict': {
                 'id': '808675',
@@ -33,26 +39,60 @@ class PlaySuisseIE(InfoExtractor):
                 'title': 'Der Läufer',
                 'description': 'md5:9f61265c7e6dcc3e046137a792b275fd',
                 'duration': 5280,
-                'episode': 'Der Läufer',
-                'thumbnail': 'md5:44af7d65ee02bbba4576b131868bb783'
+                'thumbnail': 're:https://playsuisse-img.akamaized.net/',
             }
-        },
-        {
-            'url': 'https://www.playsuisse.ch/watch/817193/0',
-            'md5': '1d6c066f92cd7fffd8b28a53526d6b59',
+        }, {
+            # series (treated as a playlist)
+            'url': 'https://www.playsuisse.ch/detail/1115687',
             'info_dict': {
-                'id': '817193',
-                'ext': 'mp4',
-                'title': 'Die Einweihungsparty',
-                'description': 'md5:91ebf04d3a42cb3ab70666acf750a930',
-                'duration': 1380,
-                'series': 'Nr. 47',
-                'season': 'Season 1',
-                'season_number': 1,
-                'episode': 'Die Einweihungsparty',
-                'episode_number': 1,
-                'thumbnail': 'md5:637585fb106e3a4bcd991958924c7e44'
-            }
+                'description': 'md5:e4a2ae29a8895823045b5c3145a02aa3',
+                'id': '1115687',
+                'series': 'They all came out to Montreux',
+                'title': 'They all came out to Montreux',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'description': 'md5:f2462744834b959a31adc6292380cda2',
+                    'duration': 3180,
+                    'episode': 'Folge 1',
+                    'episode_number': 1,
+                    'id': '1112663',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 1',
+                    'ext': 'mp4'
+                },
+            }, {
+                'info_dict': {
+                    'description': 'md5:9dfd308699fe850d3bce12dc1bad9b27',
+                    'duration': 2935,
+                    'episode': 'Folge 2',
+                    'episode_number': 2,
+                    'id': '1112661',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 2',
+                    'ext': 'mp4'
+                },
+            }, {
+                'info_dict': {
+                    'description': 'md5:14a93a3356b2492a8f786ab2227ef602',
+                    'duration': 2994,
+                    'episode': 'Folge 3',
+                    'episode_number': 3,
+                    'id': '1112664',
+                    'season': 'Season 1',
+                    'season_number': 1,
+                    'series': 'They all came out to Montreux',
+                    'thumbnail': 're:https://playsuisse-img.akamaized.net/',
+                    'title': 'Folge 3',
+                    'ext': 'mp4'
+                }
+            }],
         }
     ]
 
@@ -142,6 +182,6 @@ class PlaySuisseIE(InfoExtractor):
             'subtitles': subtitles,
             'series': media_data.get('seriesName'),
             'season_number': int_or_none(media_data.get('seasonNumber')),
-            'episode': media_data.get('name'),
+            'episode': media_data.get('name') if media_data.get('episodeNumber') else None,
             'episode_number': int_or_none(media_data.get('episodeNumber')),
         }

yt_dlp/extractor/polskieradio.py

@@ -2,26 +2,24 @@ import itertools
 import json
 import math
 import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_unquote,
-    compat_urlparse
-)
+from ..compat import compat_str
 from ..utils import (
-    determine_ext,
-    extract_attributes,
     ExtractorError,
     InAdvancePagedList,
+    determine_ext,
+    extract_attributes,
     int_or_none,
     js_to_json,
     parse_iso8601,
     strip_or_none,
     traverse_obj,
-    unified_timestamp,
     unescapeHTML,
+    unified_timestamp,
     url_or_none,
+    urljoin,
 )
 
 
@@ -44,7 +42,7 @@ class PolskieRadioBaseExtractor(InfoExtractor):
                 'duration': int_or_none(media.get('length')),
                 'vcodec': 'none' if media.get('provider') == 'audio' else None,
             })
-            entry_title = compat_urllib_parse_unquote(media['desc'])
+            entry_title = urllib.parse.unquote(media['desc'])
             if entry_title:
                 entry['title'] = entry_title
             yield entry
@@ -130,10 +128,11 @@ class PolskieRadioLegacyIE(PolskieRadioBaseExtractor):
         return self.playlist_result(entries, playlist_id, title, description)
 
 
-class PolskieRadioIE(InfoExtractor):
-    # new next.js sites, excluding radiokierowcow.pl
-    _VALID_URL = r'https?://(?:[^/]+\.)?polskieradio(?:24)?\.pl/artykul/(?P<id>\d+)'
+class PolskieRadioIE(PolskieRadioBaseExtractor):
+    # new next.js sites
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:polskieradio(?:24)?|radiokierowcow)\.pl/artykul/(?P<id>\d+)'
     _TESTS = [{
+        # articleData, attachments
         'url': 'https://jedynka.polskieradio.pl/artykul/1587943',
         'info_dict': {
             'id': '1587943',
@@ -148,6 +147,31 @@ class PolskieRadioIE(InfoExtractor):
                 'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
             },
         }],
+    }, {
+        # post, legacy html players
+        'url': 'https://trojka.polskieradio.pl/artykul/2589163,Czy-wciaz-otrzymujemy-zdjecia-z-sond-Voyager',
+        'info_dict': {
+            'id': '2589163',
+            'title': 'Czy wciąż otrzymujemy zdjęcia z sond Voyager?',
+            'description': 'md5:cf1a7f348d63a2db9c0d7a63d1669473',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '2577880',
+                'ext': 'mp3',
+                'title': 'md5:a57d10a0c02abd34dd675cb33707ad5a',
+                'duration': 321,
+            },
+        }],
+    }, {
+        # data, legacy
+        'url': 'https://radiokierowcow.pl/artykul/2694529',
+        'info_dict': {
+            'id': '2694529',
+            'title': 'Zielona fala reliktem przeszłości?',
+            'description': 'md5:f20a9a7ed9cb58916c54add94eae3bc0',
+        },
+        'playlist_count': 3,
     }, {
         'url': 'https://trojka.polskieradio.pl/artykul/1632955',
         'only_matching': True,
@@ -166,7 +190,8 @@ class PolskieRadioIE(InfoExtractor):
         webpage = self._download_webpage(url, playlist_id)
 
         article_data = traverse_obj(
-            self._search_nextjs_data(webpage, playlist_id), ('props', 'pageProps', 'data', 'articleData'))
+            self._search_nextjs_data(webpage, playlist_id), (
+                'props', 'pageProps', (('data', 'articleData'), 'post', 'data')), get_all=False)
 
         title = strip_or_none(article_data['title'])
 
@@ -178,7 +203,13 @@ class PolskieRadioIE(InfoExtractor):
             'id': self._search_regex(
                 r'([a-f\d]{8}-(?:[a-f\d]{4}-){3}[a-f\d]{12})', entry['file'], 'entry id'),
             'title': strip_or_none(entry.get('description')) or title,
-        } for entry in article_data.get('attachments') or () if entry['fileType'] in ('Audio', )]
+        } for entry in article_data.get('attachments') or () if entry.get('fileType') in ('Audio', )]
+
+        if not entries:
+            # some legacy articles have no json attachments, but players in body
+            entries = self._extract_webpage_player_entries(article_data['content'], playlist_id, {
+                'title': title,
+            })
 
         return self.playlist_result(entries, playlist_id, title, description)
 
@@ -214,6 +245,15 @@ class PolskieRadioAuditionIE(InfoExtractor):
             'thumbnail': r're:https://static\.prsa\.pl/images/.+',
         },
         'playlist_mincount': 722,
+    }, {
+        # some articles were "promoted to main page" and thus link to old frontend
+        'url': 'https://trojka.polskieradio.pl/audycja/305',
+        'info_dict': {
+            'id': '305',
+            'title': 'Co w mowie piszczy?',
+            'thumbnail': r're:https://static\.prsa\.pl/images/.+',
+        },
+        'playlist_count': 1523,
     }]
 
     def _call_lp3(self, path, query, video_id, note):
@@ -254,7 +294,6 @@ class PolskieRadioAuditionIE(InfoExtractor):
             for article in page['data']:
                 yield {
                     '_type': 'url_transparent',
-                    'ie_key': PolskieRadioIE.ie_key(),
                     'id': str(article['id']),
                     'url': article['url'],
                     'title': article.get('shortTitle'),
@@ -282,11 +321,8 @@ class PolskieRadioAuditionIE(InfoExtractor):
 class PolskieRadioCategoryIE(InfoExtractor):
     # legacy sites
     IE_NAME = 'polskieradio:category'
-    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/(?:\d+(?:,[^/]+)?/|[^/]+/Tag)(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
-        'only_matching': True
-    }, {
         'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
         'info_dict': {
             'id': '4143',
@@ -300,6 +336,36 @@ class PolskieRadioCategoryIE(InfoExtractor):
             'title': 'Muzyka',
         },
         'playlist_mincount': 61
+    }, {
+        # billennium tabs
+        'url': 'https://www.polskieradio.pl/8/2385',
+        'info_dict': {
+            'id': '2385',
+            'title': 'Droga przez mąkę',
+        },
+        'playlist_mincount': 111,
+    }, {
+        'url': 'https://www.polskieradio.pl/10/4930',
+        'info_dict': {
+            'id': '4930',
+            'title': 'Teraz K-pop!',
+        },
+        'playlist_mincount': 392,
+    }, {
+        # post back pages, audio content directly without articles
+        'url': 'https://www.polskieradio.pl/8,dwojka/7376,nowa-mowa',
+        'info_dict': {
+            'id': '7376',
+            'title': 'Nowa mowa',
+        },
+        'playlist_mincount': 244,
+    }, {
+        'url': 'https://www.polskieradio.pl/Krzysztof-Dziuba/Tag175458',
+        'info_dict': {
+            'id': '175458',
+            'title': 'Krzysztof Dziuba',
+        },
+        'playlist_mincount': 420,
     }, {
         'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
         'only_matching': True,
@@ -311,25 +377,61 @@ class PolskieRadioCategoryIE(InfoExtractor):
 
     def _entries(self, url, page, category_id):
         content = page
+        is_billennium_tabs = 'onclick="TB_LoadTab(' in page
+        is_post_back = 'onclick="__doPostBack(' in page
+        pagination = page if is_billennium_tabs else None
         for page_num in itertools.count(2):
             for a_entry, entry_id in re.findall(
-                    r'(?s)<article[^>]+>.*?(<a[^>]+href=["\']/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
+                    r'(?s)<article[^>]+>.*?(<a[^>]+href=["\'](?:(?:https?)?://[^/]+)?/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
                     content):
                 entry = extract_attributes(a_entry)
-                href = entry.get('href')
-                if not href:
-                    continue
-                yield self.url_result(
-                    compat_urlparse.urljoin(url, href), PolskieRadioLegacyIE,
-                    entry_id, entry.get('title'))
-            mobj = re.search(
-                r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
-                content)
-            if not mobj:
-                break
-            next_url = compat_urlparse.urljoin(url, mobj.group('url'))
-            content = self._download_webpage(
-                next_url, category_id, 'Downloading page %s' % page_num)
+                if entry.get('href'):
+                    yield self.url_result(
+                        urljoin(url, entry['href']), PolskieRadioLegacyIE, entry_id, entry.get('title'))
+            for a_entry in re.findall(r'<span data-media=({[^ ]+})', content):
+                yield traverse_obj(self._parse_json(a_entry, category_id), {
+                    'url': 'file',
+                    'id': 'uid',
+                    'duration': 'length',
+                    'title': ('title', {urllib.parse.unquote}),
+                    'description': ('desc', {urllib.parse.unquote}),
+                })
+            if is_billennium_tabs:
+                params = self._search_json(
+                    r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+onclick=["\']TB_LoadTab\(',
+                    pagination, 'next page params', category_id, default=None, close_objects=1,
+                    contains_pattern='.+', transform_source=lambda x: '[%s' % js_to_json(unescapeHTML(x)))
+                if not params:
+                    break
+                tab_content = self._download_json(
+                    'https://www.polskieradio.pl/CMS/TemplateBoxesManagement/TemplateBoxTabContent.aspx/GetTabContent',
+                    category_id, f'Downloading page {page_num}', headers={'content-type': 'application/json'},
+                    data=json.dumps(dict(zip((
+                        'boxInstanceId', 'tabId', 'categoryType', 'sectionId', 'categoryId', 'pagerMode',
+                        'subjectIds', 'tagIndexId', 'queryString', 'name', 'openArticlesInParentTemplate',
+                        'idSectionFromUrl', 'maxDocumentAge', 'showCategoryForArticle', 'pageNumber'
+                    ), params))).encode())['d']
+                content, pagination = tab_content['Content'], tab_content.get('PagerContent')
+            elif is_post_back:
+                target = self._search_regex(
+                    r'onclick=(?:["\'])__doPostBack\((?P<q1>["\'])(?P<target>[\w$]+)(?P=q1)\s*,\s*(?P<q2>["\'])Next(?P=q2)',
+                    content, 'pagination postback target', group='target', default=None)
+                if not target:
+                    break
+                content = self._download_webpage(
+                    url, category_id, f'Downloading page {page_num}',
+                    data=urllib.parse.urlencode({
+                        **self._hidden_inputs(content),
+                        '__EVENTTARGET': target,
+                        '__EVENTARGUMENT': 'Next',
+                    }).encode())
+            else:
+                next_url = urljoin(url, self._search_regex(
+                    r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
+                    content, 'next page url', group='url', default=None))
+                if not next_url:
+                    break
+                content = self._download_webpage(next_url, category_id, f'Downloading page {page_num}')
 
     def _real_extract(self, url):
         category_id = self._match_id(url)
@@ -337,7 +439,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
         if PolskieRadioAuditionIE.suitable(urlh.url):
             return self.url_result(urlh.url, PolskieRadioAuditionIE, category_id)
         title = self._html_search_regex(
-            r'<title>([^<]+) - [^<]+ - [^<]+</title>',
+            r'<title>([^<]+)(?: - [^<]+ - [^<]+| w [Pp]olskie[Rr]adio\.pl\s*)</title>',
             webpage, 'title', fatal=False)
         return self.playlist_result(
             self._entries(url, webpage, category_id),
@@ -506,39 +608,3 @@ class PolskieRadioPodcastIE(PolskieRadioPodcastBaseExtractor):
                 'Content-Type': 'application/json',
             })
         return self._parse_episode(data[0])
-
-
-class PolskieRadioRadioKierowcowIE(PolskieRadioBaseExtractor):
-    _VALID_URL = r'https?://(?:www\.)?radiokierowcow\.pl/artykul/(?P<id>[0-9]+)'
-    IE_NAME = 'polskieradio:kierowcow'
-
-    _TESTS = [{
-        'url': 'https://radiokierowcow.pl/artykul/2694529',
-        'info_dict': {
-            'id': '2694529',
-            'title': 'Zielona fala reliktem przeszłości?',
-            'description': 'md5:343950a8717c9818fdfd4bd2b8ca9ff2',
-        },
-        'playlist_count': 3,
-    }]
-
-    def _real_extract(self, url):
-        media_id = self._match_id(url)
-        webpage = self._download_webpage(url, media_id)
-        nextjs_build = self._search_nextjs_data(webpage, media_id)['buildId']
-        article = self._download_json(
-            f'https://radiokierowcow.pl/_next/data/{nextjs_build}/artykul/{media_id}.json?articleId={media_id}',
-            media_id)
-        data = article['pageProps']['data']
-        title = data['title']
-        entries = self._extract_webpage_player_entries(data['content'], media_id, {
-            'title': title,
-        })
-
-        return {
-            '_type': 'playlist',
-            'id': media_id,
-            'entries': entries,
-            'title': title,
-            'description': data.get('lead'),
-        }

yt_dlp/extractor/rai.py

@@ -1,19 +1,12 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
 from ..utils import (
     clean_html,
     determine_ext,
     ExtractorError,
     filter_dict,
-    find_xpath_attr,
-    fix_xml_ampersands,
     GeoRestrictedError,
-    HEADRequest,
     int_or_none,
     join_nonempty,
     parse_duration,
@@ -35,82 +28,70 @@ class RaiBaseIE(InfoExtractor):
     _GEO_BYPASS = False
 
     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
+        def fix_cdata(s):
+            # remove \r\n\t before and after <![CDATA[ ]]> to avoid
+            # polluted text with xpath_text
+            s = re.sub(r'(\]\]>)[\r\n\t]+(</)', '\\1\\2', s)
+            return re.sub(r'(>)[\r\n\t]+(<!\[CDATA\[)', '\\1\\2', s)
+
         if not re.match(r'https?://', relinker_url):
             return {'formats': [{'url': relinker_url}]}
 
+        # set User-Agent to generic 'Rai' to avoid quality filtering from
+        # the media server and get the maximum qualities available
+        relinker = self._download_xml(
+            relinker_url, video_id, note='Downloading XML metadata',
+            transform_source=fix_cdata, query={'output': 64},
+            headers={**self.geo_verification_headers(), 'User-Agent': 'Rai'})
+
+        if xpath_text(relinker, './license_url', default='{}') != '{}':
+            self.report_drm(video_id)
+
+        is_live = xpath_text(relinker, './is_live', default='N') == 'Y'
+        duration = parse_duration(xpath_text(relinker, './duration', default=None))
+        media_url = xpath_text(relinker, './url[@type="content"]', default=None)
+
+        if not media_url:
+            self.raise_no_formats('The relinker returned no media url')
+
+        # geo flag is a bit unreliable and not properly set all the time
+        geoprotection = xpath_text(relinker, './geoprotection', default='N') == 'Y'
+
+        ext = determine_ext(media_url)
         formats = []
-        geoprotection = None
-        is_live = None
-        duration = None
 
-        for platform in ('mon', 'flash', 'native'):
-            relinker = self._download_xml(
-                relinker_url, video_id,
-                note=f'Downloading XML metadata for platform {platform}',
-                transform_source=fix_xml_ampersands,
-                query={'output': 45, 'pl': platform},
-                headers=self.geo_verification_headers())
+        if ext == 'mp3':
+            formats.append({
+                'url': media_url,
+                'vcodec': 'none',
+                'acodec': 'mp3',
+                'format_id': 'https-mp3',
+            })
+        elif ext == 'm3u8' or 'format=m3u8' in media_url:
+            formats.extend(self._extract_m3u8_formats(
+                media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            # very likely no longer needed. Cannot find any url that uses it.
+            manifest_url = update_url_query(
+                media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
+                {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
+            formats.extend(self._extract_f4m_formats(
+                manifest_url, video_id, f4m_id='hds', fatal=False))
+        elif ext == 'mp4':
+            bitrate = int_or_none(xpath_text(relinker, './bitrate'))
+            formats.append({
+                'url': media_url,
+                'tbr': bitrate if bitrate > 0 else None,
+                'format_id': join_nonempty('https', bitrate, delim='-'),
+            })
+        else:
+            raise ExtractorError('Unrecognized media file found')
 
-            if xpath_text(relinker, './license_url', default='{}') != '{}':
-                self.report_drm(video_id)
-
-            if not geoprotection:
-                geoprotection = xpath_text(
-                    relinker, './geoprotection', default=None) == 'Y'
-
-            if not is_live:
-                is_live = xpath_text(
-                    relinker, './is_live', default=None) == 'Y'
-            if not duration:
-                duration = parse_duration(xpath_text(
-                    relinker, './duration', default=None))
-
-            url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
-            if url_elem is None:
-                continue
-
-            media_url = url_elem.text
-
-            # This does not imply geo restriction (e.g.
-            # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
-            if '/video_no_available.mp4' in media_url:
-                continue
-
-            ext = determine_ext(media_url)
-            if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
-                continue
-
-            if ext == 'mp3':
-                formats.append({
-                    'url': media_url,
-                    'vcodec': 'none',
-                    'acodec': 'mp3',
-                    'format_id': 'http-mp3',
-                })
-                break
-            elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
-                formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif ext == 'f4m' or platform == 'flash':
-                manifest_url = update_url_query(
-                    media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
-                    {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
-                formats.extend(self._extract_f4m_formats(
-                    manifest_url, video_id, f4m_id='hds', fatal=False))
-            else:
-                bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
-                formats.append({
-                    'url': media_url,
-                    'tbr': bitrate if bitrate > 0 else None,
-                    'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
-                })
-
-        if not formats and geoprotection is True:
+        if (not formats and geoprotection is True) or '/video_no_available.mp4' in media_url:
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 
-        if not audio_only:
-            formats.extend(self._create_http_urls(relinker_url, formats))
+        if not audio_only and not is_live:
+            formats.extend(self._create_http_urls(media_url, relinker_url, formats))
 
         return filter_dict({
             'is_live': is_live,
@@ -118,38 +99,31 @@ class RaiBaseIE(InfoExtractor):
             'formats': formats,
         })
 
-    def _create_http_urls(self, relinker_url, fmts):
-        _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
+    def _create_http_urls(self, manifest_url, relinker_url, fmts):
+        _MANIFEST_REG = r'/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8'
         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
         _QUALITY = {
             # tbr: w, h
-            '250': [352, 198],
-            '400': [512, 288],
-            '700': [512, 288],
-            '800': [700, 394],
-            '1200': [736, 414],
-            '1800': [1024, 576],
-            '2400': [1280, 720],
-            '3200': [1440, 810],
-            '3600': [1440, 810],
-            '5000': [1920, 1080],
-            '10000': [1920, 1080],
+            250: [352, 198],
+            400: [512, 288],
+            600: [512, 288],
+            700: [512, 288],
+            800: [700, 394],
+            1200: [736, 414],
+            1500: [920, 518],
+            1800: [1024, 576],
+            2400: [1280, 720],
+            3200: [1440, 810],
+            3600: [1440, 810],
+            5000: [1920, 1080],
+            10000: [1920, 1080],
         }
 
-        def test_url(url):
-            resp = self._request_webpage(
-                HEADRequest(url), None, headers={'User-Agent': 'Rai'},
-                fatal=False, errnote=False, note=False)
-
-            if resp is False:
+        def percentage(number, target, pc=20, roof=125):
+            '''check if the target is in the range of number +/- percent'''
+            if not number or number < 0:
                 return False
-
-            if resp.code == 200:
-                return False if resp.url == url else resp.url
-            return None
-
-        # filter out audio-only formats
-        fmts = [f for f in fmts if not f.get('vcodec') == 'none']
+            return abs(target - number) < min(float(number) * float(pc) / 100.0, roof)
 
         def get_format_info(tbr):
             import math
@@ -157,67 +131,78 @@ class RaiBaseIE(InfoExtractor):
             if len(fmts) == 1 and not br:
                 br = fmts[0].get('tbr')
             if br and br > 300:
-                tbr = compat_str(math.floor(br / 100) * 100)
+                tbr = math.floor(br / 100) * 100
             else:
-                tbr = '250'
+                tbr = 250
 
             # try extracting info from available m3u8 formats
-            format_copy = None
+            format_copy = [None, None]
             for f in fmts:
                 if f.get('tbr'):
-                    br_limit = math.floor(br / 100)
-                    if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
-                        format_copy = f.copy()
+                    if percentage(tbr, f['tbr']):
+                        format_copy[0] = f.copy()
+                if [f.get('width'), f.get('height')] == _QUALITY.get(tbr):
+                    format_copy[1] = f.copy()
+                    format_copy[1]['tbr'] = tbr
+
+            # prefer format with similar bitrate because there might be
+            # multiple video with the same resolution but different bitrate
+            format_copy = format_copy[0] or format_copy[1] or {}
             return {
+                'format_id': f'https-{tbr}',
                 'width': format_copy.get('width'),
                 'height': format_copy.get('height'),
                 'tbr': format_copy.get('tbr'),
                 'vcodec': format_copy.get('vcodec'),
                 'acodec': format_copy.get('acodec'),
                 'fps': format_copy.get('fps'),
-                'format_id': f'https-{tbr}',
             } if format_copy else {
+                'format_id': f'https-{tbr}',
                 'width': _QUALITY[tbr][0],
                 'height': _QUALITY[tbr][1],
-                'format_id': f'https-{tbr}',
-                'tbr': int(tbr),
+                'tbr': tbr,
+                'vcodec': 'avc1',
+                'acodec': 'mp4a',
+                'fps': 25,
             }
 
-        loc = test_url(_MP4_TMPL % (relinker_url, '*'))
-        if not isinstance(loc, compat_str):
-            return []
+        # filter out single-stream formats
+        fmts = [f for f in fmts
+                if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none']
 
-        mobj = re.match(
-            _RELINKER_REG,
-            test_url(relinker_url) or '')
+        mobj = re.search(_MANIFEST_REG, manifest_url)
         if not mobj:
             return []
-
         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
-        available_qualities = [i for i in available_qualities if i]
 
         formats = []
-        for q in available_qualities:
-            fmt = {
+        for q in filter(None, available_qualities):
+            self.write_debug(f'Creating https format for quality {q}')
+            formats.append({
                 'url': _MP4_TMPL % (relinker_url, q),
                 'protocol': 'https',
                 'ext': 'mp4',
                 **get_format_info(q)
-            }
-            formats.append(fmt)
+            })
         return formats
 
+    @staticmethod
+    def _get_thumbnails_list(thumbs, url):
+        return [{
+            'url': urljoin(url, thumb_url),
+        } for thumb_url in (thumbs or {}).values() if thumb_url]
+
     @staticmethod
     def _extract_subtitles(url, video_data):
         STL_EXT = 'stl'
         SRT_EXT = 'srt'
         subtitles = {}
-        subtitles_array = video_data.get('subtitlesArray') or []
+        subtitles_array = video_data.get('subtitlesArray') or video_data.get('subtitleList') or []
         for k in ('subtitles', 'subtitlesUrl'):
             subtitles_array.append({'url': video_data.get(k)})
         for subtitle in subtitles_array:
             sub_url = subtitle.get('url')
-            if sub_url and isinstance(sub_url, compat_str):
+            if sub_url and isinstance(sub_url, str):
                 sub_lang = subtitle.get('language') or 'it'
                 sub_url = urljoin(url, sub_url)
                 sub_ext = determine_ext(sub_url, SRT_EXT)
@@ -236,7 +221,7 @@ class RaiBaseIE(InfoExtractor):
 class RaiPlayIE(RaiBaseIE):
     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
     _TESTS = [{
-        'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
+        'url': 'https://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
         'info_dict': {
             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
@@ -244,22 +229,20 @@ class RaiPlayIE(RaiBaseIE):
             'title': 'Report del 07/04/2014',
             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Rai Gulp',
+            'thumbnail': r're:^https?://www\.raiplay\.it/.+\.jpg',
+            'uploader': 'Rai 3',
+            'creator': 'Rai 3',
             'duration': 6160,
             'series': 'Report',
             'season': '2013/14',
-            'subtitles': {
-                'it': 'count:4',
-            },
+            'subtitles': {'it': 'count:4'},
             'release_year': 2022,
             'episode': 'Espresso nel caffè - 07/04/2014',
             'timestamp': 1396919880,
             'upload_date': '20140408',
+            'formats': 'count:4',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }, {
         # 1080p direct mp4 url
         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
@@ -270,8 +253,9 @@ class RaiPlayIE(RaiBaseIE):
             'title': 'Blanca - S1E1 - Senza occhi',
             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'Rai 1',
+            'thumbnail': r're:^https://www\.raiplay\.it/dl/img/.+\.jpg',
+            'uploader': 'Rai Premium',
+            'creator': 'Rai Fiction',
             'duration': 6493,
             'series': 'Blanca',
             'season': 'Season 1',
@@ -281,6 +265,30 @@ class RaiPlayIE(RaiBaseIE):
             'episode': 'Senza occhi',
             'timestamp': 1637318940,
             'upload_date': '20211119',
+            'formats': 'count:12',
+        },
+        'params': {'skip_download': True},
+        'expected_warnings': ['Video not available. Likely due to geo-restriction.']
+    }, {
+        # 1500 quality
+        'url': 'https://www.raiplay.it/video/2012/09/S1E11---Tutto-cio-che-luccica-0cab3323-732e-45d6-8e86-7704acab6598.html',
+        'md5': 'a634d20e8ab2d43724c273563f6bf87a',
+        'info_dict': {
+            'id': '0cab3323-732e-45d6-8e86-7704acab6598',
+            'ext': 'mp4',
+            'title': 'Mia and Me - S1E11 - Tutto ciò che luccica',
+            'alt_title': 'St 1 Ep 11 - Mia and Me - Tutto ciò che luccica',
+            'description': 'md5:4969e594184b1920c4c1f2b704da9dea',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Rai Gulp',
+            'series': 'Mia and Me',
+            'season': 'Season 1',
+            'episode_number': 11,
+            'release_year': 2015,
+            'season_number': 1,
+            'episode': 'Tutto ciò che luccica',
+            'timestamp': 1348495020,
+            'upload_date': '20120924',
         },
     }, {
         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
@@ -299,57 +307,40 @@ class RaiPlayIE(RaiBaseIE):
         base, video_id = self._match_valid_url(url).groups()
 
         media = self._download_json(
-            base + '.json', video_id, 'Downloading video JSON')
+            f'{base}.json', video_id, 'Downloading video JSON')
 
         if not self.get_param('allow_unplayable_formats'):
-            if try_get(
-                    media,
-                    (lambda x: x['rights_management']['rights']['drm'],
-                     lambda x: x['program_info']['rights_management']['rights']['drm']),
-                    dict):
+            if traverse_obj(media, (('program_info', None), 'rights_management', 'rights', 'drm')):
                 self.report_drm(video_id)
 
-        title = media['name']
         video = media['video']
-
         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
-
-        thumbnails = []
-        for _, value in media.get('images', {}).items():
-            if value:
-                thumbnails.append({
-                    'url': urljoin(url, value),
-                })
-
-        date_published = media.get('date_published')
-        time_published = media.get('time_published')
-        if date_published and time_published:
-            date_published += ' ' + time_published
-
-        subtitles = self._extract_subtitles(url, video)
-
-        program_info = media.get('program_info') or {}
+        date_published = join_nonempty(
+            media.get('date_published'), media.get('time_published'), delim=' ')
         season = media.get('season')
-
         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 
         return {
             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
             'display_id': video_id,
-            'title': title,
+            'title': media.get('name'),
             'alt_title': strip_or_none(alt_title or None),
             'description': media.get('description'),
-            'uploader': strip_or_none(media.get('channel') or None),
-            'creator': strip_or_none(media.get('editor') or None),
+            'uploader': strip_or_none(
+                traverse_obj(media, ('program_info', 'channel'))
+                or media.get('channel') or None),
+            'creator': strip_or_none(
+                traverse_obj(media, ('program_info', 'editor'))
+                or media.get('editor') or None),
             'duration': parse_duration(video.get('duration')),
             'timestamp': unified_timestamp(date_published),
-            'thumbnails': thumbnails,
-            'series': program_info.get('name'),
+            'thumbnails': self._get_thumbnails_list(media.get('images'), url),
+            'series': traverse_obj(media, ('program_info', 'name')),
             'season_number': int_or_none(season),
             'season': season if (season and not season.isdigit()) else None,
             'episode': media.get('episode_title'),
             'episode_number': int_or_none(media.get('episode')),
-            'subtitles': subtitles,
+            'subtitles': self._extract_subtitles(url, video),
             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
             **relinker_info
         }
@@ -371,38 +362,39 @@ class RaiPlayLiveIE(RaiPlayIE):  # XXX: Do not subclass from concrete IE
             'live_status': 'is_live',
             'upload_date': '20090502',
             'timestamp': 1241276220,
+            'formats': 'count:3',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }]
 
 
 class RaiPlayPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{
+        # entire series episodes + extras...
         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
         'info_dict': {
             'id': 'nondirloalmiocapo',
             'title': 'Non dirlo al mio capo',
             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
         },
-        'playlist_mincount': 12,
+        'playlist_mincount': 30,
     }, {
+        # single season
         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
         'info_dict': {
             'id': 'nondirloalmiocapo',
             'title': 'Non dirlo al mio capo - Stagione 2',
             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
         },
-        'playlist_mincount': 12,
+        'playlist_count': 12,
     }]
 
     def _real_extract(self, url):
         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 
         program = self._download_json(
-            base + '.json', playlist_id, 'Downloading program JSON')
+            f'{base}.json', playlist_id, 'Downloading program JSON')
 
         if extra_id:
             extra_id = extra_id.upper().rstrip('/')
@@ -450,7 +442,7 @@ class RaiPlaySoundIE(RaiBaseIE):
             'title': 'Il Ruggito del Coniglio del 10/12/2021',
             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.+\.jpg$',
             'uploader': 'rai radio 2',
             'duration': 5685,
             'series': 'Il Ruggito del Coniglio',
@@ -459,9 +451,7 @@ class RaiPlaySoundIE(RaiBaseIE):
             'timestamp': 1638346620,
             'upload_date': '20211201',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
     }]
 
     def _real_extract(self, url):
@@ -480,9 +470,6 @@ class RaiPlaySoundIE(RaiBaseIE):
                                          lambda x: x['live']['create_date']))
 
         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
-        thumbnails = [{
-            'url': urljoin(url, thumb_url),
-        } for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
 
         return {
             **info,
@@ -494,7 +481,7 @@ class RaiPlaySoundIE(RaiBaseIE):
             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
             'timestamp': unified_timestamp(date_published),
-            'thumbnails': thumbnails,
+            'thumbnails': self._get_thumbnails_list(podcast_info.get('images'), url),
             'series': podcast_info.get('title'),
             'season_number': int_or_none(media.get('season')),
             'episode': media.get('episode_title'),
@@ -512,30 +499,30 @@ class RaiPlaySoundLiveIE(RaiPlaySoundIE):  # XXX: Do not subclass from concrete
             'display_id': 'radio2',
             'ext': 'mp4',
             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
-            'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
+            'thumbnail': r're:^https://www\.raiplaysound\.it/dl/img/.+\.png',
             'uploader': 'rai radio 2',
             'series': 'Rai Radio 2',
             'creator': 'raiplaysound',
             'is_live': True,
             'live_status': 'is_live',
         },
-        'params': {
-            'skip_download': 'live',
-        },
+        'params': {'skip_download': True},
     }]
 
 
 class RaiPlaySoundPlaylistIE(InfoExtractor):
     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
     _TESTS = [{
+        # entire show
         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
         'info_dict': {
             'id': 'ilruggitodelconiglio',
             'title': 'Il Ruggito del Coniglio',
-            'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
+            'description': 'md5:48cff6972435964284614d70474132e6',
         },
         'playlist_mincount': 65,
     }, {
+        # single season
         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
         'info_dict': {
             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
@@ -568,22 +555,19 @@ class RaiPlaySoundPlaylistIE(InfoExtractor):
 class RaiIE(RaiBaseIE):
     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
     _TESTS = [{
-        # var uniquename = "ContentItem-..."
-        # data-id="ContentItem-..."
         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
         'info_dict': {
             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
             'ext': 'mp4',
             'title': 'TG PRIMO TEMPO',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 1758,
             'upload_date': '20140612',
         },
-        'skip': 'This content is available only in Italy',
+        'params': {'skip_download': True},
+        'expected_warnings': ['Video not available. Likely due to geo-restriction.']
     }, {
-        # with ContentItem in og:url
         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
-        'md5': '06345bd97c932f19ffb129973d07a020',
         'info_dict': {
             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
             'ext': 'mp4',
@@ -592,123 +576,51 @@ class RaiIE(RaiBaseIE):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 2214,
             'upload_date': '20161103'
-        }
+        },
+        'params': {'skip_download': True},
     }, {
-        # Direct MMS URL
+        # Direct MMS: Media URL no longer works.
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
         'only_matching': True,
     }]
 
-    def _extract_from_content_id(self, content_id, url):
+    def _real_extract(self, url):
+        content_id = self._match_id(url)
         media = self._download_json(
             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
-            content_id, 'Downloading video JSON')
+            content_id, 'Downloading video JSON', fatal=False, expected_status=404)
 
-        title = media['name'].strip()
+        if media is None:
+            return None
 
-        media_type = media['type']
-        if 'Audio' in media_type:
+        if 'Audio' in media['type']:
             relinker_info = {
                 'formats': [{
-                    'format_id': media.get('formatoAudio'),
+                    'format_id': join_nonempty('https', media.get('formatoAudio'), delim='-'),
                     'url': media['audioUrl'],
                     'ext': media.get('formatoAudio'),
+                    'vcodec': 'none',
+                    'acodec': media.get('formatoAudio'),
                 }]
             }
-        elif 'Video' in media_type:
+        elif 'Video' in media['type']:
             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
         else:
             raise ExtractorError('not a media file')
 
-        thumbnails = []
-        for image_type in ('image', 'image_medium', 'image_300'):
-            thumbnail_url = media.get(image_type)
-            if thumbnail_url:
-                thumbnails.append({
-                    'url': compat_urlparse.urljoin(url, thumbnail_url),
-                })
-
-        subtitles = self._extract_subtitles(url, media)
+        thumbnails = self._get_thumbnails_list(
+            {image_type: media.get(image_type) for image_type in (
+                'image', 'image_medium', 'image_300')}, url)
 
         return {
             'id': content_id,
-            'title': title,
-            'description': strip_or_none(media.get('desc') or None),
+            'title': strip_or_none(media.get('name') or media.get('title')),
+            'description': strip_or_none(media.get('desc')) or None,
             'thumbnails': thumbnails,
-            'uploader': strip_or_none(media.get('author') or None),
+            'uploader': strip_or_none(media.get('author')) or None,
             'upload_date': unified_strdate(media.get('date')),
             'duration': parse_duration(media.get('length')),
-            'subtitles': subtitles,
-            **relinker_info
-        }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        content_item_id = None
-
-        content_item_url = self._html_search_meta(
-            ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
-             'twitter:player', 'jsonlink'), webpage, default=None)
-        if content_item_url:
-            content_item_id = self._search_regex(
-                rf'ContentItem-({self._UUID_RE})', content_item_url,
-                'content item id', default=None)
-
-        if not content_item_id:
-            content_item_id = self._search_regex(
-                rf'''(?x)
-                    (?:
-                        (?:initEdizione|drawMediaRaiTV)\(|
-                        <(?:[^>]+\bdata-id|var\s+uniquename)=|
-                        <iframe[^>]+\bsrc=
-                    )
-                    (["\'])
-                    (?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
-                ''',
-                webpage, 'content item id', default=None, group='id')
-
-        content_item_ids = set()
-        if content_item_id:
-            content_item_ids.add(content_item_id)
-        if video_id not in content_item_ids:
-            content_item_ids.add(video_id)
-
-        for content_item_id in content_item_ids:
-            try:
-                return self._extract_from_content_id(content_item_id, url)
-            except GeoRestrictedError:
-                raise
-            except ExtractorError:
-                pass
-
-        relinker_url = self._proto_relative_url(self._search_regex(
-            r'''(?x)
-                (?:
-                    var\s+videoURL|
-                    mediaInfo\.mediaUri
-                )\s*=\s*
-                ([\'"])
-                (?P<url>
-                    (?:https?:)?
-                    //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
-                    (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
-            ''',
-            webpage, 'relinker URL', group='url'))
-
-        relinker_info = self._extract_relinker_info(
-            urljoin(url, relinker_url), video_id)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title',
-            default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
+            'subtitles': self._extract_subtitles(url, media),
             **relinker_info
         }
 
@@ -726,7 +638,8 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
             'duration': 1589,
             'upload_date': '20220529',
             'uploader': 'rainews',
-        }
+        },
+        'params': {'skip_download': True},
     }, {
         # old content with fallback method to extract media urls
         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
@@ -739,12 +652,14 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
             'duration': 833,
             'upload_date': '20161103'
         },
+        'params': {'skip_download': True},
         'expected_warnings': ['unable to extract player_data'],
     }, {
         # iframe + drm
         'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
         'only_matching': True,
     }]
+    _PLAYER_TAG = 'news'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -752,8 +667,8 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
         webpage = self._download_webpage(url, video_id)
 
         player_data = self._search_json(
-            r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
-            transform_source=clean_html, fatal=False)
+            rf'<rai{self._PLAYER_TAG}-player\s*data=\'', webpage, 'player_data', video_id,
+            transform_source=clean_html, default={})
         track_info = player_data.get('track_info')
         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 
@@ -770,16 +685,36 @@ class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
 
         return {
             'id': video_id,
-            'title': track_info.get('title') or self._og_search_title(webpage),
+            'title': player_data.get('title') or track_info.get('title') or self._og_search_title(webpage),
             'upload_date': unified_strdate(track_info.get('date')),
             'uploader': strip_or_none(track_info.get('editor') or None),
             **relinker_info
         }
 
 
-class RaiSudtirolIE(RaiBaseIE):
-    _VALID_URL = r'https?://raisudtirol\.rai\.it/.+?media=(?P<id>[TP]tv\d+)'
+class RaiCulturaIE(RaiNewsIE):  # XXX: Do not subclass from concrete IE
+    _VALID_URL = rf'https?://(www\.)?raicultura\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
+    _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
     _TESTS = [{
+        'url': 'https://www.raicultura.it/letteratura/articoli/2018/12/Alberto-Asor-Rosa-Letteratura-e-potere-05ba8775-82b5-45c5-a89d-dd955fbde1fb.html',
+        'info_dict': {
+            'id': '05ba8775-82b5-45c5-a89d-dd955fbde1fb',
+            'ext': 'mp4',
+            'title': 'Alberto Asor Rosa: Letteratura e potere',
+            'duration': 1756,
+            'upload_date': '20181206',
+            'uploader': 'raicultura',
+            'formats': 'count:2',
+        },
+        'params': {'skip_download': True},
+    }]
+    _PLAYER_TAG = 'cultura'
+
+
+class RaiSudtirolIE(RaiBaseIE):
+    _VALID_URL = r'https?://raisudtirol\.rai\.it/.+media=(?P<id>\w+)'
+    _TESTS = [{
+        # mp4 file
         'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
         'info_dict': {
             'id': 'Ptv1619729460',
@@ -787,34 +722,62 @@ class RaiSudtirolIE(RaiBaseIE):
             'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
             'series': 'Euro: trasmisciun d\'economia',
             'upload_date': '20210429',
-            'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+?\.jpg',
+            'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+\.jpg',
             'uploader': 'raisudtirol',
-        }
+            'formats': 'count:1',
+        },
+        'params': {'skip_download': True},
+    }, {
+        # m3u manifest
+        'url': 'https://raisudtirol.rai.it/it/kidsplayer.php?lang=it&media=GUGGUG_P1.smil',
+        'info_dict': {
+            'id': 'GUGGUG_P1',
+            'ext': 'mp4',
+            'title': 'GUGGUG! La Prospettiva - Die Perspektive',
+            'uploader': 'raisudtirol',
+            'formats': 'count:6',
+        },
+        'params': {'skip_download': True},
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_date = self._html_search_regex(r'<span class="med_data">(.+?)</span>', webpage, 'video_date', fatal=False)
-        video_title = self._html_search_regex(r'<span class="med_title">(.+?)</span>', webpage, 'video_title', fatal=False)
-        video_url = self._html_search_regex(r'sources:\s*\[\{file:\s*"(.+?)"\}\]', webpage, 'video_url')
-        video_thumb = self._html_search_regex(r'image: \'(.+?)\'', webpage, 'video_thumb', fatal=False)
+        video_date = self._html_search_regex(
+            r'<span class="med_data">(.+?)</span>', webpage, 'video_date', default=None)
+        video_title = self._html_search_regex([
+            r'<span class="med_title">(.+?)</span>', r'title: \'(.+?)\','],
+            webpage, 'video_title', default=None)
+        video_url = self._html_search_regex([
+            r'sources:\s*\[\{file:\s*"(.+?)"\}\]',
+            r'<source\s+src="(.+?)"\s+type="application/x-mpegURL"'],
+            webpage, 'video_url', default=None)
 
-        return {
-            'id': video_id,
-            'title': join_nonempty(video_title, video_date, delim=' - '),
-            'series': video_title,
-            'upload_date': unified_strdate(video_date),
-            'thumbnail': urljoin('https://raisudtirol.rai.it/', video_thumb),
-            'uploader': 'raisudtirol',
-            'formats': [{
+        ext = determine_ext(video_url)
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, video_id)
+        elif ext == 'mp4':
+            formats = [{
                 'format_id': 'https-mp4',
                 'url': self._proto_relative_url(video_url),
                 'width': 1024,
                 'height': 576,
                 'fps': 25,
-                'vcodec': 'h264',
-                'acodec': 'aac',
-            }],
+                'vcodec': 'avc1',
+                'acodec': 'mp4a',
+            }]
+        else:
+            formats = []
+            self.raise_no_formats(f'Unrecognized media file: {video_url}')
+
+        return {
+            'id': video_id,
+            'title': join_nonempty(video_title, video_date, delim=' - '),
+            'series': video_title if video_date else None,
+            'upload_date': unified_strdate(video_date),
+            'thumbnail': urljoin('https://raisudtirol.rai.it/', self._html_search_regex(
+                r'image: \'(.+?)\'', webpage, 'video_thumb', default=None)),
+            'uploader': 'raisudtirol',
+            'formats': formats,
         }

yt_dlp/extractor/recurbate.py

@@ -0,0 +1,43 @@
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, merge_dicts
+
+
+class RecurbateIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?recurbate\.com/play\.php\?video=(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://recurbate.com/play.php?video=39161415',
+        'md5': 'dd2b4ec57aa3e3572cb5cf0997fca99f',
+        'info_dict': {
+            'id': '39161415',
+            'ext': 'mp4',
+            'description': 'md5:db48d09e4d93fc715f47fd3d6b7edd51',
+            'title': 'Performer zsnicole33 show on 2022-10-25 20:23, Chaturbate Archive – Recurbate',
+            'age_limit': 18,
+        },
+        'skip': 'Website require membership.',
+    }]
+
+    def _real_extract(self, url):
+        SUBSCRIPTION_MISSING_MESSAGE = 'This video is only available for registered users; Set your authenticated browser user agent via the --user-agent parameter.'
+        video_id = self._match_id(url)
+        try:
+            webpage = self._download_webpage(url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
+            raise
+        token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
+        video_url = f'https://recurbate.com/api/get.php?video={video_id}&token={token}'
+
+        video_webpage = self._download_webpage(video_url, video_id)
+        if video_webpage == 'shall_subscribe':
+            self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
+        entries = self._parse_html5_media_entries(video_url, video_webpage, video_id)
+        return merge_dicts({
+            'id': video_id,
+            'title': self._html_extract_title(webpage, 'title'),
+            'description': self._og_search_description(webpage),
+            'age_limit': self._rta_search(webpage),
+        }, entries[0])

yt_dlp/extractor/rottentomatoes.py

@@ -1,30 +1,80 @@
 from .common import InfoExtractor
-from .internetvideoarchive import InternetVideoArchiveIE
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    float_or_none,
+    get_element_by_class,
+    join_nonempty,
+    traverse_obj,
+    url_or_none,
+)
 
 
 class RottenTomatoesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/(?P<playlist>[^/]+)(?:/(?P<tr>trailers)(?:/(?P<id>\w+))?)?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
         'info_dict': {
             'id': '11028566',
             'ext': 'mp4',
             'title': 'Toy Story 3',
-            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.'
         },
-    }
+        'skip': 'No longer available',
+    }, {
+        'url': 'https://www.rottentomatoes.com/m/toy_story_3/trailers/VycaVoBKhGuk',
+        'info_dict': {
+            'id': 'VycaVoBKhGuk',
+            'ext': 'mp4',
+            'title': 'Toy Story 3: Trailer 2',
+            'description': '',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 149.941
+        },
+    }, {
+        'url': 'http://www.rottentomatoes.com/m/toy_story_3',
+        'info_dict': {
+            'id': 'toy_story_3',
+            'title': 'Toy Story 3',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers',
+        'info_dict': {
+            'id': 'toy_story_3-trailers',
+        },
+        'playlist_mincount': 5,
+    }]
+
+    def _extract_videos(self, data, display_id):
+        for video in traverse_obj(data, (lambda _, v: v['publicId'] and v['file'] and v['type'] == 'hls')):
+            yield {
+                'formats': self._extract_m3u8_formats(
+                    video['file'], display_id, 'mp4', m3u8_id='hls', fatal=False),
+                **traverse_obj(video, {
+                    'id': 'publicId',
+                    'title': 'title',
+                    'description': 'description',
+                    'duration': ('durationInSeconds', {float_or_none}),
+                    'thumbnail': ('image', {url_or_none}),
+                }),
+            }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        iva_id = self._search_regex(r'publishedid=(\d+)', webpage, 'internet video archive id')
+        playlist_id, trailers, video_id = self._match_valid_url(url).group('playlist', 'tr', 'id')
+        playlist_id = join_nonempty(playlist_id, trailers)
+        webpage = self._download_webpage(url, playlist_id)
+        data = self._search_json(
+            r'<script[^>]+\bid=["\'](?:heroV|v)ideos["\'][^>]*>', webpage,
+            'data', playlist_id, contains_pattern=r'\[{(?s:.+)}\]')
 
-        return {
-            '_type': 'url_transparent',
-            'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?domain=www.videodetective.com&customerid=69249&playerid=641&publishedid=' + iva_id,
-            'ie_key': InternetVideoArchiveIE.ie_key(),
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-        }
+        if video_id:
+            video_data = traverse_obj(data, lambda _, v: v['publicId'] == video_id)
+            if not video_data:
+                raise ExtractorError('Unable to extract video from webpage')
+            return next(self._extract_videos(video_data, video_id))
+
+        return self.playlist_result(
+            self._extract_videos(data, playlist_id), playlist_id,
+            clean_html(get_element_by_class('scoreboard__title', webpage)))

yt_dlp/extractor/tvplay.py

@@ -30,10 +30,7 @@ class TVPlayIE(InfoExtractor):
                             (?:
                                 tvplay(?:\.skaties)?\.lv(?:/parraides)?|
                                 (?:tv3play|play\.tv3)\.lt(?:/programos)?|
-                                tv3play(?:\.tv3)?\.ee/sisu|
-                                (?:tv(?:3|6|8|10)play)\.se/program|
-                                (?:(?:tv3play|viasat4play|tv6play)\.no|(?:tv3play)\.dk)/programmer|
-                                play\.nova(?:tv)?\.bg/programi
+                                tv3play(?:\.tv3)?\.ee/sisu
                             )
                             /(?:[^/]+/)+
                         )
@@ -92,117 +89,6 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        {
-            'url': 'http://www.tv3play.se/program/husraddarna/395385?autostart=true',
-            'info_dict': {
-                'id': '395385',
-                'ext': 'mp4',
-                'title': 'Husräddarna S02E07',
-                'description': 'md5:f210c6c89f42d4fc39faa551be813777',
-                'duration': 2574,
-                'timestamp': 1400596321,
-                'upload_date': '20140520',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv6play.se/program/den-sista-dokusapan/266636?autostart=true',
-            'info_dict': {
-                'id': '266636',
-                'ext': 'mp4',
-                'title': 'Den sista dokusåpan S01E08',
-                'description': 'md5:295be39c872520221b933830f660b110',
-                'duration': 1492,
-                'timestamp': 1330522854,
-                'upload_date': '20120229',
-                'age_limit': 18,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv8play.se/program/antikjakten/282756?autostart=true',
-            'info_dict': {
-                'id': '282756',
-                'ext': 'mp4',
-                'title': 'Antikjakten S01E10',
-                'description': 'md5:1b201169beabd97e20c5ad0ad67b13b8',
-                'duration': 2646,
-                'timestamp': 1348575868,
-                'upload_date': '20120925',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv3play.no/programmer/anna-anka-soker-assistent/230898?autostart=true',
-            'info_dict': {
-                'id': '230898',
-                'ext': 'mp4',
-                'title': 'Anna Anka søker assistent - Ep. 8',
-                'description': 'md5:f80916bf5bbe1c5f760d127f8dd71474',
-                'duration': 2656,
-                'timestamp': 1277720005,
-                'upload_date': '20100628',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.viasat4play.no/programmer/budbringerne/21873?autostart=true',
-            'info_dict': {
-                'id': '21873',
-                'ext': 'mp4',
-                'title': 'Budbringerne program 10',
-                'description': 'md5:4db78dc4ec8a85bb04fd322a3ee5092d',
-                'duration': 1297,
-                'timestamp': 1254205102,
-                'upload_date': '20090929',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.tv6play.no/programmer/hotelinspektor-alex-polizzi/361883?autostart=true',
-            'info_dict': {
-                'id': '361883',
-                'ext': 'mp4',
-                'title': 'Hotelinspektør Alex Polizzi - Ep. 10',
-                'description': 'md5:3ecf808db9ec96c862c8ecb3a7fdaf81',
-                'duration': 2594,
-                'timestamp': 1393236292,
-                'upload_date': '20140224',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
-            'info_dict': {
-                'id': '624952',
-                'ext': 'flv',
-                'title': 'Здравей, България (12.06.2015 г.) ',
-                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
-                'duration': 8838,
-                'timestamp': 1434100372,
-                'upload_date': '20150612',
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'https://play.nova.bg/programi/zdravei-bulgariya/764300?autostart=true',
-            'only_matching': True,
-        },
         {
             'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
             'only_matching': True,
@@ -327,103 +213,6 @@ class TVPlayIE(InfoExtractor):
         }
 
 
-class ViafreeIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        viafree\.(?P<country>dk|no|se|fi)
-                        /(?P<id>(?:program(?:mer)?|ohjelmat)?/(?:[^/]+/)+[^/?#&]+)
-                    '''
-    _TESTS = [{
-        'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
-        'info_dict': {
-            'id': '757786',
-            'ext': 'mp4',
-            'title': 'Det beste vorspielet - Sesong 2 - Episode 1',
-            'description': 'md5:b632cb848331404ccacd8cd03e83b4c3',
-            'series': 'Det beste vorspielet',
-            'season_number': 2,
-            'duration': 1116,
-            'timestamp': 1471200600,
-            'upload_date': '20160814',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.viafree.dk/programmer/humor/comedy-central-roast-of-charlie-sheen/film/1047660',
-        'info_dict': {
-            'id': '1047660',
-            'ext': 'mp4',
-            'title': 'Comedy Central Roast of Charlie Sheen - Comedy Central Roast of Charlie Sheen',
-            'description': 'md5:ec956d941ae9fd7c65a48fd64951dc6d',
-            'series': 'Comedy Central Roast of Charlie Sheen',
-            'season_number': 1,
-            'duration': 3747,
-            'timestamp': 1608246060,
-            'upload_date': '20201217'
-        },
-        'params': {
-            'skip_download': True
-        }
-    }, {
-        # with relatedClips
-        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-1',
-        'only_matching': True,
-    }, {
-        # Different og:image URL schema
-        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.se/program/livsstil/husraddarna/sasong-2/avsnitt-2',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.dk/programmer/reality/paradise-hotel/saeson-7/episode-5',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.viafree.se/program/underhallning/i-like-radio-live/sasong-1/676869',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.viafree.fi/ohjelmat/entertainment/amazing-makeovers/kausi-7/jakso-2',
-        'only_matching': True,
-    }]
-    _GEO_BYPASS = False
-
-    def _real_extract(self, url):
-        country, path = self._match_valid_url(url).groups()
-        content = self._download_json(
-            'https://viafree-content.mtg-api.com/viafree-content/v1/%s/path/%s' % (country, path), path)
-        program = content['_embedded']['viafreeBlocks'][0]['_embedded']['program']
-        guid = program['guid']
-        meta = content['meta']
-        title = meta['title']
-
-        try:
-            stream_href = self._download_json(
-                program['_links']['streamLink']['href'], guid,
-                headers=self.geo_verification_headers())['embedded']['prioritizedStreams'][0]['links']['stream']['href']
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self.raise_geo_restricted(countries=[country])
-            raise
-
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_href, guid, 'mp4')
-        episode = program.get('episode') or {}
-        return {
-            'id': guid,
-            'title': title,
-            'thumbnail': meta.get('image'),
-            'description': meta.get('description'),
-            'series': episode.get('seriesTitle'),
-            'subtitles': subtitles,
-            'episode_number': int_or_none(episode.get('episodeNumber')),
-            'season_number': int_or_none(episode.get('seasonNumber')),
-            'duration': int_or_none(try_get(program, lambda x: x['video']['duration']['milliseconds']), 1000),
-            'timestamp': parse_iso8601(try_get(program, lambda x: x['availability']['start'])),
-            'formats': formats,
-        }
-
-
 class TVPlayHomeIE(InfoExtractor):
     _VALID_URL = r'''(?x)
             https?://

yt_dlp/extractor/twitch.py

@@ -41,7 +41,6 @@ class TwitchBaseIE(InfoExtractor):
     _USHER_BASE = 'https://usher.ttvnw.net'
     _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
     _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
-    _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
     _NETRC_MACHINE = 'twitch'
 
     _OPERATION_HASHES = {
@@ -58,6 +57,11 @@ class TwitchBaseIE(InfoExtractor):
         'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
     }
 
+    @property
+    def _CLIENT_ID(self):
+        return self._configuration_arg(
+            'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key=TwitchStreamIE, casesense=True)[0]
+
     def _perform_login(self, username, password):
         def fail(message):
             raise ExtractorError(
@@ -194,7 +198,8 @@ class TwitchVodIE(TwitchBaseIE):
                     https?://
                         (?:
                             (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
-                            player\.twitch\.tv/\?.*?\bvideo=v?
+                            player\.twitch\.tv/\?.*?\bvideo=v?|
+                            www\.twitch\.tv/[^/]+/schedule\?vodID=
                         )
                         (?P<id>\d+)
                     '''
@@ -363,6 +368,9 @@ class TwitchVodIE(TwitchBaseIE):
             'skip_download': True
         },
         'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
+    }, {
+        'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
+        'only_matching': True,
     }]
 
     def _download_info(self, item_id):
@@ -1075,7 +1083,7 @@ class TwitchClipsIE(TwitchBaseIE):
                     https?://
                         (?:
                             clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
-                            (?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
+                            (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/)?clip/
                         )
                         (?P<id>[^/?#&]+)
                     '''
@@ -1111,6 +1119,9 @@ class TwitchClipsIE(TwitchBaseIE):
     }, {
         'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
         'only_matching': True,
+    }, {
+        'url': 'https://m.twitch.tv/clip/FaintLightGullWholeWheat',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/twitter.py

@@ -705,6 +705,7 @@ class TwitterIE(TwitterBaseIE):
             'uploader': r're:Monique Camarra.+?',
             'uploader_id': 'MoniqueCamarra',
             'live_status': 'was_live',
+            'release_timestamp': 1658417414,
             'description': 'md5:acce559345fd49f129c20dbcda3f1201',
             'timestamp': 1658407771464,
         },
@@ -1327,6 +1328,8 @@ class TwitterSpacesIE(TwitterBaseIE):
             'uploader_id': traverse_obj(
                 metadata, ('creator_results', 'result', 'legacy', 'screen_name')),
             'live_status': live_status,
+            'release_timestamp': try_call(
+                lambda: int_or_none(metadata['scheduled_start'], scale=1000)),
             'timestamp': metadata.get('created_at'),
             'formats': formats,
         }

yt_dlp/extractor/unsupported.py

@@ -131,8 +131,9 @@ class KnownPiracyIE(UnsupportedInfoExtractor):
     URLS = (
         r'dood\.(?:to|watch|so|pm|wf|re)',
         # Sites youtube-dl supports, but we won't
-        r'https://viewsb\.com',
-        r'https://filemoon\.sx',
+        r'viewsb\.com',
+        r'filemoon\.sx',
+        r'hentai\.animestigma\.com',
     )
 
     _TESTS = [{

yt_dlp/extractor/vrt.py

@@ -1,45 +1,137 @@
-from .common import InfoExtractor
+import functools
+import json
+import time
+import urllib.error
+import urllib.parse
+
+from .gigya import GigyaBaseIE
 from ..utils import (
+    ExtractorError,
+    clean_html,
     extract_attributes,
     float_or_none,
     get_element_by_class,
+    get_element_html_by_class,
+    int_or_none,
+    join_nonempty,
+    jwt_encode_hs256,
+    make_archive_id,
+    parse_age_limit,
+    parse_iso8601,
+    str_or_none,
     strip_or_none,
-    unified_timestamp,
+    traverse_obj,
+    url_or_none,
+    urlencode_postdata,
 )
 
 
-class VRTIE(InfoExtractor):
+class VRTBaseIE(GigyaBaseIE):
+    _GEO_BYPASS = False
+    _PLAYER_INFO = {
+        'platform': 'desktop',
+        'app': {
+            'type': 'browser',
+            'name': 'Chrome',
+        },
+        'device': 'undefined (undefined)',
+        'os': {
+            'name': 'Windows',
+            'version': 'x86_64'
+        },
+        'player': {
+            'name': 'VRT web player',
+            'version': '2.7.4-prod-2023-04-19T06:05:45'
+        }
+    }
+    # From https://player.vrt.be/vrtnws/js/main.js & https://player.vrt.be/ketnet/js/main.fd1de01a40a1e3d842ea.js
+    _JWT_KEY_ID = '0-0Fp51UZykfaiCJrfTE3+oMI8zvDteYfPtR+2n1R+z8w='
+    _JWT_SIGNING_KEY = '2a9251d782700769fb856da5725daf38661874ca6f80ae7dc2b05ec1a81a24ae'
+
+    def _extract_formats_and_subtitles(self, data, video_id):
+        if traverse_obj(data, 'drm'):
+            self.report_drm(video_id)
+
+        formats, subtitles = [], {}
+        for target in traverse_obj(data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type'])):
+            format_type = target['type'].upper()
+            format_url = target['url']
+            if format_type in ('HLS', 'HLS_AES'):
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_type, fatal=False))
+            elif format_type == 'MPEG_DASH':
+                fmts, subs = self._extract_mpd_formats_and_subtitles(
+                    format_url, video_id, mpd_id=format_type, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_type == 'HSS':
+                fmts, subs = self._extract_ism_formats_and_subtitles(
+                    format_url, video_id, ism_id='mss', fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
+
+        for sub in traverse_obj(data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED')):
+            subtitles.setdefault('nl', []).append({'url': sub['url']})
+
+        return formats, subtitles
+
+    def _call_api(self, video_id, client='null', id_token=None, version='v2'):
+        player_info = {'exp': (round(time.time(), 3) + 900), **self._PLAYER_INFO}
+        player_token = self._download_json(
+            'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2/tokens',
+            video_id, 'Downloading player token', headers={
+                **self.geo_verification_headers(),
+                'Content-Type': 'application/json',
+            }, data=json.dumps({
+                'identityToken': id_token or {},
+                'playerInfo': jwt_encode_hs256(player_info, self._JWT_SIGNING_KEY, headers={
+                    'kid': self._JWT_KEY_ID
+                }).decode()
+            }, separators=(',', ':')).encode())['vrtPlayerToken']
+
+        return self._download_json(
+            f'https://media-services-public.vrt.be/media-aggregator/{version}/media-items/{video_id}',
+            video_id, 'Downloading API JSON', query={
+                'vrtPlayerToken': player_token,
+                'client': client,
+            }, expected_status=400)
+
+
+class VRTIE(VRTBaseIE):
     IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
     _TESTS = [{
         'url': 'https://www.vrt.be/vrtnws/nl/2019/05/15/beelden-van-binnenkant-notre-dame-een-maand-na-de-brand/',
-        'md5': 'e1663accf5cf13f375f3cd0d10476669',
         'info_dict': {
             'id': 'pbs-pub-7855fc7b-1448-49bc-b073-316cb60caa71$vid-2ca50305-c38a-4762-9890-65cbd098b7bd',
             'ext': 'mp4',
             'title': 'Beelden van binnenkant Notre-Dame, één maand na de brand',
-            'description': 'Op maandagavond 15 april ging een deel van het dakgebinte van de Parijse kathedraal in vlammen op.',
-            'timestamp': 1557924660,
-            'upload_date': '20190515',
+            'description': 'md5:6fd85f999b2d1841aa5568f4bf02c3ff',
             'duration': 31.2,
+            'thumbnail': 'https://images.vrt.be/orig/2019/05/15/2d914d61-7710-11e9-abcc-02b7b76bf47f.jpg',
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://sporza.be/nl/2019/05/15/de-belgian-cats-zijn-klaar-voor-het-ek/',
-        'md5': '910bba927566e9ab992278f647eb4b75',
         'info_dict': {
             'id': 'pbs-pub-f2c86a46-8138-413a-a4b9-a0015a16ce2c$vid-1f112b31-e58e-4379-908d-aca6d80f8818',
             'ext': 'mp4',
-            'title': 'De Belgian Cats zijn klaar voor het EK mét Ann Wauters',
-            'timestamp': 1557923760,
-            'upload_date': '20190515',
+            'title': 'De Belgian Cats zijn klaar voor het EK',
+            'description': 'Video: De Belgian Cats zijn klaar voor het EK mét Ann Wauters | basketbal, sport in het journaal',
             'duration': 115.17,
+            'thumbnail': 'https://images.vrt.be/orig/2019/05/15/11c0dba3-770e-11e9-abcc-02b7b76bf47f.jpg',
         },
-    }, {
-        'url': 'https://www.vrt.be/vrtnws/en/2019/05/15/belgium_s-eurovision-entry-falls-at-the-first-hurdle/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.vrt.be/vrtnws/de/2019/05/15/aus-fuer-eliott-im-halbfinale-des-eurosongfestivals/',
-        'only_matching': True,
+        'params': {'skip_download': 'm3u8'},
     }]
     _CLIENT_MAP = {
         'vrt.be/vrtnws': 'vrtnieuws',
@@ -49,34 +141,285 @@ class VRTIE(InfoExtractor):
     def _real_extract(self, url):
         site, display_id = self._match_valid_url(url).groups()
         webpage = self._download_webpage(url, display_id)
-        attrs = extract_attributes(self._search_regex(
-            r'(<[^>]+class="vrtvideo( [^"]*)?"[^>]*>)', webpage, 'vrt video'))
+        attrs = extract_attributes(get_element_html_by_class('vrtvideo', webpage) or '')
 
-        asset_id = attrs['data-video-id']
-        publication_id = attrs.get('data-publication-id')
+        asset_id = attrs.get('data-video-id') or attrs['data-videoid']
+        publication_id = traverse_obj(attrs, 'data-publication-id', 'data-publicationid')
         if publication_id:
-            asset_id = publication_id + '$' + asset_id
-        client = attrs.get('data-client-code') or self._CLIENT_MAP[site]
+            asset_id = f'{publication_id}${asset_id}'
+        client = traverse_obj(attrs, 'data-client-code', 'data-client') or self._CLIENT_MAP[site]
+
+        data = self._call_api(asset_id, client)
+        formats, subtitles = self._extract_formats_and_subtitles(data, asset_id)
 
-        title = strip_or_none(get_element_by_class(
-            'vrt-title', webpage) or self._html_search_meta(
-            ['og:title', 'twitter:title', 'name'], webpage))
         description = self._html_search_meta(
             ['og:description', 'twitter:description', 'description'], webpage)
         if description == '…':
             description = None
-        timestamp = unified_timestamp(self._html_search_meta(
-            'article:published_time', webpage))
 
         return {
-            '_type': 'url_transparent',
             'id': asset_id,
-            'display_id': display_id,
-            'title': title,
+            'formats': formats,
+            'subtitles': subtitles,
             'description': description,
-            'thumbnail': attrs.get('data-posterimage'),
-            'timestamp': timestamp,
+            'thumbnail': url_or_none(attrs.get('data-posterimage')),
             'duration': float_or_none(attrs.get('data-duration'), 1000),
-            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (client, asset_id),
-            'ie_key': 'Canvas',
+            '_old_archive_ids': [make_archive_id('Canvas', asset_id)],
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('shortDescription', {str}),
+                'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
+                'thumbnail': ('posterImageUrl', {url_or_none}),
+            }),
+        }
+
+
+class VrtNUIE(VRTBaseIE):
+    IE_DESC = 'VRT MAX'
+    _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        # CONTENT_IS_AGE_RESTRICTED
+        'url': 'https://www.vrt.be/vrtnu/a-z/de-ideale-wereld/2023-vj/de-ideale-wereld-d20230116/',
+        'info_dict': {
+            'id': 'pbs-pub-855b00a8-6ce2-4032-ac4f-1fcf3ae78524$vid-d2243aa1-ec46-4e34-a55b-92568459906f',
+            'ext': 'mp4',
+            'title': 'Tom Waes',
+            'description': 'Satirisch actualiteitenmagazine met Ella Leyers. Tom Waes is te gast.',
+            'timestamp': 1673905125,
+            'release_timestamp': 1673905125,
+            'series': 'De ideale wereld',
+            'season_id': '1672830988794',
+            'episode': 'Aflevering 1',
+            'episode_number': 1,
+            'episode_id': '1672830988861',
+            'display_id': 'de-ideale-wereld-d20230116',
+            'channel': 'VRT',
+            'duration': 1939.0,
+            'thumbnail': 'https://images.vrt.be/orig/2023/01/10/1bb39cb3-9115-11ed-b07d-02b7b76bf47f.jpg',
+            'release_date': '20230116',
+            'upload_date': '20230116',
+            'age_limit': 12,
+        },
+    }, {
+        'url': 'https://www.vrt.be/vrtnu/a-z/buurman--wat-doet-u-nu-/6/buurman--wat-doet-u-nu--s6-trailer/',
+        'info_dict': {
+            'id': 'pbs-pub-ad4050eb-d9e5-48c2-9ec8-b6c355032361$vid-0465537a-34a8-4617-8352-4d8d983b4eee',
+            'ext': 'mp4',
+            'title': 'Trailer seizoen 6 \'Buurman, wat doet u nu?\'',
+            'description': 'md5:197424726c61384b4e5c519f16c0cf02',
+            'timestamp': 1652940000,
+            'release_timestamp': 1652940000,
+            'series': 'Buurman, wat doet u nu?',
+            'season': 'Seizoen 6',
+            'season_number': 6,
+            'season_id': '1652344200907',
+            'episode': 'Aflevering 0',
+            'episode_number': 0,
+            'episode_id': '1652951873524',
+            'display_id': 'buurman--wat-doet-u-nu--s6-trailer',
+            'channel': 'VRT',
+            'duration': 33.13,
+            'thumbnail': 'https://images.vrt.be/orig/2022/05/23/3c234d21-da83-11ec-b07d-02b7b76bf47f.jpg',
+            'release_date': '20220519',
+            'upload_date': '20220519',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+    _NETRC_MACHINE = 'vrtnu'
+    _authenticated = False
+
+    def _perform_login(self, username, password):
+        auth_info = self._gigya_login({
+            'APIKey': '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy',
+            'targetEnv': 'jssdk',
+            'loginID': username,
+            'password': password,
+            'authMode': 'cookie',
+        })
+
+        if auth_info.get('errorDetails'):
+            raise ExtractorError(f'Unable to login. VrtNU said: {auth_info["errorDetails"]}', expected=True)
+
+        # Sometimes authentication fails for no good reason, retry
+        for retry in self.RetryManager():
+            if retry.attempt > 1:
+                self._sleep(1, None)
+            try:
+                self._request_webpage(
+                    'https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token',
+                    errnote='Could not get XSRF Token', query={
+                        'provider': 'site',
+                        'destination': 'https://www.vrt.be/vrtnu/',
+                    })
+                self._request_webpage(
+                    'https://login.vrt.be/perform_login', None,
+                    note='Performing login', errnote='Login failed',
+                    query={'client_id': 'vrtnu-site'}, data=urlencode_postdata({
+                        'UID': auth_info['UID'],
+                        'UIDSignature': auth_info['UIDSignature'],
+                        'signatureTimestamp': auth_info['signatureTimestamp'],
+                        '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
+                    }))
+            except ExtractorError as e:
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                    retry.error = e
+                    continue
+                raise
+
+        self._authenticated = True
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        parsed_url = urllib.parse.urlparse(url)
+        details = self._download_json(
+            f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip("/")}.model.json',
+            display_id, 'Downloading asset JSON', 'Unable to download asset JSON')['details']
+
+        watch_info = traverse_obj(details, (
+            'actions', lambda _, v: v['type'] == 'watch-episode', {dict}), get_all=False) or {}
+        video_id = join_nonempty(
+            'episodePublicationId', 'episodeVideoId', delim='$', from_dict=watch_info)
+        if '$' not in video_id:
+            raise ExtractorError('Unable to extract video ID')
+
+        vrtnutoken = self._download_json(
+            'https://token.vrt.be/refreshtoken', video_id, note='Retrieving vrtnutoken',
+            errnote='Token refresh failed')['vrtnutoken'] if self._authenticated else None
+
+        video_info = self._call_api(video_id, 'vrtnu-web@PROD', vrtnutoken)
+
+        if 'title' not in video_info:
+            code = video_info.get('code')
+            if code in ('AUTHENTICATION_REQUIRED', 'CONTENT_IS_AGE_RESTRICTED'):
+                self.raise_login_required(code, method='password')
+            elif code in ('INVALID_LOCATION', 'CONTENT_AVAILABLE_ONLY_IN_BE'):
+                self.raise_geo_restricted(countries=['BE'])
+            elif code == 'CONTENT_AVAILABLE_ONLY_FOR_BE_RESIDENTS_AND_EXPATS':
+                if not self._authenticated:
+                    self.raise_login_required(code, method='password')
+                self.raise_geo_restricted(countries=['BE'])
+            raise ExtractorError(code, expected=True)
+
+        formats, subtitles = self._extract_formats_and_subtitles(video_info, video_id)
+
+        return {
+            **traverse_obj(details, {
+                'title': 'title',
+                'description': ('description', {clean_html}),
+                'timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
+                'release_timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
+                'series': ('data', 'program', 'title'),
+                'season': ('data', 'season', 'title', 'value'),
+                'season_number': ('data', 'season', 'title', 'raw', {int_or_none}),
+                'season_id': ('data', 'season', 'id', {str_or_none}),
+                'episode': ('data', 'episode', 'number', 'value', {str_or_none}),
+                'episode_number': ('data', 'episode', 'number', 'raw', {int_or_none}),
+                'episode_id': ('data', 'episode', 'id', {str_or_none}),
+                'age_limit': ('data', 'episode', 'age', 'raw', {parse_age_limit}),
+            }),
+            'id': video_id,
+            'display_id': display_id,
+            'channel': 'VRT',
+            'formats': formats,
+            'duration': float_or_none(video_info.get('duration'), 1000),
+            'thumbnail': url_or_none(video_info.get('posterImageUrl')),
+            'subtitles': subtitles,
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
+        }
+
+
+class KetnetIE(VRTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.ketnet.be/kijken/m/meisjes/6/meisjes-s6a5',
+        'info_dict': {
+            'id': 'pbs-pub-39f8351c-a0a0-43e6-8394-205d597d6162$vid-5e306921-a9aa-4fa9-9f39-5b82c8f1028e',
+            'ext': 'mp4',
+            'title': 'Meisjes',
+            'episode': 'Reeks 6: Week 5',
+            'season': 'Reeks 6',
+            'series': 'Meisjes',
+            'timestamp': 1685251800,
+            'upload_date': '20230528',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        video = self._download_json(
+            'https://senior-bff.ketnet.be/graphql', display_id, query={
+                'query': '''{
+  video(id: "content/ketnet/nl/%s.model.json") {
+    description
+    episodeNr
+    imageUrl
+    mediaReference
+    programTitle
+    publicationDate
+    seasonTitle
+    subtitleVideodetail
+    titleVideodetail
+  }
+}''' % display_id,
+            })['data']['video']
+
+        video_id = urllib.parse.unquote(video['mediaReference'])
+        data = self._call_api(video_id, 'ketnet@PROD', version='v1')
+        formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
+            **traverse_obj(video, {
+                'title': ('titleVideodetail', {str}),
+                'description': ('description', {str}),
+                'thumbnail': ('thumbnail', {url_or_none}),
+                'timestamp': ('publicationDate', {parse_iso8601}),
+                'series': ('programTitle', {str}),
+                'season': ('seasonTitle', {str}),
+                'episode': ('subtitleVideodetail', {str}),
+                'episode_number': ('episodeNr', {int_or_none}),
+            }),
+        }
+
+
+class DagelijkseKostIE(VRTBaseIE):
+    IE_DESC = 'dagelijksekost.een.be'
+    _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
+        'info_dict': {
+            'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
+            'ext': 'mp4',
+            'title': 'Hachis parmentier met witloof',
+            'description': 'md5:9960478392d87f63567b5b117688cdc5',
+            'display_id': 'hachis-parmentier-met-witloof',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(
+            r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
+
+        data = self._call_api(video_id, 'dako@prod', version='v1')
+        formats, subtitles = self._extract_formats_and_subtitles(data, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'display_id': display_id,
+            'title': strip_or_none(get_element_by_class(
+                'dish-metadata__title', webpage) or self._html_search_meta('twitter:title', webpage)),
+            'description': clean_html(get_element_by_class(
+                'dish-description', webpage)) or self._html_search_meta(
+                ['description', 'twitter:description', 'og:description'], webpage),
+            '_old_archive_ids': [make_archive_id('Canvas', video_id)],
         }

yt_dlp/extractor/weverse.py

@@ -0,0 +1,607 @@
+import base64
+import hashlib
+import hmac
+import itertools
+import json
+import re
+import time
+import urllib.error
+import urllib.parse
+import uuid
+
+from .common import InfoExtractor
+from .naver import NaverBaseIE
+from .youtube import YoutubeIE
+from ..utils import (
+    ExtractorError,
+    UserNotLive,
+    float_or_none,
+    int_or_none,
+    str_or_none,
+    traverse_obj,
+    try_call,
+    update_url_query,
+    url_or_none,
+)
+
+
+class WeverseBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'weverse'
+    _ACCOUNT_API_BASE = 'https://accountapi.weverse.io/web/api/v2'
+    _API_HEADERS = {
+        'Referer': 'https://weverse.io/',
+        'WEV-device-Id': str(uuid.uuid4()),
+    }
+
+    def _perform_login(self, username, password):
+        if self._API_HEADERS.get('Authorization'):
+            return
+
+        headers = {
+            'x-acc-app-secret': '5419526f1c624b38b10787e5c10b2a7a',
+            'x-acc-app-version': '2.2.6',
+            'x-acc-language': 'en',
+            'x-acc-service-id': 'weverse',
+            'x-acc-trace-id': str(uuid.uuid4()),
+            'x-clog-user-device-id': str(uuid.uuid4()),
+        }
+        check_username = self._download_json(
+            f'{self._ACCOUNT_API_BASE}/signup/email/status', None,
+            note='Checking username', query={'email': username}, headers=headers)
+        if not check_username.get('hasPassword'):
+            raise ExtractorError('Invalid username provided', expected=True)
+
+        headers['content-type'] = 'application/json'
+        try:
+            auth = self._download_json(
+                f'{self._ACCOUNT_API_BASE}/auth/token/by-credentials', None, data=json.dumps({
+                    'email': username,
+                    'password': password,
+                }, separators=(',', ':')).encode(), headers=headers, note='Logging in')
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                raise ExtractorError('Invalid password provided', expected=True)
+            raise
+
+        WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {auth["accessToken"]}'
+
+    def _real_initialize(self):
+        if self._API_HEADERS.get('Authorization'):
+            return
+
+        token = try_call(lambda: self._get_cookies('https://weverse.io/')['we2_access_token'].value)
+        if not token:
+            self.raise_login_required()
+
+        WeverseBaseIE._API_HEADERS['Authorization'] = f'Bearer {token}'
+
+    def _call_api(self, ep, video_id, data=None, note='Downloading API JSON'):
+        # Ref: https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/2488.a09b41ff.chunk.js
+        # From https://ssl.pstatic.net/static/wevweb/2_3_2_11101725/public/static/js/main.e206f7c1.js:
+        key = b'1b9cb6378d959b45714bec49971ade22e6e24e42'
+        api_path = update_url_query(ep, {
+            'appId': 'be4d79eb8fc7bd008ee82c8ec4ff6fd4',
+            'language': 'en',
+            'platform': 'WEB',
+            'wpf': 'pc',
+        })
+        wmsgpad = int(time.time() * 1000)
+        wmd = base64.b64encode(hmac.HMAC(
+            key, f'{api_path[:255]}{wmsgpad}'.encode(), digestmod=hashlib.sha1).digest()).decode()
+        headers = {'Content-Type': 'application/json'} if data else {}
+        try:
+            return self._download_json(
+                f'https://global.apis.naver.com/weverse/wevweb{api_path}', video_id, note=note,
+                data=data, headers={**self._API_HEADERS, **headers}, query={
+                    'wmsgpad': wmsgpad,
+                    'wmd': wmd,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                self.raise_login_required(
+                    'Session token has expired. Log in again or refresh cookies in browser')
+            elif isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Your account does not have access to this content', expected=True)
+            raise
+
+    def _call_post_api(self, video_id):
+        return self._call_api(f'/post/v1.0/post-{video_id}?fieldSet=postV1', video_id)
+
+    def _get_community_id(self, channel):
+        return str(self._call_api(
+            f'/community/v1.0/communityIdUrlPathByUrlPathArtistCode?keyword={channel}',
+            channel, note='Fetching community ID')['communityId'])
+
+    def _get_formats(self, data, video_id):
+        formats = traverse_obj(data, ('videos', 'list', lambda _, v: url_or_none(v['source']), {
+            'url': 'source',
+            'width': ('encodingOption', 'width', {int_or_none}),
+            'height': ('encodingOption', 'height', {int_or_none}),
+            'vcodec': 'type',
+            'vbr': ('bitrate', 'video', {int_or_none}),
+            'abr': ('bitrate', 'audio', {int_or_none}),
+            'filesize': ('size', {int_or_none}),
+            'format_id': ('encodingOption', 'id', {str_or_none}),
+        }))
+
+        for stream in traverse_obj(data, ('streams', lambda _, v: v['type'] == 'HLS' and url_or_none(v['source']))):
+            query = {}
+            for param in traverse_obj(stream, ('keys', lambda _, v: v['type'] == 'param' and v['name'])):
+                query[param['name']] = param.get('value', '')
+            fmts = self._extract_m3u8_formats(
+                stream['source'], video_id, 'mp4', m3u8_id='hls', fatal=False, query=query)
+            if query:
+                for fmt in fmts:
+                    fmt['url'] = update_url_query(fmt['url'], query)
+                    fmt['extra_param_to_segment_url'] = urllib.parse.urlencode(query)
+            formats.extend(fmts)
+
+        return formats
+
+    def _get_subs(self, caption_url):
+        subs_ext_re = r'\.(?:ttml|vtt)'
+        replace_ext = lambda x, y: re.sub(subs_ext_re, y, x)
+        if re.search(subs_ext_re, caption_url):
+            return [replace_ext(caption_url, '.ttml'), replace_ext(caption_url, '.vtt')]
+        return [caption_url]
+
+    def _parse_post_meta(self, metadata):
+        return traverse_obj(metadata, {
+            'title': ((('extension', 'mediaInfo', 'title'), 'title'), {str}),
+            'description': ((('extension', 'mediaInfo', 'body'), 'body'), {str}),
+            'uploader': ('author', 'profileName', {str}),
+            'uploader_id': ('author', 'memberId', {str}),
+            'creator': ('community', 'communityName', {str}),
+            'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
+            'duration': ('extension', 'video', 'playTime', {float_or_none}),
+            'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
+            'release_timestamp': ('extension', 'video', 'onAirStartAt', {lambda x: int_or_none(x, 1000)}),
+            'thumbnail': ('extension', (('mediaInfo', 'thumbnail', 'url'), ('video', 'thumb')), {url_or_none}),
+            'view_count': ('extension', 'video', 'playCount', {int_or_none}),
+            'like_count': ('extension', 'video', 'likeCount', {int_or_none}),
+            'comment_count': ('commentCount', {int_or_none}),
+        }, get_all=False)
+
+    def _extract_availability(self, data):
+        return self._availability(**traverse_obj(data, ((('extension', 'video'), None), {
+            'needs_premium': 'paid',
+            'needs_subscription': 'membershipOnly',
+        }), get_all=False, expected_type=bool), needs_auth=True)
+
+    def _extract_live_status(self, data):
+        data = traverse_obj(data, ('extension', 'video', {dict})) or {}
+        if data.get('type') == 'LIVE':
+            return traverse_obj({
+                'ONAIR': 'is_live',
+                'DONE': 'post_live',
+                'STANDBY': 'is_upcoming',
+                'DELAY': 'is_upcoming',
+            }, (data.get('status'), {str})) or 'is_live'
+        return 'was_live' if data.get('liveToVod') else 'not_live'
+
+
+class WeverseIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/live/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/live/0-107323480',
+        'md5': '1fa849f00181eef9100d3c8254c47979',
+        'info_dict': {
+            'id': '0-107323480',
+            'ext': 'mp4',
+            'title': '행복한 평이루💜',
+            'description': '',
+            'uploader': 'Billlie',
+            'uploader_id': '5ae14aed7b7cdc65fa87c41fe06cc936',
+            'channel': 'billlie',
+            'channel_id': '72',
+            'channel_url': 'https://weverse.io/billlie',
+            'creator': 'Billlie',
+            'timestamp': 1666262062,
+            'upload_date': '20221020',
+            'release_timestamp': 1666262058,
+            'release_date': '20221020',
+            'duration': 3102,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'was_live',
+        },
+    }, {
+        'url': 'https://weverse.io/lesserafim/live/2-102331763',
+        'md5': 'e46125c08b13a6c8c1f4565035cca987',
+        'info_dict': {
+            'id': '2-102331763',
+            'ext': 'mp4',
+            'title': '🎂김채원 생신🎂',
+            'description': '🎂김채원 생신🎂',
+            'uploader': 'LE SSERAFIM ',
+            'uploader_id': 'd26ddc1e258488a0a2b795218d14d59d',
+            'channel': 'lesserafim',
+            'channel_id': '47',
+            'channel_url': 'https://weverse.io/lesserafim',
+            'creator': 'LE SSERAFIM',
+            'timestamp': 1659353400,
+            'upload_date': '20220801',
+            'release_timestamp': 1659353400,
+            'release_date': '20220801',
+            'duration': 3006,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'was_live',
+            'subtitles': {
+                'id_ID': 'count:2',
+                'en_US': 'count:2',
+                'es_ES': 'count:2',
+                'vi_VN': 'count:2',
+                'th_TH': 'count:2',
+                'zh_CN': 'count:2',
+                'zh_TW': 'count:2',
+                'ja_JP': 'count:2',
+                'ko_KR': 'count:2',
+            },
+        },
+    }, {
+        'url': 'https://weverse.io/treasure/live/2-117230416',
+        'info_dict': {
+            'id': '2-117230416',
+            'ext': 'mp4',
+            'title': r're:스껄도려님 첫 스무살 생파🦋',
+            'description': '',
+            'uploader': 'TREASURE',
+            'uploader_id': '77eabbc449ca37f7970054a136f60082',
+            'channel': 'treasure',
+            'channel_id': '20',
+            'channel_url': 'https://weverse.io/treasure',
+            'creator': 'TREASURE',
+            'timestamp': 1680667651,
+            'upload_date': '20230405',
+            'release_timestamp': 1680667639,
+            'release_date': '20230405',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'is_live',
+        },
+        'skip': 'Livestream has ended',
+    }]
+
+    def _real_extract(self, url):
+        channel, video_id = self._match_valid_url(url).group('artist', 'id')
+        post = self._call_post_api(video_id)
+        api_video_id = post['extension']['video']['videoId']
+        availability = self._extract_availability(post)
+        live_status = self._extract_live_status(post)
+        video_info, formats = {}, []
+
+        if live_status == 'is_upcoming':
+            self.raise_no_formats('Livestream has not yet started', expected=True)
+
+        elif live_status == 'is_live':
+            video_info = self._call_api(
+                f'/video/v1.0/lives/{api_video_id}/playInfo?preview.format=json&preview.version=v2',
+                video_id, note='Downloading live JSON')
+            playback = self._parse_json(video_info['lipPlayback'], video_id)
+            m3u8_url = traverse_obj(playback, (
+                'media', lambda _, v: v['protocol'] == 'HLS', 'path', {url_or_none}), get_all=False)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', live=True)
+
+        elif live_status == 'post_live':
+            if availability in ('premium_only', 'subscriber_only'):
+                self.report_drm(video_id)
+            self.raise_no_formats(
+                'Livestream has ended and downloadable VOD is not available', expected=True)
+
+        else:
+            infra_video_id = post['extension']['video']['infraVideoId']
+            in_key = self._call_api(
+                f'/video/v1.0/vod/{api_video_id}/inKey?preview=false', video_id,
+                data=b'{}', note='Downloading VOD API key')['inKey']
+
+            video_info = self._download_json(
+                f'https://global.apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{infra_video_id}',
+                video_id, note='Downloading VOD JSON', query={
+                    'key': in_key,
+                    'sid': traverse_obj(post, ('extension', 'video', 'serviceId')) or '2070',
+                    'pid': str(uuid.uuid4()),
+                    'nonce': int(time.time() * 1000),
+                    'devt': 'html5_pc',
+                    'prv': 'Y' if post.get('membershipOnly') else 'N',
+                    'aup': 'N',
+                    'stpb': 'N',
+                    'cpl': 'en',
+                    'env': 'prod',
+                    'lc': 'en',
+                    'adi': '[{"adSystem":"null"}]',
+                    'adu': '/',
+                })
+
+            formats = self._get_formats(video_info, video_id)
+            has_drm = traverse_obj(video_info, ('meta', 'provider', 'name', {str.lower})) == 'drm'
+            if has_drm and formats:
+                self.report_warning(
+                    'Requested content is DRM-protected, only a 30-second preview is available', video_id)
+            elif has_drm and not formats:
+                self.report_drm(video_id)
+
+        return {
+            'id': video_id,
+            'channel': channel,
+            'channel_url': f'https://weverse.io/{channel}',
+            'formats': formats,
+            'availability': availability,
+            'live_status': live_status,
+            **self._parse_post_meta(post),
+            **NaverBaseIE.process_subtitles(video_info, self._get_subs),
+        }
+
+
+class WeverseMediaIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/media/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/media/4-116372884',
+        'md5': '8efc9cfd61b2f25209eb1a5326314d28',
+        'info_dict': {
+            'id': 'e-C9wLSQs6o',
+            'ext': 'mp4',
+            'title': 'Billlie | \'EUNOIA\' Performance Video (heartbeat ver.)',
+            'description': 'md5:6181caaf2a2397bca913ffe368c104e5',
+            'channel': 'Billlie',
+            'channel_id': 'UCyc9sUCxELTDK9vELO5Fzeg',
+            'channel_url': 'https://www.youtube.com/channel/UCyc9sUCxELTDK9vELO5Fzeg',
+            'uploader': 'Billlie',
+            'uploader_id': '@Billlie',
+            'uploader_url': 'http://www.youtube.com/@Billlie',
+            'upload_date': '20230403',
+            'duration': 211,
+            'age_limit': 0,
+            'playable_in_embed': True,
+            'live_status': 'not_live',
+            'availability': 'public',
+            'view_count': int,
+            'comment_count': int,
+            'like_count': int,
+            'channel_follower_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/e-C9wLSQs6o/maxresdefault.jpg',
+            'categories': ['Entertainment'],
+            'tags': 'count:7',
+        },
+    }, {
+        'url': 'https://weverse.io/billlie/media/3-102914520',
+        'md5': '031551fcbd716bc4f080cb6174a43d8a',
+        'info_dict': {
+            'id': '3-102914520',
+            'ext': 'mp4',
+            'title': 'From. SUHYEON🌸',
+            'description': 'Billlie 멤버별 독점 영상 공개💙💜',
+            'uploader': 'Billlie_official',
+            'uploader_id': 'f569c6e92f7eaffef0a395037dcaa54f',
+            'channel': 'billlie',
+            'channel_id': '72',
+            'channel_url': 'https://weverse.io/billlie',
+            'creator': 'Billlie',
+            'timestamp': 1662174000,
+            'upload_date': '20220903',
+            'release_timestamp': 1662174000,
+            'release_date': '20220903',
+            'duration': 17.0,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'not_live',
+        },
+    }]
+
+    def _real_extract(self, url):
+        channel, video_id = self._match_valid_url(url).group('artist', 'id')
+        post = self._call_post_api(video_id)
+        media_type = traverse_obj(post, ('extension', 'mediaInfo', 'mediaType', {str.lower}))
+        youtube_id = traverse_obj(post, ('extension', 'youtube', 'youtubeVideoId', {str}))
+
+        if media_type == 'vod':
+            return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)
+        elif media_type == 'youtube' and youtube_id:
+            return self.url_result(youtube_id, YoutubeIE)
+        elif media_type == 'image':
+            self.raise_no_formats('No video content found in webpage', expected=True)
+        elif media_type:
+            raise ExtractorError(f'Unsupported media type "{media_type}"')
+
+        self.raise_no_formats('No video content found in webpage')
+
+
+class WeverseMomentIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<artist>[^/?#]+)/moment/(?P<uid>[\da-f]+)/post/(?P<id>[\d-]+)'
+    _TESTS = [{
+        'url': 'https://weverse.io/secretnumber/moment/66a07e164b56a696ee71c99315ffe27b/post/1-117229444',
+        'md5': '87733ac19a54081b7dfc2442036d282b',
+        'info_dict': {
+            'id': '1-117229444',
+            'ext': 'mp4',
+            'title': '今日もめっちゃいい天気☀️🌤️',
+            'uploader': '레아',
+            'uploader_id': '66a07e164b56a696ee71c99315ffe27b',
+            'channel': 'secretnumber',
+            'channel_id': '56',
+            'creator': 'SECRET NUMBER',
+            'duration': 10,
+            'upload_date': '20230405',
+            'timestamp': 1680653968,
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+        },
+        'skip': 'Moment has expired',
+    }]
+
+    def _real_extract(self, url):
+        channel, uploader_id, video_id = self._match_valid_url(url).group('artist', 'uid', 'id')
+        post = self._call_post_api(video_id)
+        api_video_id = post['extension']['moment']['video']['videoId']
+        video_info = self._call_api(
+            f'/cvideo/v1.0/cvideo-{api_video_id}/playInfo?videoId={api_video_id}', video_id,
+            note='Downloading moment JSON')['playInfo']
+
+        return {
+            'id': video_id,
+            'channel': channel,
+            'uploader_id': uploader_id,
+            'formats': self._get_formats(video_info, video_id),
+            'availability': self._extract_availability(post),
+            **traverse_obj(post, {
+                'title': ((('extension', 'moment', 'body'), 'body'), {str}),
+                'uploader': ('author', 'profileName', {str}),
+                'creator': (('community', 'author'), 'communityName', {str}),
+                'channel_id': (('community', 'author'), 'communityId', {str_or_none}),
+                'duration': ('extension', 'moment', 'video', 'uploadInfo', 'playTime', {float_or_none}),
+                'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}),
+                'thumbnail': ('extension', 'moment', 'video', 'uploadInfo', 'imageUrl', {url_or_none}),
+                'like_count': ('emotionCount', {int_or_none}),
+                'comment_count': ('commentCount', {int_or_none}),
+            }, get_all=False),
+            **NaverBaseIE.process_subtitles(video_info, self._get_subs),
+        }
+
+
+class WeverseTabBaseIE(WeverseBaseIE):
+    _ENDPOINT = None
+    _PATH = None
+    _QUERY = {}
+    _RESULT_IE = None
+
+    def _entries(self, channel_id, channel, first_page):
+        query = self._QUERY.copy()
+
+        for page in itertools.count(1):
+            posts = first_page if page == 1 else self._call_api(
+                update_url_query(self._ENDPOINT % channel_id, query), channel,
+                note=f'Downloading {self._PATH} tab page {page}')
+
+            for post in traverse_obj(posts, ('data', lambda _, v: v['postId'])):
+                yield self.url_result(
+                    f'https://weverse.io/{channel}/{self._PATH}/{post["postId"]}',
+                    self._RESULT_IE, post['postId'], **self._parse_post_meta(post),
+                    channel=channel, channel_url=f'https://weverse.io/{channel}',
+                    availability=self._extract_availability(post),
+                    live_status=self._extract_live_status(post))
+
+            query['after'] = traverse_obj(posts, ('paging', 'nextParams', 'after', {str}))
+            if not query['after']:
+                break
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+        channel_id = self._get_community_id(channel)
+
+        first_page = self._call_api(
+            update_url_query(self._ENDPOINT % channel_id, self._QUERY), channel,
+            note=f'Downloading {self._PATH} tab page 1')
+
+        return self.playlist_result(
+            self._entries(channel_id, channel, first_page), f'{channel}-{self._PATH}',
+            **traverse_obj(first_page, ('data', ..., {
+                'playlist_title': ('community', 'communityName', {str}),
+                'thumbnail': ('author', 'profileImageUrl', {url_or_none}),
+            }), get_all=False))
+
+
+class WeverseLiveTabIE(WeverseTabBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/live/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/live/',
+        'playlist_mincount': 55,
+        'info_dict': {
+            'id': 'billlie-live',
+            'title': 'Billlie',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+        },
+    }]
+
+    _ENDPOINT = '/post/v1.0/community-%s/liveTabPosts'
+    _PATH = 'live'
+    _QUERY = {'fieldSet': 'postsV1'}
+    _RESULT_IE = WeverseIE
+
+
+class WeverseMediaTabIE(WeverseTabBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/media(?:/|/all|/new)?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/billlie/media/',
+        'playlist_mincount': 231,
+        'info_dict': {
+            'id': 'billlie-media',
+            'title': 'Billlie',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
+        },
+    }, {
+        'url': 'https://weverse.io/lesserafim/media/all',
+        'only_matching': True,
+    }, {
+        'url': 'https://weverse.io/lesserafim/media/new',
+        'only_matching': True,
+    }]
+
+    _ENDPOINT = '/media/v1.0/community-%s/more'
+    _PATH = 'media'
+    _QUERY = {'fieldSet': 'postsV1', 'filterType': 'RECENT'}
+    _RESULT_IE = WeverseMediaIE
+
+
+class WeverseLiveIE(WeverseBaseIE):
+    _VALID_URL = r'https?://(?:www\.|m\.)?weverse.io/(?P<id>[^/?#]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://weverse.io/purplekiss',
+        'info_dict': {
+            'id': '3-116560493',
+            'ext': 'mp4',
+            'title': r're:모하냥🫶🏻',
+            'description': '내일은 금요일~><',
+            'uploader': '채인',
+            'uploader_id': '1ffb1d9d904d6b3db2783f876eb9229d',
+            'channel': 'purplekiss',
+            'channel_id': '35',
+            'channel_url': 'https://weverse.io/purplekiss',
+            'creator': 'PURPLE KISS',
+            'timestamp': 1680780892,
+            'upload_date': '20230406',
+            'release_timestamp': 1680780883,
+            'release_date': '20230406',
+            'thumbnail': 'https://weverse-live.pstatic.net/v1.0/live/62044/thumb',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'availability': 'needs_auth',
+            'live_status': 'is_live',
+        },
+        'skip': 'Livestream has ended',
+    }, {
+        'url': 'https://weverse.io/billlie/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+        channel_id = self._get_community_id(channel)
+
+        video_id = traverse_obj(
+            self._call_api(update_url_query(f'/post/v1.0/community-{channel_id}/liveTab', {
+                'debugMessage': 'true',
+                'fields': 'onAirLivePosts.fieldSet(postsV1).limit(10),reservedLivePosts.fieldSet(postsV1).limit(10)',
+            }), channel, note='Downloading live JSON'), (
+                ('onAirLivePosts', 'reservedLivePosts'), 'data',
+                lambda _, v: self._extract_live_status(v) in ('is_live', 'is_upcoming'), 'postId', {str}),
+            get_all=False)
+
+        if not video_id:
+            raise UserNotLive(video_id=channel)
+
+        return self.url_result(f'https://weverse.io/{channel}/live/{video_id}', WeverseIE)

yt_dlp/extractor/weyyak.py

@@ -0,0 +1,86 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_age_limit,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class WeyyakIE(InfoExtractor):
+    _VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/(?:player/)?(?P<type>episode|movie)/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'https://weyyak.com/en/player/episode/1341952/Ribat-Al-Hob-Episode49',
+            'md5': '0caf55c1a615531c8fe60f146ae46849',
+            'info_dict': {
+                'id': '1341952',
+                'ext': 'mp4',
+                'title': 'Ribat Al Hob',
+                'duration': 2771,
+                'alt_title': 'رباط الحب',
+                'season': 'Season 1',
+                'season_number': 1,
+                'episode': 'Episode 49',
+                'episode_number': 49,
+                'timestamp': 1485907200,
+                'upload_date': '20170201',
+                'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
+                'categories': ['Drama', 'Thrillers', 'Romance'],
+                'tags': 'count:8',
+            },
+        },
+        {
+            'url': 'https://weyyak.com/en/movie/233255/8-Seconds',
+            'md5': 'fe740ae0f63e4d1c8a7fc147a410c564',
+            'info_dict': {
+                'id': '233255',
+                'ext': 'mp4',
+                'title': '8 Seconds',
+                'duration': 6490,
+                'alt_title': '8 ثواني',
+                'description': 'md5:45b83a155c30b49950624c7e99600b9d',
+                'age_limit': 15,
+                'release_year': 2015,
+                'timestamp': 1683106031,
+                'upload_date': '20230503',
+                'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
+                'categories': ['Drama', 'Social'],
+                'cast': ['Ceylin Adiyaman', 'Esra Inal'],
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id, lang, type_ = self._match_valid_url(url).group('id', 'lang', 'type')
+
+        path = 'episode/' if type_ == 'episode' else 'contents/moviedetails?contentkey='
+        data = self._download_json(
+            f'https://msapifo-prod-me.weyyak.z5.com/v1/{lang}/{path}{video_id}', video_id)['data']
+        m3u8_url = self._download_json(
+            f'https://api-weyyak.akamaized.net/get_info/{data["video_id"]}',
+            video_id, 'Extracting video details')['url_video']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'alt_title': ('translated_title', {str}),
+                'description': ('synopsis', {str}),
+                'duration': ('length', {float_or_none}),
+                'age_limit': ('age_rating', {parse_age_limit}),
+                'season_number': ('season_number', {int_or_none}),
+                'episode_number': ('episode_number', {int_or_none}),
+                'thumbnail': ('imagery', 'thumbnail', {url_or_none}),
+                'categories': ('genres', ..., {str}),
+                'tags': ('tags', ..., {str}),
+                'cast': (('main_actor', 'main_actress'), {str}),
+                'timestamp': ('insertedAt', {unified_timestamp}),
+                'release_year': ('production_year', {int_or_none}),
+            }),
+        }

yt_dlp/extractor/wrestleuniverse.py

@@ -2,6 +2,7 @@ import base64
 import binascii
 import json
 import time
+import uuid
 
 from .common import InfoExtractor
 from ..dependencies import Cryptodome
@@ -12,30 +13,95 @@ from ..utils import (
     traverse_obj,
     try_call,
     url_or_none,
+    urlencode_postdata,
 )
 
 
 class WrestleUniverseBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'wrestleuniverse'
     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
     _API_PATH = None
-    _TOKEN = None
+    _REAL_TOKEN = None
     _TOKEN_EXPIRY = None
+    _REFRESH_TOKEN = None
+    _DEVICE_ID = None
+    _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
+    _LOGIN_HEADERS = {
+        'Accept': '*/*',
+        'Content-Type': 'application/json',
+        'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
+        'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
+        'Referer': 'https://www.wrestle-universe.com/',
+        'Origin': 'https://www.wrestle-universe.com',
+    }
 
-    def _get_token_cookie(self):
-        if not self._TOKEN or not self._TOKEN_EXPIRY:
-            self._TOKEN = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
-            if not self._TOKEN:
+    @property
+    def _TOKEN(self):
+        if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
+            token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
+            if not token and not self._REFRESH_TOKEN:
                 self.raise_login_required()
-            expiry = traverse_obj(jwt_decode_hs256(self._TOKEN), ('exp', {int_or_none}))
-            if not expiry:
-                raise ExtractorError('There was a problem with the token cookie')
-            self._TOKEN_EXPIRY = expiry
+            self._REAL_TOKEN = token
 
-        if self._TOKEN_EXPIRY <= int(time.time()):
-            raise ExtractorError(
-                'Expired token. Refresh your cookies in browser and try again', expected=True)
+        if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
+            if not self._REFRESH_TOKEN:
+                raise ExtractorError(
+                    'Expired token. Refresh your cookies in browser and try again', expected=True)
+            self._refresh_token()
 
-        return self._TOKEN
+        return self._REAL_TOKEN
+
+    @_TOKEN.setter
+    def _TOKEN(self, value):
+        self._REAL_TOKEN = value
+
+        expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
+        if not expiry:
+            raise ExtractorError('There was a problem with the auth token')
+        self._TOKEN_EXPIRY = expiry
+
+    def _perform_login(self, username, password):
+        login = self._download_json(
+            'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
+            'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
+                'returnSecureToken': True,
+                'email': username,
+                'password': password,
+            }, separators=(',', ':')).encode())
+        self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
+        if not self._REFRESH_TOKEN:
+            self.report_warning('No refresh token was granted')
+        self._TOKEN = traverse_obj(login, ('idToken', {str}))
+
+    def _real_initialize(self):
+        if WrestleUniverseBaseIE._DEVICE_ID:
+            return
+
+        WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
+        if not WrestleUniverseBaseIE._DEVICE_ID:
+            WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
+            if WrestleUniverseBaseIE._DEVICE_ID:
+                return
+            WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
+
+        self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
+
+    def _refresh_token(self):
+        refresh = self._download_json(
+            'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
+            query=self._LOGIN_QUERY, data=urlencode_postdata({
+                'grant_type': 'refresh_token',
+                'refresh_token': self._REFRESH_TOKEN,
+            }), headers={
+                **self._LOGIN_HEADERS,
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+        if traverse_obj(refresh, ('refresh_token', {str})):
+            self._REFRESH_TOKEN = refresh['refresh_token']
+        token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
+        if not token:
+            raise ExtractorError('No auth token returned from refresh request')
+        self._TOKEN = token
 
     def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
         headers = {'CA-CID': ''}
@@ -43,7 +109,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
             headers['Content-Type'] = 'application/json;charset=utf-8'
             data = json.dumps(data, separators=(',', ':')).encode()
         if auth:
-            headers['Authorization'] = f'Bearer {self._get_token_cookie()}'
+            headers['Authorization'] = f'Bearer {self._TOKEN}'
         return self._download_json(
             f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
@@ -65,7 +131,7 @@ class WrestleUniverseBaseIE(InfoExtractor):
 
         token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
         api_json = self._call_api(video_id, param, msg, data={
-            # 'deviceId' (random uuid4 generated at login) is not required yet
+            'deviceId': self._DEVICE_ID,
             'token': token,
             **data,
         }, query=query, fatal=fatal)
@@ -105,7 +171,7 @@ class WrestleUniverseVODIE(WrestleUniverseBaseIE):
             'upload_date': '20230129',
             'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
             'chapters': 'count:7',
-            'cast': 'count:18',
+            'cast': 'count:21',
         },
         'params': {
             'skip_download': 'm3u8',
@@ -169,6 +235,7 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
         'params': {
             'skip_download': 'm3u8',
         },
+        'skip': 'No longer available',
     }, {
         'note': 'unencrypted HLS',
         'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
@@ -196,14 +263,17 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
         lang, video_id = self._match_valid_url(url).group('lang', 'id')
         metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
 
-        info = traverse_obj(metadata, {
-            'title': ('displayName', {str}),
-            'description': ('description', {str}),
-            'channel': ('labels', 'group', {str}),
-            'location': ('labels', 'venue', {str}),
-            'timestamp': ('startTime', {int_or_none}),
-            'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
-        })
+        info = {
+            'id': video_id,
+            **traverse_obj(metadata, {
+                'title': ('displayName', {str}),
+                'description': ('description', {str}),
+                'channel': ('labels', 'group', {str}),
+                'location': ('labels', 'venue', {str}),
+                'timestamp': ('startTime', {int_or_none}),
+                'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
+            }),
+        }
 
         ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
         if info.get('timestamp') and ended_time:
@@ -211,23 +281,20 @@ class WrestleUniversePPVIE(WrestleUniverseBaseIE):
 
         video_data, decrypt = self._call_encrypted_api(
             video_id, ':watchArchive', 'watch archive', data={'method': 1})
-        formats = self._get_formats(video_data, (
+        info['formats'] = self._get_formats(video_data, (
             ('hls', None), ('urls', 'chromecastUrls'), ..., {url_or_none}), video_id)
-        for f in formats:
+        for f in info['formats']:
             # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
             if f.get('tbr'):
                 f['tbr'] = int(f['tbr'] / 2.5)
 
         hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
-        if not hls_aes_key and traverse_obj(video_data, ('hls', 'encryptType', {int}), default=0) > 0:
-            self.report_warning('HLS AES-128 key was not found in API response')
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'hls_aes': {
+        if hls_aes_key:
+            info['hls_aes'] = {
                 'key': hls_aes_key,
                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
             },
-            **info,
-        }
+        elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
+            self.report_warning('HLS AES-128 key was not found in API response')
+
+        return info

yt_dlp/extractor/wykop.py

@@ -0,0 +1,268 @@
+import json
+import urllib.error
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    format_field,
+    parse_iso8601,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class WykopBaseExtractor(InfoExtractor):
+    def _get_token(self, force_refresh=False):
+        if not force_refresh:
+            maybe_cached = self.cache.load('wykop', 'bearer')
+            if maybe_cached:
+                return maybe_cached
+
+        new_token = traverse_obj(
+            self._do_call_api('auth', None, 'Downloading anonymous auth token', data={
+                # hardcoded in frontend
+                'key': 'w53947240748',
+                'secret': 'd537d9e0a7adc1510842059ae5316419',
+            }), ('data', 'token'))
+
+        self.cache.store('wykop', 'bearer', new_token)
+        return new_token
+
+    def _do_call_api(self, path, video_id, note='Downloading JSON metadata', data=None, headers={}):
+        if data:
+            data = json.dumps({'data': data}).encode()
+            headers['Content-Type'] = 'application/json'
+
+        return self._download_json(
+            f'https://wykop.pl/api/v3/{path}', video_id,
+            note=note, data=data, headers=headers)
+
+    def _call_api(self, path, video_id, note='Downloading JSON metadata'):
+        token = self._get_token()
+        for retrying in range(2):
+            try:
+                return self._do_call_api(path, video_id, note, headers={'Authorization': f'Bearer {token}'})
+            except ExtractorError as e:
+                if not retrying and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    token = self._get_token(True)
+                    continue
+                raise
+
+    def _common_data_extract(self, data):
+        author = traverse_obj(data, ('author', 'username'), expected_type=str)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': data.get('slug'),
+            'url': traverse_obj(data,
+                                ('media', 'embed', 'url'),  # what gets an iframe embed
+                                ('source', 'url'),  # clickable url (dig only)
+                                expected_type=url_or_none),
+            'thumbnail': traverse_obj(
+                data, ('media', 'photo', 'url'), ('media', 'embed', 'thumbnail'), expected_type=url_or_none),
+            'uploader': author,
+            'uploader_id': author,
+            'uploader_url': format_field(author, None, 'https://wykop.pl/ludzie/%s'),
+            'timestamp': parse_iso8601(data.get('created_at'), delimiter=' '),  # time it got submitted
+            'like_count': traverse_obj(data, ('votes', 'up'), expected_type=int),
+            'dislike_count': traverse_obj(data, ('votes', 'down'), expected_type=int),
+            'comment_count': traverse_obj(data, ('comments', 'count'), expected_type=int),
+            'age_limit': 18 if data.get('adult') else 0,
+            'tags': data.get('tags'),
+        }
+
+
+class WykopDigIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:dig'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/link/6912923/najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
+        'info_dict': {
+            'id': 'rlSTBvViflc',
+            'ext': 'mp4',
+            'title': 'Najbardziej zrzędliwy kot na świecie I Frozen Planet II I BBC Earth',
+            'display_id': 'najbardziej-zrzedliwy-kot-na-swiecie-i-frozen-planet-ii-i-bbc-earth',
+            'description': 'md5:ac0f87dea1cdcb6b0c53f3612a095c87',
+            'tags': ['zwierzaczki', 'koty', 'smiesznykotek', 'humor', 'rozrywka', 'ciekawostki'],
+            'age_limit': 0,
+            'timestamp': 1669154480,
+            'release_timestamp': 1669194241,
+            'release_date': '20221123',
+            'uploader': 'starnak',
+            'uploader_id': 'starnak',
+            'uploader_url': 'https://wykop.pl/ludzie/starnak',
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'view_count': int,
+            'channel': 'BBC Earth',
+            'channel_id': 'UCwmZiChSryoWQCZMIQezgTg',
+            'channel_url': 'https://www.youtube.com/channel/UCwmZiChSryoWQCZMIQezgTg',
+            'categories': ['Pets & Animals'],
+            'upload_date': '20220923',
+            'duration': 191,
+            'channel_follower_count': int,
+            'availability': 'public',
+            'live_status': 'not_live',
+            'playable_in_embed': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return cls._match_valid_url(url) and not WykopDigCommentIE.suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._call_api(f'links/{video_id}', video_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': video_id,
+            'title': data['title'],
+            'description': data.get('description'),
+            # time it got "digged" to the homepage
+            'release_timestamp': parse_iso8601(data.get('published_at'), delimiter=' '),
+        }
+
+
+class WykopDigCommentIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:dig:comment'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/link/(?P<dig_id>\d+)/[^/]+/komentarz/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/link/6992589/strollowal-oszusta-przez-ponad-24-minuty-udawal-naiwniaka-i-nagral-rozmowe/komentarz/114540527/podobna-sytuacja-ponizej-ciekawa-dyskusja-z-oszustem-na-sam-koniec-sam-bylem-w-biurze-swiadkiem-podobnej-rozmowy-niemal-zakonczonej-sukcesem-bandyty-g',
+        'info_dict': {
+            'id': 'u6tEi2FmKZY',
+            'ext': 'mp4',
+            'title': 'md5:e7c741c5baa7ed6478000caf72865577',
+            'display_id': 'md5:45b2d12bd0e262d09cc7cf7abc8412db',
+            'description': 'md5:bcec7983429f9c0630f9deb9d3d1ba5e',
+            'timestamp': 1674476945,
+            'uploader': 'Bartholomew',
+            'uploader_id': 'Bartholomew',
+            'uploader_url': 'https://wykop.pl/ludzie/Bartholomew',
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'tags': [],
+            'availability': 'public',
+            'duration': 1838,
+            'upload_date': '20230117',
+            'categories': ['Entertainment'],
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'channel_follower_count': int,
+            'playable_in_embed': True,
+            'live_status': 'not_live',
+            'age_limit': 0,
+            'chapters': 'count:3',
+            'channel': 'Poszukiwacze Okazji',
+            'channel_id': 'UCzzvJDZThwv06dR4xmzrZBw',
+            'channel_url': 'https://www.youtube.com/channel/UCzzvJDZThwv06dR4xmzrZBw',
+        },
+    }]
+
+    def _real_extract(self, url):
+        dig_id, comment_id = self._search_regex(self._VALID_URL, url, 'dig and comment ids', group=('dig_id', 'id'))
+        data = self._call_api(f'links/{dig_id}/comments/{comment_id}', comment_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': comment_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }
+
+
+class WykopPostIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:post'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/wpis/68893343/kot-koty-smiesznykotek',
+        'info_dict': {
+            'id': 'PL8JMjiUPHUhwc9ZlKa_5IFeBwBV8Xe7jI',
+            'title': 'PawelW124 - #kot #koty #smiesznykotek',
+            'description': '#kot #koty #smiesznykotek',
+            'display_id': 'kot-koty-smiesznykotek',
+            'tags': ['kot', 'koty', 'smiesznykotek'],
+            'uploader': 'PawelW124',
+            'uploader_id': 'PawelW124',
+            'uploader_url': 'https://wykop.pl/ludzie/PawelW124',
+            'timestamp': 1668938142,
+            'age_limit': 0,
+            'like_count': int,
+            'dislike_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+            'comment_count': int,
+            'channel': 'Revan',
+            'channel_id': 'UCW9T_-uZoiI7ROARQdTDyOw',
+            'channel_url': 'https://www.youtube.com/channel/UCW9T_-uZoiI7ROARQdTDyOw',
+            'upload_date': '20221120',
+            'modified_date': '20220814',
+            'availability': 'public',
+            'view_count': int,
+        },
+        'playlist_mincount': 15,
+        'params': {
+            'flat_playlist': True,
+        }
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return cls._match_valid_url(url) and not WykopPostCommentIE.suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._call_api(f'entries/{video_id}', video_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': video_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }
+
+
+class WykopPostCommentIE(WykopBaseExtractor):
+    IE_NAME = 'wykop:post:comment'
+    _VALID_URL = r'https?://(?:www\.)?wykop\.pl/wpis/(?P<post_id>\d+)/[^/#]+#(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://wykop.pl/wpis/70084873/test-test-test#249303979',
+        'info_dict': {
+            'id': 'confusedquickarmyant',
+            'ext': 'mp4',
+            'title': 'tpap - treść komentarza',
+            'display_id': 'tresc-komentarza',
+            'description': 'treść komentarza',
+            'uploader': 'tpap',
+            'uploader_id': 'tpap',
+            'uploader_url': 'https://wykop.pl/ludzie/tpap',
+            'timestamp': 1675349470,
+            'upload_date': '20230202',
+            'tags': [],
+            'duration': 2.12,
+            'age_limit': 0,
+            'categories': [],
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'thumbnail': r're:https?://wykop\.pl/cdn/.+',
+        },
+    }]
+
+    def _real_extract(self, url):
+        post_id, comment_id = self._search_regex(self._VALID_URL, url, 'post and comment ids', group=('post_id', 'id'))
+        data = self._call_api(f'entries/{post_id}/comments/{comment_id}', comment_id)['data']
+
+        return {
+            **self._common_data_extract(data),
+            'id': comment_id,
+            'title': f"{traverse_obj(data, ('author', 'username'))} - {data.get('content') or ''}",
+            'description': data.get('content'),
+        }

yt_dlp/extractor/youtube.py

@@ -66,7 +66,6 @@ from ..utils import (
     variadic,
 )
 
-
 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
 # any clients starting with _ cannot be explicitly requested by the user
 INNERTUBE_CLIENTS = {
@@ -894,9 +893,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def extract_relative_time(relative_time_text):
         """
         Extracts a relative time from string and converts to dt object
-        e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
+        e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
         """
-        mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
+
+        # XXX: this could be moved to a general function in utils.py
+        # The relative time text strings are roughly the same as what
+        # Javascript's Intl.RelativeTimeFormat function generates.
+        # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
+        mobj = re.search(
+            r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
+            relative_time_text)
         if mobj:
             start = mobj.group('start')
             if start:
@@ -1039,6 +1045,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                       else self._get_count({'simpleText': view_count_text}))
         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
 
+        channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
+                   or self._get_text(reel_header_renderer, 'channelTitleText'))
+
+        channel_handle = traverse_obj(renderer, (
+            'shortBylineText', 'runs', ..., 'navigationEndpoint',
+            (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
+            expected_type=self.handle_from_url, get_all=False)
         return {
             '_type': 'url',
             'ie_key': YoutubeIE.ie_key(),
@@ -1048,9 +1061,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'description': description,
             'duration': duration,
             'channel_id': channel_id,
-            'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
-                        or self._get_text(reel_header_renderer, 'channelTitleText')),
+            'channel': channel,
             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
+            'uploader': channel,
+            'uploader_id': channel_handle,
+            'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
             'timestamp': (self._parse_time_text(time_text)
                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
@@ -1274,6 +1289,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
                 'uploader_id': '@PhilippHagemeister',
+                'heatmap': 'count:100',
             }
         },
         {
@@ -1427,6 +1443,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'FlyingKitty',
                 'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
                 'uploader_id': '@FlyingKitty900',
+                'comment_count': int,
             },
         },
         {
@@ -3023,17 +3040,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
-             r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
@@ -3277,42 +3291,66 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                           chapter_time, chapter_title, duration)
             for contents in content_list)), [])
 
+    def _extract_heatmap_from_player_overlay(self, data):
+        content_list = traverse_obj(data, (
+            'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer', 'decoratedPlayerBarRenderer', 'playerBar',
+            'multiMarkersPlayerBarRenderer', 'markersMap', ..., 'value', 'heatmap', 'heatmapRenderer', 'heatMarkers', {list}))
+        return next(filter(None, (
+            traverse_obj(contents, (..., 'heatMarkerRenderer', {
+                'start_time': ('timeRangeStartMillis', {functools.partial(float_or_none, scale=1000)}),
+                'end_time': {lambda x: (x['timeRangeStartMillis'] + x['markerDurationMillis']) / 1000},
+                'value': ('heatMarkerIntensityScoreNormalized', {float_or_none}),
+            })) for contents in content_list)), None)
+
     def _extract_comment(self, comment_renderer, parent=None):
         comment_id = comment_renderer.get('commentId')
         if not comment_id:
             return
 
-        text = self._get_text(comment_renderer, 'contentText')
+        info = {
+            'id': comment_id,
+            'text': self._get_text(comment_renderer, 'contentText'),
+            'like_count': self._get_count(comment_renderer, 'voteCount'),
+            'author_id': traverse_obj(comment_renderer, ('authorEndpoint', 'browseEndpoint', 'browseId', {self.ucid_or_none})),
+            'author': self._get_text(comment_renderer, 'authorText'),
+            'author_thumbnail': traverse_obj(comment_renderer, ('authorThumbnail', 'thumbnails', -1, 'url', {url_or_none})),
+            'parent': parent or 'root',
+        }
 
         # Timestamp is an estimate calculated from the current time and time_text
         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
         timestamp = self._parse_time_text(time_text)
 
-        author = self._get_text(comment_renderer, 'authorText')
-        author_id = try_get(comment_renderer,
-                            lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
-
-        votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
-                                                       lambda x: x['likeCount']), str)) or 0
-        author_thumbnail = try_get(comment_renderer,
-                                   lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
-
-        author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
-        is_favorited = 'creatorHeart' in (try_get(
-            comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
-        return {
-            'id': comment_id,
-            'text': text,
+        info.update({
+            # FIXME: non-standard, but we need a way of showing that it is an estimate.
+            '_time_text': time_text,
             'timestamp': timestamp,
-            'time_text': time_text,
-            'like_count': votes,
-            'is_favorited': is_favorited,
-            'author': author,
-            'author_id': author_id,
-            'author_thumbnail': author_thumbnail,
-            'author_is_uploader': author_is_uploader,
-            'parent': parent or 'root'
-        }
+        })
+
+        info['author_url'] = urljoin(
+            'https://www.youtube.com', traverse_obj(comment_renderer, ('authorEndpoint', (
+                ('browseEndpoint', 'canonicalBaseUrl'), ('commandMetadata', 'webCommandMetadata', 'url'))),
+                expected_type=str, get_all=False))
+
+        author_is_uploader = traverse_obj(comment_renderer, 'authorIsChannelOwner')
+        if author_is_uploader is not None:
+            info['author_is_uploader'] = author_is_uploader
+
+        comment_abr = traverse_obj(
+            comment_renderer, ('actionsButtons', 'commentActionButtonsRenderer'), expected_type=dict)
+        if comment_abr is not None:
+            info['is_favorited'] = 'creatorHeart' in comment_abr
+
+        comment_ab_icontype = traverse_obj(
+            comment_renderer, ('authorCommentBadge', 'authorCommentBadgeRenderer', 'icon', 'iconType'))
+        if comment_ab_icontype is not None:
+            info['author_is_verified'] = comment_ab_icontype in ('CHECK_CIRCLE_THICK', 'OFFICIAL_ARTIST_BADGE')
+
+        is_pinned = traverse_obj(comment_renderer, 'pinnedCommentBadge')
+        if is_pinned:
+            info['is_pinned'] = True
+
+        return info
 
     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
 
@@ -3325,7 +3363,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 expected_comment_count = self._get_count(
                     comments_header_renderer, 'countText', 'commentsCount')
 
-                if expected_comment_count:
+                if expected_comment_count is not None:
                     tracker['est_total'] = expected_comment_count
                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
@@ -3360,14 +3398,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 comment = self._extract_comment(comment_renderer, parent)
                 if not comment:
                     continue
-                is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
                 comment_id = comment['id']
-                if is_pinned:
+                if comment.get('is_pinned'):
                     tracker['pinned_comment_ids'].add(comment_id)
                 # Sometimes YouTube may break and give us infinite looping comments.
                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
                 if comment_id in tracker['seen_comment_ids']:
-                    if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
+                    if comment_id in tracker['pinned_comment_ids'] and not comment.get('is_pinned'):
                         # Pinned comments may appear a second time in newest first sort
                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
                         continue
@@ -3396,7 +3433,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if not tracker:
             tracker = dict(
                 running_total=0,
-                est_total=0,
+                est_total=None,
                 current_page_thread=0,
                 total_parent_comments=0,
                 total_reply_comments=0,
@@ -3429,11 +3466,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
             is_forced_continuation = True
 
+        continuation_items_path = (
+            'onResponseReceivedEndpoints', ..., ('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems')
         for page_num in itertools.count(0):
             if not continuation:
                 break
             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
-            comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
+            comment_prog_str = f"({tracker['running_total']}/~{tracker['est_total']})"
             if page_num == 0:
                 if is_first_continuation:
                     note_prefix = 'Downloading comment section API JSON'
@@ -3444,11 +3483,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
                     '       ' if parent else '', ' replies' if parent else '',
                     page_num, comment_prog_str)
+
+            # Do a deep check for incomplete data as sometimes YouTube may return no comments for a continuation
+            # Ignore check if YouTube says the comment count is 0.
+            check_get_keys = None
+            if not is_forced_continuation and not (tracker['est_total'] == 0 and tracker['running_total'] == 0):
+                check_get_keys = [[*continuation_items_path, ..., (
+                    'commentsHeaderRenderer' if is_first_continuation else ('commentThreadRenderer', 'commentRenderer'))]]
             try:
                 response = self._extract_response(
                     item_id=None, query=continuation,
                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
-                    check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
+                    check_get_keys=check_get_keys)
             except ExtractorError as e:
                 # Ignore incomplete data error for replies if retries didn't work.
                 # This is to allow any other parent comments and comment threads to be downloaded.
@@ -3460,15 +3506,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 else:
                     raise
             is_forced_continuation = False
-            continuation_contents = traverse_obj(
-                response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
-
             continuation = None
-            for continuation_section in continuation_contents:
-                continuation_items = traverse_obj(
-                    continuation_section,
-                    (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
-                    get_all=False, expected_type=list) or []
+            for continuation_items in traverse_obj(response, continuation_items_path, expected_type=list, default=[]):
                 if is_first_continuation:
                     continuation = extract_header(continuation_items)
                     is_first_continuation = False
@@ -4349,6 +4388,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 or self._extract_chapters_from_description(video_description, duration)
                 or None)
 
+            info['heatmap'] = self._extract_heatmap_from_player_overlay(initial_data)
+
         contents = traverse_obj(
             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
             expected_type=list, default=[])
@@ -4611,8 +4652,11 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
     def _music_reponsive_list_entry(self, renderer):
         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
         if video_id:
+            title = traverse_obj(renderer, (
+                'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
+                'text', 'runs', 0, 'text'))
             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
-                                   ie=YoutubeIE.ie_key(), video_id=video_id)
+                                   ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
         if playlist_id:
             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
@@ -4671,11 +4715,19 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
 
     def _rich_entries(self, rich_grid_renderer):
         renderer = traverse_obj(
-            rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
+            rich_grid_renderer,
+            ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer')), get_all=False) or {}
         video_id = renderer.get('videoId')
-        if not video_id:
+        if video_id:
+            yield self._extract_video(renderer)
+            return
+        playlist_id = renderer.get('playlistId')
+        if playlist_id:
+            yield self.url_result(
+                f'https://www.youtube.com/playlist?list={playlist_id}',
+                ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+                video_title=self._get_text(renderer, 'title'))
             return
-        yield self._extract_video(renderer)
 
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
@@ -4904,7 +4956,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
         if metadata_renderer:
             channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
-                                                         ('channelUrl', {self.ucid_from_url}))
+                                      ('channelUrl', {self.ucid_from_url}))
             info.update({
                 'channel': metadata_renderer.get('title'),
                 'channel_id': channel_id,
@@ -5861,7 +5913,25 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader_id': '@colethedj1894',
             'uploader': 'colethedj',
         },
+        'playlist': [{
+            'info_dict': {
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'id': 'BaW_jenozKc',
+                '_type': 'url',
+                'ie_key': 'Youtube',
+                'duration': 10,
+                'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
+                'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
+                'view_count': int,
+                'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
+                'channel': 'Philipp Hagemeister',
+                'uploader_id': '@PhilippHagemeister',
+                'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
+                'uploader': 'Philipp Hagemeister',
+            }
+        }],
         'playlist_count': 1,
+        'params': {'extract_flat': True},
     }, {
         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
         'url': 'https://www.youtube.com/feed/recommended',
@@ -6162,6 +6232,9 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
                 'channel_url': str,
                 'concurrent_view_count': int,
                 'channel': str,
+                'uploader': str,
+                'uploader_url': str,
+                'uploader_id': str
             }
         }],
         'params': {'extract_flat': True, 'playlist_items': '1'},
@@ -6217,6 +6290,40 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader': '3Blue1Brown',
         },
         'playlist_count': 0,
+    }, {
+        # Podcasts tab, with rich entry playlistRenderers
+        'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
+        'info_dict': {
+            'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
+            'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
+            'title': '99 Percent Invisible - Podcasts',
+            'uploader': '99 Percent Invisible',
+            'channel_follower_count': int,
+            'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+            'tags': [],
+            'channel': '99 Percent Invisible',
+            'uploader_id': '@99percentinvisiblepodcast',
+        },
+        'playlist_count': 1,
+    }, {
+        # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
+        'url': 'https://www.youtube.com/@AHimitsu/releases',
+        'info_dict': {
+            'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+            'channel': 'A Himitsu',
+            'uploader_url': 'https://www.youtube.com/@AHimitsu',
+            'title': 'A Himitsu - Releases',
+            'uploader_id': '@AHimitsu',
+            'uploader': 'A Himitsu',
+            'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+            'tags': 'count:16',
+            'description': 'I make music',
+            'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
+            'channel_follower_count': int,
+        },
+        'playlist_mincount': 10,
     }]
 
     @classmethod

yt_dlp/extractor/zingmp3.py

@@ -1,16 +1,11 @@
-import functools
 import hashlib
 import hmac
+import itertools
 import json
 import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import (
-    OnDemandPagedList,
-    int_or_none,
-    traverse_obj,
-    urljoin,
-)
+from ..utils import int_or_none, traverse_obj, try_call, urljoin
 
 
 class ZingMp3BaseIE(InfoExtractor):
@@ -37,6 +32,7 @@ class ZingMp3BaseIE(InfoExtractor):
         'info-artist': '/api/v2/page/get/artist',
         'user-list-song': '/api/v2/song/get/list',
         'user-list-video': '/api/v2/video/get/list',
+        'hub': '/api/v2/page/get/hub-detail',
     }
 
     def _api_url(self, url_type, params):
@@ -46,9 +42,9 @@ class ZingMp3BaseIE(InfoExtractor):
             ''.join(f'{k}={v}' for k, v in sorted(params.items())).encode()).hexdigest()
         data = {
             **params,
-            'apiKey': '88265e23d4284f25963e6eedac8fbfa3',
-            'sig': hmac.new(
-                b'2aa2d1c561e809b267f3638c4a307aab', f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
+            'apiKey': 'X5BM3w8N7MKozC0B85o4KMlzLZKhV00y',
+            'sig': hmac.new(b'acOrvUS15XRW2o9JksiK1KgQ6Vbds8ZW',
+                            f'{api_slug}{sha256}'.encode(), hashlib.sha512).hexdigest(),
         }
         return f'{self._DOMAIN}{api_slug}?{urllib.parse.urlencode(data)}'
 
@@ -67,6 +63,19 @@ class ZingMp3BaseIE(InfoExtractor):
         for url in traverse_obj(items, (..., 'link')) or []:
             yield self.url_result(urljoin(self._DOMAIN, url))
 
+    def _fetch_page(self, id_, url_type, page):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _paged_list(self, _id, url_type):
+        count = 0
+        for page in itertools.count(1):
+            data = self._fetch_page(_id, url_type, page)
+            entries = list(self._parse_items(data.get('items')))
+            count += len(entries)
+            yield from entries
+            if not data.get('hasMore') or try_call(lambda: count > data['total']):
+                break
+
 
 class ZingMp3IE(ZingMp3BaseIE):
     _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip|embed'
@@ -166,8 +175,11 @@ class ZingMp3IE(ZingMp3BaseIE):
                     'height': int_or_none(res),
                 })
 
-        if not formats and item.get('msg') == 'Sorry, this content is not available in your country.':
-            self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+        if not formats:
+            if item.get('msg') == 'Sorry, this content is not available in your country.':
+                self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+            else:
+                self.raise_no_formats('The song is only for VIP accounts.')
 
         lyric = item.get('lyric') or self._call_api('lyric', {'id': item_id}, fatal=False).get('file')
 
@@ -200,7 +212,7 @@ class ZingMp3AlbumIE(ZingMp3BaseIE):
             'id': 'ZWZAEZZD',
             'title': 'Những Bài Hát Hay Nhất Của Mr. Siro',
         },
-        'playlist_mincount': 49,
+        'playlist_mincount': 20,
     }, {
         'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
@@ -305,22 +317,20 @@ class ZingMp3ChartMusicVideoIE(ZingMp3BaseIE):
             'id': 'IWZ9Z086',
             'title': 'the-loai-video_Khong-Loi',
         },
-        'playlist_mincount': 10,
+        'playlist_mincount': 1,
     }]
 
     def _fetch_page(self, song_id, url_type, page):
-        return self._parse_items(self._call_api(url_type, {
+        return self._call_api(url_type, {
             'id': song_id,
             'type': 'genre',
-            'page': page + 1,
+            'page': page,
             'count': self._PER_PAGE
-        }).get('items'))
+        })
 
     def _real_extract(self, url):
         song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
-        return self.playlist_result(
-            OnDemandPagedList(functools.partial(self._fetch_page, song_id, url_type), self._PER_PAGE),
-            song_id, f'{url_type}_{regions}')
+        return self.playlist_result(self._paged_list(song_id, url_type), song_id, f'{url_type}_{regions}')
 
 
 class ZingMp3UserIE(ZingMp3BaseIE):
@@ -331,7 +341,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - bai-hat',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 91,
     }, {
@@ -339,7 +349,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - album',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 3,
     }, {
@@ -347,7 +357,7 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - single',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 20,
     }, {
@@ -355,19 +365,19 @@ class ZingMp3UserIE(ZingMp3BaseIE):
         'info_dict': {
             'id': 'IWZ98609',
             'title': 'Mr. Siro - video',
-            'description': 'md5:85ab29bd7b21725c12bf76fd1d6922e5',
+            'description': 'md5:5bdcf45e955dc1b8d7f518f322ffef36',
         },
         'playlist_mincount': 15,
     }]
 
     def _fetch_page(self, user_id, url_type, page):
         url_type = 'user-list-song' if url_type == 'bai-hat' else 'user-list-video'
-        return self._parse_items(self._call_api(url_type, {
+        return self._call_api(url_type, {
             'id': user_id,
             'type': 'artist',
-            'page': page + 1,
+            'page': page,
             'count': self._PER_PAGE
-        }, query={'sort': 'new', 'sectionId': 'aSong'}).get('items'))
+        })
 
     def _real_extract(self, url):
         user_alias, url_type = self._match_valid_url(url).group('user', 'type')
@@ -376,10 +386,41 @@ class ZingMp3UserIE(ZingMp3BaseIE):
 
         user_info = self._call_api('info-artist', {}, user_alias, query={'alias': user_alias})
         if url_type in ('bai-hat', 'video'):
-            entries = OnDemandPagedList(
-                functools.partial(self._fetch_page, user_info['id'], url_type), self._PER_PAGE)
+            entries = self._paged_list(user_info['id'], url_type)
         else:
             entries = self._parse_items(traverse_obj(user_info, (
-                'sections', lambda _, v: v['link'] == f'/{user_alias}/{url_type}', 'items', ...)))
+                'sections',
+                lambda _, v: v['sectionId'] == 'aAlbum' if url_type == 'album' else v['sectionId'] == 'aSingle',
+                'items', ...)))
         return self.playlist_result(
             entries, user_info['id'], f'{user_info.get("name")} - {url_type}', user_info.get('biography'))
+
+
+class ZingMp3HubIE(ZingMp3BaseIE):
+    IE_NAME = 'zingmp3:hub'
+    _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?P<type>hub)/(?P<regions>[^/]+)/(?P<id>[^\.]+)'
+    _TESTS = [{
+        'url': 'https://zingmp3.vn/hub/Nhac-Moi/IWZ9Z0CA.html',
+        'info_dict': {
+            'id': 'IWZ9Z0CA',
+            'title': 'Nhạc Mới',
+            'description': 'md5:1cc31b68a6f746427b07b2756c22a558',
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'https://zingmp3.vn/hub/Nhac-Viet/IWZ9Z087.html',
+        'info_dict': {
+            'id': 'IWZ9Z087',
+            'title': 'Nhạc Việt',
+            'description': 'md5:acc976c8bdde64d5c6ee4a92c39f7a77',
+        },
+        'playlist_mincount': 30,
+    }]
+
+    def _real_extract(self, url):
+        song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type')
+        hub_detail = self._call_api(url_type, {'id': song_id})
+        entries = self._parse_items(traverse_obj(hub_detail, (
+            'sections', lambda _, v: v['sectionId'] == 'hub', 'items', ...)))
+        return self.playlist_result(
+            entries, song_id, hub_detail.get('title'), hub_detail.get('description'))

yt_dlp/jsinterp.py

@@ -20,7 +20,12 @@ from .utils import (
 
 def _js_bit_op(op):
     def zeroise(x):
-        return 0 if x in (None, JS_Undefined) else x
+        if x in (None, JS_Undefined):
+            return 0
+        with contextlib.suppress(TypeError):
+            if math.isnan(x):  # NB: NaN cannot be checked by membership
+                return 0
+        return x
 
     def wrapped(a, b):
         return op(zeroise(a), zeroise(b)) & 0xffffffff
@@ -243,7 +248,7 @@ class JSInterpreter:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
-        in_quote, escaping, after_op, in_regex_char_group, in_unary_op = None, False, True, False, False
+        in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
         for idx, char in enumerate(expr):
             if not in_quote and char in _MATCHING_PARENS:
                 counters[_MATCHING_PARENS[char]] += 1
@@ -347,8 +352,10 @@ class JSInterpreter:
             inner, outer = self._separate(expr, expr[0], 1)
             if expr[0] == '/':
                 flags, outer = self._regex_flags(outer)
+                # We don't support regex methods yet, so no point compiling it
+                inner = f'{inner}/{flags}'
                 # Avoid https://github.com/python/cpython/issues/74534
-                inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
+                # inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
             else:
                 inner = json.loads(js_to_json(f'{inner}{expr[0]}', strict=True))
             if not outer:
@@ -438,7 +445,7 @@ class JSInterpreter:
                 err = e
 
             pending = (None, False)
-            m = re.match(r'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{'.format(**globals()), expr)
+            m = re.match(fr'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{', expr)
             if m:
                 sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
                 if err:

yt_dlp/options.py

@@ -34,6 +34,7 @@ from .utils import (
     join_nonempty,
     orderedSet_from_options,
     remove_end,
+    variadic,
     write_string,
 )
 from .version import CHANNEL, __version__
@@ -250,7 +251,7 @@ def create_parser():
             if multiple_args:
                 val = [val, *value[1:]]
         elif default_key is not None:
-            keys, val = [default_key], value
+            keys, val = variadic(default_key), value
         else:
             raise optparse.OptionValueError(
                 f'wrong {opt_str} formatting; it should be {option.metavar}, not "{value}"')
@@ -323,7 +324,7 @@ def create_parser():
         help='Print program version and exit')
     general.add_option(
         '-U', '--update',
-        action='store_true', dest='update_self',
+        action='store_const', dest='update_self', const=CHANNEL,
         help=format_field(
             is_non_updateable(), None, 'Check if updates are available. %s',
             default=f'Update this program to the latest {CHANNEL} version'))
@@ -335,9 +336,9 @@ def create_parser():
         '--update-to',
         action='store', dest='update_self', metavar='[CHANNEL]@[TAG]',
         help=(
-            'Upgrade/downgrade to a specific version. CHANNEL and TAG defaults to '
-            f'"{CHANNEL}" and "latest" respectively if omitted; See "UPDATE" for details. '
-            f'Supported channels: {", ".join(UPDATE_SOURCES)}'))
+            'Upgrade/downgrade to a specific version. CHANNEL can be a repository as well. '
+            f'CHANNEL and TAG default to "{CHANNEL.partition("@")[0]}" and "latest" respectively if omitted; '
+            f'See "UPDATE" for details. Supported channels: {", ".join(UPDATE_SOURCES)}'))
     general.add_option(
         '-i', '--ignore-errors',
         action='store_true', dest='ignoreerrors',
@@ -411,7 +412,7 @@ def create_parser():
     general.add_option(
         '--no-flat-playlist',
         action='store_false', dest='extract_flat',
-        help='Extract the videos of a playlist')
+        help='Fully extract the videos of a playlist (default)')
     general.add_option(
         '--live-from-start',
         action='store_true', dest='live_from_start',
@@ -447,8 +448,25 @@ def create_parser():
         help='Do not mark videos watched (default)')
     general.add_option(
         '--no-colors', '--no-colours',
-        action='store_true', dest='no_color', default=False,
-        help='Do not emit color codes in output (Alias: --no-colours)')
+        action='store_const', dest='color', const={
+            'stdout': 'no_color',
+            'stderr': 'no_color',
+        },
+        help=optparse.SUPPRESS_HELP)
+    general.add_option(
+        '--color',
+        dest='color', metavar='[STREAM:]POLICY', default={}, type='str',
+        action='callback', callback=_dict_from_options_callback,
+        callback_kwargs={
+            'allowed_keys': 'stdout|stderr',
+            'default_key': ['stdout', 'stderr'],
+            'process': str.strip,
+        }, help=(
+            'Whether to emit color codes in output, optionally prefixed by '
+            'the STREAM (stdout or stderr) to apply the setting to. '
+            'Can be one of "always", "auto" (default), "never", or '
+            '"no_color" (use non color terminal sequences). '
+            'Can be used multiple times'))
     general.add_option(
         '--compat-options',
         metavar='OPTS', dest='compat_opts', default=set(), type='str',
@@ -528,11 +546,11 @@ def create_parser():
         help=optparse.SUPPRESS_HELP)
     geo.add_option(
         '--xff', metavar='VALUE',
-        dest='geo_bypass', default="default",
+        dest='geo_bypass', default='default',
         help=(
             'How to fake X-Forwarded-For HTTP header to try bypassing geographic restriction. '
-            'One of "default" (Only when known to be useful), "never", '
-            'a two-letter ISO 3166-2 country code, or an IP block in CIDR notation'))
+            'One of "default" (only when known to be useful), "never", '
+            'an IP block in CIDR notation, or a two-letter ISO 3166-2 country code'))
     geo.add_option(
         '--geo-bypass',
         action='store_const', dest='geo_bypass', const='default',
@@ -624,7 +642,7 @@ def create_parser():
             'that contains the phrase "cats & dogs" (caseless). '
             'Use "--match-filter -" to interactively ask whether to download each video'))
     selection.add_option(
-        '--no-match-filter',
+        '--no-match-filters',
         dest='match_filter', action='store_const', const=None,
         help='Do not use any --match-filter (default)')
     selection.add_option(

yt_dlp/update.py

@@ -16,6 +16,7 @@ from .utils import (
     Popen,
     cached_method,
     deprecation_warning,
+    network_exceptions,
     remove_end,
     remove_start,
     sanitized_Request,
@@ -128,27 +129,36 @@ class Updater:
         self.ydl = ydl
 
         self.target_channel, sep, self.target_tag = (target or CHANNEL).rpartition('@')
-        if not sep and self.target_tag in UPDATE_SOURCES:  # stable => stable@latest
-            self.target_channel, self.target_tag = self.target_tag, None
+        # stable => stable@latest
+        if not sep and ('/' in self.target_tag or self.target_tag in UPDATE_SOURCES):
+            self.target_channel = self.target_tag
+            self.target_tag = None
         elif not self.target_channel:
-            self.target_channel = CHANNEL
+            self.target_channel = CHANNEL.partition('@')[0]
 
         if not self.target_tag:
-            self.target_tag, self._exact = 'latest', False
+            self.target_tag = 'latest'
+            self._exact = False
         elif self.target_tag != 'latest':
             self.target_tag = f'tags/{self.target_tag}'
 
-    @property
-    def _target_repo(self):
-        try:
-            return UPDATE_SOURCES[self.target_channel]
-        except KeyError:
-            return self._report_error(
-                f'Invalid update channel {self.target_channel!r} requested. '
-                f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
+        if '/' in self.target_channel:
+            self._target_repo = self.target_channel
+            if self.target_channel not in (CHANNEL, *UPDATE_SOURCES.values()):
+                self.ydl.report_warning(
+                    f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
+                    f'from {self.ydl._format_err(self._target_repo, self.ydl.Styles.EMPHASIS)}. '
+                    f'Run {self.ydl._format_err("at your own risk", "light red")}')
+                self.restart = self._blocked_restart
+        else:
+            self._target_repo = UPDATE_SOURCES.get(self.target_channel)
+            if not self._target_repo:
+                self._report_error(
+                    f'Invalid update channel {self.target_channel!r} requested. '
+                    f'Valid channels are {", ".join(UPDATE_SOURCES)}', True)
 
     def _version_compare(self, a, b, channel=CHANNEL):
-        if channel != self.target_channel:
+        if self._exact and channel != self.target_channel:
             return False
 
         if _VERSION_RE.fullmatch(f'{a}.{b}'):
@@ -258,8 +268,8 @@ class Updater:
             self.ydl.to_screen((
                 f'Available version: {self._label(self.target_channel, self.latest_version)}, ' if self.target_tag == 'latest' else ''
             ) + f'Current version: {self._label(CHANNEL, self.current_version)}')
-        except Exception:
-            return self._report_network_error('obtain version info', delim='; Please try again later or')
+        except network_exceptions as e:
+            return self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
 
         if not is_non_updateable():
             self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
@@ -303,7 +313,7 @@ class Updater:
 
         try:
             newcontent = self._download(self.release_name, self._tag)
-        except Exception as e:
+        except network_exceptions as e:
             if isinstance(e, urllib.error.HTTPError) and e.code == 404:
                 return self._report_error(
                     f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)
@@ -371,6 +381,12 @@ class Updater:
         _, _, returncode = Popen.run(self.cmd)
         return returncode
 
+    def _blocked_restart(self):
+        self._report_error(
+            'Automatically restarting into custom builds is disabled for security reasons. '
+            'Restart yt-dlp to use the updated version', expected=True)
+        return self.ydl._download_retcode
+
 
 def run_update(ydl):
     """Update the program file with the latest version from the repository

yt_dlp/utils/__init__.py

@@ -0,0 +1,14 @@
+import warnings
+
+from ..compat.compat_utils import passthrough_module
+
+# XXX: Implement this the same way as other DeprecationWarnings without circular import
+passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=5))
+del passthrough_module
+
+# isort: off
+from .traversal import *
+from ._utils import *
+from ._utils import _configuration_args, _get_exe_version_output
+from ._deprecated import *

yt_dlp/utils/_deprecated.py

@@ -0,0 +1,30 @@
+"""Deprecated - New code should avoid these"""
+
+from ._utils import preferredencoding
+
+
+def encodeFilename(s, for_subprocess=False):
+    assert isinstance(s, str)
+    return s
+
+
+def decodeFilename(b, for_subprocess=False):
+    return b
+
+
+def decodeArgument(b):
+    return b
+
+
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, str)
+    return optval
+
+
+def error_to_compat_str(err):
+    return str(err)

yt_dlp/utils/_legacy.py

@@ -0,0 +1,176 @@
+"""No longer used and new code should not use. Exists only for API compat."""
+
+import platform
+import struct
+import sys
+import urllib.parse
+import zlib
+
+from ._utils import decode_base_n, preferredencoding
+from .traversal import traverse_obj
+from ..dependencies import certifi, websockets
+
+# isort: split
+from ..cookies import YoutubeDLCookieJar  # noqa: F401
+
+has_certifi = bool(certifi)
+has_websockets = bool(websockets)
+
+
+def load_plugins(name, suffix, namespace):
+    from ..plugins import load_plugins
+    ret = load_plugins(name, suffix)
+    namespace.update(ret)
+    return ret
+
+
+def traverse_dict(dictn, keys, casesense=True):
+    return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
+
+
+def decode_base(value, digits):
+    return decode_base_n(value, table=digits)
+
+
+def platform_name():
+    """ Returns the platform name as a str """
+    return platform.platform()
+
+
+def get_subprocess_encoding():
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # For subprocess calls, encode with locale encoding
+        # Refer to http://stackoverflow.com/a/9951851/35070
+        encoding = preferredencoding()
+    else:
+        encoding = sys.getfilesystemencoding()
+    if encoding is None:
+        encoding = 'utf-8'
+    return encoding
+
+
+# UNUSED
+# Based on png2str() written by @gdkchan and improved by @yokrysty
+# Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
+def decode_png(png_data):
+    # Reference: https://www.w3.org/TR/PNG/
+    header = png_data[8:]
+
+    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
+        raise OSError('Not a valid PNG file.')
+
+    int_map = {1: '>B', 2: '>H', 4: '>I'}
+    unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
+
+    chunks = []
+
+    while header:
+        length = unpack_integer(header[:4])
+        header = header[4:]
+
+        chunk_type = header[:4]
+        header = header[4:]
+
+        chunk_data = header[:length]
+        header = header[length:]
+
+        header = header[4:]  # Skip CRC
+
+        chunks.append({
+            'type': chunk_type,
+            'length': length,
+            'data': chunk_data
+        })
+
+    ihdr = chunks[0]['data']
+
+    width = unpack_integer(ihdr[:4])
+    height = unpack_integer(ihdr[4:8])
+
+    idat = b''
+
+    for chunk in chunks:
+        if chunk['type'] == b'IDAT':
+            idat += chunk['data']
+
+    if not idat:
+        raise OSError('Unable to read PNG data.')
+
+    decompressed_data = bytearray(zlib.decompress(idat))
+
+    stride = width * 3
+    pixels = []
+
+    def _get_pixel(idx):
+        x = idx % stride
+        y = idx // stride
+        return pixels[y][x]
+
+    for y in range(height):
+        basePos = y * (1 + stride)
+        filter_type = decompressed_data[basePos]
+
+        current_row = []
+
+        pixels.append(current_row)
+
+        for x in range(stride):
+            color = decompressed_data[1 + basePos + x]
+            basex = y * stride + x
+            left = 0
+            up = 0
+
+            if x > 2:
+                left = _get_pixel(basex - 3)
+            if y > 0:
+                up = _get_pixel(basex - stride)
+
+            if filter_type == 1:  # Sub
+                color = (color + left) & 0xff
+            elif filter_type == 2:  # Up
+                color = (color + up) & 0xff
+            elif filter_type == 3:  # Average
+                color = (color + ((left + up) >> 1)) & 0xff
+            elif filter_type == 4:  # Paeth
+                a = left
+                b = up
+                c = 0
+
+                if x > 2 and y > 0:
+                    c = _get_pixel(basex - stride - 3)
+
+                p = a + b - c
+
+                pa = abs(p - a)
+                pb = abs(p - b)
+                pc = abs(p - c)
+
+                if pa <= pb and pa <= pc:
+                    color = (color + a) & 0xff
+                elif pb <= pc:
+                    color = (color + b) & 0xff
+                else:
+                    color = (color + c) & 0xff
+
+            current_row.append(color)
+
+    return width, height, pixels
+
+
+def register_socks_protocols():
+    # "Register" SOCKS protocols
+    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
+    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
+    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if scheme not in urllib.parse.uses_netloc:
+            urllib.parse.uses_netloc.append(scheme)
+
+
+def handle_youtubedl_headers(headers):
+    filtered_headers = headers
+
+    if 'Youtubedl-no-compression' in filtered_headers:
+        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
+        del filtered_headers['Youtubedl-no-compression']
+
+    return filtered_headers

yt_dlp/utils/_utils.py

@@ -47,26 +47,20 @@ import urllib.request
 import xml.etree.ElementTree
 import zlib
 
-from .compat import functools  # isort: split
-from .compat import (
+from . import traversal
+
+from ..compat import functools  # isort: split
+from ..compat import (
     compat_etree_fromstring,
     compat_expanduser,
     compat_HTMLParseError,
     compat_os_name,
     compat_shlex_quote,
 )
-from .dependencies import brotli, certifi, websockets, xattr
-from .socks import ProxyType, sockssocket
-
-
-def register_socks_protocols():
-    # "Register" SOCKS protocols
-    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
-    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
-    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
-        if scheme not in urllib.parse.uses_netloc:
-            urllib.parse.uses_netloc.append(scheme)
+from ..dependencies import brotli, certifi, websockets, xattr
+from ..socks import ProxyType, sockssocket
 
+__name__ = __name__.rsplit('.', 1)[0]  # Pretend to be the parent module
 
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
@@ -136,8 +130,13 @@ USER_AGENTS = {
 }
 
 
-NO_DEFAULT = object()
-IDENTITY = lambda x: x
+class NO_DEFAULT:
+    pass
+
+
+def IDENTITY(x):
+    return x
+
 
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
@@ -224,6 +223,7 @@ DATE_FORMATS_DAY_FIRST.extend([
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
     '%d-%m-%Y %H:%M',
+    '%H:%M %d/%m/%Y',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
@@ -928,27 +928,6 @@ class Popen(subprocess.Popen):
             return stdout or default, stderr or default, proc.returncode
 
 
-def get_subprocess_encoding():
-    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # For subprocess calls, encode with locale encoding
-        # Refer to http://stackoverflow.com/a/9951851/35070
-        encoding = preferredencoding()
-    else:
-        encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = 'utf-8'
-    return encoding
-
-
-def encodeFilename(s, for_subprocess=False):
-    assert isinstance(s, str)
-    return s
-
-
-def decodeFilename(b, for_subprocess=False):
-    return b
-
-
 def encodeArgument(s):
     # Legacy code that uses byte strings
     # Uncomment the following line after fixing all post processors
@@ -956,20 +935,6 @@ def encodeArgument(s):
     return s if isinstance(s, str) else s.decode('ascii')
 
 
-def decodeArgument(b):
-    return b
-
-
-def decodeOption(optval):
-    if optval is None:
-        return optval
-    if isinstance(optval, bytes):
-        optval = optval.decode(preferredencoding())
-
-    assert isinstance(optval, str)
-    return optval
-
-
 _timetuple = collections.namedtuple('Time', ('hours', 'minutes', 'seconds', 'milliseconds'))
 
 
@@ -1034,7 +999,7 @@ def make_HTTPS_handler(params, **kwargs):
 
     context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
     if opts_check_certificate:
-        if has_certifi and 'no-certifi' not in params.get('compat_opts', []):
+        if certifi and 'no-certifi' not in params.get('compat_opts', []):
             context.load_verify_locations(cafile=certifi.where())
         else:
             try:
@@ -1068,7 +1033,7 @@ def make_HTTPS_handler(params, **kwargs):
 
 
 def bug_reports_message(before=';'):
-    from .update import REPOSITORY
+    from ..update import REPOSITORY
 
     msg = (f'please report this issue on  https://github.com/{REPOSITORY}/issues?q= , '
            'filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U')
@@ -1351,25 +1316,12 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     return hc
 
 
-def handle_youtubedl_headers(headers):
-    filtered_headers = headers
-
-    if 'Youtubedl-no-compression' in filtered_headers:
-        filtered_headers = {k: v for k, v in filtered_headers.items() if k.lower() != 'accept-encoding'}
-        del filtered_headers['Youtubedl-no-compression']
-
-    return filtered_headers
-
-
 class YoutubeDLHandler(urllib.request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
-    the standard headers to every HTTP request and handles gzipped and
-    deflated responses from web servers. If compression is to be avoided in
-    a particular request, the original request in the program code only has
-    to include the HTTP header "Youtubedl-no-compression", which will be
-    removed before making the real request.
+    the standard headers to every HTTP request and handles gzipped, deflated and
+    brotli responses from web servers.
 
     Part of this code was copied from:
 
@@ -1410,6 +1362,23 @@ class YoutubeDLHandler(urllib.request.HTTPHandler):
             return data
         return brotli.decompress(data)
 
+    @staticmethod
+    def gz(data):
+        gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
+        try:
+            return gz.read()
+        except OSError as original_oserror:
+            # There may be junk add the end of the file
+            # See http://stackoverflow.com/q/4928560/35070 for details
+            for i in range(1, 1024):
+                try:
+                    gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
+                    return gz.read()
+                except OSError:
+                    continue
+            else:
+                raise original_oserror
+
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
         # always respected by websites, some tend to give out URLs with non percent-encoded
@@ -1432,44 +1401,32 @@ class YoutubeDLHandler(urllib.request.HTTPHandler):
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
 
+        if 'Youtubedl-no-compression' in req.headers:  # deprecated
+            req.headers.pop('Youtubedl-no-compression', None)
+            req.add_header('Accept-encoding', 'identity')
+
         if 'Accept-encoding' not in req.headers:
             req.add_header('Accept-encoding', ', '.join(SUPPORTED_ENCODINGS))
 
-        req.headers = handle_youtubedl_headers(req.headers)
-
         return super().do_request_(req)
 
     def http_response(self, req, resp):
         old_resp = resp
-        # gzip
-        if resp.headers.get('Content-encoding', '') == 'gzip':
-            content = resp.read()
-            gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
-            try:
-                uncompressed = io.BytesIO(gz.read())
-            except OSError as original_ioerror:
-                # There may be junk add the end of the file
-                # See http://stackoverflow.com/q/4928560/35070 for details
-                for i in range(1, 1024):
-                    try:
-                        gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
-                        uncompressed = io.BytesIO(gz.read())
-                    except OSError:
-                        continue
-                    break
-                else:
-                    raise original_ioerror
-            resp = urllib.request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        # deflate
-        if resp.headers.get('Content-encoding', '') == 'deflate':
-            gz = io.BytesIO(self.deflate(resp.read()))
-            resp = urllib.request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        # brotli
-        if resp.headers.get('Content-encoding', '') == 'br':
-            resp = urllib.request.addinfourl(
-                io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
+
+        # Content-Encoding header lists the encodings in order that they were applied [1].
+        # To decompress, we simply do the reverse.
+        # [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
+        decoded_response = None
+        for encoding in (e.strip() for e in reversed(resp.headers.get('Content-encoding', '').split(','))):
+            if encoding == 'gzip':
+                decoded_response = self.gz(decoded_response or resp.read())
+            elif encoding == 'deflate':
+                decoded_response = self.deflate(decoded_response or resp.read())
+            elif encoding == 'br' and brotli:
+                decoded_response = self.brotli(decoded_response or resp.read())
+
+        if decoded_response is not None:
+            resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/ytdl-org/youtube-dl/issues/6457).
@@ -1565,136 +1522,6 @@ def is_path_like(f):
     return isinstance(f, (str, bytes, os.PathLike))
 
 
-class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
-    """
-    See [1] for cookie file format.
-
-    1. https://curl.haxx.se/docs/http-cookies.html
-    """
-    _HTTPONLY_PREFIX = '#HttpOnly_'
-    _ENTRY_LEN = 7
-    _HEADER = '''# Netscape HTTP Cookie File
-# This file is generated by yt-dlp.  Do not edit.
-
-'''
-    _CookieFileEntry = collections.namedtuple(
-        'CookieFileEntry',
-        ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
-
-    def __init__(self, filename=None, *args, **kwargs):
-        super().__init__(None, *args, **kwargs)
-        if is_path_like(filename):
-            filename = os.fspath(filename)
-        self.filename = filename
-
-    @staticmethod
-    def _true_or_false(cndn):
-        return 'TRUE' if cndn else 'FALSE'
-
-    @contextlib.contextmanager
-    def open(self, file, *, write=False):
-        if is_path_like(file):
-            with open(file, 'w' if write else 'r', encoding='utf-8') as f:
-                yield f
-        else:
-            if write:
-                file.truncate(0)
-            yield file
-
-    def _really_save(self, f, ignore_discard=False, ignore_expires=False):
-        now = time.time()
-        for cookie in self:
-            if (not ignore_discard and cookie.discard
-                    or not ignore_expires and cookie.is_expired(now)):
-                continue
-            name, value = cookie.name, cookie.value
-            if value is None:
-                # cookies.txt regards 'Set-Cookie: foo' as a cookie
-                # with no name, whereas http.cookiejar regards it as a
-                # cookie with no value.
-                name, value = '', name
-            f.write('%s\n' % '\t'.join((
-                cookie.domain,
-                self._true_or_false(cookie.domain.startswith('.')),
-                cookie.path,
-                self._true_or_false(cookie.secure),
-                str_or_none(cookie.expires, default=''),
-                name, value
-            )))
-
-    def save(self, filename=None, *args, **kwargs):
-        """
-        Save cookies to a file.
-        Code is taken from CPython 3.6
-        https://github.com/python/cpython/blob/8d999cbf4adea053be6dbb612b9844635c4dfb8e/Lib/http/cookiejar.py#L2091-L2117 """
-
-        if filename is None:
-            if self.filename is not None:
-                filename = self.filename
-            else:
-                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
-
-        # Store session cookies with `expires` set to 0 instead of an empty string
-        for cookie in self:
-            if cookie.expires is None:
-                cookie.expires = 0
-
-        with self.open(filename, write=True) as f:
-            f.write(self._HEADER)
-            self._really_save(f, *args, **kwargs)
-
-    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
-        """Load cookies from a file."""
-        if filename is None:
-            if self.filename is not None:
-                filename = self.filename
-            else:
-                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
-
-        def prepare_line(line):
-            if line.startswith(self._HTTPONLY_PREFIX):
-                line = line[len(self._HTTPONLY_PREFIX):]
-            # comments and empty lines are fine
-            if line.startswith('#') or not line.strip():
-                return line
-            cookie_list = line.split('\t')
-            if len(cookie_list) != self._ENTRY_LEN:
-                raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
-            cookie = self._CookieFileEntry(*cookie_list)
-            if cookie.expires_at and not cookie.expires_at.isdigit():
-                raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
-            return line
-
-        cf = io.StringIO()
-        with self.open(filename) as f:
-            for line in f:
-                try:
-                    cf.write(prepare_line(line))
-                except http.cookiejar.LoadError as e:
-                    if f'{line.strip()} '[0] in '[{"':
-                        raise http.cookiejar.LoadError(
-                            'Cookies file must be Netscape formatted, not JSON. See  '
-                            'https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp')
-                    write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
-                    continue
-        cf.seek(0)
-        self._really_load(cf, filename, ignore_discard, ignore_expires)
-        # Session cookies are denoted by either `expires` field set to
-        # an empty string or 0. MozillaCookieJar only recognizes the former
-        # (see [1]). So we need force the latter to be recognized as session
-        # cookies on our own.
-        # Session cookies may be important for cookies-based authentication,
-        # e.g. usually, when user does not check 'Remember me' check box while
-        # logging in on a site, some important cookies are stored as session
-        # cookies so that not recognizing them will result in failed login.
-        # 1. https://bugs.python.org/issue17164
-        for cookie in self:
-            # Treat `expires=0` cookies as session cookies
-            if cookie.expires == 0:
-                cookie.expires = None
-                cookie.discard = True
-
-
 class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
     def __init__(self, cookiejar=None):
         urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
@@ -1711,61 +1538,44 @@ class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
 
     The code is based on HTTPRedirectHandler implementation from CPython [1].
 
-    This redirect handler solves two issues:
-     - ensures redirect URL is always unicode under python 2
-     - introduces support for experimental HTTP response status code
-       308 Permanent Redirect [2] used by some sites [3]
+    This redirect handler fixes and improves the logic to better align with RFC7261
+     and what browsers tend to do [2][3]
 
     1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
-    2. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
-    3. https://github.com/ytdl-org/youtube-dl/issues/28768
+    2. https://datatracker.ietf.org/doc/html/rfc7231
+    3. https://github.com/python/cpython/issues/91306
     """
 
     http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
 
     def redirect_request(self, req, fp, code, msg, headers, newurl):
-        """Return a Request or None in response to a redirect.
-
-        This is called by the http_error_30x methods when a
-        redirection response is received.  If a redirection should
-        take place, return a new Request to allow http_error_30x to
-        perform the redirect.  Otherwise, raise HTTPError if no-one
-        else should try to handle this url.  Return None if you can't
-        but another Handler might.
-        """
-        m = req.get_method()
-        if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
-                 or code in (301, 302, 303) and m == "POST")):
+        if code not in (301, 302, 303, 307, 308):
             raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
-        # Strictly (according to RFC 2616), 301 or 302 in response to
-        # a POST MUST NOT cause a redirection without confirmation
-        # from the user (of urllib.request, in this case).  In practice,
-        # essentially all clients do redirect in this case, so we do
-        # the same.
-
-        # Be conciliant with URIs containing a space.  This is mainly
-        # redundant with the more complete encoding done in http_error_302(),
-        # but it is kept for compatibility with other callers.
-        newurl = newurl.replace(' ', '%20')
-
-        CONTENT_HEADERS = ("content-length", "content-type")
-        # NB: don't use dict comprehension for python 2.6 compatibility
-        newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS}
 
+        new_method = req.get_method()
+        new_data = req.data
+        remove_headers = []
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
-        if code == 303 and m != 'HEAD':
-            m = 'GET'
+        if code == 303 and req.get_method() != 'HEAD':
+            new_method = 'GET'
         # 301 and 302 redirects are commonly turned into a GET from a POST
         # for subsequent requests by browsers, so we'll do the same.
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.2
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.3
-        if code in (301, 302) and m == 'POST':
-            m = 'GET'
+        elif code in (301, 302) and req.get_method() == 'POST':
+            new_method = 'GET'
+
+        # only remove payload if method changed (e.g. POST to GET)
+        if new_method != req.get_method():
+            new_data = None
+            remove_headers.extend(['Content-Length', 'Content-Type'])
+
+        new_headers = {k: v for k, v in req.headers.items() if k.lower() not in remove_headers}
 
         return urllib.request.Request(
-            newurl, headers=newheaders, origin_req_host=req.origin_req_host,
-            unverifiable=True, method=m)
+            newurl, headers=new_headers, origin_req_host=req.origin_req_host,
+            unverifiable=True, method=new_method, data=new_data)
 
 
 def extract_timezone(date_str):
@@ -2011,20 +1821,14 @@ class DateRange:
             date = date_from_str(date)
         return self.start <= date <= self.end
 
-    def __str__(self):
-        return f'{self.start.isoformat()} - {self.end.isoformat()}'
+    def __repr__(self):
+        return f'{__name__}.{type(self).__name__}({self.start.isoformat()!r}, {self.end.isoformat()!r})'
 
     def __eq__(self, other):
         return (isinstance(other, DateRange)
                 and self.start == other.start and self.end == other.end)
 
 
-def platform_name():
-    """ Returns the platform name as a str """
-    deprecation_warning(f'"{__name__}.platform_name" is deprecated, use "platform.platform" instead')
-    return platform.platform()
-
-
 @functools.cache
 def system_identifier():
     python_implementation = platform.python_implementation()
@@ -2076,7 +1880,7 @@ def write_string(s, out=None, encoding=None):
 
 
 def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
-    from . import _IN_CLI
+    from .. import _IN_CLI
     if _IN_CLI:
         if msg in deprecation_warning._cache:
             return
@@ -3286,14 +3090,10 @@ def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO
 
 
 def variadic(x, allowed_types=NO_DEFAULT):
-    return x if is_iterable_like(x, blocked_types=allowed_types) else (x,)
-
-
-def dict_get(d, key_or_keys, default=None, skip_false_values=True):
-    for val in map(d.get, variadic(key_or_keys)):
-        if val is not None and (val or not skip_false_values):
-            return val
-    return default
+    if not isinstance(allowed_types, (tuple, type)):
+        deprecation_warning('allowed_types should be a tuple or a type')
+        allowed_types = tuple(allowed_types)
+    return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
 
 
 def try_call(*funcs, expected_type=None, args=[], kwargs={}):
@@ -3533,7 +3333,7 @@ def is_outdated_version(version, limit, assume_new=True):
 def ytdl_is_updateable():
     """ Returns if yt-dlp can be updated with -U """
 
-    from .update import is_non_updateable
+    from ..update import is_non_updateable
 
     return not is_non_updateable()
 
@@ -3543,10 +3343,6 @@ def args_to_str(args):
     return ' '.join(compat_shlex_quote(a) for a in args)
 
 
-def error_to_compat_str(err):
-    return str(err)
-
-
 def error_to_str(err):
     return f'{type(err).__name__}: {err}'
 
@@ -3633,7 +3429,7 @@ def mimetype2ext(mt, default=NO_DEFAULT):
     mimetype = mt.partition(';')[0].strip().lower()
     _, _, subtype = mimetype.rpartition('/')
 
-    ext = traverse_obj(MAP, mimetype, subtype, subtype.rsplit('+')[-1])
+    ext = traversal.traverse_obj(MAP, mimetype, subtype, subtype.rsplit('+')[-1])
     if ext:
         return ext
     elif default is not NO_DEFAULT:
@@ -3665,7 +3461,7 @@ def parse_codecs(codecs_str):
             vcodec = full_codec
             if parts[0] in ('dvh1', 'dvhe'):
                 hdr = 'DV'
-            elif parts[0] == 'av1' and traverse_obj(parts, 3) == '10':
+            elif parts[0] == 'av1' and traversal.traverse_obj(parts, 3) == '10':
                 hdr = 'HDR10'
             elif parts[:2] == ['vp9', '2']:
                 hdr = 'HDR10'
@@ -3711,8 +3507,7 @@ def get_compatible_ext(*, vcodecs, acodecs, vexts, aexts, preferences=None):
         },
     }
 
-    sanitize_codec = functools.partial(
-        try_get, getter=lambda x: x[0].split('.')[0].replace('0', '').lower())
+    sanitize_codec = functools.partial(try_get, getter=lambda x: x[0].split('.')[0].replace('0', ''))
     vcodec, acodec = sanitize_codec(vcodecs), sanitize_codec(acodecs)
 
     for ext in preferences or COMPATIBLE_CODECS.keys():
@@ -5093,12 +4888,6 @@ def decode_base_n(string, n=None, table=None):
     return result
 
 
-def decode_base(value, digits):
-    deprecation_warning(f'{__name__}.decode_base is deprecated and may be removed '
-                        f'in a future version. Use {__name__}.decode_base_n instead')
-    return decode_base_n(value, table=digits)
-
-
 def decode_packed_codes(code):
     mobj = re.search(PACKED_CODES_RE, code)
     obfuscated_code, base, count, symbols = mobj.groups()
@@ -5143,113 +4932,6 @@ def urshift(val, n):
     return val >> n if val >= 0 else (val + 0x100000000) >> n
 
 
-# Based on png2str() written by @gdkchan and improved by @yokrysty
-# Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
-def decode_png(png_data):
-    # Reference: https://www.w3.org/TR/PNG/
-    header = png_data[8:]
-
-    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
-        raise OSError('Not a valid PNG file.')
-
-    int_map = {1: '>B', 2: '>H', 4: '>I'}
-    unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
-
-    chunks = []
-
-    while header:
-        length = unpack_integer(header[:4])
-        header = header[4:]
-
-        chunk_type = header[:4]
-        header = header[4:]
-
-        chunk_data = header[:length]
-        header = header[length:]
-
-        header = header[4:]  # Skip CRC
-
-        chunks.append({
-            'type': chunk_type,
-            'length': length,
-            'data': chunk_data
-        })
-
-    ihdr = chunks[0]['data']
-
-    width = unpack_integer(ihdr[:4])
-    height = unpack_integer(ihdr[4:8])
-
-    idat = b''
-
-    for chunk in chunks:
-        if chunk['type'] == b'IDAT':
-            idat += chunk['data']
-
-    if not idat:
-        raise OSError('Unable to read PNG data.')
-
-    decompressed_data = bytearray(zlib.decompress(idat))
-
-    stride = width * 3
-    pixels = []
-
-    def _get_pixel(idx):
-        x = idx % stride
-        y = idx // stride
-        return pixels[y][x]
-
-    for y in range(height):
-        basePos = y * (1 + stride)
-        filter_type = decompressed_data[basePos]
-
-        current_row = []
-
-        pixels.append(current_row)
-
-        for x in range(stride):
-            color = decompressed_data[1 + basePos + x]
-            basex = y * stride + x
-            left = 0
-            up = 0
-
-            if x > 2:
-                left = _get_pixel(basex - 3)
-            if y > 0:
-                up = _get_pixel(basex - stride)
-
-            if filter_type == 1:  # Sub
-                color = (color + left) & 0xff
-            elif filter_type == 2:  # Up
-                color = (color + up) & 0xff
-            elif filter_type == 3:  # Average
-                color = (color + ((left + up) >> 1)) & 0xff
-            elif filter_type == 4:  # Paeth
-                a = left
-                b = up
-                c = 0
-
-                if x > 2 and y > 0:
-                    c = _get_pixel(basex - stride - 3)
-
-                p = a + b - c
-
-                pa = abs(p - a)
-                pb = abs(p - b)
-                pc = abs(p - c)
-
-                if pa <= pb and pa <= pc:
-                    color = (color + a) & 0xff
-                elif pb <= pc:
-                    color = (color + b) & 0xff
-                else:
-                    color = (color + c) & 0xff
-
-            current_row.append(color)
-
-    return width, height, pixels
-
-
 def write_xattr(path, key, value):
     # Windows: Write xattrs to NTFS Alternate Data Streams:
     # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
@@ -5408,8 +5090,8 @@ def to_high_limit_path(path):
 
 
 def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
-    val = traverse_obj(obj, *variadic(field))
-    if (not val and val != 0) if ignore is NO_DEFAULT else val in variadic(ignore):
+    val = traversal.traverse_obj(obj, *variadic(field))
+    if not val if ignore is NO_DEFAULT else val in variadic(ignore):
         return default
     return template % func(val)
 
@@ -5446,12 +5128,12 @@ def make_dir(path, to_screen=None):
         return True
     except OSError as err:
         if callable(to_screen) is not None:
-            to_screen('unable to create directory ' + error_to_compat_str(err))
+            to_screen(f'unable to create directory {err}')
         return False
 
 
 def get_executable_path():
-    from .update import _get_variant_and_executable_path
+    from ..update import _get_variant_and_executable_path
 
     return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
 
@@ -5475,244 +5157,6 @@ def get_system_config_dirs(package_name):
     yield os.path.join('/etc', package_name)
 
 
-def traverse_obj(
-        obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
-        casesense=True, is_user_input=False, traverse_string=False):
-    """
-    Safely traverse nested `dict`s and `Iterable`s
-
-    >>> obj = [{}, {"key": "value"}]
-    >>> traverse_obj(obj, (1, "key"))
-    "value"
-
-    Each of the provided `paths` is tested and the first producing a valid result will be returned.
-    The next path will also be tested if the path branched but no results could be found.
-    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
-    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
-
-    The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
-
-    The keys in the path can be one of:
-        - `None`:           Return the current object.
-        - `set`:            Requires the only item in the set to be a type or function,
-                            like `{type}`/`{func}`. If a `type`, returns only values
-                            of this type. If a function, returns `func(obj)`.
-        - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
-        - `slice`:          Branch out and return all values in `obj[key]`.
-        - `Ellipsis`:       Branch out and return a list of all values.
-        - `tuple`/`list`:   Branch out and return a list of all matching values.
-                            Read as: `[traverse_obj(obj, branch) for branch in branches]`.
-        - `function`:       Branch out and return values filtered by the function.
-                            Read as: `[value for key, value in obj if function(key, value)]`.
-                            For `Iterable`s, `key` is the index of the value.
-                            For `re.Match`es, `key` is the group number (0 = full match)
-                            as well as additionally any group names, if given.
-        - `dict`            Transform the current object and return a matching dict.
-                            Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
-
-        `tuple`, `list`, and `dict` all support nested paths and branches.
-
-    @params paths           Paths which to traverse by.
-    @param default          Value to return if the paths do not match.
-                            If the last key in the path is a `dict`, it will apply to each value inside
-                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
-    @param expected_type    If a `type`, only accept final values of this type.
-                            If any other callable, try to call the function on each result.
-                            If the last key in the path is a `dict`, it will apply to each value inside
-                            the dict instead, recursively. This does respect branching paths.
-    @param get_all          If `False`, return the first matching result, otherwise all matching ones.
-    @param casesense        If `False`, consider string dictionary keys as case insensitive.
-
-    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
-
-    @param is_user_input    Whether the keys are generated from user input.
-                            If `True` strings get converted to `int`/`slice` if needed.
-    @param traverse_string  Whether to traverse into objects as strings.
-                            If `True`, any non-compatible object will first be
-                            converted into a string and then traversed into.
-                            The return value of that path will be a string instead,
-                            not respecting any further branching.
-
-
-    @returns                The result of the object traversal.
-                            If successful, `get_all=True`, and the path branches at least once,
-                            then a list of results is returned instead.
-                            If no `default` is given and the last path branches, a `list` of results
-                            is always returned. If a path ends on a `dict` that result will always be a `dict`.
-    """
-    casefold = lambda k: k.casefold() if isinstance(k, str) else k
-
-    if isinstance(expected_type, type):
-        type_test = lambda val: val if isinstance(val, expected_type) else None
-    else:
-        type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
-
-    def apply_key(key, obj, is_last):
-        branching = False
-        result = None
-
-        if obj is None and traverse_string:
-            if key is ... or callable(key) or isinstance(key, slice):
-                branching = True
-                result = ()
-
-        elif key is None:
-            result = obj
-
-        elif isinstance(key, set):
-            assert len(key) == 1, 'Set should only be used to wrap a single item'
-            item = next(iter(key))
-            if isinstance(item, type):
-                if isinstance(obj, item):
-                    result = obj
-            else:
-                result = try_call(item, args=(obj,))
-
-        elif isinstance(key, (list, tuple)):
-            branching = True
-            result = itertools.chain.from_iterable(
-                apply_path(obj, branch, is_last)[0] for branch in key)
-
-        elif key is ...:
-            branching = True
-            if isinstance(obj, collections.abc.Mapping):
-                result = obj.values()
-            elif is_iterable_like(obj):
-                result = obj
-            elif isinstance(obj, re.Match):
-                result = obj.groups()
-            elif traverse_string:
-                branching = False
-                result = str(obj)
-            else:
-                result = ()
-
-        elif callable(key):
-            branching = True
-            if isinstance(obj, collections.abc.Mapping):
-                iter_obj = obj.items()
-            elif is_iterable_like(obj):
-                iter_obj = enumerate(obj)
-            elif isinstance(obj, re.Match):
-                iter_obj = itertools.chain(
-                    enumerate((obj.group(), *obj.groups())),
-                    obj.groupdict().items())
-            elif traverse_string:
-                branching = False
-                iter_obj = enumerate(str(obj))
-            else:
-                iter_obj = ()
-
-            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
-            if not branching:  # string traversal
-                result = ''.join(result)
-
-        elif isinstance(key, dict):
-            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
-            result = {
-                k: v if v is not None else default for k, v in iter_obj
-                if v is not None or default is not NO_DEFAULT
-            } or None
-
-        elif isinstance(obj, collections.abc.Mapping):
-            result = (try_call(obj.get, args=(key,)) if casesense or try_call(obj.__contains__, args=(key,)) else
-                      next((v for k, v in obj.items() if casefold(k) == key), None))
-
-        elif isinstance(obj, re.Match):
-            if isinstance(key, int) or casesense:
-                with contextlib.suppress(IndexError):
-                    result = obj.group(key)
-
-            elif isinstance(key, str):
-                result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
-
-        elif isinstance(key, (int, slice)):
-            if is_iterable_like(obj, collections.abc.Sequence):
-                branching = isinstance(key, slice)
-                with contextlib.suppress(IndexError):
-                    result = obj[key]
-            elif traverse_string:
-                with contextlib.suppress(IndexError):
-                    result = str(obj)[key]
-
-        return branching, result if branching else (result,)
-
-    def lazy_last(iterable):
-        iterator = iter(iterable)
-        prev = next(iterator, NO_DEFAULT)
-        if prev is NO_DEFAULT:
-            return
-
-        for item in iterator:
-            yield False, prev
-            prev = item
-
-        yield True, prev
-
-    def apply_path(start_obj, path, test_type):
-        objs = (start_obj,)
-        has_branched = False
-
-        key = None
-        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
-            if is_user_input and isinstance(key, str):
-                if key == ':':
-                    key = ...
-                elif ':' in key:
-                    key = slice(*map(int_or_none, key.split(':')))
-                elif int_or_none(key) is not None:
-                    key = int(key)
-
-            if not casesense and isinstance(key, str):
-                key = key.casefold()
-
-            if __debug__ and callable(key):
-                # Verify function signature
-                inspect.signature(key).bind(None, None)
-
-            new_objs = []
-            for obj in objs:
-                branching, results = apply_key(key, obj, last)
-                has_branched |= branching
-                new_objs.append(results)
-
-            objs = itertools.chain.from_iterable(new_objs)
-
-        if test_type and not isinstance(key, (dict, list, tuple)):
-            objs = map(type_test, objs)
-
-        return objs, has_branched, isinstance(key, dict)
-
-    def _traverse_obj(obj, path, allow_empty, test_type):
-        results, has_branched, is_dict = apply_path(obj, path, test_type)
-        results = LazyList(item for item in results if item not in (None, {}))
-        if get_all and has_branched:
-            if results:
-                return results.exhaust()
-            if allow_empty:
-                return [] if default is NO_DEFAULT else default
-            return None
-
-        return results[0] if results else {} if allow_empty and is_dict else None
-
-    for index, path in enumerate(paths, 1):
-        result = _traverse_obj(obj, path, index == len(paths), True)
-        if result is not None:
-            return result
-
-    return None if default is NO_DEFAULT else default
-
-
-def traverse_dict(dictn, keys, casesense=True):
-    deprecation_warning(f'"{__name__}.traverse_dict" is deprecated and may be removed '
-                        f'in a future version. Use "{__name__}.traverse_obj" instead')
-    return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
-
-
-def get_first(obj, keys, **kwargs):
-    return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
-
-
 def time_seconds(**kwargs):
     """
     Returns TZ-aware time in seconds since the epoch (1970-01-01T00:00:00Z)
@@ -5808,7 +5252,7 @@ def number_of_digits(number):
 
 def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
-        values = (traverse_obj(from_dict, variadic(v)) for v in values)
+        values = (traversal.traverse_obj(from_dict, variadic(v)) for v in values)
     return delim.join(map(str, filter(None, values)))
 
 
@@ -6519,15 +5963,3 @@ class FormatSorter:
                 format['abr'] = format.get('tbr') - format.get('vbr', 0)
 
         return tuple(self._calculate_field_preference(format, field) for field in self._order)
-
-
-# Deprecated
-has_certifi = bool(certifi)
-has_websockets = bool(websockets)
-
-
-def load_plugins(name, suffix, namespace):
-    from .plugins import load_plugins
-    ret = load_plugins(name, suffix)
-    namespace.update(ret)
-    return ret

yt_dlp/utils/traversal.py

@@ -0,0 +1,254 @@
+import collections.abc
+import contextlib
+import inspect
+import itertools
+import re
+
+from ._utils import (
+    IDENTITY,
+    NO_DEFAULT,
+    LazyList,
+    int_or_none,
+    is_iterable_like,
+    try_call,
+    variadic,
+)
+
+
+def traverse_obj(
+        obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
+        casesense=True, is_user_input=False, traverse_string=False):
+    """
+    Safely traverse nested `dict`s and `Iterable`s
+
+    >>> obj = [{}, {"key": "value"}]
+    >>> traverse_obj(obj, (1, "key"))
+    "value"
+
+    Each of the provided `paths` is tested and the first producing a valid result will be returned.
+    The next path will also be tested if the path branched but no results could be found.
+    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
+    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
+
+    The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
+
+    The keys in the path can be one of:
+        - `None`:           Return the current object.
+        - `set`:            Requires the only item in the set to be a type or function,
+                            like `{type}`/`{func}`. If a `type`, returns only values
+                            of this type. If a function, returns `func(obj)`.
+        - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
+        - `slice`:          Branch out and return all values in `obj[key]`.
+        - `Ellipsis`:       Branch out and return a list of all values.
+        - `tuple`/`list`:   Branch out and return a list of all matching values.
+                            Read as: `[traverse_obj(obj, branch) for branch in branches]`.
+        - `function`:       Branch out and return values filtered by the function.
+                            Read as: `[value for key, value in obj if function(key, value)]`.
+                            For `Iterable`s, `key` is the index of the value.
+                            For `re.Match`es, `key` is the group number (0 = full match)
+                            as well as additionally any group names, if given.
+        - `dict`            Transform the current object and return a matching dict.
+                            Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
+
+        `tuple`, `list`, and `dict` all support nested paths and branches.
+
+    @params paths           Paths which to traverse by.
+    @param default          Value to return if the paths do not match.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
+    @param expected_type    If a `type`, only accept final values of this type.
+                            If any other callable, try to call the function on each result.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, recursively. This does respect branching paths.
+    @param get_all          If `False`, return the first matching result, otherwise all matching ones.
+    @param casesense        If `False`, consider string dictionary keys as case insensitive.
+
+    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+
+    @param is_user_input    Whether the keys are generated from user input.
+                            If `True` strings get converted to `int`/`slice` if needed.
+    @param traverse_string  Whether to traverse into objects as strings.
+                            If `True`, any non-compatible object will first be
+                            converted into a string and then traversed into.
+                            The return value of that path will be a string instead,
+                            not respecting any further branching.
+
+
+    @returns                The result of the object traversal.
+                            If successful, `get_all=True`, and the path branches at least once,
+                            then a list of results is returned instead.
+                            If no `default` is given and the last path branches, a `list` of results
+                            is always returned. If a path ends on a `dict` that result will always be a `dict`.
+    """
+    casefold = lambda k: k.casefold() if isinstance(k, str) else k
+
+    if isinstance(expected_type, type):
+        type_test = lambda val: val if isinstance(val, expected_type) else None
+    else:
+        type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
+
+    def apply_key(key, obj, is_last):
+        branching = False
+        result = None
+
+        if obj is None and traverse_string:
+            if key is ... or callable(key) or isinstance(key, slice):
+                branching = True
+                result = ()
+
+        elif key is None:
+            result = obj
+
+        elif isinstance(key, set):
+            assert len(key) == 1, 'Set should only be used to wrap a single item'
+            item = next(iter(key))
+            if isinstance(item, type):
+                if isinstance(obj, item):
+                    result = obj
+            else:
+                result = try_call(item, args=(obj,))
+
+        elif isinstance(key, (list, tuple)):
+            branching = True
+            result = itertools.chain.from_iterable(
+                apply_path(obj, branch, is_last)[0] for branch in key)
+
+        elif key is ...:
+            branching = True
+            if isinstance(obj, collections.abc.Mapping):
+                result = obj.values()
+            elif is_iterable_like(obj):
+                result = obj
+            elif isinstance(obj, re.Match):
+                result = obj.groups()
+            elif traverse_string:
+                branching = False
+                result = str(obj)
+            else:
+                result = ()
+
+        elif callable(key):
+            branching = True
+            if isinstance(obj, collections.abc.Mapping):
+                iter_obj = obj.items()
+            elif is_iterable_like(obj):
+                iter_obj = enumerate(obj)
+            elif isinstance(obj, re.Match):
+                iter_obj = itertools.chain(
+                    enumerate((obj.group(), *obj.groups())),
+                    obj.groupdict().items())
+            elif traverse_string:
+                branching = False
+                iter_obj = enumerate(str(obj))
+            else:
+                iter_obj = ()
+
+            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
+            if not branching:  # string traversal
+                result = ''.join(result)
+
+        elif isinstance(key, dict):
+            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
+            result = {
+                k: v if v is not None else default for k, v in iter_obj
+                if v is not None or default is not NO_DEFAULT
+            } or None
+
+        elif isinstance(obj, collections.abc.Mapping):
+            result = (try_call(obj.get, args=(key,)) if casesense or try_call(obj.__contains__, args=(key,)) else
+                      next((v for k, v in obj.items() if casefold(k) == key), None))
+
+        elif isinstance(obj, re.Match):
+            if isinstance(key, int) or casesense:
+                with contextlib.suppress(IndexError):
+                    result = obj.group(key)
+
+            elif isinstance(key, str):
+                result = next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
+
+        elif isinstance(key, (int, slice)):
+            if is_iterable_like(obj, collections.abc.Sequence):
+                branching = isinstance(key, slice)
+                with contextlib.suppress(IndexError):
+                    result = obj[key]
+            elif traverse_string:
+                with contextlib.suppress(IndexError):
+                    result = str(obj)[key]
+
+        return branching, result if branching else (result,)
+
+    def lazy_last(iterable):
+        iterator = iter(iterable)
+        prev = next(iterator, NO_DEFAULT)
+        if prev is NO_DEFAULT:
+            return
+
+        for item in iterator:
+            yield False, prev
+            prev = item
+
+        yield True, prev
+
+    def apply_path(start_obj, path, test_type):
+        objs = (start_obj,)
+        has_branched = False
+
+        key = None
+        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
+            if is_user_input and isinstance(key, str):
+                if key == ':':
+                    key = ...
+                elif ':' in key:
+                    key = slice(*map(int_or_none, key.split(':')))
+                elif int_or_none(key) is not None:
+                    key = int(key)
+
+            if not casesense and isinstance(key, str):
+                key = key.casefold()
+
+            if __debug__ and callable(key):
+                # Verify function signature
+                inspect.signature(key).bind(None, None)
+
+            new_objs = []
+            for obj in objs:
+                branching, results = apply_key(key, obj, last)
+                has_branched |= branching
+                new_objs.append(results)
+
+            objs = itertools.chain.from_iterable(new_objs)
+
+        if test_type and not isinstance(key, (dict, list, tuple)):
+            objs = map(type_test, objs)
+
+        return objs, has_branched, isinstance(key, dict)
+
+    def _traverse_obj(obj, path, allow_empty, test_type):
+        results, has_branched, is_dict = apply_path(obj, path, test_type)
+        results = LazyList(item for item in results if item not in (None, {}))
+        if get_all and has_branched:
+            if results:
+                return results.exhaust()
+            if allow_empty:
+                return [] if default is NO_DEFAULT else default
+            return None
+
+        return results[0] if results else {} if allow_empty and is_dict else None
+
+    for index, path in enumerate(paths, 1):
+        result = _traverse_obj(obj, path, index == len(paths), True)
+        if result is not None:
+            return result
+
+    return None if default is NO_DEFAULT else default
+
+
+def get_first(obj, *paths, **kwargs):
+    return traverse_obj(obj, *((..., *variadic(keys)) for keys in paths), **kwargs, get_all=False)
+
+
+def dict_get(d, key_or_keys, default=None, skip_false_values=True):
+    for val in map(d.get, variadic(key_or_keys)):
+        if val is not None and (val or not skip_false_values):
+            return val
+    return default