[fd/dash] Re-extract if using --load-info-json with --live-from-start (#13922)

Closes #13906
Authored by: bashonly

.github/workflows/build.yml

@@ -208,7 +208,7 @@ jobs:
             python3.9 -m pip install -U pip wheel 'setuptools>=71.0.2'
             # XXX: Keep this in sync with pyproject.toml (it can't be accessed at this stage) and exclude secretstorage
             python3.9 -m pip install -U Pyinstaller mutagen pycryptodomex brotli certifi cffi \
-              'requests>=2.32.2,<3' 'urllib3>=1.26.17,<3' 'websockets>=13.0'
+              'requests>=2.32.2,<3' 'urllib3>=2.0.2,<3' 'websockets>=13.0'
 
           run: |
             cd repo

README.md

@@ -171,6 +171,9 @@ yt-dlp --update-to nightly
 python3 -m pip install -U --pre "yt-dlp[default]"
 ```
 
+When running a yt-dlp version that is older than 90 days, you will see a warning message suggesting to update to the latest version.
+You can suppress this warning by adding `--no-update` to your command or configuration file.
+
 ## DEPENDENCIES
 Python versions 3.9+ (CPython) and 3.11+ (PyPy) are supported. Other versions and implementations may or may not work correctly.
 

devscripts/cli_to_api.py

@@ -20,6 +20,7 @@ def parse_patched_options(opts):
         'fragment_retries': 0,
         'extract_flat': False,
         'concat_playlist': 'never',
+        'update_self': False,
     })
     yt_dlp.options.create_parser = lambda: patched_parser
     try:

pyproject.toml

@@ -51,7 +51,7 @@ default = [
     "mutagen",
     "pycryptodomex",
     "requests>=2.32.2,<3",
-    "urllib3>=1.26.17,<3",
+    "urllib3>=2.0.2,<3",
     "websockets>=13.0",
 ]
 curl-cffi = [

yt_dlp/YoutubeDL.py

@@ -73,6 +73,7 @@ from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
 from .update import (
     REPOSITORY,
     _get_system_deprecation,
+    _get_outdated_warning,
     _make_label,
     current_git_head,
     detect_variant,
@@ -504,6 +505,7 @@ class YoutubeDL:
     force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
     noprogress:        Do not print the progress bar
     live_from_start:   Whether to download livestreams videos from the start
+    warn_when_outdated: Emit a warning if the yt-dlp version is older than 90 days
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see yt_dlp/downloader/common.py):
@@ -703,6 +705,9 @@ class YoutubeDL:
         system_deprecation = _get_system_deprecation()
         if system_deprecation:
             self.deprecated_feature(system_deprecation.replace('\n', '\n                    '))
+        elif self.params.get('warn_when_outdated'):
+            if outdated_warning := _get_outdated_warning():
+                self.report_warning(outdated_warning)
 
         if self.params.get('allow_unplayable_formats'):
             self.report_warning(

yt_dlp/__init__.py

@@ -971,6 +971,7 @@ def parse_options(argv=None):
         'geo_bypass': opts.geo_bypass,
         'geo_bypass_country': opts.geo_bypass_country,
         'geo_bypass_ip_block': opts.geo_bypass_ip_block,
+        'warn_when_outdated': opts.update_self is None,
         '_warnings': warnings,
         '_deprecation_warnings': deprecation_warnings,
         'compat_opts': opts.compat_opts,

yt_dlp/downloader/dash.py

@@ -3,7 +3,7 @@ import urllib.parse
 
 from . import get_suitable_downloader
 from .fragment import FragmentFD
-from ..utils import update_url_query, urljoin
+from ..utils import ReExtractInfo, update_url_query, urljoin
 
 
 class DashSegmentsFD(FragmentFD):
@@ -28,6 +28,11 @@ class DashSegmentsFD(FragmentFD):
         requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
         args = []
         for fmt in requested_formats or [info_dict]:
+            # Re-extract if --load-info-json is used and 'fragments' was originally a generator
+            # See https://github.com/yt-dlp/yt-dlp/issues/13906
+            if isinstance(fmt['fragments'], str):
+                raise ReExtractInfo('the stream needs to be re-extracted', expected=True)
+
             try:
                 fragment_count = 1 if self.params.get('test') else len(fmt['fragments'])
             except TypeError:

yt_dlp/networking/_requests.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import contextlib
 import functools
 import http.client
 import logging
@@ -20,9 +19,9 @@ if urllib3 is None:
 
 urllib3_version = tuple(int_or_none(x, default=0) for x in urllib3.__version__.split('.'))
 
-if urllib3_version < (1, 26, 17):
+if urllib3_version < (2, 0, 2):
     urllib3._yt_dlp__version = f'{urllib3.__version__} (unsupported)'
-    raise ImportError('Only urllib3 >= 1.26.17 is supported')
+    raise ImportError('Only urllib3 >= 2.0.2 is supported')
 
 if requests.__build__ < 0x023202:
     requests._yt_dlp__version = f'{requests.__version__} (unsupported)'
@@ -101,27 +100,10 @@ class Urllib3PercentREOverride:
 # https://github.com/urllib3/urllib3/commit/a2697e7c6b275f05879b60f593c5854a816489f0
 import urllib3.util.url
 
-if hasattr(urllib3.util.url, 'PERCENT_RE'):
-    urllib3.util.url.PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url.PERCENT_RE)
-elif hasattr(urllib3.util.url, '_PERCENT_RE'):  # urllib3 >= 2.0.0
+if hasattr(urllib3.util.url, '_PERCENT_RE'):  # was 'PERCENT_RE' in urllib3 < 2.0.0
     urllib3.util.url._PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url._PERCENT_RE)
 else:
-    warnings.warn('Failed to patch PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
-
-'''
-Workaround for issue in urllib.util.ssl_.py: ssl_wrap_context does not pass
-server_hostname to SSLContext.wrap_socket if server_hostname is an IP,
-however this is an issue because we set check_hostname to True in our SSLContext.
-
-Monkey-patching IS_SECURETRANSPORT forces ssl_wrap_context to pass server_hostname regardless.
-
-This has been fixed in urllib3 2.0+.
-See: https://github.com/urllib3/urllib3/issues/517
-'''
-
-if urllib3_version < (2, 0, 0):
-    with contextlib.suppress(Exception):
-        urllib3.util.IS_SECURETRANSPORT = urllib3.util.ssl_.IS_SECURETRANSPORT = True
+    warnings.warn('Failed to patch _PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
 
 
 # Requests will not automatically handle no_proxy by default

yt_dlp/postprocessor/xattrpp.py

@@ -54,6 +54,9 @@ class XAttrMetadataPP(PostProcessor):
                     if infoname == 'upload_date':
                         value = hyphenate_date(value)
                     elif xattrname == 'com.apple.metadata:kMDItemWhereFroms':
+                        # NTFS ADS doesn't support colons in names
+                        if os.name == 'nt':
+                            continue
                         value = self.APPLE_PLIST_TEMPLATE % value
                     write_xattr(info['filepath'], xattrname, value.encode())
 

yt_dlp/update.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import atexit
 import contextlib
+import datetime as dt
 import functools
 import hashlib
 import json
@@ -171,6 +172,22 @@ def _get_system_deprecation():
     return f'Support for Python version {major}.{minor} has been deprecated. {PYTHON_MSG}'
 
 
+def _get_outdated_warning():
+    # Only yt-dlp guarantees a stable release at least every 90 days
+    if not ORIGIN.startswith('yt-dlp/'):
+        return None
+
+    with contextlib.suppress(Exception):
+        last_updated = dt.date(*version_tuple(__version__)[:3])
+        if last_updated < dt.datetime.now(dt.timezone.utc).date() - dt.timedelta(days=90):
+            return ('\n         '.join((
+                f'Your yt-dlp version ({__version__}) is older than 90 days!',
+                'It is strongly recommended to always use the latest version.',
+                f'{is_non_updateable() or """Run "yt-dlp --update" or "yt-dlp -U" to update"""}.',
+                'To suppress this warning, add --no-update to your command/config.')))
+    return None
+
+
 def _sha256_file(path):
     h = hashlib.sha256()
     mv = memoryview(bytearray(128 * 1024))