From 5faffa999fd33b373d47773e8ee639d072accec2 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 6 Jun 2026 16:24:53 -0500 Subject: [PATCH] [pp/exec] Restrict `--exec` template usage to safe conversions (#16883) Authored by: bashonly --- README.md | 15 +++++++---- test/test_postprocessors.py | 50 +++++++++++++++++++++++++++++++++++- yt_dlp/YoutubeDL.py | 26 ++++++++++++++++--- yt_dlp/__init__.py | 3 ++- yt_dlp/options.py | 9 ++++--- yt_dlp/postprocessor/exec.py | 11 +++++++- yt_dlp/utils/_utils.py | 4 +++ 7 files changed, 103 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 09e290b89d..e334bec8ed 100644 --- a/README.md +++ b/README.md @@ -2332,8 +2332,8 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu * `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi` * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior * ~~yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [aria2c](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is~~ -* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filters` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this -* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values +* yt-dlp versions from 2021.09.01 to 2022.11.11 (inclusive) applied `--match-filters` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this +* yt-dlp versions from 2021.11.10 to 2023.06.21 (inclusive) estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values * yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests. * The sub-modules `swfinterp`, `casefold` are removed. * Passing `--simulate` (or calling `extract_info` with `download=False`) no longer alters the default format selection. See [#9843](https://github.com/yt-dlp/yt-dlp/issues/9843) for details. @@ -2342,8 +2342,8 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu For convenience, there are some compat option aliases available to use: * `--compat-options all`: Use all compat options (**Do NOT use this!**) -* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort` -* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort` +* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort,-allow-unsafe-exec-expansion` +* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort,-allow-unsafe-exec-expansion` * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization` * `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx` * `--compat-options 2023`: Same as `--compat-options 2024,prefer-vp9-sort` @@ -2358,7 +2358,12 @@ The following compat options restore vulnerable behavior from before security pa > :warning: Only use if a valid file download is rejected because its extension is detected as uncommon > - > **This option can enable remote code execution! Consider [opening an issue]() instead!** + > **This option can enable remote code execution!** Consider [opening an issue]() instead! + +* `--compat-options allow-unsafe-exec-expansion`: The `--exec` option allows output template syntax to be used in its commands; however, for security reasons the conversions that can be used are restricted to `i`/`d` (signed integer decimal), `f` (floating-point decimal) and `q` (shell-quoted). yt-dlp versions from 2021.04.11 to 2026.03.17 (inclusive) did not apply this restriction. This option reverts this restriction + + > :warning: **This option can enable remote code execution!** Consider using `%()q` conversions in your exec command templates for any string values. + ### Deprecated options diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py index d58a97fc6e..ad595d6b70 100644 --- a/test/test_postprocessors.py +++ b/test/test_postprocessors.py @@ -11,7 +11,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import subprocess from yt_dlp import YoutubeDL -from yt_dlp.utils import shell_quote +from yt_dlp.utils import ( + UnsafeExecExpansionError, + shell_quote, +) from yt_dlp.postprocessor import ( ExecPP, FFmpegThumbnailsConvertorPP, @@ -91,6 +94,51 @@ class TestExec(unittest.TestCase): self.assertEqual(pp.parse_cmd('echo {}', info), cmd) self.assertEqual(pp.parse_cmd('echo %(filepath)q', info), cmd) + def test_unsafe_exec_expansion(self): + # Test unsafe placeholder + ydl = YoutubeDL({'outtmpl_na_placeholder': ';'}) + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe placeholder', ExecPP, ydl, 'echo %(title)q') + + ydl = YoutubeDL() + # Test unsafe commands + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)s"') + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title).100B"') + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)S"') + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)#j"') + self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe default', ExecPP, ydl, 'echo %(title|;)q') + # Test safe commands + self.assertIsInstance( + ExecPP(ydl, [ + 'echo', + 'echo {}', + 'echo %(title)q', + 'echo %(title|NA)q', + 'echo %(title)#q', + 'echo %(view_count)i', + 'echo %(view_count)02d', + 'echo %(aspect_ratio)f', + 'echo %(aspect_ratio).2f', + ]), + ExecPP) + + # Test compat opt + ydl = YoutubeDL({ + 'outtmpl_na_placeholder': ';', + 'compat_opts': {'allow-unsafe-exec-expansion'}, + }) + self.assertIsInstance( + ExecPP(ydl, [ + 'echo "%(title)s"', + 'echo %(title)q', + 'echo "%(title).100B"', + 'echo "%(title)S"', + 'echo "%(title)#S"', + 'echo "%(title)j"', + 'echo "%(title)#j"', + 'echo %(title|;)q', + ]), + ExecPP) + class TestModifyChaptersPP(unittest.TestCase): def setUp(self): diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 7db6303d4f..8ce01d5233 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -112,6 +112,7 @@ from .utils import ( RejectedVideoReached, SameFileError, UnavailableVideoError, + UnsafeExecExpansionError, UserNotLive, YoutubeDLError, age_restricted, @@ -826,9 +827,14 @@ class YoutubeDL: for pp_def_raw in self.params.get('postprocessors', []): pp_def = dict(pp_def_raw) when = pp_def.pop('when', 'post_process') - self.add_post_processor( - get_postprocessor(pp_def.pop('key'))(self, **pp_def), - when=when) + # Handle errors for ExecPP command validation + try: + self.add_post_processor( + get_postprocessor(pp_def.pop('key'))(self, **pp_def), + when=when) + except UnsafeExecExpansionError as e: + self.report_error(e) + raise def preload_download_archive(fn): """Preload the archive, if any is specified""" @@ -1254,7 +1260,7 @@ class YoutubeDL: info_dict.pop('__pending_error', None) return info_dict - def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False): + def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False): """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict @param sanitize Whether to sanitize the output as a filename """ @@ -1305,6 +1311,8 @@ class YoutubeDL: (?:&(?P.*?))? (?:\|(?P.*?))? )$''') + SAFE_EXEC_CONVERSIONS = 'difq' + UNSAFE_DEFAULT_CHARS = '"\' \n\t;&|^$%*<>{}()[]`#\\' def _from_user_input(field): if field == ':': @@ -1429,6 +1437,16 @@ class YoutubeDL: if fmt == 's' and last_field in field_size_compat_map and isinstance(value, int): fmt = f'0{field_size_compat_map[last_field]:d}d' + # Validate safety of exec commands + if _exec: + if fmt[-1] not in SAFE_EXEC_CONVERSIONS: + raise UnsafeExecExpansionError(f'Unsafe conversion(s) in exec command: {outtmpl!r}') + elif any(unsafe_char in default for unsafe_char in UNSAFE_DEFAULT_CHARS): + if default == na: + raise UnsafeExecExpansionError(f'Unsafe placeholder for exec command: {na!r}') + else: + raise UnsafeExecExpansionError(f'Unsafe default(s) in exec command: {outtmpl!r}') + flags = outer_mobj.group('conversion') or '' str_fmt = f'{fmt[:-1]}s' if value is None: diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 2f6ba47832..18cf4c8d2c 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -44,6 +44,7 @@ from .utils import ( GeoUtils, PlaylistEntries, SameFileError, + UnsafeExecExpansionError, download_range_func, expand_path, float_or_none, @@ -1077,7 +1078,7 @@ def main(argv=None): IN_CLI.value = True try: _exit(*variadic(_real_main(argv))) - except (CookieLoadError, DownloadError): + except (CookieLoadError, DownloadError, UnsafeExecExpansionError): _exit(1) except SameFileError as e: _exit(f'ERROR: {e}') diff --git a/yt_dlp/options.py b/yt_dlp/options.py index ab25540642..c64f4c4363 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -568,9 +568,10 @@ def create_parser(): 'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi', 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date', 'prefer-legacy-http-handler', 'manifest-filesize-approx', 'allow-unsafe-ext', 'prefer-vp9-sort', 'mtime-by-default', + 'allow-unsafe-exec-expansion', }, 'aliases': { - 'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort'], - 'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort'], + 'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort', '-allow-unsafe-exec-expansion'], + 'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort', '-allow-unsafe-exec-expansion'], '2021': ['2022', 'no-certifi', 'filename-sanitization'], '2022': ['2023', 'no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler', 'manifest-filesize-approx'], '2023': ['2024', 'prefer-vp9-sort'], @@ -1769,7 +1770,9 @@ def create_parser(): help=( 'Execute a command, optionally prefixed with when to execute it, separated by a ":". ' 'Supported values of "WHEN" are the same as that of --use-postprocessor (default: after_move). ' - 'The same syntax as the output template can be used to pass any field as arguments to the command. ' + 'The same syntax as the output template can be used to pass any field as arguments to the command; ' + 'however, for security reasons the only allowed conversions are: ' + '"i"/"d" (signed integer decimal), "f" (floating-point decimal) and "q" (shell-quoted). ' 'If no fields are passed, %(filepath,_filename|)q is appended to the end of the command. ' 'This option can be used multiple times')) postproc.add_option( diff --git a/yt_dlp/postprocessor/exec.py b/yt_dlp/postprocessor/exec.py index 243487dd25..ac5708b10e 100644 --- a/yt_dlp/postprocessor/exec.py +++ b/yt_dlp/postprocessor/exec.py @@ -5,8 +5,17 @@ from ..utils import Popen, PostProcessingError, shell_quote, variadic class ExecPP(PostProcessor): def __init__(self, downloader, exec_cmd): - PostProcessor.__init__(self, downloader) + # Need to set exec_cmd attribute before set_downloader is called by PostProcessor.__init__ self.exec_cmd = variadic(exec_cmd) + PostProcessor.__init__(self, downloader) + + def set_downloader(self, downloader): + super().set_downloader(downloader) + # Validate safety of exec commands + params = getattr(self._downloader, 'params', None) + if params and 'allow-unsafe-exec-expansion' not in params['compat_opts']: + for cmd in self.exec_cmd: + _ = self._downloader.prepare_outtmpl(cmd, {}, _exec=True) def parse_cmd(self, cmd, info): tmpl, tmpl_dict = self._downloader.prepare_outtmpl(cmd, info) diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index 74bb6dcdb1..34388e6fe6 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -1182,6 +1182,10 @@ class XAttrUnavailableError(YoutubeDLError): pass +class UnsafeExecExpansionError(YoutubeDLError): + pass + + def is_path_like(f): return isinstance(f, (str, bytes, os.PathLike))