[pp/exec] Restrict --exec template usage to safe conversions (#16883)

Authored by: bashonly

README.md

@@ -2332,8 +2332,8 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi`
 * yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 * ~~yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [aria2c](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is~~
-* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filters` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
+* yt-dlp versions from 2021.09.01 to 2022.11.11 (inclusive) applied `--match-filters` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
-* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
+* yt-dlp versions from 2021.11.10 to 2023.06.21 (inclusive) estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
 * yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
 * The sub-modules `swfinterp`, `casefold` are removed.
 * Passing `--simulate` (or calling `extract_info` with `download=False`) no longer alters the default format selection. See [#9843](https://github.com/yt-dlp/yt-dlp/issues/9843) for details.
@@ -2342,8 +2342,8 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 For convenience, there are some compat option aliases available to use:
 
 * `--compat-options all`: Use all compat options (**Do NOT use this!**)
-* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort`
+* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort,-allow-unsafe-exec-expansion`
-* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort`
+* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx,-allow-unsafe-ext,-prefer-vp9-sort,-allow-unsafe-exec-expansion`
 * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization`
 * `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`
 * `--compat-options 2023`: Same as `--compat-options 2024,prefer-vp9-sort`
@@ -2358,7 +2358,12 @@ The following compat options restore vulnerable behavior from before security pa
 
     > :warning: Only use if a valid file download is rejected because its extension is detected as uncommon
     >
-    > **This option can enable remote code execution! Consider [opening an issue](<https://github.com/yt-dlp/yt-dlp/issues/new/choose>) instead!**
+    > **This option can enable remote code execution!** Consider [opening an issue](<https://github.com/yt-dlp/yt-dlp/issues/new/choose>) instead!
+
+* `--compat-options allow-unsafe-exec-expansion`: The `--exec` option allows output template syntax to be used in its commands; however, for security reasons the conversions that can be used are restricted to `i`/`d` (signed integer decimal), `f` (floating-point decimal) and `q` (shell-quoted). yt-dlp versions from 2021.04.11 to 2026.03.17 (inclusive) did not apply this restriction. This option reverts this restriction
+
+    > :warning: **This option can enable remote code execution!** Consider using `%()q` conversions in your exec command templates for any string values.
+
 
 ### Deprecated options
 

test/test_postprocessors.py

@@ -11,7 +11,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import subprocess
 
 from yt_dlp import YoutubeDL
-from yt_dlp.utils import shell_quote
+from yt_dlp.utils import (
+    UnsafeExecExpansionError,
+    shell_quote,
+)
 from yt_dlp.postprocessor import (
     ExecPP,
     FFmpegThumbnailsConvertorPP,
@@ -91,6 +94,51 @@ class TestExec(unittest.TestCase):
         self.assertEqual(pp.parse_cmd('echo {}', info), cmd)
         self.assertEqual(pp.parse_cmd('echo %(filepath)q', info), cmd)
 
+    def test_unsafe_exec_expansion(self):
+        # Test unsafe placeholder
+        ydl = YoutubeDL({'outtmpl_na_placeholder': ';'})
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe placeholder', ExecPP, ydl, 'echo %(title)q')
+
+        ydl = YoutubeDL()
+        # Test unsafe commands
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)s"')
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title).100B"')
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)S"')
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe conversion', ExecPP, ydl, 'echo "%(title)#j"')
+        self.assertRaisesRegex(UnsafeExecExpansionError, r'Unsafe default', ExecPP, ydl, 'echo %(title|;)q')
+        # Test safe commands
+        self.assertIsInstance(
+            ExecPP(ydl, [
+                'echo',
+                'echo {}',
+                'echo %(title)q',
+                'echo %(title|NA)q',
+                'echo %(title)#q',
+                'echo %(view_count)i',
+                'echo %(view_count)02d',
+                'echo %(aspect_ratio)f',
+                'echo %(aspect_ratio).2f',
+            ]),
+            ExecPP)
+
+        # Test compat opt
+        ydl = YoutubeDL({
+            'outtmpl_na_placeholder': ';',
+            'compat_opts': {'allow-unsafe-exec-expansion'},
+        })
+        self.assertIsInstance(
+            ExecPP(ydl, [
+                'echo "%(title)s"',
+                'echo %(title)q',
+                'echo "%(title).100B"',
+                'echo "%(title)S"',
+                'echo "%(title)#S"',
+                'echo "%(title)j"',
+                'echo "%(title)#j"',
+                'echo %(title|;)q',
+            ]),
+            ExecPP)
+
 
 class TestModifyChaptersPP(unittest.TestCase):
     def setUp(self):

yt_dlp/YoutubeDL.py

@@ -112,6 +112,7 @@ from .utils import (
     RejectedVideoReached,
     SameFileError,
     UnavailableVideoError,
+    UnsafeExecExpansionError,
     UserNotLive,
     YoutubeDLError,
     age_restricted,
@@ -826,9 +827,14 @@ class YoutubeDL:
         for pp_def_raw in self.params.get('postprocessors', []):
             pp_def = dict(pp_def_raw)
             when = pp_def.pop('when', 'post_process')
-            self.add_post_processor(
+            # Handle errors for ExecPP command validation
-                get_postprocessor(pp_def.pop('key'))(self, **pp_def),
+            try:
-                when=when)
+                self.add_post_processor(
+                    get_postprocessor(pp_def.pop('key'))(self, **pp_def),
+                    when=when)
+            except UnsafeExecExpansionError as e:
+                self.report_error(e)
+                raise
 
         def preload_download_archive(fn):
             """Preload the archive, if any is specified"""
@@ -1254,7 +1260,7 @@ class YoutubeDL:
         info_dict.pop('__pending_error', None)
         return info_dict
 
-    def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
+    def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False):
         """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
         @param sanitize    Whether to sanitize the output as a filename
         """
@@ -1305,6 +1311,8 @@ class YoutubeDL:
                 (?:&(?P<replacement>.*?))?
                 (?:\|(?P<default>.*?))?
             )$''')
+        SAFE_EXEC_CONVERSIONS = 'difq'
+        UNSAFE_DEFAULT_CHARS = '"\' \n\t;&|^$%*<>{}()[]`#\\'
 
         def _from_user_input(field):
             if field == ':':
@@ -1429,6 +1437,16 @@ class YoutubeDL:
             if fmt == 's' and last_field in field_size_compat_map and isinstance(value, int):
                 fmt = f'0{field_size_compat_map[last_field]:d}d'
 
+            # Validate safety of exec commands
+            if _exec:
+                if fmt[-1] not in SAFE_EXEC_CONVERSIONS:
+                    raise UnsafeExecExpansionError(f'Unsafe conversion(s) in exec command: {outtmpl!r}')
+                elif any(unsafe_char in default for unsafe_char in UNSAFE_DEFAULT_CHARS):
+                    if default == na:
+                        raise UnsafeExecExpansionError(f'Unsafe placeholder for exec command: {na!r}')
+                    else:
+                        raise UnsafeExecExpansionError(f'Unsafe default(s) in exec command: {outtmpl!r}')
+
             flags = outer_mobj.group('conversion') or ''
             str_fmt = f'{fmt[:-1]}s'
             if value is None:

yt_dlp/__init__.py

@@ -44,6 +44,7 @@ from .utils import (
     GeoUtils,
     PlaylistEntries,
     SameFileError,
+    UnsafeExecExpansionError,
     download_range_func,
     expand_path,
     float_or_none,
@@ -1077,7 +1078,7 @@ def main(argv=None):
     IN_CLI.value = True
     try:
         _exit(*variadic(_real_main(argv)))
-    except (CookieLoadError, DownloadError):
+    except (CookieLoadError, DownloadError, UnsafeExecExpansionError):
         _exit(1)
     except SameFileError as e:
         _exit(f'ERROR: {e}')

yt_dlp/options.py

@@ -568,9 +568,10 @@ def create_parser():
                 'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
                 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date',
                 'prefer-legacy-http-handler', 'manifest-filesize-approx', 'allow-unsafe-ext', 'prefer-vp9-sort', 'mtime-by-default',
+                'allow-unsafe-exec-expansion',
             }, 'aliases': {
-                'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort'],
+                'youtube-dl': ['all', '-multistreams', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort', '-allow-unsafe-exec-expansion'],
-                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort'],
+                'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat', '-playlist-match-filter', '-manifest-filesize-approx', '-allow-unsafe-ext', '-prefer-vp9-sort', '-allow-unsafe-exec-expansion'],
                 '2021': ['2022', 'no-certifi', 'filename-sanitization'],
                 '2022': ['2023', 'no-external-downloader-progress', 'playlist-match-filter', 'prefer-legacy-http-handler', 'manifest-filesize-approx'],
                 '2023': ['2024', 'prefer-vp9-sort'],
@@ -1769,7 +1770,9 @@ def create_parser():
         help=(
             'Execute a command, optionally prefixed with when to execute it, separated by a ":". '
             'Supported values of "WHEN" are the same as that of --use-postprocessor (default: after_move). '
-            'The same syntax as the output template can be used to pass any field as arguments to the command. '
+            'The same syntax as the output template can be used to pass any field as arguments to the command; '
+            'however, for security reasons the only allowed conversions are: '
+            '"i"/"d" (signed integer decimal), "f" (floating-point decimal) and "q" (shell-quoted). '
             'If no fields are passed, %(filepath,_filename|)q is appended to the end of the command. '
             'This option can be used multiple times'))
     postproc.add_option(

yt_dlp/postprocessor/exec.py

@@ -5,8 +5,17 @@ from ..utils import Popen, PostProcessingError, shell_quote, variadic
 class ExecPP(PostProcessor):
 
     def __init__(self, downloader, exec_cmd):
-        PostProcessor.__init__(self, downloader)
+        # Need to set exec_cmd attribute before set_downloader is called by PostProcessor.__init__
         self.exec_cmd = variadic(exec_cmd)
+        PostProcessor.__init__(self, downloader)
+
+    def set_downloader(self, downloader):
+        super().set_downloader(downloader)
+        # Validate safety of exec commands
+        params = getattr(self._downloader, 'params', None)
+        if params and 'allow-unsafe-exec-expansion' not in params['compat_opts']:
+            for cmd in self.exec_cmd:
+                _ = self._downloader.prepare_outtmpl(cmd, {}, _exec=True)
 
     def parse_cmd(self, cmd, info):
         tmpl, tmpl_dict = self._downloader.prepare_outtmpl(cmd, info)

yt_dlp/utils/_utils.py

@@ -1182,6 +1182,10 @@ class XAttrUnavailableError(YoutubeDLError):
     pass
 
 
+class UnsafeExecExpansionError(YoutubeDLError):
+    pass
+
+
 def is_path_like(f):
     return isinstance(f, (str, bytes, os.PathLike))