[networking] Add support for zstandard content-encoding

Supported by urllib/requests/curl_cffi

Authored-by: coletdjnz

yt_dlp/networking/_curlcffi.py

@@ -29,7 +29,7 @@ if curl_cffi is None:
     raise ImportError('curl_cffi is not installed')
 
 
-curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3]))
+curl_cffi_version = tuple(map(int, re.split(r'\D+', curl_cffi.__version__)[:3]))
 
 if curl_cffi_version != (0, 5, 10) and not ((0, 7, 0) <= curl_cffi_version < (0, 8, 0)):
     curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)'

yt_dlp/networking/_requests.py

@@ -8,7 +8,7 @@ import re
 import socket
 import warnings
 
-from ..dependencies import brotli, requests, urllib3
+from ..dependencies import requests, urllib3
 from ..utils import bug_reports_message, int_or_none, variadic
 from ..utils.networking import normalize_url
 
@@ -59,12 +59,7 @@ from .exceptions import (
 )
 from ..socks import ProxyError as SocksProxyError
 
-SUPPORTED_ENCODINGS = [
-    'gzip', 'deflate',
-]
-
-if brotli is not None:
-    SUPPORTED_ENCODINGS.append('br')
+SUPPORTED_ENCODINGS = urllib3.util.request.ACCEPT_ENCODING.split(',')
 
 '''
 Override urllib3's behavior to not convert lower-case percent-encoded characters
@@ -259,7 +254,6 @@ class RequestsRH(RequestHandler, InstanceStoreMixin):
     https://github.com/psf/requests
     """
     _SUPPORTED_URL_SCHEMES = ('http', 'https')
-    _SUPPORTED_ENCODINGS = tuple(SUPPORTED_ENCODINGS)
     _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
     _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
     RH_NAME = 'requests'

yt_dlp/networking/_urllib.py

@@ -38,7 +38,7 @@ from .exceptions import (
     SSLError,
     TransportError,
 )
-from ..dependencies import brotli
+from ..dependencies import brotli, zstandard
 from ..socks import ProxyError as SocksProxyError
 from ..utils import update_url_query
 from ..utils.networking import normalize_url
@@ -50,6 +50,10 @@ if brotli:
     SUPPORTED_ENCODINGS.append('br')
     CONTENT_DECODE_ERRORS.append(brotli.error)
 
+if zstandard:
+    SUPPORTED_ENCODINGS.append('zstd')
+    CONTENT_DECODE_ERRORS.append(zstandard.ZstdError)
+
 
 def _create_http_connection(http_class, source_address, *args, **kwargs):
     hc = http_class(*args, **kwargs)
@@ -118,6 +122,12 @@ class HTTPHandler(urllib.request.AbstractHTTPHandler):
             return data
         return brotli.decompress(data)
 
+    @staticmethod
+    def zstd(data):
+        if not data:
+            return data
+        return zstandard.ZstdDecompressor().decompress(data)
+
     @staticmethod
     def gz(data):
         # There may be junk added the end of the file
@@ -158,6 +168,8 @@ class HTTPHandler(urllib.request.AbstractHTTPHandler):
                 decoded_response = self.deflate(decoded_response or resp.read())
             elif encoding == 'br' and brotli:
                 decoded_response = self.brotli(decoded_response or resp.read())
+            elif encoding == 'zstd' and zstandard:
+                decoded_response = self.zstd(decoded_response or resp.read())
 
         if decoded_response is not None:
             resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)