[rh:curl_cffi] Support curl_cffi 0.16.x (#17439)

Closes #17341
Authored by: bashonly, coletdjnz

Co-authored-by: coletdjnz <coletdjnz@protonmail.com>

devscripts/update_requirements.py

@@ -97,7 +97,6 @@ BUNDLE_TARGETS = {
         extras=['curl-cffi'],
         # Only need curl-cffi+cffi in this requirements file; their deps are installed directly
         # XXX: Try to keep these in sync with curl-cffi's and cffi's transitive dependencies
-        prune_packages=['rich'],
         omit_packages=['certifi', 'pycparser'],
     ),
 }

pyproject.toml

@@ -58,7 +58,7 @@ default = [
     "yt-dlp-ejs==0.8.0",
 ]
 curl-cffi = [
-    "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.16 ; implementation_name == 'cpython'",
+    "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.17 ; implementation_name == 'cpython'",
 ]
 secretstorage = [
     "secretstorage",

test/test_http_proxy.py

@@ -83,6 +83,12 @@ class HTTPProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin):
 
         self.server.close_request(self.request)
 
+    def finish(self):
+        try:
+            super().finish()
+        finally:
+            self.server.close_request(self.request)
+
 
 if urllib3:
     import urllib3.util.ssltransport
@@ -132,7 +138,11 @@ class HTTPSProxyHandler(HTTPProxyHandler):
             request = SSLTransport(request, ssl_context=sslctx, server_side=True)
         else:
             request = sslctx.wrap_socket(request, server_side=True)
-        super().__init__(request, *args, **kwargs)
+        try:
+            super().__init__(request, *args, **kwargs)
+        except Exception:
+            request.close()
+            raise
 
 
 class HTTPConnectProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin):
@@ -163,6 +173,12 @@ class HTTPConnectProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin):
         self.request_handler(self.request, self.client_address, self.server, proxy_info=proxy_info)
         self.server.close_request(self.request)
 
+    def finish(self):
+        try:
+            super().finish()
+        finally:
+            self.server.close_request(self.request)
+
 
 class HTTPSConnectProxyHandler(HTTPConnectProxyHandler):
     def __init__(self, request, *args, **kwargs):
@@ -171,7 +187,11 @@ class HTTPSConnectProxyHandler(HTTPConnectProxyHandler):
         sslctx.load_cert_chain(certfn, None)
         request = sslctx.wrap_socket(request, server_side=True)
         self._original_request = request
-        super().__init__(request, *args, **kwargs)
+        try:
+            super().__init__(request, *args, **kwargs)
+        except Exception:
+            request.close()
+            raise
 
     def do_CONNECT(self):
         super().do_CONNECT()

test/test_networking.py

@@ -388,13 +388,23 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
             assert res.status == 200
             res.close()
 
-    def test_percent_encode(self, handler):
+    def test_percent_encode_unicode(self, handler):
+        # RFC 3986 §6.2.2.1 defines that percent-encoding SHOULD be normalized to uppercase.
         with handler() as rh:
             # Unicode characters should be encoded with uppercase percent-encoding
             res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/中文.html'))
             assert res.status == 200
             res.close()
-            # don't normalize existing percent encodings
+
+    @pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi (non-standard)')
+    def test_percent_encode_keep_existing(self, handler):
+        # NOTE: RFC 3986 §6.2.2.1 defines that percent-encoding SHOULD be normalized to uppercase.
+        #  For compatibility with legacy sites (e.g., redirects using lowercase encodings and only accept that),
+        #  our default handlers (urllib/requests) preserve existing percent-encoding instead of normalizing it.
+        #
+        # CurlCFFI is excluded because it forces uppercase encodings and is hard to change. This is acceptable
+        # since CurlCFFI is used only for impersonation. https://github.com/curl/curl/pull/21592
+        with handler() as rh:
             res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/%c7%9f'))
             assert res.status == 200
             res.close()

yt_dlp/networking/_curlcffi.py

@@ -33,9 +33,9 @@ if curl_cffi is None:
 
 curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3]))
 
-if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version < (0, 16):
+if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version < (0, 17):
     curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)'
-    raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x through 0.15.x are supported')
+    raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x through 0.16.x are supported')
 
 import curl_cffi.requests
 from curl_cffi.const import CurlECode, CurlOpt
@@ -175,6 +175,9 @@ BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
         'firefox144': ImpersonateTarget('firefox', '144', 'macos', '26'),
         'firefox147': ImpersonateTarget('firefox', '147', 'macos', '26'),
     },
+    (0, 16, 1): {
+        'chrome150': ImpersonateTarget('chrome', '150', 'macos', '26'),
+    },
 }
 
 # Needed for curl_cffi < 0.11
@@ -327,7 +330,8 @@ class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
 
             elif (
                 e.code == CurlECode.PROXY
-                or (e.code == CurlECode.RECV_ERROR and 'CONNECT' in str(e))
+                # curl_cffi >= 0.16.0: changed to CurlECode.COULDNT_CONNECT https://github.com/curl/curl/pull/21084
+                or (e.code in (CurlECode.RECV_ERROR, CurlECode.COULDNT_CONNECT) and 'CONNECT' in str(e))
             ):
                 raise ProxyError(cause=e) from e
             else: