diff --git a/devscripts/update_requirements.py b/devscripts/update_requirements.py index 23676c9450..3a5e2e5002 100755 --- a/devscripts/update_requirements.py +++ b/devscripts/update_requirements.py @@ -97,7 +97,6 @@ BUNDLE_TARGETS = { extras=['curl-cffi'], # Only need curl-cffi+cffi in this requirements file; their deps are installed directly # XXX: Try to keep these in sync with curl-cffi's and cffi's transitive dependencies - prune_packages=['rich'], omit_packages=['certifi', 'pycparser'], ), } diff --git a/pyproject.toml b/pyproject.toml index e403ee0e27..5bc3cb204c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ default = [ "yt-dlp-ejs==0.8.0", ] curl-cffi = [ - "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.16 ; implementation_name == 'cpython'", + "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.17 ; implementation_name == 'cpython'", ] secretstorage = [ "secretstorage", diff --git a/test/test_http_proxy.py b/test/test_http_proxy.py index 22ce3ca5d7..c80d41b927 100644 --- a/test/test_http_proxy.py +++ b/test/test_http_proxy.py @@ -83,6 +83,12 @@ class HTTPProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin): self.server.close_request(self.request) + def finish(self): + try: + super().finish() + finally: + self.server.close_request(self.request) + if urllib3: import urllib3.util.ssltransport @@ -132,7 +138,11 @@ class HTTPSProxyHandler(HTTPProxyHandler): request = SSLTransport(request, ssl_context=sslctx, server_side=True) else: request = sslctx.wrap_socket(request, server_side=True) - super().__init__(request, *args, **kwargs) + try: + super().__init__(request, *args, **kwargs) + except Exception: + request.close() + raise class HTTPConnectProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin): @@ -163,6 +173,12 @@ class HTTPConnectProxyHandler(BaseHTTPRequestHandler, HTTPProxyAuthMixin): self.request_handler(self.request, self.client_address, self.server, proxy_info=proxy_info) self.server.close_request(self.request) + def finish(self): + try: + super().finish() + finally: + self.server.close_request(self.request) + class HTTPSConnectProxyHandler(HTTPConnectProxyHandler): def __init__(self, request, *args, **kwargs): @@ -171,7 +187,11 @@ class HTTPSConnectProxyHandler(HTTPConnectProxyHandler): sslctx.load_cert_chain(certfn, None) request = sslctx.wrap_socket(request, server_side=True) self._original_request = request - super().__init__(request, *args, **kwargs) + try: + super().__init__(request, *args, **kwargs) + except Exception: + request.close() + raise def do_CONNECT(self): super().do_CONNECT() diff --git a/test/test_networking.py b/test/test_networking.py index ee648938cf..9dbe263b6f 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -388,13 +388,23 @@ class TestHTTPRequestHandler(TestRequestHandlerBase): assert res.status == 200 res.close() - def test_percent_encode(self, handler): + def test_percent_encode_unicode(self, handler): + # RFC 3986 §6.2.2.1 defines that percent-encoding SHOULD be normalized to uppercase. with handler() as rh: # Unicode characters should be encoded with uppercase percent-encoding res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/中文.html')) assert res.status == 200 res.close() - # don't normalize existing percent encodings + + @pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi (non-standard)') + def test_percent_encode_keep_existing(self, handler): + # NOTE: RFC 3986 §6.2.2.1 defines that percent-encoding SHOULD be normalized to uppercase. + # For compatibility with legacy sites (e.g., redirects using lowercase encodings and only accept that), + # our default handlers (urllib/requests) preserve existing percent-encoding instead of normalizing it. + # + # CurlCFFI is excluded because it forces uppercase encodings and is hard to change. This is acceptable + # since CurlCFFI is used only for impersonation. https://github.com/curl/curl/pull/21592 + with handler() as rh: res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/%c7%9f')) assert res.status == 200 res.close() diff --git a/yt_dlp/networking/_curlcffi.py b/yt_dlp/networking/_curlcffi.py index 1a34b269ff..3a054c7042 100644 --- a/yt_dlp/networking/_curlcffi.py +++ b/yt_dlp/networking/_curlcffi.py @@ -33,9 +33,9 @@ if curl_cffi is None: curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3])) -if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version < (0, 16): +if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version < (0, 17): curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)' - raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x through 0.15.x are supported') + raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x through 0.16.x are supported') import curl_cffi.requests from curl_cffi.const import CurlECode, CurlOpt @@ -175,6 +175,9 @@ BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = { 'firefox144': ImpersonateTarget('firefox', '144', 'macos', '26'), 'firefox147': ImpersonateTarget('firefox', '147', 'macos', '26'), }, + (0, 16, 1): { + 'chrome150': ImpersonateTarget('chrome', '150', 'macos', '26'), + }, } # Needed for curl_cffi < 0.11 @@ -327,7 +330,8 @@ class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin): elif ( e.code == CurlECode.PROXY - or (e.code == CurlECode.RECV_ERROR and 'CONNECT' in str(e)) + # curl_cffi >= 0.16.0: changed to CurlECode.COULDNT_CONNECT https://github.com/curl/curl/pull/21084 + or (e.code in (CurlECode.RECV_ERROR, CurlECode.COULDNT_CONNECT) and 'CONNECT' in str(e)) ): raise ProxyError(cause=e) from e else: