From 5506357315802856914979d5ace8bb6e14abc346 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Fri, 16 May 2025 20:42:57 +1200 Subject: [PATCH] improve docs --- yt_dlp/extractor/youtube/pot/README.md | 44 +++++++++++++++-------- yt_dlp/extractor/youtube/pot/provider.py | 45 ++++++++++++++++-------- yt_dlp/extractor/youtube/pot/utils.py | 2 +- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/yt_dlp/extractor/youtube/pot/README.md b/yt_dlp/extractor/youtube/pot/README.md index 79e1578e41..1b662e0a08 100644 --- a/yt_dlp/extractor/youtube/pot/README.md +++ b/yt_dlp/extractor/youtube/pot/README.md @@ -51,15 +51,18 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP # Innertube Client Name. # For example, "WEB", "ANDROID", "TVHTML5". - # For a list of WebPO client names, see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. - # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS for a list of client names currently supported by the YouTube extractor. + # For a list of WebPO client names, + # see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. + # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS + # for a list of client names currently supported by the YouTube extractor. _SUPPORTED_CLIENTS = ('WEB', 'TVHTML5') _SUPPORTED_CONTEXTS = ( PoTokenContext.GVS, ) - # If your provider makes external requests to websites (i.e. to youtube.com) using another library or service (i.e., not _request_webpage), + # If your provider makes external requests to websites (i.e. to youtube.com) + # using another library or service (i.e., not _request_webpage), # set the request features that are supported here. # If only using _request_webpage to make external requests, set this to None. _SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( @@ -84,16 +87,18 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP pass def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: - # ℹ️ If you need to validate the request before making the request to the external source - # Raise yt_dlp.extractor.youtube.pot.provider.PoTokenProviderRejectedRequest if the request is not supported + # ℹ️ If you need to validate the request before making the request to the external source. + # Raise yt_dlp.extractor.youtube.pot.provider.PoTokenProviderRejectedRequest if the request is not supported. if request.is_authenticated: raise PoTokenProviderRejectedRequest( 'This provider does not support authenticated requests' ) # ℹ️ Settings are pulled from extractor args passed to yt-dlp with the key `youtubepot-`. - # For this example, the extractor arg would be `--extractor-args "youtubepot-mypotokenprovider:url=https://custom.example.com/get_pot"` - external_provider_url = self._configuration_arg('url', default=['https://provider.example.com/get_pot'])[0] + # For this example, the extractor arg would be: + # `--extractor-args "youtubepot-mypotokenprovider:url=https://custom.example.com/get_pot"` + external_provider_url = self._configuration_arg( + 'url', default=['https://provider.example.com/get_pot'])[0] # See below for logging guidelines self.logger.trace(f'Using external provider URL: {external_provider_url}') @@ -114,12 +119,15 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP 'do_not_cache': request.bypass_cache, }).encode(), proxies={'all': None}), pot_request=request, - note=f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider', + note=( + f'Requesting {request.context.value} PO Token ' + f'for {request.internal_client_name} client from external provider'), ) except RequestError as e: # ℹ️ If there is an error, raise PoTokenProviderError. - # You can specify whether it is expected or not. If it is unexpected, the log will include a link to the bug report location (BUG_REPORT_LOCATION). + # You can specify whether it is expected or not. If it is unexpected, + # the log will include a link to the bug report location (BUG_REPORT_LOCATION). raise PoTokenProviderError( 'Networking error while fetching to get PO Token from external provider', expected=True @@ -153,18 +161,26 @@ def my_provider_preference(provider: PoTokenProvider, request: PoTokenRequest) - ## Logging Guidelines - Use the `self.logger` object to log messages. -- When making HTTP requests, use `self.logger.info` to log a message to standard non-verbose output. This lets users know what is happening when a time-expensive operation is taking place. - - For example, `self.logger.info(f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider')` -- Use `self.logger.debug` to log a message to the verbose output (`--verbose`). Try to keep this to a minimum. -- Use `self.logger.trace` to log a message to the PO Token debug output (`--extractor-args "youtube:pot_debug=true"`). Log as much as you like here as needed for debugging your provider. +- When making HTTP requests or any other expensive operation, use `self.logger.info` to log a message to standard non-verbose output. + - This lets users know what is happening when a time-expensive operation is taking place. + - It is recommended to include the PO Token context and internal client name in the message if possible. + - For example, `self.logger.info(f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider')`. +- Use `self.logger.debug` to log a message to the verbose output (`--verbose`). + - For debugging information visible to users posting verbose logs. + - Try to not log too much, prefer using trace logging for detailed debug messages. +- Use `self.logger.trace` to log a message to the PO Token debug output (`--extractor-args "youtube:pot_debug=true"`). + - Log as much as you like here as needed for debugging your provider. - Avoid logging PO Tokens or any sensitive information to debug or info output. ## Debugging -- Use `-v --extractor-args "youtube:pot_debug=true"` to enable PO Token debug output. +- Use `-v --extractor-args "youtube:pot_trace=true"` to enable PO Token debug output. ## Caching +> [!WARNING] +> The following describes more advance features that most users/developers will not need to use. + > [!IMPORTANT] > yt-dlp currently has a built-in LRU Memory Cache Provider and a cache spec provider for WebPO Tokens. > You should only need to implement cache providers if you want an external cache, or a cache spec if you are handling non-WebPO Tokens. diff --git a/yt_dlp/extractor/youtube/pot/provider.py b/yt_dlp/extractor/youtube/pot/provider.py index 80b21bfc77..1a28070561 100644 --- a/yt_dlp/extractor/youtube/pot/provider.py +++ b/yt_dlp/extractor/youtube/pot/provider.py @@ -110,12 +110,14 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'): # Innertube Client Name. # For example, "WEB", "ANDROID", "TVHTML5". - # For a list of WebPO client names, see yt_dlp.extractor.youtube.pot._builtin.utils.WEBPO_CLIENTS. - # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS for a list of client names currently supported by the YouTube extractor. + # For a list of WebPO client names, see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. + # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS + # for a list of client names currently supported by the YouTube extractor. _SUPPORTED_CLIENTS: tuple[str] | None = () - # If making external requests to websites (i.e. to youtube.com) using another library or service (i.e., not _request_webpage), - # add the request features that are supported. + # If making external requests to websites (i.e. to youtube.com) + # using another library or service (i.e., not _request_webpage), + # add the request features that are supported. # If only using _request_webpage to make external requests, set this to None. _SUPPORTED_EXTERNAL_REQUEST_FEATURES: tuple[ExternalRequestFeature] | None = () @@ -124,14 +126,20 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'): raise PoTokenProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available') # Validate request using built-in settings - if self._SUPPORTED_CONTEXTS is not None and request.context not in self._SUPPORTED_CONTEXTS: - raise PoTokenProviderRejectedRequest(f'PO Token Context "{request.context}" is not supported by {self.PROVIDER_NAME}') + if ( + self._SUPPORTED_CONTEXTS is not None + and request.context not in self._SUPPORTED_CONTEXTS + ): + raise PoTokenProviderRejectedRequest( + f'PO Token Context "{request.context}" is not supported by {self.PROVIDER_NAME}') if self._SUPPORTED_CLIENTS is not None: - client_name = traverse_obj(request.innertube_context, ('client', 'clientName')) + client_name = traverse_obj( + request.innertube_context, ('client', 'clientName')) if client_name not in self._SUPPORTED_CLIENTS: raise PoTokenProviderRejectedRequest( - f'Client "{client_name}" is not supported by {self.PROVIDER_NAME}. Supported clients: {", ".join(self._SUPPORTED_CLIENTS) or "none"}') + f'Client "{client_name}" is not supported by {self.PROVIDER_NAME}. ' + f'Supported clients: {", ".join(self._SUPPORTED_CLIENTS) or "none"}') self.__validate_external_request_features(request) @@ -158,15 +166,25 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'): scheme = urllib.parse.urlparse(request.request_proxy).scheme if scheme.lower() not in self._supported_proxy_schemes: raise PoTokenProviderRejectedRequest( - f'External requests by "{self.PROVIDER_NAME}" provider do not support proxy scheme "{scheme}". Supported proxy schemes: {", ".join(self._supported_proxy_schemes) or "none"}') + f'External requests by "{self.PROVIDER_NAME}" provider do not ' + f'support proxy scheme "{scheme}". Supported proxy schemes: ' + f'{", ".join(self._supported_proxy_schemes) or "none"}') - if request.request_source_address and ExternalRequestFeature.SOURCE_ADDRESS not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES: + if ( + request.request_source_address + and ExternalRequestFeature.SOURCE_ADDRESS not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES + ): raise PoTokenProviderRejectedRequest( - f'External requests by "{self.PROVIDER_NAME}" provider do not support setting source address') + f'External requests by "{self.PROVIDER_NAME}" provider ' + f'do not support setting source address') - if not request.request_verify_tls and ExternalRequestFeature.DISABLE_TLS_VERIFICATION not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES: + if ( + not request.request_verify_tls + and ExternalRequestFeature.DISABLE_TLS_VERIFICATION not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES + ): raise PoTokenProviderRejectedRequest( - f'External requests by "{self.PROVIDER_NAME}" provider do not support ignoring TLS certificate failures') + f'External requests by "{self.PROVIDER_NAME}" provider ' + f'do not support ignoring TLS certificate failures') def request_pot(self, request: PoTokenRequest) -> PoTokenResponse: self.__validate_request(request) @@ -198,7 +216,6 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'): # Merge some ctx request settings into the request # Most of these will already be used by the configured ydl instance, # however, the YouTube extractor may override some. - # TODO: add _Request_webpage to IeContentProvider if pot_request is not None: req.headers = HTTPHeaderDict(pot_request.request_headers, req.headers) req.proxies = req.proxies or ({'all': pot_request.request_proxy} if pot_request.request_proxy else {}) diff --git a/yt_dlp/extractor/youtube/pot/utils.py b/yt_dlp/extractor/youtube/pot/utils.py index 68d1d6f02d..1c0db243bf 100644 --- a/yt_dlp/extractor/youtube/pot/utils.py +++ b/yt_dlp/extractor/youtube/pot/utils.py @@ -59,7 +59,7 @@ def get_webpo_content_binding( def _extract_visitor_id(visitor_data): if not visitor_data: - return + return None # Attempt to extract the visitor ID from the visitor_data protobuf # xxx: ideally should use a protobuf parser