artem/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2026-06-29 14:14:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve docs

Browse Source
This commit is contained in:
coletdjnz 2025-05-16 20:42:57 +12:00
parent 50d12bbc5e
commit 5506357315
No known key found for this signature in database
GPG Key ID: 91984263BB39894A
3 changed files with 62 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
yt_dlp/extractor/youtube/pot/README.md
View File

@ -51,15 +51,18 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP
# Innertube Client Name. # Innertube Client Name.
# For example, "WEB", "ANDROID", "TVHTML5". # For example, "WEB", "ANDROID", "TVHTML5".
# For a list of WebPO client names, see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS. # For a list of WebPO client names,
# Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS for a list of client names currently supported by the YouTube extractor. # see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS.
# Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS
# for a list of client names currently supported by the YouTube extractor.
_SUPPORTED_CLIENTS = ('WEB', 'TVHTML5') _SUPPORTED_CLIENTS = ('WEB', 'TVHTML5')
_SUPPORTED_CONTEXTS = ( _SUPPORTED_CONTEXTS = (
PoTokenContext.GVS, PoTokenContext.GVS,
) )
# If your provider makes external requests to websites (i.e. to youtube.com) using another library or service (i.e., not _request_webpage), # If your provider makes external requests to websites (i.e. to youtube.com)
# using another library or service (i.e., not _request_webpage),
# set the request features that are supported here. # set the request features that are supported here.
# If only using _request_webpage to make external requests, set this to None. # If only using _request_webpage to make external requests, set this to None.
_SUPPORTED_EXTERNAL_REQUEST_FEATURES = ( _SUPPORTED_EXTERNAL_REQUEST_FEATURES = (
@ -84,16 +87,18 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP
pass pass
def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse: def _real_request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
# ℹ️ If you need to validate the request before making the request to the external source # ℹ️ If you need to validate the request before making the request to the external source.
# Raise yt_dlp.extractor.youtube.pot.provider.PoTokenProviderRejectedRequest if the request is not supported # Raise yt_dlp.extractor.youtube.pot.provider.PoTokenProviderRejectedRequest if the request is not supported.
if request.is_authenticated: if request.is_authenticated:
raise PoTokenProviderRejectedRequest( raise PoTokenProviderRejectedRequest(
'This provider does not support authenticated requests' 'This provider does not support authenticated requests'
) )
# ℹ️ Settings are pulled from extractor args passed to yt-dlp with the key `youtubepot-<PROVIDER_KEY>`. # ℹ️ Settings are pulled from extractor args passed to yt-dlp with the key `youtubepot-<PROVIDER_KEY>`.
# For this example, the extractor arg would be `--extractor-args "youtubepot-mypotokenprovider:url=https://custom.example.com/get_pot"` # For this example, the extractor arg would be:
external_provider_url = self._configuration_arg('url', default=['https://provider.example.com/get_pot'])[0] # `--extractor-args "youtubepot-mypotokenprovider:url=https://custom.example.com/get_pot"`
external_provider_url = self._configuration_arg(
'url', default=['https://provider.example.com/get_pot'])[0]
# See below for logging guidelines # See below for logging guidelines
self.logger.trace(f'Using external provider URL: {external_provider_url}') self.logger.trace(f'Using external provider URL: {external_provider_url}')
@ -114,12 +119,15 @@ class MyPoTokenProviderPTP(PoTokenProvider): # Provider name must end with "PTP
'do_not_cache': request.bypass_cache, 'do_not_cache': request.bypass_cache,
}).encode(), proxies={'all': None}), }).encode(), proxies={'all': None}),
pot_request=request, pot_request=request,
note=f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider', note=(
f'Requesting {request.context.value} PO Token '
f'for {request.internal_client_name} client from external provider'),
) )
except RequestError as e: except RequestError as e:
# ℹ️ If there is an error, raise PoTokenProviderError. # ℹ️ If there is an error, raise PoTokenProviderError.
# You can specify whether it is expected or not. If it is unexpected, the log will include a link to the bug report location (BUG_REPORT_LOCATION). # You can specify whether it is expected or not. If it is unexpected,
# the log will include a link to the bug report location (BUG_REPORT_LOCATION).
raise PoTokenProviderError( raise PoTokenProviderError(
'Networking error while fetching to get PO Token from external provider', 'Networking error while fetching to get PO Token from external provider',
expected=True expected=True
@ -153,18 +161,26 @@ def my_provider_preference(provider: PoTokenProvider, request: PoTokenRequest) -
## Logging Guidelines ## Logging Guidelines
- Use the `self.logger` object to log messages. - Use the `self.logger` object to log messages.
- When making HTTP requests, use `self.logger.info` to log a message to standard non-verbose output. This lets users know what is happening when a time-expensive operation is taking place. - When making HTTP requests or any other expensive operation, use `self.logger.info` to log a message to standard non-verbose output.
- For example, `self.logger.info(f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider')` - This lets users know what is happening when a time-expensive operation is taking place.
- Use `self.logger.debug` to log a message to the verbose output (`--verbose`). Try to keep this to a minimum. - It is recommended to include the PO Token context and internal client name in the message if possible.
- Use `self.logger.trace` to log a message to the PO Token debug output (`--extractor-args "youtube:pot_debug=true"`). Log as much as you like here as needed for debugging your provider. - For example, `self.logger.info(f'Requesting {request.context.value} PO Token for {request.internal_client_name} client from external provider')`.
- Use `self.logger.debug` to log a message to the verbose output (`--verbose`).
- For debugging information visible to users posting verbose logs.
- Try to not log too much, prefer using trace logging for detailed debug messages.
- Use `self.logger.trace` to log a message to the PO Token debug output (`--extractor-args "youtube:pot_debug=true"`).
- Log as much as you like here as needed for debugging your provider.
- Avoid logging PO Tokens or any sensitive information to debug or info output. - Avoid logging PO Tokens or any sensitive information to debug or info output.
## Debugging ## Debugging
- Use `-v --extractor-args "youtube:pot_debug=true"` to enable PO Token debug output. - Use `-v --extractor-args "youtube:pot_trace=true"` to enable PO Token debug output.
## Caching ## Caching
> [!WARNING]
> The following describes more advance features that most users/developers will not need to use.
> [!IMPORTANT] > [!IMPORTANT]
> yt-dlp currently has a built-in LRU Memory Cache Provider and a cache spec provider for WebPO Tokens. > yt-dlp currently has a built-in LRU Memory Cache Provider and a cache spec provider for WebPO Tokens.
> You should only need to implement cache providers if you want an external cache, or a cache spec if you are handling non-WebPO Tokens. > You should only need to implement cache providers if you want an external cache, or a cache spec if you are handling non-WebPO Tokens.

45
yt_dlp/extractor/youtube/pot/provider.py
View File

@ -110,12 +110,14 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'):
# Innertube Client Name. # Innertube Client Name.
# For example, "WEB", "ANDROID", "TVHTML5". # For example, "WEB", "ANDROID", "TVHTML5".
# For a list of WebPO client names, see yt_dlp.extractor.youtube.pot._builtin.utils.WEBPO_CLIENTS. # For a list of WebPO client names, see yt_dlp.extractor.youtube.pot.utils.WEBPO_CLIENTS.
# Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS for a list of client names currently supported by the YouTube extractor. # Also see yt_dlp.extractor.youtube._base.INNERTUBE_CLIENTS
# for a list of client names currently supported by the YouTube extractor.
_SUPPORTED_CLIENTS: tuple[str] | None = () _SUPPORTED_CLIENTS: tuple[str] | None = ()
# If making external requests to websites (i.e. to youtube.com) using another library or service (i.e., not _request_webpage), # If making external requests to websites (i.e. to youtube.com)
# add the request features that are supported. # using another library or service (i.e., not _request_webpage),
# add the request features that are supported.
# If only using _request_webpage to make external requests, set this to None. # If only using _request_webpage to make external requests, set this to None.
_SUPPORTED_EXTERNAL_REQUEST_FEATURES: tuple[ExternalRequestFeature] | None = () _SUPPORTED_EXTERNAL_REQUEST_FEATURES: tuple[ExternalRequestFeature] | None = ()
@ -124,14 +126,20 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'):
raise PoTokenProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available') raise PoTokenProviderRejectedRequest(f'{self.PROVIDER_NAME} is not available')
# Validate request using built-in settings # Validate request using built-in settings
if self._SUPPORTED_CONTEXTS is not None and request.context not in self._SUPPORTED_CONTEXTS: if (
raise PoTokenProviderRejectedRequest(f'PO Token Context "{request.context}" is not supported by {self.PROVIDER_NAME}') self._SUPPORTED_CONTEXTS is not None
and request.context not in self._SUPPORTED_CONTEXTS
):
raise PoTokenProviderRejectedRequest(
f'PO Token Context "{request.context}" is not supported by {self.PROVIDER_NAME}')
if self._SUPPORTED_CLIENTS is not None: if self._SUPPORTED_CLIENTS is not None:
client_name = traverse_obj(request.innertube_context, ('client', 'clientName')) client_name = traverse_obj(
request.innertube_context, ('client', 'clientName'))
if client_name not in self._SUPPORTED_CLIENTS: if client_name not in self._SUPPORTED_CLIENTS:
raise PoTokenProviderRejectedRequest( raise PoTokenProviderRejectedRequest(
f'Client "{client_name}" is not supported by {self.PROVIDER_NAME}. Supported clients: {", ".join(self._SUPPORTED_CLIENTS) or "none"}') f'Client "{client_name}" is not supported by {self.PROVIDER_NAME}. '
f'Supported clients: {", ".join(self._SUPPORTED_CLIENTS) or "none"}')
self.__validate_external_request_features(request) self.__validate_external_request_features(request)
@ -158,15 +166,25 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'):
scheme = urllib.parse.urlparse(request.request_proxy).scheme scheme = urllib.parse.urlparse(request.request_proxy).scheme
if scheme.lower() not in self._supported_proxy_schemes: if scheme.lower() not in self._supported_proxy_schemes:
raise PoTokenProviderRejectedRequest( raise PoTokenProviderRejectedRequest(
f'External requests by "{self.PROVIDER_NAME}" provider do not support proxy scheme "{scheme}". Supported proxy schemes: {", ".join(self._supported_proxy_schemes) or "none"}') f'External requests by "{self.PROVIDER_NAME}" provider do not '
f'support proxy scheme "{scheme}". Supported proxy schemes: '
f'{", ".join(self._supported_proxy_schemes) or "none"}')
if request.request_source_address and ExternalRequestFeature.SOURCE_ADDRESS not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES: if (
request.request_source_address
and ExternalRequestFeature.SOURCE_ADDRESS not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES
):
raise PoTokenProviderRejectedRequest( raise PoTokenProviderRejectedRequest(
f'External requests by "{self.PROVIDER_NAME}" provider do not support setting source address') f'External requests by "{self.PROVIDER_NAME}" provider '
f'do not support setting source address')
if not request.request_verify_tls and ExternalRequestFeature.DISABLE_TLS_VERIFICATION not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES: if (
not request.request_verify_tls
and ExternalRequestFeature.DISABLE_TLS_VERIFICATION not in self._SUPPORTED_EXTERNAL_REQUEST_FEATURES
):
raise PoTokenProviderRejectedRequest( raise PoTokenProviderRejectedRequest(
f'External requests by "{self.PROVIDER_NAME}" provider do not support ignoring TLS certificate failures') f'External requests by "{self.PROVIDER_NAME}" provider '
f'do not support ignoring TLS certificate failures')
def request_pot(self, request: PoTokenRequest) -> PoTokenResponse: def request_pot(self, request: PoTokenRequest) -> PoTokenResponse:
self.__validate_request(request) self.__validate_request(request)
@ -198,7 +216,6 @@ class PoTokenProvider(IEContentProvider, abc.ABC, suffix='PTP'):
# Merge some ctx request settings into the request # Merge some ctx request settings into the request
# Most of these will already be used by the configured ydl instance, # Most of these will already be used by the configured ydl instance,
# however, the YouTube extractor may override some. # however, the YouTube extractor may override some.
# TODO: add _Request_webpage to IeContentProvider
if pot_request is not None: if pot_request is not None:
req.headers = HTTPHeaderDict(pot_request.request_headers, req.headers) req.headers = HTTPHeaderDict(pot_request.request_headers, req.headers)
req.proxies = req.proxies or ({'all': pot_request.request_proxy} if pot_request.request_proxy else {}) req.proxies = req.proxies or ({'all': pot_request.request_proxy} if pot_request.request_proxy else {})

2
yt_dlp/extractor/youtube/pot/utils.py
View File

@ -59,7 +59,7 @@ def get_webpo_content_binding(
def _extract_visitor_id(visitor_data): def _extract_visitor_id(visitor_data):
if not visitor_data: if not visitor_data:
return return None
# Attempt to extract the visitor ID from the visitor_data protobuf # Attempt to extract the visitor ID from the visitor_data protobuf
# xxx: ideally should use a protobuf parser # xxx: ideally should use a protobuf parser