Compare commits

..

No commits in common. "2c28ee5d76d2c0d350407fd81dbdd71394b67993" and "a4acc4223289eeb4d32af7b798bfe6e9c38f4b8d" have entirely different histories.

5 changed files with 49 additions and 411 deletions

View File

@ -183,7 +183,7 @@ jobs:
run: | run: |
sudo apt -y install pandoc man sudo apt -y install pandoc man
python -m pip install -U --require-hashes -r "bundle/requirements/requirements-pip.txt" python -m pip install -U --require-hashes -r "bundle/requirements/requirements-pip.txt"
python -m pip install -U --require-hashes -r "bundle/requirements/requirements-build.txt" python -m pip install -U --require-hashes -r "bundle/requirements/requirements-pypi-build.txt"
- name: Prepare - name: Prepare
env: env:

View File

@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command: # This file was autogenerated by uv via the following command:
# uv export --no-python-downloads --no-progress --color=never --format=requirements.txt --frozen --refresh --no-emit-project --no-default-groups --group=build --output-file=bundle/requirements/requirements-build.txt # uv export --no-python-downloads --no-progress --color=never --format=requirements.txt --frozen --refresh --no-emit-project --no-default-groups --group=build --output-file=bundle/requirements/requirements-pypi-build.txt
build==1.4.2 \ build==1.4.2 \
--hash=sha256:35b14e1ee329c186d3f08466003521ed7685ec15ecffc07e68d706090bf161d1 \ --hash=sha256:35b14e1ee329c186d3f08466003521ed7685ec15ecffc07e68d706090bf161d1 \
--hash=sha256:7a4d8651ea877cb2a89458b1b198f2e69f536c95e89129dbf5d448045d60db88 --hash=sha256:7a4d8651ea877cb2a89458b1b198f2e69f536c95e89129dbf5d448045d60db88

View File

@ -8,16 +8,11 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import collections.abc import collections.abc
import contextlib
import dataclasses import dataclasses
import hashlib import hashlib
import io import io
import itertools
import json
import pathlib import pathlib
import re import re
import typing
import urllib.parse
import zipfile import zipfile
from devscripts.tomlparse import parse_toml from devscripts.tomlparse import parse_toml
@ -47,12 +42,6 @@ PINNED_EXTRAS = {
'pin-deno': 'deno', 'pin-deno': 'deno',
} }
WELLKNOWN_PACKAGES = {
'deno': {'owner': 'denoland', 'repo': 'deno'},
'protobug': {'owner': 'yt-dlp', 'repo': 'protobug'},
'yt-dlp-ejs': {'owner': 'yt-dlp', 'repo': 'ejs'},
}
EJS_ASSETS = { EJS_ASSETS = {
'yt.solver.lib.js': False, 'yt.solver.lib.js': False,
'yt.solver.lib.min.js': False, 'yt.solver.lib.min.js': False,
@ -120,88 +109,26 @@ PYINSTALLER_BUILDS_TARGETS = {
'win-arm64-pyinstaller': 'win_arm64', 'win-arm64-pyinstaller': 'win_arm64',
} }
PYINSTALLER_BUILDS_URL = 'https://api.github.com/repos/yt-dlp/Pyinstaller-Builds/releases/latest'
PYINSTALLER_BUILDS_TMPL = '''\ PYINSTALLER_BUILDS_TMPL = '''\
{}pyinstaller @ {} \\ {}pyinstaller @ {} \\
--hash={} --hash={}
''' '''
PYINSTALLER_VERSION_RE = re.compile(r'pyinstaller-(?P<version>[0-9]+\.[0-9]+\.[0-9]+)-')
def call_pypi_api(project: str) -> dict[str, dict[str, typing.Any]]:
print(f'Fetching package info from PyPI API: {project}', file=sys.stderr)
headers = {
'Accept': 'application/json',
'User-Agent': 'yt-dlp',
}
with request(f'https://pypi.org/pypi/{project}/json', headers=headers) as resp:
return json.load(resp)
def fetch_latest_github_release(owner: str, repo: str) -> dict[str, dict[str, typing.Any]]:
print(f'Fetching latest release from Github API: {owner}/{repo}', file=sys.stderr)
return call_github_api(f'/repos/{owner}/{repo}/releases/latest')
def fetch_github_tags(
owner: str,
repo: str,
*,
fetch_tags: list[str] | None = None,
) -> dict[str, dict[str, typing.Any]]:
needed_tags = set(fetch_tags or [])
results = {}
for page in itertools.count(1):
print(f'Fetching tags list page {page} from Github API: {owner}/{repo}', file=sys.stderr)
tags = call_github_api(
f'/repos/{owner}/{repo}/tags',
query={'per_page': '100', 'page': page})
if not tags:
break
if not fetch_tags:
# Fetch all tags
results.update({tag['name']: tag for tag in tags})
continue
for tag in tags:
clean_tag = tag['name'].removeprefix('v')
possible_matches = {tag['name'], clean_tag}
# Normalize calver tags like 2024.01.01 to 2024.1.1
with contextlib.suppress(ValueError):
possible_matches.add('.'.join(map(str, map(int, clean_tag.split('.')))))
for name in possible_matches:
if name in needed_tags:
needed_tags.remove(name)
results[name] = tag
break # from inner loop
if not needed_tags:
break
if not needed_tags:
break
return results
def generate_table_lines( def generate_table_lines(
table_name: str, table_name: str,
table: dict[str, str | bool | int | float | list[str | dict[str, str]]], table: dict[str, str | list[str | dict[str, str]]],
) -> collections.abc.Iterator[str]: ) -> collections.abc.Iterator[str]:
SUPPORTED_TYPES = (str, bool, int, float, list)
yield f'[{table_name}]\n' yield f'[{table_name}]\n'
for name, value in table.items(): for name, value in table.items():
if not isinstance(value, SUPPORTED_TYPES): assert isinstance(value, (str, list)), 'only string & array table values are supported'
raise TypeError(
f'expected {"/".join(t.__name__ for t in SUPPORTED_TYPES)} value, '
f'got {type(value).__name__}')
if not isinstance(value, list): if isinstance(value, str):
yield f'{name} = {json.dumps(value)}\n' yield f'{name} = "{value}"\n'
continue continue
yield f'{name} = [' yield f'{name} = ['
@ -210,7 +137,7 @@ def generate_table_lines(
for element in value: for element in value:
yield ' ' yield ' '
if isinstance(element, dict): if isinstance(element, dict):
yield '{ ' + ', '.join(f'{k} = {json.dumps(v)}' for k, v in element.items()) + ' }' yield '{ ' + ', '.join(f'{k} = "{v}"' for k, v in element.items()) + ' }'
else: else:
yield f'"{element}"' yield f'"{element}"'
yield ',\n' yield ',\n'
@ -221,7 +148,7 @@ def generate_table_lines(
def replace_table_in_pyproject( def replace_table_in_pyproject(
pyproject_text: str, pyproject_text: str,
table_name: str, table_name: str,
table: dict[str, str | bool | int | float | list[str | dict[str, str]]], table: dict[str, str | list[str | dict[str, str]]],
) -> collections.abc.Iterator[str]: ) -> collections.abc.Iterator[str]:
INSIDE = 1 INSIDE = 1
BEYOND = 2 BEYOND = 2
@ -242,7 +169,7 @@ def replace_table_in_pyproject(
def modify_and_write_pyproject( def modify_and_write_pyproject(
pyproject_text: str, pyproject_text: str,
table_name: str, table_name: str,
table: dict[str, str | bool | int | float | list[str | dict[str, str]]], table: dict[str, str | list[str | dict[str, str]]],
) -> None: ) -> None:
with PYPROJECT_PATH.open(mode='w') as f: with PYPROJECT_PATH.open(mode='w') as f:
f.writelines(replace_table_in_pyproject(pyproject_text, table_name, table)) f.writelines(replace_table_in_pyproject(pyproject_text, table_name, table))
@ -328,10 +255,7 @@ def makefile_variables(
if keys_only: if keys_only:
return variables return variables
if not all(arg is not None for arg in (version, name, digest, not filetypes or data)): assert all(arg is not None for arg in (version, name, digest, not filetypes or data))
raise ValueError(
'makefile_variables requires version, name, digest, '
f'{"and data, " if filetypes else ""}OR keys_only=True')
if filetypes: if filetypes:
with io.BytesIO(data) as buf, zipfile.ZipFile(buf) as zipf: with io.BytesIO(data) as buf, zipfile.ZipFile(buf) as zipf:
@ -347,11 +271,12 @@ def ejs_makefile_variables(**kwargs) -> dict[str, str | None]:
return makefile_variables('EJS', filetypes=['PY', 'JS'], **kwargs) return makefile_variables('EJS', filetypes=['PY', 'JS'], **kwargs)
def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]] | None: def update_ejs(verify: bool = False):
PACKAGE_NAME = 'yt-dlp-ejs' PACKAGE_NAME = 'yt-dlp-ejs'
PREFIX = f' "{PACKAGE_NAME}==' PREFIX = f' "{PACKAGE_NAME}=='
LIBRARY_NAME = PACKAGE_NAME.replace('-', '_') LIBRARY_NAME = PACKAGE_NAME.replace('-', '_')
PACKAGE_PATH = BASE_PATH / 'yt_dlp/extractor/youtube/jsc/_builtin/vendor' PACKAGE_PATH = BASE_PATH / 'yt_dlp/extractor/youtube/jsc/_builtin/vendor'
RELEASE_URL = 'https://api.github.com/repos/yt-dlp/ejs/releases/latest'
current_version = None current_version = None
with PYPROJECT_PATH.open() as file: with PYPROJECT_PATH.open() as file:
@ -361,7 +286,8 @@ def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]]
current_version, _, _ = line.removeprefix(PREFIX).partition('"') current_version, _, _ = line.removeprefix(PREFIX).partition('"')
if not current_version: if not current_version:
raise ValueError(f'{PACKAGE_NAME} dependency line could not be found') print(f'{PACKAGE_NAME} dependency line could not be found')
return
makefile_info = ejs_makefile_variables(keys_only=True) makefile_info = ejs_makefile_variables(keys_only=True)
prefixes = tuple(f'{key} = ' for key in makefile_info) prefixes = tuple(f'{key} = ' for key in makefile_info)
@ -372,13 +298,13 @@ def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]]
key, _, val = line.partition(' = ') key, _, val = line.partition(' = ')
makefile_info[key] = val.rstrip() makefile_info[key] = val.rstrip()
info = fetch_latest_github_release('yt-dlp', 'ejs') info = call_github_api(RELEASE_URL)
version = info['tag_name'] version = info['tag_name']
if version == current_version: if version == current_version:
print(f'{PACKAGE_NAME} is up to date! ({version})', file=sys.stderr) print(f'{PACKAGE_NAME} is up to date! ({version})')
return return
print(f'Updating {PACKAGE_NAME} from {current_version} to {version}', file=sys.stderr) print(f'Updating {PACKAGE_NAME} from {current_version} to {version}')
hashes = [] hashes = []
wheel_info = {} wheel_info = {}
for asset in info['assets']: for asset in info['assets']:
@ -395,8 +321,7 @@ def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]]
# verify digest from github # verify digest from github
algo, _, expected = digest.partition(':') algo, _, expected = digest.partition(':')
hexdigest = hashlib.new(algo, data).hexdigest() hexdigest = hashlib.new(algo, data).hexdigest()
if hexdigest != expected: assert hexdigest == expected, f'downloaded attest mismatch ({hexdigest!r} != {expected!r})'
raise ValueError(f'downloaded attest mismatch ({hexdigest!r} != {expected!r})')
if is_wheel: if is_wheel:
wheel_info = ejs_makefile_variables(version=version, name=name, digest=digest, data=data) wheel_info = ejs_makefile_variables(version=version, name=name, digest=digest, data=data)
@ -410,11 +335,10 @@ def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]]
(PACKAGE_PATH / name).write_bytes(data) (PACKAGE_PATH / name).write_bytes(data)
hash_mapping = '\n'.join(hashes) hash_mapping = '\n'.join(hashes)
if missing_assets := [asset_name for asset_name in EJS_ASSETS if asset_name not in hash_mapping]: for asset_name in EJS_ASSETS:
raise ValueError(f'asset(s) not found in release: {", ".join(missing_assets)}') assert asset_name in hash_mapping, f'{asset_name} not found in release'
if missing_fields := [key for key in makefile_info if not wheel_info.get(key)]: assert all(wheel_info.get(key) for key in makefile_info), 'wheel info not found in release'
raise ValueError(f'wheel info not found in release: {", ".join(missing_fields)}')
(PACKAGE_PATH / '_info.py').write_text(EJS_TEMPLATE.format( (PACKAGE_PATH / '_info.py').write_text(EJS_TEMPLATE.format(
version=version, version=version,
@ -430,120 +354,10 @@ def update_ejs(verify: bool = False) -> dict[str, tuple[str | None, str | None]]
makefile = makefile.replace(f'{key} = {makefile_info[key]}', f'{key} = {wheel_info[key]}') makefile = makefile.replace(f'{key} = {makefile_info[key]}', f'{key} = {wheel_info[key]}')
MAKEFILE_PATH.write_text(makefile) MAKEFILE_PATH.write_text(makefile)
return update_requirements(upgrade_only=PACKAGE_NAME, verify=verify) update_requirements(upgrade_only=PACKAGE_NAME, verify=verify)
@dataclasses.dataclass def update_requirements(upgrade_only: str | None = None, verify: bool = False):
class Dependency:
name: str
exact_version: str | None
direct_reference: str | None
specifier: str | None
markers: str | None
def parse_version_from_dist(filename: str, name: str, *, require: bool = False) -> str | None:
# Ref: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode
normalized_name = re.sub(r'[-_.]+', '-', name).lower().replace('-', '_')
# Ref: https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers
if mobj := re.match(rf'{normalized_name}-(?P<version>[^-]+)-', filename):
return mobj.group('version')
if require:
raise ValueError(f'unable to parse version from distribution filename: {filename}')
return None
def parse_dependency(line: str, *, require_version: bool = False) -> Dependency:
# Ref: https://packaging.python.org/en/latest/specifications/name-normalization/
NAME_RE = re.compile(r'^(?P<name>[A-Z0-9](?:[A-Z0-9._-]*[A-Z0-9])?)', re.IGNORECASE)
line = line.rstrip().removesuffix('\\')
mobj = NAME_RE.match(line)
if not mobj:
raise ValueError(f'unable to parse Dependency.name from line:\n {line}')
name = mobj.group('name')
rest = line[len(name):].lstrip()
specifier_or_direct_reference, _, markers = map(str.strip, rest.partition(';'))
specifier, _, direct_reference = map(str.strip, specifier_or_direct_reference.partition('@'))
exact_version = None
if ',' not in specifier and specifier.startswith('=='):
exact_version = specifier[2:]
# Ref: https://packaging.python.org/en/latest/specifications/binary-distribution-format/
if direct_reference and not exact_version:
filename = urllib.parse.urlparse(direct_reference).path.rpartition('/')[2]
if filename.endswith(('.tar.gz', '.whl')):
exact_version = parse_version_from_dist(filename, name)
if require_version and not exact_version:
raise ValueError(f'unable to parse Dependency.exact_version from line:\n {line}')
return Dependency(
name=name,
exact_version=exact_version,
direct_reference=direct_reference or None,
specifier=specifier or None,
markers=markers or None)
def package_diff_dict(
old_dict: dict[str, str],
new_dict: dict[str, str],
) -> dict[str, tuple[str | None, str | None]]:
"""
@param old_dict: Dictionary w/ package names as keys and old package versions as values
@param new_dict: Dictionary w/ package names as keys and new package versions as values
@returns Dictionary w/ package names as keys and tuples of (old_ver, new_ver) as values
"""
ret_dict = {}
for name, new_version in new_dict.items():
if name not in old_dict:
ret_dict[name] = (None, new_version)
continue
old_version = old_dict[name]
if new_version != old_version:
ret_dict[name] = (old_version, new_version)
for name, old_version in old_dict.items():
if name not in new_dict:
ret_dict[name] = (old_version, None)
return ret_dict
def get_lock_packages(lock: dict[str, typing.Any]) -> dict[str, str]:
return {
package['name']: package['version']
for package in lock['package']
if package.get('version')
}
def evaluate_requirements_txt(old_txt: str, new_txt: str) -> dict[str, tuple[str | None, str | None]]:
old_dict = {}
new_dict = {}
for txt, dct in [(old_txt, old_dict), (new_txt, new_dict)]:
for line in txt.splitlines():
if not line.strip() or line.startswith(('#', ' ')):
continue
dep = parse_dependency(line, require_version=True)
dct.update({dep.name: dep.exact_version})
return package_diff_dict(old_dict, new_dict)
def update_requirements(
upgrade_only: str | None = None,
verify: bool = False,
) -> dict[str, tuple[str | None, str | None]]:
# Are we upgrading all packages or only one (e.g. 'yt-dlp-ejs' or 'protobug')? # Are we upgrading all packages or only one (e.g. 'yt-dlp-ejs' or 'protobug')?
upgrade_arg = f'--upgrade-package={upgrade_only}' if upgrade_only else '--upgrade' upgrade_arg = f'--upgrade-package={upgrade_only}' if upgrade_only else '--upgrade'
@ -558,50 +372,30 @@ def update_requirements(
# Write an intermediate pyproject.toml to use for generating lockfile and bundle requirements # Write an intermediate pyproject.toml to use for generating lockfile and bundle requirements
modify_and_write_pyproject(pyproject_text, table_name=EXTRAS_TABLE, table=extras) modify_and_write_pyproject(pyproject_text, table_name=EXTRAS_TABLE, table=extras)
old_lock = None
if LOCKFILE_PATH.is_file():
old_lock = parse_toml(LOCKFILE_PATH.read_text())
# If verifying, set UV_EXCLUDE_NEWER env var with the last timestamp recorded in uv.lock # If verifying, set UV_EXCLUDE_NEWER env var with the last timestamp recorded in uv.lock
env = None env = None
if verify or upgrade_only in pyproject_toml['tool']['uv']['exclude-newer-package']: if verify:
env = os.environ.copy() env = os.environ.copy()
env['UV_EXCLUDE_NEWER'] = old_lock['options']['exclude-newer'] env['UV_EXCLUDE_NEWER'] = parse_toml(LOCKFILE_PATH.read_text())['options']['exclude-newer']
print(f'Setting UV_EXCLUDE_NEWER={env["UV_EXCLUDE_NEWER"]}', file=sys.stderr)
# Generate/upgrade lockfile # Generate/upgrade lockfile
print(f'Running: uv lock {upgrade_arg}', file=sys.stderr)
run_process('uv', 'lock', upgrade_arg, env=env) run_process('uv', 'lock', upgrade_arg, env=env)
# Record diff in uv.lock packages # Generate bundle requirements
old_packages = get_lock_packages(old_lock) if old_lock else {}
new_packages = get_lock_packages(parse_toml(LOCKFILE_PATH.read_text()))
all_updates = package_diff_dict(old_packages, new_packages)
# Update Windows PyInstaller requirements; need to compare before & after .txt's for reporting
if not upgrade_only or upgrade_only.lower() == 'pyinstaller': if not upgrade_only or upgrade_only.lower() == 'pyinstaller':
info = fetch_latest_github_release('yt-dlp', 'Pyinstaller-Builds') info = call_github_api(PYINSTALLER_BUILDS_URL)
for target_suffix, asset_tag in PYINSTALLER_BUILDS_TARGETS.items(): for target_suffix, asset_tag in PYINSTALLER_BUILDS_TARGETS.items():
asset_info = next(asset for asset in info['assets'] if asset_tag in asset['name']) asset_info = next(asset for asset in info['assets'] if asset_tag in asset['name'])
pyinstaller_version = parse_version_from_dist( pyinstaller_version = PYINSTALLER_VERSION_RE.match(asset_info['name']).group('version')
asset_info['name'], 'pyinstaller', require=True)
pyinstaller_builds_deps = run_pip_compile( pyinstaller_builds_deps = run_pip_compile(
'--no-emit-package=pyinstaller', '--no-emit-package=pyinstaller',
upgrade_arg, upgrade_arg,
input_line=f'pyinstaller=={pyinstaller_version}', input_line=f'pyinstaller=={pyinstaller_version}',
env=env) env=env)
requirements_path = REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(target_suffix) requirements_path = REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(target_suffix)
if requirements_path.is_file(): requirements_path.write_text(PYINSTALLER_BUILDS_TMPL.format(
old_requirements_txt = requirements_path.read_text() pyinstaller_builds_deps, asset_info['browser_download_url'], asset_info['digest']))
else:
old_requirements_txt = ''
new_requirements_txt = PYINSTALLER_BUILDS_TMPL.format(
pyinstaller_builds_deps, asset_info['browser_download_url'], asset_info['digest'])
requirements_path.write_text(new_requirements_txt)
all_updates.update(evaluate_requirements_txt(old_requirements_txt, new_requirements_txt))
# Export bundle requirements; any updates to these are already recorded w/ uv.lock package diff
for target_suffix, target in BUNDLE_TARGETS.items(): for target_suffix, target in BUNDLE_TARGETS.items():
run_uv_export( run_uv_export(
extras=target.extras, extras=target.extras,
@ -610,150 +404,23 @@ def update_requirements(
omit_packages=target.omit_packages, omit_packages=target.omit_packages,
output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(target_suffix)) output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(target_suffix))
# Export group requirements; any updates to these are already recorded w/ uv.lock package diff run_uv_export(
for group in ('build',): groups=['build'],
run_uv_export( output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format('pypi-build'))
groups=[group],
output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(group))
# Compile requirements for single packages; need to compare before & after .txt's for reporting run_pip_compile(
for package in ('pip',): upgrade_arg,
requirements_path = REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(package) input_line='pip',
if requirements_path.is_file(): output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format('pip'),
old_requirements_txt = requirements_path.read_text() env=env)
else:
old_requirements_txt = ''
run_pip_compile( # Generate pinned extras
upgrade_arg,
input_line=package,
output_file=REQUIREMENTS_PATH / REQS_OUTPUT_TMPL.format(package),
env=env)
new_requirements_txt = requirements_path.read_text()
all_updates.update(evaluate_requirements_txt(old_requirements_txt, new_requirements_txt))
# Generate new pinned extras; any updates to these are already recorded w/ uv.lock package diff
for pinned_name, extra_name in PINNED_EXTRAS.items(): for pinned_name, extra_name in PINNED_EXTRAS.items():
extras[pinned_name] = run_uv_export(extras=[extra_name], bare=True).splitlines() extras[pinned_name] = run_uv_export(extras=[extra_name], bare=True).splitlines()
# Write the finalized pyproject.toml # Write the finalized pyproject.toml
modify_and_write_pyproject(pyproject_text, table_name=EXTRAS_TABLE, table=extras) modify_and_write_pyproject(pyproject_text, table_name=EXTRAS_TABLE, table=extras)
return all_updates
def generate_report(
all_updates: dict[str, tuple[str | None, str | None]],
) -> collections.abc.Iterator[str]:
GITHUB_RE = re.compile(r'https://github\.com/(?P<owner>[0-9a-zA-Z_-]+)/(?P<repo>[0-9a-zA-Z_-]+)')
yield 'package | old | new | diff | changelog'
yield '--------|-----|-----|------|----------'
for package, (old, new) in sorted(all_updates.items()):
if package in WELLKNOWN_PACKAGES:
github_info = WELLKNOWN_PACKAGES[package]
changelog = ''
else:
project_urls = call_pypi_api(package)['info']['project_urls']
github_info = next((
mobj.groupdict() for url in project_urls.values()
if (mobj := GITHUB_RE.match(url))), None)
changelog = next((
url for key, url in project_urls.items()
if key.lower().startswith(('change', 'history', 'release '))), '')
if changelog:
name = urllib.parse.urlparse(changelog).path.rstrip('/').rpartition('/')[2] or 'changelog'
changelog = f'[{name}](<{changelog}>)'
md_old = old = old or ''
md_new = new = new or ''
if old and new:
# bolden and italicize the differing parts
old_parts = old.split('.')
new_parts = new.split('.')
offset = None
for index, (old_part, new_part) in enumerate(zip(old_parts, new_parts, strict=False)):
if old_part != new_part:
offset = index
break
if offset is not None:
md_old = '.'.join(old_parts[:offset]) + '.***' + '.'.join(old_parts[offset:]) + '***'
md_new = '.'.join(new_parts[:offset]) + '.***' + '.'.join(new_parts[offset:]) + '***'
compare = ''
if github_info:
tags_info = fetch_github_tags(
github_info['owner'], github_info['repo'], fetch_tags=[old, new])
old_tag = tags_info.get(old) and tags_info[old]['name']
new_tag = tags_info.get(new) and tags_info[new]['name']
github_url = 'https://github.com/{owner}/{repo}'.format(**github_info)
if new_tag:
md_new = f'[{md_new}]({github_url}/releases/tag/{new_tag})'
if old_tag:
md_old = f'[{md_old}]({github_url}/releases/tag/{old_tag})'
if new_tag and old_tag:
compare = f'[`{old_tag}...{new_tag}`]({github_url}/compare/{old_tag}...{new_tag})'
yield ' | '.join((
f'[**`{package}`**](https://pypi.org/project/{package})',
md_old,
md_new,
compare,
changelog,
))
def make_pull_request_description(all_updates: dict[str, tuple[str | None, str | None]]) -> str:
return '\n'.join((
'<!-- BEGIN update_requirements generated section -->\n',
*generate_report(all_updates),
'\n<!-- END update_requirements generated section -->\n\n',
))
def make_commit_message(all_updates: dict[str, tuple[str | None, str | None]]) -> str:
return '\n\n'.join((
make_commit_title(all_updates),
make_commit_body(all_updates),
))
def make_commit_title(all_updates: dict[str, tuple[str | None, str | None]]) -> str:
count = len(all_updates)
return f'[build] Update {count} dependenc{"ies" if count > 1 else "y"}'
def make_commit_body(all_updates: dict[str, tuple[str | None, str | None]]) -> str:
return '\n'.join(make_commit_line(package, old, new) for package, (old, new) in all_updates.items())
def make_commit_line(package: str, old: str | None, new: str | None) -> str:
if old is None:
return f'* Add {package} {new}'
if new is None:
return f'* Remove {package} {old}'
return f'* Bump {package} {old} => {new}'
def table_a_raza(header: tuple[str, ...], rows: list[tuple[str, ...]]) -> collections.abc.Generator[str]:
widths = [len(col) for col in header]
for row in rows:
for index, (width, col) in enumerate(zip(widths, row, strict=True)):
if len(col) > width:
widths[index] = len(col)
yield ' | '.join(col.ljust(width) for width, col in zip(widths, header, strict=True))
yield '-|-'.join(''.ljust(width, '-') for width in widths)
for row in rows:
yield ' | '.join(col.ljust(width) for width, col in zip(widths, row, strict=True))
def parse_args(): def parse_args():
import argparse import argparse
@ -764,9 +431,6 @@ def parse_args():
parser.add_argument( parser.add_argument(
'--verify', action='store_true', '--verify', action='store_true',
help='only verify the update(s) using the previously recorded cooldown timestamp') help='only verify the update(s) using the previously recorded cooldown timestamp')
parser.add_argument(
'--no-markdown-reports', action='store_true',
help='do not generate markdown PR descriptions; avoids optional PyPI/GitHub API calls')
return parser.parse_args() return parser.parse_args()
@ -774,31 +438,10 @@ def main():
args = parse_args() args = parse_args()
if args.upgrade_only in ('ejs', 'yt-dlp-ejs'): if args.upgrade_only in ('ejs', 'yt-dlp-ejs'):
all_updates = update_ejs(verify=args.verify) update_ejs(verify=args.verify)
else: else:
all_updates = update_requirements(upgrade_only=args.upgrade_only, verify=args.verify) update_requirements(upgrade_only=args.upgrade_only, verify=args.verify)
if all_updates is None:
return 1
elif not all_updates:
print('All requirements are up-to-date', file=sys.stderr)
return 0
if args.verify:
print('Verification failed! Updates were made:', file=sys.stderr)
for row in table_a_raza(('package', 'old', 'new'), [
(package, old or '', new or '')
for package, (old, new) in all_updates.items()
]):
print(row)
return 1
else:
if not args.no_markdown_reports:
print(make_pull_request_description(all_updates))
print(make_commit_message(all_updates))
return 0
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) main()

View File

@ -43,10 +43,10 @@ class TwitchBaseIE(InfoExtractor):
_OPERATION_HASHES = { _OPERATION_HASHES = {
'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a', 'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a',
'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f', 'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f',
'ClipsCards__User': '1cd671bfa12cec480499c087319f26d21925e9695d1f80225aae6a4354f23088', 'ClipsCards__User': '90c33f5e6465122fba8f9371e2a97076f9ed06c6fed3788d002ab9eba8f91d88',
'ShareClipRenderStatus': '0a02bb974443b576f5579aab0fef1d4b7f44e58a8a256f0c5adfead0db70640f', 'ShareClipRenderStatus': '1844261bb449fa51e6167040311da4a7a5f1c34fe71c71a3e0c4f551bc30c698',
'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077', 'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077',
'StreamMetadata': 'ad022ca32220d5523d03a23cbcb5beaa1e0999889c1f8f78f9f2520dafb5cae6', 'StreamMetadata': 'b57f9b910f8cd1a4659d894fe7550ccc81ec9052c01e438b290fd66a040b9b93',
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01', 'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
'VideoPreviewOverlay': '9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f', 'VideoPreviewOverlay': '9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f',
'VideoMetadata': '45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d', 'VideoMetadata': '45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d',

View File

@ -6,7 +6,6 @@ import dataclasses
import datetime as dt import datetime as dt
import hashlib import hashlib
import json import json
import re
import traceback import traceback
import typing import typing
import urllib.parse import urllib.parse
@ -434,13 +433,9 @@ def provider_display_list(providers: Iterable[IEContentProvider]):
def clean_pot(po_token: str): def clean_pot(po_token: str):
# Clean and validate the PO Token. This will strip invalid characters off # Clean and validate the PO Token. This will strip invalid characters off
# (e.g. additional url params the user may accidentally include) # (e.g. additional url params the user may accidentally include)
mobj = re.match(r'([^?&#]+)', urllib.parse.unquote(po_token))
if not mobj:
raise ValueError('Invalid PO Token')
try: try:
return base64.urlsafe_b64encode( return base64.urlsafe_b64encode(
base64.urlsafe_b64decode(mobj.group(1))).decode() base64.urlsafe_b64decode(urllib.parse.unquote(po_token))).decode()
except (binascii.Error, ValueError): except (binascii.Error, ValueError):
raise ValueError('Invalid PO Token') raise ValueError('Invalid PO Token')