diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml index 3b0ef323d7..20e5e944fc 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml @@ -63,14 +63,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml index c8702c3569..4aeff7dc64 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml @@ -75,14 +75,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml index 5a6d2b0fbd..2f516ebb71 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml @@ -71,14 +71,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml index a17770f614..201586e9dc 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml @@ -56,14 +56,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml index c600a9dcb6..765de86a29 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml @@ -52,14 +52,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml index 57bc9daf51..198e21bec2 100644 --- a/.github/ISSUE_TEMPLATE/6_question.yml +++ b/.github/ISSUE_TEMPLATE/6_question.yml @@ -58,14 +58,15 @@ body: placeholder: | [debug] Command-line config: ['-vU', 'https://www.youtube.com/watch?v=BaW_jenozKc'] [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 - [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp [b634ba742] (win_exe) - [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 - [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 - [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] yt-dlp version nightly@... from yt-dlp/yt-dlp-nightly-builds [1a176d874] (win_exe) + [debug] Python 3.10.11 (CPython AMD64 64bit) - Windows-10-10.0.20348-SP0 (OpenSSL 1.1.1t 7 Feb 2023) + [debug] exe versions: ffmpeg 7.0.2 (setts), ffprobe 7.0.2 + [debug] Optional libraries: Cryptodome-3.21.0, brotli-1.1.0, certifi-2024.08.30, curl_cffi-0.5.10, mutagen-1.47.0, requests-2.32.3, sqlite3-3.40.1, urllib3-2.2.3, websockets-13.1 [debug] Proxy map: {} - [debug] Request Handlers: urllib, requests - [debug] Loaded 1893 extractors - [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp-nightly-builds/releases/latest + [debug] Request Handlers: urllib, requests, websockets, curl_cffi + [debug] Loaded 1838 extractors + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: nightly@... from yt-dlp/yt-dlp-nightly-builds yt-dlp is up to date (nightly@... from yt-dlp/yt-dlp-nightly-builds) [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d062d7720d..a211ae1652 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -411,7 +411,7 @@ jobs: run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds python devscripts/install_deps.py -o --include build python devscripts/install_deps.py --include curl-cffi - python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.10.0-py3-none-any.whl" + python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.11.1-py3-none-any.whl" - name: Prepare run: | @@ -460,7 +460,7 @@ jobs: run: | python devscripts/install_deps.py -o --include build python devscripts/install_deps.py - python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.10.0-py3-none-any.whl" + python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.11.1-py3-none-any.whl" - name: Prepare run: | @@ -504,7 +504,8 @@ jobs: - windows32 runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - name: Download artifacts + uses: actions/download-artifact@v4 with: path: artifact pattern: build-bin-* diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index c49319b171..78445e417e 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -28,3 +28,20 @@ jobs: actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit + + publish_pypi: + needs: [release] + if: vars.MASTER_PYPI_PROJECT != '' + runs-on: ubuntu-latest + permissions: + id-token: write # mandatory for trusted publishing + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: dist + name: build-pypi + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index b536c50669..8f72844058 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -41,3 +41,20 @@ jobs: actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit + + publish_pypi: + needs: [release] + if: vars.NIGHTLY_PYPI_PROJECT != '' + runs-on: ubuntu-latest + permissions: + id-token: write # mandatory for trusted publishing + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: dist + name: build-pypi + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bc09c64d0..26b93e429c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,6 @@ name: Release on: workflow_call: inputs: - prerelease: - required: false - default: true - type: boolean source: required: false default: '' @@ -18,6 +14,10 @@ on: required: false default: '' type: string + prerelease: + required: false + default: true + type: boolean workflow_dispatch: inputs: source: @@ -278,11 +278,20 @@ jobs: make clean-cache python -m build --no-isolation . + - name: Upload artifacts + if: github.event_name != 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: build-pypi + path: | + dist/* + compression-level: 0 + - name: Publish to PyPI + if: github.event_name == 'workflow_dispatch' uses: pypa/gh-action-pypi-publish@release/v1 with: verbose: true - attestations: false # Currently doesn't work w/ reusable workflows (breaks nightly) publish: needs: [prepare, build] diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 949bc89c47..a9a0557426 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -688,3 +688,10 @@ KarboniteKream mikkovedru pktiuk rubyevadestaxes +avagordon01 +CounterPillow +JoseAngelB +KBelmin +kesor +MellowKyler +Wesley107772 diff --git a/Changelog.md b/Changelog.md index 0efccadd10..2648b9fe22 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,62 @@ # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2024.11.04 + +#### Important changes +- **Beginning with this release, yt-dlp's Python dependencies *must* be installed using the `default` group** +If you're installing yt-dlp with pip/pipx or requiring yt-dlp in your own Python project, you'll need to specify `yt-dlp[default]` if you want to also install yt-dlp's optional dependencies (which were previously included by default). [Read more](https://github.com/yt-dlp/yt-dlp/pull/11255) +- **The minimum *required* Python version has been raised to 3.9** +Python 3.8 reached its end-of-life on 2024.10.07, and yt-dlp has now removed support for it. As an unfortunate side effect, the official `yt-dlp.exe` and `yt-dlp_x86.exe` binaries are no longer supported on Windows 7. [Read more](https://github.com/yt-dlp/yt-dlp/issues/10086) + +#### Core changes +- [Allow thumbnails with `.jpe` extension](https://github.com/yt-dlp/yt-dlp/commit/5bc5fb2835ea59bdf326bd12176d74d2c7348a95) ([#11408](https://github.com/yt-dlp/yt-dlp/issues/11408)) by [bashonly](https://github.com/bashonly) +- [Expand paths in `--plugin-dirs`](https://github.com/yt-dlp/yt-dlp/commit/914af9a0cf51c9a3f74aa88d952bee8334c67511) ([#11334](https://github.com/yt-dlp/yt-dlp/issues/11334)) by [bashonly](https://github.com/bashonly) +- [Fix `--netrc` empty string parsing for Python <=3.10](https://github.com/yt-dlp/yt-dlp/commit/88402b714ec124633933737bc156b172a3dec3d6) ([#11414](https://github.com/yt-dlp/yt-dlp/issues/11414)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) +- [Populate format sorting fields before dependent fields](https://github.com/yt-dlp/yt-dlp/commit/5c880ef42e9c2b2fc412f6d69dad37d34fb75a62) ([#11353](https://github.com/yt-dlp/yt-dlp/issues/11353)) by [Grub4K](https://github.com/Grub4K) +- [Prioritize AV1](https://github.com/yt-dlp/yt-dlp/commit/3945677a75e94a1fecc085432d791e1c21220cd3) ([#11153](https://github.com/yt-dlp/yt-dlp/issues/11153)) by [seproDev](https://github.com/seproDev) +- [Remove Python 3.8 support](https://github.com/yt-dlp/yt-dlp/commit/d784464399b600ba9516bbcec6286f11d68974dd) ([#11321](https://github.com/yt-dlp/yt-dlp/issues/11321)) by [bashonly](https://github.com/bashonly) +- **aes**: [Fix GCM pad length calculation](https://github.com/yt-dlp/yt-dlp/commit/beae2db127d3b5017cbcf685da9de7a9ef496541) ([#11438](https://github.com/yt-dlp/yt-dlp/issues/11438)) by [seproDev](https://github.com/seproDev) +- **cookies**: [Support chrome table version 24](https://github.com/yt-dlp/yt-dlp/commit/4613096f2e6eab9dcbac0e98b6cec760bbc99375) ([#11425](https://github.com/yt-dlp/yt-dlp/issues/11425)) by [kesor](https://github.com/kesor), [seproDev](https://github.com/seproDev) +- **utils** + - [Allow partial application for more functions](https://github.com/yt-dlp/yt-dlp/commit/b6dc2c49e8793c6dfa21275e61caf49ec1148b81) ([#11391](https://github.com/yt-dlp/yt-dlp/issues/11391)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) (With fixes in [422195e](https://github.com/yt-dlp/yt-dlp/commit/422195ec70a00b0d2002b238cacbae7790c57fdf) by [Grub4K](https://github.com/Grub4K)) + - [Fix `find_element` by class](https://github.com/yt-dlp/yt-dlp/commit/f93c16395cea1fe9ffc3c594d3e019c3b214544c) ([#11402](https://github.com/yt-dlp/yt-dlp/issues/11402)) by [bashonly](https://github.com/bashonly) + - [Fix and improve `find_element` and `find_elements`](https://github.com/yt-dlp/yt-dlp/commit/b103aca24d35b72b405c340357dc01a0ed534281) ([#11443](https://github.com/yt-dlp/yt-dlp/issues/11443)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) + +#### Extractor changes +- [Resolve `language` to ISO639-2 for ISM formats](https://github.com/yt-dlp/yt-dlp/commit/21cdcf03a237a0c4979c941d5a5385cae44c7906) ([#11359](https://github.com/yt-dlp/yt-dlp/issues/11359)) by [bashonly](https://github.com/bashonly) +- **ardmediathek**: [Extract chapters](https://github.com/yt-dlp/yt-dlp/commit/59f8dd8239c31f00b708da53b39b1e2e9409b6e6) ([#11442](https://github.com/yt-dlp/yt-dlp/issues/11442)) by [iw0nderhow](https://github.com/iw0nderhow) +- **bfmtv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/754940e9a558565d6bd3c0c529802569b1d0ae4e) ([#11444](https://github.com/yt-dlp/yt-dlp/issues/11444)) by [seproDev](https://github.com/seproDev) +- **bluesky**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/5c7a5aaab27e9c3cb367b663a6136ca58866e547) ([#11055](https://github.com/yt-dlp/yt-dlp/issues/11055)) by [MellowKyler](https://github.com/MellowKyler), [seproDev](https://github.com/seproDev) +- **ccma**: [Support new 3cat.cat domain](https://github.com/yt-dlp/yt-dlp/commit/330335386d4f7603d92d6796798375336005275e) ([#11222](https://github.com/yt-dlp/yt-dlp/issues/11222)) by [JoseAngelB](https://github.com/JoseAngelB) +- **chzzk**: video: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/9c6534da81e485b2325b3489ee4128943e6d3e4b) ([#11228](https://github.com/yt-dlp/yt-dlp/issues/11228)) by [hui1601](https://github.com/hui1601) +- **cnn**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/9acf79c91a8c6c55ca972747c6858e784e2da351) ([#10185](https://github.com/yt-dlp/yt-dlp/issues/10185)) by [kylegustavo](https://github.com/kylegustavo), [seproDev](https://github.com/seproDev) +- **dailymotion** + - [Improve embed extraction](https://github.com/yt-dlp/yt-dlp/commit/a403dcf9be20b49cbb3017328f4aaa352fb6d685) ([#10843](https://github.com/yt-dlp/yt-dlp/issues/10843)) by [bashonly](https://github.com/bashonly), [pzhlkj6612](https://github.com/pzhlkj6612) + - [Support shortened URLs](https://github.com/yt-dlp/yt-dlp/commit/d1358231371f20fa23020fa9176be3b56119873e) ([#11374](https://github.com/yt-dlp/yt-dlp/issues/11374)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) +- **facebook**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/ec9b25043f399de6a591d8370d32bf0e66c117f2) ([#11343](https://github.com/yt-dlp/yt-dlp/issues/11343)) by [kclauhk](https://github.com/kclauhk) +- **generic**: [Do not impersonate by default](https://github.com/yt-dlp/yt-dlp/commit/c29f5a7fae93a08f3cfbb6127b2faa75145b06a0) ([#11336](https://github.com/yt-dlp/yt-dlp/issues/11336)) by [bashonly](https://github.com/bashonly) +- **nfl**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/838f4385de8300a4dd4e7ffbbf0e5b7b85fb52c2) ([#11409](https://github.com/yt-dlp/yt-dlp/issues/11409)) by [bashonly](https://github.com/bashonly) +- **niconicouser**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6abef74232c0fc695cd803c18ae446cacb129389) ([#11324](https://github.com/yt-dlp/yt-dlp/issues/11324)) by [Wesley107772](https://github.com/Wesley107772) +- **soundcloud**: [Extract artists](https://github.com/yt-dlp/yt-dlp/commit/f101e5d34c97c608156ad5396714c2a2edca966a) ([#11377](https://github.com/yt-dlp/yt-dlp/issues/11377)) by [seproDev](https://github.com/seproDev) +- **tumblr**: [Support more URLs](https://github.com/yt-dlp/yt-dlp/commit/b03267bf0675eeb8df5baf1daac7cf67840c91a5) ([#6057](https://github.com/yt-dlp/yt-dlp/issues/6057)) by [selfisekai](https://github.com/selfisekai), [seproDev](https://github.com/seproDev) +- **twitter**: [Remove cookies migration workaround](https://github.com/yt-dlp/yt-dlp/commit/76802f461332d444e596437c42374fa237fa5174) ([#11392](https://github.com/yt-dlp/yt-dlp/issues/11392)) by [bashonly](https://github.com/bashonly) +- **vimeo**: [Fix API retries](https://github.com/yt-dlp/yt-dlp/commit/57212a5f97ce367590aaa5c3e9a135eead8f81f7) ([#11351](https://github.com/yt-dlp/yt-dlp/issues/11351)) by [bashonly](https://github.com/bashonly) +- **yle_areena**: [Support live events](https://github.com/yt-dlp/yt-dlp/commit/a6783a3b9905e547f6c1d4df9d7c7999feda8afa) ([#11358](https://github.com/yt-dlp/yt-dlp/issues/11358)) by [bashonly](https://github.com/bashonly), [CounterPillow](https://github.com/CounterPillow) +- **youtube**: [Adjust OAuth refresh token handling](https://github.com/yt-dlp/yt-dlp/commit/d569a8845254d90ce13ad74ae76695e8d6441068) ([#11414](https://github.com/yt-dlp/yt-dlp/issues/11414)) by [bashonly](https://github.com/bashonly) + +#### Misc. changes +- **build** + - [Disable attestations for trusted publishing](https://github.com/yt-dlp/yt-dlp/commit/428ffb75aa3534b275cf54de42693a4d261519da) ([#11418](https://github.com/yt-dlp/yt-dlp/issues/11418)) by [bashonly](https://github.com/bashonly) + - [Move optional dependencies to the `default` group](https://github.com/yt-dlp/yt-dlp/commit/87884f15580910e4e0fe0e1db73508debc657471) ([#11255](https://github.com/yt-dlp/yt-dlp/issues/11255)) by [bashonly](https://github.com/bashonly) + - [Use Ubuntu 20.04 and Python 3.9 for Linux ARM builds](https://github.com/yt-dlp/yt-dlp/commit/dd2e24446954246a2ec4d4a7e95531f52a14b351) ([#8638](https://github.com/yt-dlp/yt-dlp/issues/8638)) by [bashonly](https://github.com/bashonly) +- **cleanup** + - Miscellaneous + - [ea9e35d](https://github.com/yt-dlp/yt-dlp/commit/ea9e35d85fba5eab341cdcaf1eaed69b57f7e465) by [bashonly](https://github.com/bashonly) + - [c998238](https://github.com/yt-dlp/yt-dlp/commit/c998238c2e76c62d1d29962c6e8ebe916cc7913b) by [bashonly](https://github.com/bashonly), [KBelmin](https://github.com/KBelmin) + - [197d0b0](https://github.com/yt-dlp/yt-dlp/commit/197d0b03b6a3c8fe4fa5ace630eeffec629bf72c) by [avagordon01](https://github.com/avagordon01), [bashonly](https://github.com/bashonly), [grqz](https://github.com/grqz), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) +- **devscripts**: `make_changelog`: [Parse full commit message for fixes](https://github.com/yt-dlp/yt-dlp/commit/0a3991edae0e10f2ea41ece9fdea5e48f789f1de) ([#11366](https://github.com/yt-dlp/yt-dlp/issues/11366)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) + ### 2024.10.22 #### Important changes diff --git a/README.md b/README.md index 103256f8ae..2df72b7498 100644 --- a/README.md +++ b/README.md @@ -479,7 +479,8 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git --no-download-archive Do not use archive file (default) --max-downloads NUMBER Abort after downloading NUMBER files --break-on-existing Stop the download process when encountering - a file that is in the archive + a file that is in the archive supplied with + the --download-archive option --no-break-on-existing Do not stop the download process when encountering a file that is in the archive (default) diff --git a/pyproject.toml b/pyproject.toml index 2e62e58dc4..92d399e319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ default = [ "pycryptodomex", "requests>=2.32.2,<3", "urllib3>=1.26.17,<3", - "websockets>=13.0", + "websockets>=13.0,<14", ] curl-cffi = [ "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'", @@ -83,7 +83,7 @@ test = [ "pytest-rerunfailures~=14.0", ] pyinstaller = [ - "pyinstaller>=6.10.0", # Windows temp cleanup fixed in 6.10.0 + "pyinstaller>=6.11.1", # Windows temp cleanup fixed in 6.11.1 ] [project.urls] diff --git a/supportedsites.md b/supportedsites.md index 7b22e8c6fa..fc79e4ae61 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -190,6 +190,7 @@ - **blerp** - **blogger.com** - **Bloomberg** + - **Bluesky** - **BokeCC** - **BongaCams** - **Boosty** @@ -247,7 +248,7 @@ - **cbsnews:livevideo**: CBS News Live Videos - **cbssports**: (**Currently broken**) - **cbssports:embed**: (**Currently broken**) - - **CCMA** + - **CCMA**: 3Cat, TV3 and Catalunya Ràdio - **CCTV**: 央视网 - **CDA**: [*cdapl*](## "netrc machine") - **CDAFolder** @@ -280,8 +281,6 @@ - **cmt.com**: (**Currently broken**) - **CNBCVideo** - **CNN** - - **CNNArticle** - - **CNNBlogs** - **CNNIndonesia** - **ComedyCentral** - **ComedyCentralTV** @@ -685,9 +684,9 @@ - **LastFMPlaylist** - **LastFMUser** - **LaXarxaMes**: [*laxarxames*](## "netrc machine") - - **lbry** - - **lbry:channel** - - **lbry:playlist** + - **lbry**: odysee.com + - **lbry:channel**: odysee.com channels + - **lbry:playlist**: odysee.com playlists - **LCI** - **Lcp** - **LcpPlay** @@ -1446,7 +1445,7 @@ - **TeleQuebecSquat** - **TeleQuebecVideo** - **TeleTask**: (**Currently broken**) - - **Telewebion** + - **Telewebion**: (**Currently broken**) - **Tempo** - **TennisTV**: [*tennistv*](## "netrc machine") - **TenPlay**: [*10play*](## "netrc machine") diff --git a/test/test_cookies.py b/test/test_cookies.py index e1271f67eb..4b9b9b5a91 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -105,6 +105,13 @@ class TestCookies(unittest.TestCase): decryptor = LinuxChromeCookieDecryptor('Chrome', Logger()) self.assertEqual(decryptor.decrypt(encrypted_value), value) + def test_chrome_cookie_decryptor_linux_v10_meta24(self): + with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}): + encrypted_value = b'v10\x1f\xe4\x0e[\x83\x0c\xcc*kPi \xce\x8d\x1d\xbb\x80\r\x11\t\xbb\x9e^Hy\x94\xf4\x963\x9f\x82\xba\xfe\xa1\xed\xb9\xf1)\x00710\x92\xc8/<\x96B' + value = 'DE' + decryptor = LinuxChromeCookieDecryptor('Chrome', Logger(), meta_version=24) + self.assertEqual(decryptor.decrypt(encrypted_value), value) + def test_chrome_cookie_decryptor_windows_v10(self): with MonkeyPatch(cookies, { '_get_windows_v10_key': lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2= 24) elif version == b'v11': self._cookie_counts['v11'] += 1 if self._v11_key is None: self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True) return None - return _decrypt_aes_cbc_multi(ciphertext, (self._v11_key, self._empty_key), self._logger) + return _decrypt_aes_cbc_multi( + ciphertext, (self._v11_key, self._empty_key), self._logger, + hash_prefix=self._meta_version >= 24) else: self._logger.warning(f'unknown cookie version: "{version}"', only_once=True) @@ -464,11 +475,12 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): class MacChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_keyring_name, logger): + def __init__(self, browser_keyring_name, logger, meta_version=None): self._logger = logger password = _get_mac_keyring_password(browser_keyring_name, logger) self._v10_key = None if password is None else self.derive_key(password) self._cookie_counts = {'v10': 0, 'other': 0} + self._meta_version = meta_version or 0 @staticmethod def derive_key(password): @@ -486,7 +498,8 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True) return None - return _decrypt_aes_cbc_multi(ciphertext, (self._v10_key,), self._logger) + return _decrypt_aes_cbc_multi( + ciphertext, (self._v10_key,), self._logger, hash_prefix=self._meta_version >= 24) else: self._cookie_counts['other'] += 1 @@ -496,10 +509,11 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_root, logger): + def __init__(self, browser_root, logger, meta_version=None): self._logger = logger self._v10_key = _get_windows_v10_key(browser_root, logger) self._cookie_counts = {'v10': 0, 'other': 0} + self._meta_version = meta_version or 0 def decrypt(self, encrypted_value): version = encrypted_value[:3] @@ -523,7 +537,9 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] authentication_tag = raw_ciphertext[-authentication_tag_length:] - return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger) + return _decrypt_aes_gcm( + ciphertext, self._v10_key, nonce, authentication_tag, self._logger, + hash_prefix=self._meta_version >= 24) else: self._cookie_counts['other'] += 1 @@ -1009,10 +1025,12 @@ def pbkdf2_sha1(password, salt, iterations, key_length): return hashlib.pbkdf2_hmac('sha1', password, salt, iterations, key_length) -def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16): +def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' * 16, hash_prefix=False): for key in keys: plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)) try: + if hash_prefix: + return plaintext[32:].decode() return plaintext.decode() except UnicodeDecodeError: pass @@ -1020,7 +1038,7 @@ def _decrypt_aes_cbc_multi(ciphertext, keys, logger, initialization_vector=b' ' return None -def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger): +def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger, hash_prefix=False): try: plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce) except ValueError: @@ -1028,6 +1046,8 @@ def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger): return None try: + if hash_prefix: + return plaintext[32:].decode() return plaintext.decode() except UnicodeDecodeError: logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True) diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py index 2cfa4c9522..0e4404d49e 100644 --- a/yt_dlp/dependencies/Cryptodome.py +++ b/yt_dlp/dependencies/Cryptodome.py @@ -24,7 +24,7 @@ try: from Crypto.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5 # noqa: F401 from Crypto.Hash import CMAC, SHA1 # noqa: F401 from Crypto.PublicKey import RSA # noqa: F401 -except ImportError: +except (ImportError, OSError): __version__ = f'broken {__version__}'.strip() diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index ff5ddb2493..51caefd4d7 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -708,6 +708,7 @@ from .gab import ( GabTVIE, ) from .gaia import GaiaIE +from .gamedevtv import GameDevTVDashboardIE from .gamejolt import ( GameJoltCommunityIE, GameJoltGameIE, @@ -1155,6 +1156,7 @@ from .mitele import MiTeleIE from .mixch import ( MixchArchiveIE, MixchIE, + MixchMovieIE, ) from .mixcloud import ( MixcloudIE, @@ -1938,9 +1940,7 @@ from .spotify import ( ) from .spreaker import ( SpreakerIE, - SpreakerPageIE, SpreakerShowIE, - SpreakerShowPageIE, ) from .springboardplatform import SpringboardPlatformIE from .sprout import SproutIE diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py index 7cc15ec7b6..f1b8779271 100644 --- a/yt_dlp/extractor/adobepass.py +++ b/yt_dlp/extractor/adobepass.py @@ -1362,7 +1362,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en def _download_webpage_handle(self, *args, **kwargs): headers = self.geo_verification_headers() - headers.update(kwargs.get('headers', {})) + headers.update(kwargs.get('headers') or {}) kwargs['headers'] = headers return super()._download_webpage_handle( *args, **kwargs) diff --git a/yt_dlp/extractor/afreecatv.py b/yt_dlp/extractor/afreecatv.py index 83e510d1a2..6682a89817 100644 --- a/yt_dlp/extractor/afreecatv.py +++ b/yt_dlp/extractor/afreecatv.py @@ -154,7 +154,7 @@ class AfreecaTVIE(AfreecaTVBaseIE): 'title': ('title', {str}), 'uploader': ('writer_nick', {str}), 'uploader_id': ('bj_id', {str}), - 'duration': ('total_file_duration', {functools.partial(int_or_none, scale=1000)}), + 'duration': ('total_file_duration', {int_or_none(scale=1000)}), 'thumbnail': ('thumb', {url_or_none}), }) @@ -178,7 +178,7 @@ class AfreecaTVIE(AfreecaTVBaseIE): 'title': f'{common_info.get("title") or "Untitled"} (part {file_num})', 'formats': formats, **traverse_obj(file_element, { - 'duration': ('duration', {functools.partial(int_or_none, scale=1000)}), + 'duration': ('duration', {int_or_none(scale=1000)}), 'timestamp': ('file_start', {unified_timestamp}), }), }) @@ -234,7 +234,7 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE): 'catch_list', lambda _, v: v['files'][0]['file'], { 'id': ('files', 0, 'file_info_key', {str}), 'url': ('files', 0, 'file', {url_or_none}), - 'duration': ('files', 0, 'duration', {functools.partial(int_or_none, scale=1000)}), + 'duration': ('files', 0, 'duration', {int_or_none(scale=1000)}), 'title': ('title', {str}), 'uploader': ('writer_nick', {str}), 'uploader_id': ('writer_id', {str}), diff --git a/yt_dlp/extractor/allstar.py b/yt_dlp/extractor/allstar.py index 5ea1c30e3d..697d83c1e5 100644 --- a/yt_dlp/extractor/allstar.py +++ b/yt_dlp/extractor/allstar.py @@ -71,7 +71,7 @@ class AllstarBaseIE(InfoExtractor): 'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}), 'duration': ('clipLength', {int_or_none}), 'filesize': ('clipSizeBytes', {int_or_none}), - 'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}), + 'timestamp': ('createdDate', {int_or_none(scale=1000)}), 'uploader': ('username', {str}), 'uploader_id': ('user', '_id', {str}), 'view_count': ('views', {int_or_none}), diff --git a/yt_dlp/extractor/anvato.py b/yt_dlp/extractor/anvato.py index 109783b7e1..2dc959aa43 100644 --- a/yt_dlp/extractor/anvato.py +++ b/yt_dlp/extractor/anvato.py @@ -31,24 +31,6 @@ class AnvatoIE(InfoExtractor): _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce' # from anvplayer.min.js _TESTS = [{ - # from https://www.nfl.com/videos/baker-mayfield-s-game-changing-plays-from-3-td-game-week-14 - 'url': 'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:899441', - 'md5': '921919dab3cd0b849ff3d624831ae3e2', - 'info_dict': { - 'id': '899441', - 'ext': 'mp4', - 'title': 'Baker Mayfield\'s game-changing plays from 3-TD game Week 14', - 'description': 'md5:85e05a3cc163f8c344340f220521136d', - 'upload_date': '20201215', - 'timestamp': 1608009755, - 'thumbnail': r're:^https?://.*\.jpg', - 'uploader': 'NFL', - 'tags': ['Baltimore Ravens at Cleveland Browns (2020-REG-14)', 'Baker Mayfield', 'Game Highlights', - 'Player Highlights', 'Cleveland Browns', 'league'], - 'duration': 157, - 'categories': ['Entertainment', 'Game', 'Highlights'], - }, - }, { # from https://ktla.com/news/99-year-old-woman-learns-to-fly-in-torrance-checks-off-bucket-list-dream/ 'url': 'anvato:X8POa4zpGZMmeiq0wqiO8IP5rMqQM9VN:8032455', 'md5': '837718bcfb3a7778d022f857f7a9b19e', @@ -239,31 +221,6 @@ class AnvatoIE(InfoExtractor): 'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582', } - def _generate_nfl_token(self, anvack, mcp_id): - reroute = self._download_json( - 'https://api.nfl.com/v1/reroute', mcp_id, data=b'grant_type=client_credentials', - headers={'X-Domain-Id': 100}, note='Fetching token info') - token_type = reroute.get('token_type') or 'Bearer' - auth_token = f'{token_type} {reroute["access_token"]}' - response = self._download_json( - 'https://api.nfl.com/v3/shield/', mcp_id, data=json.dumps({ - 'query': '''{ - viewer { - mediaToken(anvack: "%s", id: %s) { - token - } - } -}''' % (anvack, mcp_id), # noqa: UP031 - }).encode(), headers={ - 'Authorization': auth_token, - 'Content-Type': 'application/json', - }, note='Fetching NFL API token') - return traverse_obj(response, ('data', 'viewer', 'mediaToken', 'token')) - - _TOKEN_GENERATORS = { - 'GXvEgwyJeWem8KCYXfeoHWknwP48Mboj': _generate_nfl_token, - } - def _server_time(self, access_key, video_id): return int_or_none(traverse_obj(self._download_json( f'{self._API_BASE_URL}/server_time', video_id, query={'anvack': access_key}, @@ -288,8 +245,6 @@ class AnvatoIE(InfoExtractor): } if extracted_token is not None: api['anvstk2'] = extracted_token - elif self._TOKEN_GENERATORS.get(access_key) is not None: - api['anvstk2'] = self._TOKEN_GENERATORS[access_key](self, access_key, video_id) elif self._ANVACK_TABLE.get(access_key) is not None: api['anvstk'] = md5_text(f'{access_key}|{anvrid}|{server_time}|{self._ANVACK_TABLE[access_key]}') else: diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py index efc79dd141..89d3299213 100644 --- a/yt_dlp/extractor/ard.py +++ b/yt_dlp/extractor/ard.py @@ -299,7 +299,7 @@ class ARDBetaMediathekIE(InfoExtractor): 'info_dict': { 'id': '94834686', 'ext': 'mp4', - 'duration': 2700, + 'duration': 2670, 'episode': '7 Tage ... unter harten Jungs', 'description': 'md5:0f215470dcd2b02f59f4bd10c963f072', 'upload_date': '20231005', @@ -307,10 +307,28 @@ class ARDBetaMediathekIE(InfoExtractor): 'display_id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3', 'series': '7 Tage ...', 'channel': 'HR', - 'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f6e6d5ffac41925c?w=960&ch=fa32ba69bc87989a', + 'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:430c86d233afa42d?w=960&ch=fa32ba69bc87989a', 'title': '7 Tage ... unter harten Jungs', '_old_archive_ids': ['ardbetamediathek N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3'], }, + }, { + 'url': 'https://www.ardmediathek.de/video/lokalzeit-aus-duesseldorf/lokalzeit-aus-duesseldorf-oder-31-10-2024/wdr-duesseldorf/Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtOWFkMTc0ZWMtMDA5ZS00ZDEwLWFjYjctMGNmNTdhNzVmNzUz', + 'info_dict': { + 'id': '13847165', + 'chapters': 'count:8', + 'ext': 'mp4', + 'channel': 'WDR', + 'display_id': 'Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtOWFkMTc0ZWMtMDA5ZS00ZDEwLWFjYjctMGNmNTdhNzVmNzUz', + 'episode': 'Lokalzeit aus Düsseldorf | 31.10.2024', + 'series': 'Lokalzeit aus Düsseldorf', + 'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f02ec9bd9b7bd5f6?w=960&ch=612491dcd5e09b0c', + 'title': 'Lokalzeit aus Düsseldorf | 31.10.2024', + 'upload_date': '20241031', + 'timestamp': 1730399400, + 'description': 'md5:12db30b3b706314efe3778b8df1a7058', + 'duration': 1759, + '_old_archive_ids': ['ardbetamediathek Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtOWFkMTc0ZWMtMDA5ZS00ZDEwLWFjYjctMGNmNTdhNzVmNzUz'], + }, }, { 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE', 'only_matching': True, @@ -455,6 +473,12 @@ class ARDBetaMediathekIE(InfoExtractor): 'subtitles': subtitles, 'is_live': is_live, 'age_limit': age_limit, + **traverse_obj(media_data, { + 'chapters': ('pluginData', 'jumpmarks@all', 'chapterArray', lambda _, v: int_or_none(v['chapterTime']), { + 'start_time': ('chapterTime', {int_or_none}), + 'title': ('chapterTitle', {str}), + }), + }), **traverse_obj(media_data, ('meta', { 'title': 'title', 'description': 'synopsis', diff --git a/yt_dlp/extractor/bandcamp.py b/yt_dlp/extractor/bandcamp.py index 0abe059829..939c2800e6 100644 --- a/yt_dlp/extractor/bandcamp.py +++ b/yt_dlp/extractor/bandcamp.py @@ -1,4 +1,3 @@ -import functools import json import random import re @@ -10,7 +9,6 @@ from ..utils import ( ExtractorError, extract_attributes, float_or_none, - get_element_html_by_id, int_or_none, parse_filesize, str_or_none, @@ -21,7 +19,7 @@ from ..utils import ( url_or_none, urljoin, ) -from ..utils.traversal import traverse_obj +from ..utils.traversal import find_element, traverse_obj class BandcampIE(InfoExtractor): @@ -45,6 +43,8 @@ class BandcampIE(InfoExtractor): 'uploader_url': 'https://youtube-dl.bandcamp.com', 'uploader_id': 'youtube-dl', 'thumbnail': 'https://f4.bcbits.com/img/a3216802731_5.jpg', + 'artists': ['youtube-dl "\'/\\ä↭'], + 'album_artists': ['youtube-dl "\'/\\ä↭'], }, 'skip': 'There is a limit of 200 free downloads / month for the test song', }, { @@ -271,6 +271,18 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE 'timestamp': 1311756226, 'upload_date': '20110727', 'uploader': 'Blazo', + 'thumbnail': 'https://f4.bcbits.com/img/a1721150828_5.jpg', + 'album_artists': ['Blazo'], + 'uploader_url': 'https://blazo.bandcamp.com', + 'release_date': '20110727', + 'release_timestamp': 1311724800.0, + 'track': 'Intro', + 'uploader_id': 'blazo', + 'track_number': 1, + 'album': 'Jazz Format Mixtape vol.1', + 'artists': ['Blazo'], + 'duration': 19.335, + 'track_id': '1353101989', }, }, { @@ -282,6 +294,18 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE 'timestamp': 1311757238, 'upload_date': '20110727', 'uploader': 'Blazo', + 'track': 'Kero One - Keep It Alive (Blazo remix)', + 'release_date': '20110727', + 'track_id': '38097443', + 'track_number': 2, + 'duration': 181.467, + 'uploader_url': 'https://blazo.bandcamp.com', + 'album': 'Jazz Format Mixtape vol.1', + 'uploader_id': 'blazo', + 'album_artists': ['Blazo'], + 'artists': ['Blazo'], + 'thumbnail': 'https://f4.bcbits.com/img/a1721150828_5.jpg', + 'release_timestamp': 1311724800.0, }, }, ], @@ -289,6 +313,7 @@ class BandcampAlbumIE(BandcampIE): # XXX: Do not subclass from concrete IE 'title': 'Jazz Format Mixtape vol.1', 'id': 'jazz-format-mixtape-vol-1', 'uploader_id': 'blazo', + 'description': 'md5:38052a93217f3ffdc033cd5dbbce2989', }, 'params': { 'playlistend': 2, @@ -363,10 +388,10 @@ class BandcampWeeklyIE(BandcampIE): # XXX: Do not subclass from concrete IE _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P\d+)' _TESTS = [{ 'url': 'https://bandcamp.com/?show=224', - 'md5': 'b00df799c733cf7e0c567ed187dea0fd', + 'md5': '61acc9a002bed93986b91168aa3ab433', 'info_dict': { 'id': '224', - 'ext': 'opus', + 'ext': 'mp3', 'title': 'BC Weekly April 4th 2017 - Magic Moments', 'description': 'md5:5d48150916e8e02d030623a48512c874', 'duration': 5829.77, @@ -376,7 +401,7 @@ class BandcampWeeklyIE(BandcampIE): # XXX: Do not subclass from concrete IE 'episode_id': '224', }, 'params': { - 'format': 'opus-lo', + 'format': 'mp3-128', }, }, { 'url': 'https://bandcamp.com/?blah/blah@&show=228', @@ -484,7 +509,7 @@ class BandcampUserIE(InfoExtractor): or re.findall(r']+trackTitle["\'][^"\']+["\']([^"\']+)', webpage)) yield from traverse_obj(webpage, ( - {functools.partial(get_element_html_by_id, 'music-grid')}, {extract_attributes}, + {find_element(id='music-grid', html=True)}, {extract_attributes}, 'data-client-items', {json.loads}, ..., 'page_url', {str})) def _real_extract(self, url): @@ -493,4 +518,4 @@ class BandcampUserIE(InfoExtractor): return self.playlist_from_matches( self._yield_items(webpage), uploader, f'Discography of {uploader}', - getter=functools.partial(urljoin, url)) + getter=urljoin(url)) diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py index 3af923f958..89fcf4425d 100644 --- a/yt_dlp/extractor/bbc.py +++ b/yt_dlp/extractor/bbc.py @@ -1284,9 +1284,9 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE **traverse_obj(model, { 'title': ('title', {str}), 'thumbnail': ('imageUrl', {lambda u: urljoin(url, u.replace('$recipe', 'raw'))}), - 'description': ('synopses', ('long', 'medium', 'short'), {str}, {lambda x: x or None}, any), + 'description': ('synopses', ('long', 'medium', 'short'), {str}, filter, any), 'duration': ('versions', 0, 'duration', {int}), - 'timestamp': ('versions', 0, 'availableFrom', {functools.partial(int_or_none, scale=1000)}), + 'timestamp': ('versions', 0, 'availableFrom', {int_or_none(scale=1000)}), }), } @@ -1386,7 +1386,7 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE formats = traverse_obj(media_data, ('playlist', lambda _, v: url_or_none(v['url']), { 'url': ('url', {url_or_none}), 'ext': ('format', {str}), - 'tbr': ('bitrate', {functools.partial(int_or_none, scale=1000)}), + 'tbr': ('bitrate', {int_or_none(scale=1000)}), })) if formats: entry = { @@ -1398,7 +1398,7 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE 'title': ('title', {str}), 'thumbnail': ('imageUrl', {lambda u: urljoin(url, u.replace('$recipe', 'raw'))}), 'description': ('synopses', ('long', 'medium', 'short'), {str}, any), - 'timestamp': ('firstPublished', {functools.partial(int_or_none, scale=1000)}), + 'timestamp': ('firstPublished', {int_or_none(scale=1000)}), }), } done = True @@ -1428,7 +1428,7 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE if not entry.get('timestamp'): entry['timestamp'] = traverse_obj(next_data, ( ..., 'contents', is_type('timestamp'), 'model', - 'timestamp', {functools.partial(int_or_none, scale=1000)}, any)) + 'timestamp', {int_or_none(scale=1000)}, any)) entries.append(entry) return self.playlist_result( entries, playlist_id, playlist_title, playlist_description) diff --git a/yt_dlp/extractor/bfmtv.py b/yt_dlp/extractor/bfmtv.py index 87f011783b..49d4819a3d 100644 --- a/yt_dlp/extractor/bfmtv.py +++ b/yt_dlp/extractor/bfmtv.py @@ -1,18 +1,33 @@ import re from .common import InfoExtractor -from ..utils import extract_attributes +from ..utils import ExtractorError, extract_attributes class BFMTVBaseIE(InfoExtractor): _VALID_URL_BASE = r'https?://(?:www\.|rmc\.)?bfmtv\.com/' _VALID_URL_TMPL = _VALID_URL_BASE + r'(?:[^/]+/)*[^/?&#]+_%s[A-Z]-(?P\d{12})\.html' - _VIDEO_BLOCK_REGEX = r'(]+class="video_block[^"]*"[^>]*>)' + _VIDEO_BLOCK_REGEX = r'(]+class="video_block[^"]*"[^>]*>.*?)' + _VIDEO_ELEMENT_REGEX = r'(]+>)' BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s' - def _brightcove_url_result(self, video_id, video_block): - account_id = video_block.get('accountid') or '876450612001' - player_id = video_block.get('playerid') or 'I2qBTln4u' + def _extract_video(self, video_block): + video_element = self._search_regex( + self._VIDEO_ELEMENT_REGEX, video_block, 'video element', default=None) + if video_element: + video_element_attrs = extract_attributes(video_element) + video_id = video_element_attrs.get('data-video-id') + if not video_id: + return + account_id = video_element_attrs.get('data-account') or '876450610001' + player_id = video_element_attrs.get('adjustplayer') or '19dszYXgm' + else: + video_block_attrs = extract_attributes(video_block) + video_id = video_block_attrs.get('videoid') + if not video_id: + return + account_id = video_block_attrs.get('accountid') or '876630703001' + player_id = video_block_attrs.get('playerid') or 'KbPwEbuHx' return self.url_result( self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id), 'BrightcoveNew', video_id) @@ -40,23 +55,25 @@ class BFMTVIE(BFMTVBaseIE): def _real_extract(self, url): bfmtv_id = self._match_id(url) webpage = self._download_webpage(url, bfmtv_id) - video_block = extract_attributes(self._search_regex( + video = self._extract_video(self._search_regex( self._VIDEO_BLOCK_REGEX, webpage, 'video block')) - return self._brightcove_url_result(video_block['videoid'], video_block) + if not video: + raise ExtractorError('Failed to extract video') + return video -class BFMTVLiveIE(BFMTVIE): # XXX: Do not subclass from concrete IE +class BFMTVLiveIE(BFMTVBaseIE): IE_NAME = 'bfmtv:live' _VALID_URL = BFMTVBaseIE._VALID_URL_BASE + '(?P(?:[^/]+/)?en-direct)' _TESTS = [{ 'url': 'https://www.bfmtv.com/en-direct/', 'info_dict': { - 'id': '5615950982001', + 'id': '6346069778112', 'ext': 'mp4', - 'title': r're:^le direct BFMTV WEB \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + 'title': r're:^Le Live BFM TV \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', 'uploader_id': '876450610001', - 'upload_date': '20220926', - 'timestamp': 1664207191, + 'upload_date': '20240202', + 'timestamp': 1706887572, 'live_status': 'is_live', 'thumbnail': r're:https://.+/image\.jpg', 'tags': [], @@ -69,6 +86,15 @@ class BFMTVLiveIE(BFMTVIE): # XXX: Do not subclass from concrete IE 'only_matching': True, }] + def _real_extract(self, url): + bfmtv_id = self._match_id(url) + webpage = self._download_webpage(url, bfmtv_id) + video = self._extract_video(self._search_regex( + self._VIDEO_BLOCK_REGEX, webpage, 'video block')) + if not video: + raise ExtractorError('Failed to extract video') + return video + class BFMTVArticleIE(BFMTVBaseIE): IE_NAME = 'bfmtv:article' @@ -102,18 +128,16 @@ class BFMTVArticleIE(BFMTVBaseIE): }, }] + def _entries(self, webpage): + for video_block_el in re.findall(self._VIDEO_BLOCK_REGEX, webpage): + video = self._extract_video(video_block_el) + if video: + yield video + def _real_extract(self, url): bfmtv_id = self._match_id(url) webpage = self._download_webpage(url, bfmtv_id) - entries = [] - for video_block_el in re.findall(self._VIDEO_BLOCK_REGEX, webpage): - video_block = extract_attributes(video_block_el) - video_id = video_block.get('videoid') - if not video_id: - continue - entries.append(self._brightcove_url_result(video_id, video_block)) - return self.playlist_result( - entries, bfmtv_id, self._og_search_title(webpage, fatal=False), + self._entries(webpage), bfmtv_id, self._og_search_title(webpage, fatal=False), self._html_search_meta(['og:description', 'description'], webpage)) diff --git a/yt_dlp/extractor/bibeltv.py b/yt_dlp/extractor/bibeltv.py index 666b51c56a..ad00245def 100644 --- a/yt_dlp/extractor/bibeltv.py +++ b/yt_dlp/extractor/bibeltv.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..utils import ( @@ -50,7 +49,7 @@ class BibelTVBaseIE(InfoExtractor): **traverse_obj(data, { 'title': 'title', 'description': 'description', - 'duration': ('duration', {functools.partial(int_or_none, scale=1000)}), + 'duration': ('duration', {int_or_none(scale=1000)}), 'timestamp': ('schedulingStart', {parse_iso8601}), 'season_number': 'seasonNumber', 'episode_number': 'episodeNumber', diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py index 62f68fbc6d..02ea67707f 100644 --- a/yt_dlp/extractor/bilibili.py +++ b/yt_dlp/extractor/bilibili.py @@ -109,7 +109,7 @@ class BilibiliBaseIE(InfoExtractor): fragments = traverse_obj(play_info, ('durl', lambda _, v: url_or_none(v['url']), { 'url': ('url', {url_or_none}), - 'duration': ('length', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('length', {float_or_none(scale=1000)}), 'filesize': ('size', {int_or_none}), })) if fragments: @@ -124,7 +124,7 @@ class BilibiliBaseIE(InfoExtractor): 'quality': ('quality', {int_or_none}), 'format_id': ('quality', {str_or_none}), 'format_note': ('quality', {lambda x: format_names.get(x)}), - 'duration': ('timelength', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('timelength', {float_or_none(scale=1000)}), }), **parse_resolution(format_names.get(play_info.get('quality'))), }) @@ -1585,7 +1585,7 @@ class BilibiliPlaylistIE(BilibiliSpaceListBaseIE): 'title': ('title', {str}), 'uploader': ('upper', 'name', {str}), 'uploader_id': ('upper', 'mid', {str_or_none}), - 'timestamp': ('ctime', {int_or_none}, {lambda x: x or None}), + 'timestamp': ('ctime', {int_or_none}, filter), 'thumbnail': ('cover', {url_or_none}), })), } diff --git a/yt_dlp/extractor/bluesky.py b/yt_dlp/extractor/bluesky.py index 42edd1107a..0e58a0932d 100644 --- a/yt_dlp/extractor/bluesky.py +++ b/yt_dlp/extractor/bluesky.py @@ -382,7 +382,7 @@ class BlueskyIE(InfoExtractor): 'age_limit': ( 'labels', ..., 'val', {lambda x: 18 if x in ('sexual', 'porn', 'graphic-media') else None}, any), 'description': (*record_path, 'text', {str}, filter), - 'title': (*record_path, 'text', {lambda x: x.replace('\n', '')}, {truncate_string(left=50)}), + 'title': (*record_path, 'text', {lambda x: x.replace('\n', ' ')}, {truncate_string(left=50)}), }), }) return entries diff --git a/yt_dlp/extractor/bpb.py b/yt_dlp/extractor/bpb.py index 7fe0899449..d7bf58b366 100644 --- a/yt_dlp/extractor/bpb.py +++ b/yt_dlp/extractor/bpb.py @@ -1,35 +1,20 @@ -import functools import re from .common import InfoExtractor from ..utils import ( clean_html, extract_attributes, - get_element_text_and_html_by_tag, - get_elements_by_class, join_nonempty, js_to_json, mimetype2ext, unified_strdate, url_or_none, urljoin, - variadic, ) -from ..utils.traversal import traverse_obj - - -def html_get_element(tag=None, cls=None): - assert tag or cls, 'One of tag or class is required' - - if cls: - func = functools.partial(get_elements_by_class, cls, tag=tag) - else: - func = functools.partial(get_element_text_and_html_by_tag, tag) - - def html_get_element_wrapper(html): - return variadic(func(html))[0] - - return html_get_element_wrapper +from ..utils.traversal import ( + find_element, + traverse_obj, +) class BpbIE(InfoExtractor): @@ -41,12 +26,12 @@ class BpbIE(InfoExtractor): 'info_dict': { 'id': '297', 'ext': 'mp4', - 'creator': 'Kooperative Berlin', - 'description': 'md5:f4f75885ba009d3e2b156247a8941ce6', - 'release_date': '20160115', + 'creators': ['Kooperative Berlin'], + 'description': r're:Joachim Gauck, .*\n\nKamera: .*', + 'release_date': '20150716', 'series': 'Interview auf dem Geschichtsforum 1989 | 2009', - 'tags': ['Friedliche Revolution', 'Erinnerungskultur', 'Vergangenheitspolitik', 'DDR 1949 - 1990', 'Freiheitsrecht', 'BStU', 'Deutschland'], - 'thumbnail': 'https://www.bpb.de/cache/images/7/297_teaser_16x9_1240.jpg?8839D', + 'tags': [], + 'thumbnail': r're:https?://www\.bpb\.de/cache/images/7/297_teaser_16x9_1240\.jpg.*', 'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR', 'uploader': 'Bundeszentrale für politische Bildung', }, @@ -55,11 +40,12 @@ class BpbIE(InfoExtractor): 'info_dict': { 'id': '522184', 'ext': 'mp4', - 'creator': 'Institute for Strategic Dialogue Germany gGmbH (ISD)', + 'creators': ['Institute for Strategic Dialogue Germany gGmbH (ISD)'], 'description': 'md5:f83c795ff8f825a69456a9e51fc15903', 'release_date': '20230621', - 'tags': ['Desinformation', 'Ukraine', 'Russland', 'Geflüchtete'], - 'thumbnail': 'https://www.bpb.de/cache/images/4/522184_teaser_16x9_1240.png?EABFB', + 'series': 'Narrative über den Krieg Russlands gegen die Ukraine (NUK)', + 'tags': [], + 'thumbnail': r're:https://www\.bpb\.de/cache/images/4/522184_teaser_16x9_1240\.png.*', 'title': 'md5:9b01ccdbf58dbf9e5c9f6e771a803b1c', 'uploader': 'Bundeszentrale für politische Bildung', }, @@ -68,11 +54,12 @@ class BpbIE(InfoExtractor): 'info_dict': { 'id': '518789', 'ext': 'mp4', - 'creator': 'Institute for Strategic Dialogue Germany gGmbH (ISD)', + 'creators': ['Institute for Strategic Dialogue Germany gGmbH (ISD)'], 'description': 'md5:85228aed433e84ff0ff9bc582abd4ea8', 'release_date': '20230302', - 'tags': ['Desinformation', 'Ukraine', 'Russland', 'Geflüchtete'], - 'thumbnail': 'https://www.bpb.de/cache/images/9/518789_teaser_16x9_1240.jpeg?56D0D', + 'series': 'Narrative über den Krieg Russlands gegen die Ukraine (NUK)', + 'tags': [], + 'thumbnail': r're:https://www\.bpb\.de/cache/images/9/518789_teaser_16x9_1240\.jpeg.*', 'title': 'md5:3e956f264bb501f6383f10495a401da4', 'uploader': 'Bundeszentrale für politische Bildung', }, @@ -84,12 +71,12 @@ class BpbIE(InfoExtractor): 'info_dict': { 'id': '315813', 'ext': 'mp3', - 'creator': 'Axel Schröder', + 'creators': ['Axel Schröder'], 'description': 'md5:eda9d1af34e5912efef5baf54fba4427', 'release_date': '20200921', 'series': 'Auf Endlagersuche. Der deutsche Weg zu einem sicheren Atommülllager', 'tags': ['Atomenergie', 'Endlager', 'hoch-radioaktiver Abfall', 'Endlagersuche', 'Atommüll', 'Atomendlager', 'Gorleben', 'Deutschland'], - 'thumbnail': 'https://www.bpb.de/cache/images/3/315813_teaser_16x9_1240.png?92A94', + 'thumbnail': r're:https://www\.bpb\.de/cache/images/3/315813_teaser_16x9_1240\.png.*', 'title': 'Folge 1: Eine Einführung', 'uploader': 'Bundeszentrale für politische Bildung', }, @@ -98,12 +85,12 @@ class BpbIE(InfoExtractor): 'info_dict': { 'id': '517806', 'ext': 'mp3', - 'creator': 'Bundeszentrale für politische Bildung', + 'creators': ['Bundeszentrale für politische Bildung'], 'description': 'md5:594689600e919912aade0b2871cc3fed', 'release_date': '20230127', 'series': 'Vorträge des Fachtags "Modernisierer. Grenzgänger. Anstifter. Sechs Jahrzehnte \'Neue Rechte\'"', 'tags': ['Rechtsextremismus', 'Konservatismus', 'Konservativismus', 'neue Rechte', 'Rechtspopulismus', 'Schnellroda', 'Deutschland'], - 'thumbnail': 'https://www.bpb.de/cache/images/6/517806_teaser_16x9_1240.png?7A7A0', + 'thumbnail': r're:https://www\.bpb\.de/cache/images/6/517806_teaser_16x9_1240\.png.*', 'title': 'Die Weltanschauung der "Neuen Rechten"', 'uploader': 'Bundeszentrale für politische Bildung', }, @@ -147,7 +134,7 @@ class BpbIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - title_result = traverse_obj(webpage, ({html_get_element(cls='opening-header__title')}, {self._TITLE_RE.match})) + title_result = traverse_obj(webpage, ({find_element(cls='opening-header__title')}, {self._TITLE_RE.match})) json_lds = list(self._yield_json_ld(webpage, video_id, fatal=False)) return { @@ -156,15 +143,15 @@ class BpbIE(InfoExtractor): # This metadata could be interpreted otherwise, but it fits "series" the most 'series': traverse_obj(title_result, ('series', {str.strip})) or None, 'description': join_nonempty(*traverse_obj(webpage, [( - {html_get_element(cls='opening-intro')}, - [{html_get_element(tag='bpb-accordion-item')}, {html_get_element(cls='text-content')}], + {find_element(cls='opening-intro')}, + [{find_element(tag='bpb-accordion-item')}, {find_element(cls='text-content')}], ), {clean_html}]), delim='\n\n') or None, - 'creator': self._html_search_meta('author', webpage), + 'creators': traverse_obj(self._html_search_meta('author', webpage), all), 'uploader': self._html_search_meta('publisher', webpage), 'release_date': unified_strdate(self._html_search_meta('date', webpage)), 'tags': traverse_obj(json_lds, (..., 'keywords', {lambda x: x.split(',')}, ...)), **traverse_obj(self._parse_vue_attributes('bpb-player', webpage, video_id), { 'formats': (':sources', ..., {self._process_source}), - 'thumbnail': ('poster', {lambda x: urljoin(url, x)}), + 'thumbnail': ('poster', {urljoin(url)}), }), } diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py index ec72f0d884..0b2c447987 100644 --- a/yt_dlp/extractor/bravotv.py +++ b/yt_dlp/extractor/bravotv.py @@ -145,10 +145,9 @@ class BravoTVIE(AdobePassIE): tp_metadata = self._download_json( update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False) - seconds_or_none = lambda x: float_or_none(x, 1000) chapters = traverse_obj(tp_metadata, ('chapters', ..., { - 'start_time': ('startTime', {seconds_or_none}), - 'end_time': ('endTime', {seconds_or_none}), + 'start_time': ('startTime', {float_or_none(scale=1000)}), + 'end_time': ('endTime', {float_or_none(scale=1000)}), })) # prune pointless single chapters that span the entire duration from short videos if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')): @@ -168,8 +167,8 @@ class BravoTVIE(AdobePassIE): **merge_dicts(traverse_obj(tp_metadata, { 'title': 'title', 'description': 'description', - 'duration': ('duration', {seconds_or_none}), - 'timestamp': ('pubDate', {seconds_or_none}), + 'duration': ('duration', {float_or_none(scale=1000)}), + 'timestamp': ('pubDate', {float_or_none(scale=1000)}), 'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}), 'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}), 'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}), diff --git a/yt_dlp/extractor/bundestag.py b/yt_dlp/extractor/bundestag.py index 71f7726659..3dacbbd24a 100644 --- a/yt_dlp/extractor/bundestag.py +++ b/yt_dlp/extractor/bundestag.py @@ -8,11 +8,13 @@ from ..utils import ( bug_reports_message, clean_html, format_field, - get_element_text_and_html_by_tag, int_or_none, url_or_none, ) -from ..utils.traversal import traverse_obj +from ..utils.traversal import ( + find_element, + traverse_obj, +) class BundestagIE(InfoExtractor): @@ -115,9 +117,8 @@ class BundestagIE(InfoExtractor): note='Downloading metadata overlay', fatal=False, ), { 'title': ( - {functools.partial(get_element_text_and_html_by_tag, 'h3')}, 0, - {functools.partial(re.sub, r']*>[^<]+', '')}, {clean_html}), - 'description': ({functools.partial(get_element_text_and_html_by_tag, 'p')}, 0, {clean_html}), + {find_element(tag='h3')}, {functools.partial(re.sub, r']*>[^<]+', '')}, {clean_html}), + 'description': ({find_element(tag='p')}, {clean_html}), })) return result diff --git a/yt_dlp/extractor/caffeinetv.py b/yt_dlp/extractor/caffeinetv.py index aa107f8585..ea5134d2f3 100644 --- a/yt_dlp/extractor/caffeinetv.py +++ b/yt_dlp/extractor/caffeinetv.py @@ -53,7 +53,7 @@ class CaffeineTVIE(InfoExtractor): 'like_count': ('like_count', {int_or_none}), 'view_count': ('view_count', {int_or_none}), 'comment_count': ('comment_count', {int_or_none}), - 'tags': ('tags', ..., {str}, {lambda x: x or None}), + 'tags': ('tags', ..., {str}, filter), 'uploader': ('user', 'name', {str}), 'uploader_id': (((None, 'user'), 'username'), {str}, any), 'is_live': ('is_live', {bool}), @@ -62,7 +62,7 @@ class CaffeineTVIE(InfoExtractor): 'title': ('broadcast_title', {str}), 'duration': ('content_duration', {int_or_none}), 'timestamp': ('broadcast_start_time', {parse_iso8601}), - 'thumbnail': ('preview_image_path', {lambda x: urljoin(url, x)}), + 'thumbnail': ('preview_image_path', {urljoin(url)}), }), 'age_limit': { # assume Apple Store ratings: https://en.wikipedia.org/wiki/Mobile_software_content_rating_system diff --git a/yt_dlp/extractor/cbc.py b/yt_dlp/extractor/cbc.py index b44c23fa10..c0cf3da3de 100644 --- a/yt_dlp/extractor/cbc.py +++ b/yt_dlp/extractor/cbc.py @@ -453,8 +453,8 @@ class CBCPlayerIE(InfoExtractor): chapters = traverse_obj(data, ( 'media', 'chapters', lambda _, v: float(v['startTime']) is not None, { - 'start_time': ('startTime', {functools.partial(float_or_none, scale=1000)}), - 'end_time': ('endTime', {functools.partial(float_or_none, scale=1000)}), + 'start_time': ('startTime', {float_or_none(scale=1000)}), + 'end_time': ('endTime', {float_or_none(scale=1000)}), 'title': ('name', {str}), })) # Filter out pointless single chapters with start_time==0 and no end_time @@ -465,8 +465,8 @@ class CBCPlayerIE(InfoExtractor): **traverse_obj(data, { 'title': ('title', {str}), 'description': ('description', {str.strip}), - 'thumbnail': ('image', 'url', {url_or_none}, {functools.partial(update_url, query=None)}), - 'timestamp': ('publishedAt', {functools.partial(float_or_none, scale=1000)}), + 'thumbnail': ('image', 'url', {url_or_none}, {update_url(query=None)}), + 'timestamp': ('publishedAt', {float_or_none(scale=1000)}), 'media_type': ('media', 'clipType', {str}), 'series': ('showName', {str}), 'season_number': ('media', 'season', {int_or_none}), diff --git a/yt_dlp/extractor/cbsnews.py b/yt_dlp/extractor/cbsnews.py index 972e111190..b01c0efd5d 100644 --- a/yt_dlp/extractor/cbsnews.py +++ b/yt_dlp/extractor/cbsnews.py @@ -96,7 +96,7 @@ class CBSNewsBaseIE(InfoExtractor): **traverse_obj(item, { 'title': (None, ('fulltitle', 'title')), 'description': 'dek', - 'timestamp': ('timestamp', {lambda x: float_or_none(x, 1000)}), + 'timestamp': ('timestamp', {float_or_none(scale=1000)}), 'duration': ('duration', {float_or_none}), 'subtitles': ('captions', {get_subtitles}), 'thumbnail': ('images', ('hd', 'sd'), {url_or_none}), diff --git a/yt_dlp/extractor/chaturbate.py b/yt_dlp/extractor/chaturbate.py index b49f741efa..864d61f9c2 100644 --- a/yt_dlp/extractor/chaturbate.py +++ b/yt_dlp/extractor/chaturbate.py @@ -9,7 +9,7 @@ from ..utils import ( class ChaturbateIE(InfoExtractor): - _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?:fullvideo/?\?.*?\bb=)?(?P[^/?&#]+)' + _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.(?Pcom|eu|global)/(?:fullvideo/?\?.*?\bb=)?(?P[^/?&#]+)' _TESTS = [{ 'url': 'https://www.chaturbate.com/siswet19/', 'info_dict': { @@ -29,15 +29,24 @@ class ChaturbateIE(InfoExtractor): }, { 'url': 'https://en.chaturbate.com/siswet19/', 'only_matching': True, + }, { + 'url': 'https://chaturbate.eu/siswet19/', + 'only_matching': True, + }, { + 'url': 'https://chaturbate.eu/fullvideo/?b=caylin', + 'only_matching': True, + }, { + 'url': 'https://chaturbate.global/siswet19/', + 'only_matching': True, }] _ROOM_OFFLINE = 'Room is currently offline' def _real_extract(self, url): - video_id = self._match_id(url) + video_id, tld = self._match_valid_url(url).group('id', 'tld') webpage = self._download_webpage( - f'https://chaturbate.com/{video_id}/', video_id, + f'https://chaturbate.{tld}/{video_id}/', video_id, headers=self.geo_verification_headers()) found_m3u8_urls = [] diff --git a/yt_dlp/extractor/chzzk.py b/yt_dlp/extractor/chzzk.py index e0b9980afd..aec77ac454 100644 --- a/yt_dlp/extractor/chzzk.py +++ b/yt_dlp/extractor/chzzk.py @@ -1,5 +1,3 @@ -import functools - from .common import InfoExtractor from ..utils import ( UserNotLive, @@ -77,7 +75,7 @@ class CHZZKLiveIE(InfoExtractor): 'thumbnails': thumbnails, **traverse_obj(live_detail, { 'title': ('liveTitle', {str}), - 'timestamp': ('openDate', {functools.partial(parse_iso8601, delimiter=' ')}), + 'timestamp': ('openDate', {parse_iso8601(delimiter=' ')}), 'concurrent_view_count': ('concurrentUserCount', {int_or_none}), 'view_count': ('accumulateCount', {int_or_none}), 'channel': ('channel', 'channelName', {str}), @@ -146,23 +144,37 @@ class CHZZKVideoIE(InfoExtractor): video_meta = self._download_json( f'https://api.chzzk.naver.com/service/v3/videos/{video_id}', video_id, note='Downloading video info', errnote='Unable to download video info')['content'] - formats, subtitles = self._extract_mpd_formats_and_subtitles( - f'https://apis.naver.com/neonplayer/vodplay/v1/playback/{video_meta["videoId"]}', video_id, - query={ - 'key': video_meta['inKey'], - 'env': 'real', - 'lc': 'en_US', - 'cpl': 'en_US', - }, note='Downloading video playback', errnote='Unable to download video playback') + + live_status = 'was_live' if video_meta.get('liveOpenDate') else 'not_live' + video_status = video_meta.get('vodStatus') + if video_status == 'UPLOAD': + playback = self._parse_json(video_meta['liveRewindPlaybackJson'], video_id) + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + playback['media'][0]['path'], video_id, 'mp4', m3u8_id='hls') + elif video_status == 'ABR_HLS': + formats, subtitles = self._extract_mpd_formats_and_subtitles( + f'https://apis.naver.com/neonplayer/vodplay/v1/playback/{video_meta["videoId"]}', + video_id, query={ + 'key': video_meta['inKey'], + 'env': 'real', + 'lc': 'en_US', + 'cpl': 'en_US', + }) + else: + self.raise_no_formats( + f'Unknown video status detected: "{video_status}"', expected=True, video_id=video_id) + formats, subtitles = [], {} + live_status = 'post_live' if live_status == 'was_live' else None return { 'id': video_id, 'formats': formats, 'subtitles': subtitles, + 'live_status': live_status, **traverse_obj(video_meta, { 'title': ('videoTitle', {str}), 'thumbnail': ('thumbnailImageUrl', {url_or_none}), - 'timestamp': ('publishDateAt', {functools.partial(float_or_none, scale=1000)}), + 'timestamp': ('publishDateAt', {float_or_none(scale=1000)}), 'view_count': ('readCount', {int_or_none}), 'duration': ('duration', {int_or_none}), 'channel': ('channel', 'channelName', {str}), diff --git a/yt_dlp/extractor/cineverse.py b/yt_dlp/extractor/cineverse.py index c8c6c48c27..124c874e2c 100644 --- a/yt_dlp/extractor/cineverse.py +++ b/yt_dlp/extractor/cineverse.py @@ -3,6 +3,7 @@ import re from .common import InfoExtractor from ..utils import ( filter_dict, + float_or_none, int_or_none, parse_age_limit, smuggle_url, @@ -85,7 +86,7 @@ class CineverseIE(CineverseBaseIE): 'title': 'title', 'id': ('details', 'item_id'), 'description': ('details', 'description'), - 'duration': ('duration', {lambda x: x / 1000}), + 'duration': ('duration', {float_or_none(scale=1000)}), 'cast': ('details', 'cast', {lambda x: x.split(', ')}), 'modified_timestamp': ('details', 'updated_by', 0, 'update_time', 'time', {int_or_none}), 'season_number': ('details', 'season', {int_or_none}), diff --git a/yt_dlp/extractor/cloudflarestream.py b/yt_dlp/extractor/cloudflarestream.py index 8a409461a8..9e9e89a801 100644 --- a/yt_dlp/extractor/cloudflarestream.py +++ b/yt_dlp/extractor/cloudflarestream.py @@ -8,7 +8,7 @@ class CloudflareStreamIE(InfoExtractor): _DOMAIN_RE = r'(?:cloudflarestream\.com|(?:videodelivery|bytehighway)\.net)' _EMBED_RE = rf'(?:embed\.|{_SUBDOMAIN_RE}){_DOMAIN_RE}/embed/[^/?#]+\.js\?(?:[^#]+&)?video=' _ID_RE = r'[\da-f]{32}|eyJ[\w-]+\.[\w-]+\.[\w-]+' - _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}{_DOMAIN_RE}/|{_EMBED_RE})(?P{_ID_RE})' + _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}(?P{_DOMAIN_RE})/|{_EMBED_RE})(?P{_ID_RE})' _EMBED_REGEX = [ rf']+\bsrc=(["\'])(?P(?:https?:)?//{_EMBED_RE}(?:{_ID_RE})(?:(?!\1).)*)\1', rf']+\bsrc=["\'](?Phttps?://{_SUBDOMAIN_RE}{_DOMAIN_RE}/[\da-f]{{32}})', @@ -19,7 +19,7 @@ class CloudflareStreamIE(InfoExtractor): 'id': '31c9291ab41fac05471db4e73aa11717', 'ext': 'mp4', 'title': '31c9291ab41fac05471db4e73aa11717', - 'thumbnail': 'https://videodelivery.net/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg', + 'thumbnail': 'https://cloudflarestream.com/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg', }, 'params': { 'skip_download': 'm3u8', @@ -30,7 +30,7 @@ class CloudflareStreamIE(InfoExtractor): 'id': '0e8e040aec776862e1d632a699edf59e', 'ext': 'mp4', 'title': '0e8e040aec776862e1d632a699edf59e', - 'thumbnail': 'https://videodelivery.net/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg', + 'thumbnail': 'https://cloudflarestream.com/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg', }, }, { 'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1', @@ -54,7 +54,7 @@ class CloudflareStreamIE(InfoExtractor): 'id': 'eaef9dea5159cf968be84241b5cedfe7', 'ext': 'mp4', 'title': 'eaef9dea5159cf968be84241b5cedfe7', - 'thumbnail': 'https://videodelivery.net/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg', + 'thumbnail': 'https://cloudflarestream.com/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg', }, 'params': { 'skip_download': 'm3u8', @@ -62,8 +62,9 @@ class CloudflareStreamIE(InfoExtractor): }] def _real_extract(self, url): - video_id = self._match_id(url) - domain = 'bytehighway.net' if 'bytehighway.net/' in url else 'videodelivery.net' + video_id, domain = self._match_valid_url(url).group('id', 'domain') + if domain != 'bytehighway.net': + domain = 'cloudflarestream.com' base_url = f'https://{domain}/{video_id}/' if '.' in video_id: video_id = self._parse_json(base64.urlsafe_b64decode( diff --git a/yt_dlp/extractor/cnn.py b/yt_dlp/extractor/cnn.py index cfcec9d1fd..8148762c54 100644 --- a/yt_dlp/extractor/cnn.py +++ b/yt_dlp/extractor/cnn.py @@ -1,4 +1,3 @@ -import functools import json import re @@ -199,7 +198,7 @@ class CNNIE(InfoExtractor): 'timestamp': ('data-publish-date', {parse_iso8601}), 'thumbnail': ( 'data-poster-image-override', {json.loads}, 'big', 'uri', {url_or_none}, - {functools.partial(update_url, query='c=original')}), + {update_url(query='c=original')}), 'display_id': 'data-video-slug', }), **traverse_obj(video_data, { diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index d1845b2b0f..1f8c1c84e8 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1577,7 +1577,9 @@ class InfoExtractor: if default is not NO_DEFAULT: fatal = False for mobj in re.finditer(JSON_LD_RE, html): - json_ld_item = self._parse_json(mobj.group('json_ld'), video_id, fatal=fatal) + json_ld_item = self._parse_json( + mobj.group('json_ld'), video_id, fatal=fatal, + errnote=False if default is not NO_DEFAULT else None) for json_ld in variadic(json_ld_item): if isinstance(json_ld, dict): yield json_ld diff --git a/yt_dlp/extractor/condenast.py b/yt_dlp/extractor/condenast.py index 9c02cd3429..0c84cfdab7 100644 --- a/yt_dlp/extractor/condenast.py +++ b/yt_dlp/extractor/condenast.py @@ -12,6 +12,7 @@ from ..utils import ( parse_iso8601, strip_or_none, try_get, + urljoin, ) @@ -112,8 +113,7 @@ class CondeNastIE(InfoExtractor): m_paths = re.finditer( r'(?s)

.*?playlist))= ) - [/=](?P[^/?_&]+)(?:.+?\bplaylist=(?Px[0-9a-z]+))? - ''' + ) + (?P[^/?_&#]+)(?:[\w-]*\?playlist=(?Px[0-9a-z]+))? + ''' IE_NAME = 'dailymotion' _EMBED_REGEX = [r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1'] _TESTS = [{ @@ -123,7 +134,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor): 'view_count': int, 'like_count': int, 'tags': ['hollywood', 'celeb', 'celebrity', 'movies', 'red carpet'], - 'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/K456B1aXqIx58LKWQ/x1080', + 'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/K456B1cmt4ZcZ9KiM/x1080', }, }, { 'url': 'https://geo.dailymotion.com/player.html?video=x89eyek&mute=true', @@ -142,7 +153,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor): 'view_count': int, 'like_count': int, 'tags': ['en_quete_d_esprit'], - 'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Tncwi1YNg_RUl7ueu/x1080', + 'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Tncwi1clTH6StrxMP/x1080', }, }, { 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames', @@ -217,6 +228,66 @@ class DailymotionIE(DailymotionBaseInfoExtractor): }, { 'url': 'https://geo.dailymotion.com/player/xakln.html?video=x8mjju4&customConfig%5BcustomParams%5D=%2Ffr-fr%2Ftennis%2Fwimbledon-mens-singles%2Farticles-video', 'only_matching': True, + }, { # playlist-only + 'url': 'https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj', + 'only_matching': True, + }, { + 'url': 'https://geo.dailymotion.com/player/xmyye.html?video=x93blhi', + 'only_matching': True, + }, { + 'url': 'https://www.dailymotion.com/crawler/video/x8u4owg', + 'only_matching': True, + }, { + 'url': 'https://www.dailymotion.com/embed/video/x8u4owg', + 'only_matching': True, + }, { + 'url': 'https://dai.ly/x94cnnk', + 'only_matching': True, + }] + _WEBPAGE_TESTS = [{ + # https://geo.dailymotion.com/player/xmyye.html?video=x93blhi + 'url': 'https://www.financialounge.com/video/2024/08/01/borse-europee-in-rosso-dopo-la-fed-a-milano-volano-mediobanca-e-tim-edizione-del-1-agosto/', + 'info_dict': { + 'id': 'x93blhi', + 'ext': 'mp4', + 'title': 'OnAir - 01/08/24', + 'description': '', + 'duration': 217, + 'timestamp': 1722505658, + 'upload_date': '20240801', + 'uploader': 'Financialounge', + 'uploader_id': 'x2vtgmm', + 'age_limit': 0, + 'tags': [], + 'view_count': int, + 'like_count': int, + }, + }, { + # https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj + 'url': 'https://www.cycleworld.com/blogs/ask-kevin/ducati-continues-to-evolve-with-v4/', + 'info_dict': { + 'id': 'x7wdsj', + }, + 'playlist_mincount': 50, + }, { + # https://www.dailymotion.com/crawler/video/x8u4owg + 'url': 'https://www.leparisien.fr/environnement/video-le-veloto-la-voiture-a-pedales-qui-aimerait-se-faire-une-place-sur-les-routes-09-03-2024-KCYMCPM4WFHJXMSKBUI66UNFPU.php', + 'info_dict': { + 'id': 'x8u4owg', + 'ext': 'mp4', + 'like_count': int, + 'uploader': 'Le Parisien', + 'thumbnail': 'https://www.leparisien.fr/resizer/ho_GwveeYftNkLwg_cEta--5Bv4=/1200x675/cloudfront-eu-central-1.images.arcpublishing.com/leparisien/BFXJNEBN75EUNHGYJLORUC3TX4.jpg', + 'upload_date': '20240309', + 'view_count': int, + 'timestamp': 1709997866, + 'age_limit': 0, + 'uploader_id': 'x32f7b', + 'title': 'VIDÉO. Le «\xa0véloto\xa0», la voiture à pédales qui aimerait se faire une place sur les routes', + 'duration': 428.0, + 'description': 'À bord du « véloto », l’alternative à la voiture pour la campagne', + 'tags': ['biclou', 'vélo', 'véloto', 'campagne', 'voiture', 'environnement', 'véhicules intermédiaires'], + }, }] _GEO_BYPASS = False _COMMON_MEDIA_FIELDS = '''description @@ -232,16 +303,35 @@ class DailymotionIE(DailymotionBaseInfoExtractor): for mobj in re.finditer( r'(?s)DM\.player\([^,]+,\s*{.*?video[\'"]?\s*:\s*["\']?(?P[0-9a-zA-Z]+).+?}\s*\);', webpage): yield from 'https://www.dailymotion.com/embed/video/' + mobj.group('id') + for mobj in re.finditer( + r'(?s)', webpage), + (..., {js_to_json}, {json.loads}, ..., {self._find_json}, ...)) + meta = traverse_obj(nextjs_data, ( + ..., lambda _, v: v['meta']['path'] == urllib.parse.urlparse(url).path, 'meta', any)) + + video_id = meta['uuid'] + info_dict = traverse_obj(meta, { + 'title': ('title', {str}), + 'description': ('description', {str.strip}), + }) + + if traverse_obj(meta, ('program', 'subtype')) != 'movie': + for season_data in traverse_obj(nextjs_data, (..., 'children', ..., 'playlists', ...)): + episode_data = traverse_obj( + season_data, ('videos', lambda _, v: v['videoId'] == video_id, any)) + if not episode_data: + continue + + episode_title = traverse_obj( + episode_data, 'contextualTitle', 'episodeTitle', expected_type=str) + info_dict.update({ + 'title': episode_title or info_dict.get('title'), + 'series': remove_end(info_dict.get('title'), f' - {episode_title}'), + 'season_number': traverse_obj(season_data, ('season', {int_or_none})), + 'episode_number': traverse_obj(episode_data, ('episodeNumber', {int_or_none})), + }) + break api = self._download_json( f'https://api.goplay.be/web/v1/videos/long-form/{video_id}', diff --git a/yt_dlp/extractor/ilpost.py b/yt_dlp/extractor/ilpost.py index 2868f0c62c..da203cf5ff 100644 --- a/yt_dlp/extractor/ilpost.py +++ b/yt_dlp/extractor/ilpost.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..utils import ( @@ -63,7 +62,7 @@ class IlPostIE(InfoExtractor): 'url': ('podcast_raw_url', {url_or_none}), 'thumbnail': ('image', {url_or_none}), 'timestamp': ('timestamp', {int_or_none}), - 'duration': ('milliseconds', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('milliseconds', {float_or_none(scale=1000)}), 'availability': ('free', {lambda v: 'public' if v else 'subscriber_only'}), }), } diff --git a/yt_dlp/extractor/jiocinema.py b/yt_dlp/extractor/jiocinema.py index 30d98ba796..94c85064ef 100644 --- a/yt_dlp/extractor/jiocinema.py +++ b/yt_dlp/extractor/jiocinema.py @@ -326,11 +326,11 @@ class JioCinemaIE(JioCinemaBaseIE): # fallback metadata 'title': ('name', {str}), 'description': ('fullSynopsis', {str}), - 'series': ('show', 'name', {str}, {lambda x: x or None}), + 'series': ('show', 'name', {str}, filter), 'season': ('tournamentName', {str}, {lambda x: x if x != 'Season 0' else None}), - 'season_number': ('episode', 'season', {int_or_none}, {lambda x: x or None}), + 'season_number': ('episode', 'season', {int_or_none}, filter), 'episode': ('fullTitle', {str}), - 'episode_number': ('episode', 'episodeNo', {int_or_none}, {lambda x: x or None}), + 'episode_number': ('episode', 'episodeNo', {int_or_none}, filter), 'age_limit': ('ageNemonic', {parse_age_limit}), 'duration': ('totalDuration', {float_or_none}), 'thumbnail': ('images', {url_or_none}), @@ -338,10 +338,10 @@ class JioCinemaIE(JioCinemaBaseIE): **traverse_obj(metadata, ('result', 0, { 'title': ('fullTitle', {str}), 'description': ('fullSynopsis', {str}), - 'series': ('showName', {str}, {lambda x: x or None}), - 'season': ('seasonName', {str}, {lambda x: x or None}), + 'series': ('showName', {str}, filter), + 'season': ('seasonName', {str}, filter), 'season_number': ('season', {int_or_none}), - 'season_id': ('seasonId', {str}, {lambda x: x or None}), + 'season_id': ('seasonId', {str}, filter), 'episode': ('fullTitle', {str}), 'episode_number': ('episode', {int_or_none}), 'timestamp': ('uploadTime', {int_or_none}), diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py index bd21e59501..1f001d421a 100644 --- a/yt_dlp/extractor/kick.py +++ b/yt_dlp/extractor/kick.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..networking import HEADRequest @@ -137,7 +136,7 @@ class KickVODIE(KickBaseIE): 'uploader': ('livestream', 'channel', 'user', 'username', {str}), 'uploader_id': ('livestream', 'channel', 'user_id', {int}, {str_or_none}), 'timestamp': ('created_at', {parse_iso8601}), - 'duration': ('livestream', 'duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('livestream', 'duration', {float_or_none(scale=1000)}), 'thumbnail': ('livestream', 'thumbnail', {url_or_none}), 'categories': ('livestream', 'categories', ..., 'name', {str}), 'view_count': ('views', {int_or_none}), diff --git a/yt_dlp/extractor/kika.py b/yt_dlp/extractor/kika.py index 852a4de3f2..69f4a3ce03 100644 --- a/yt_dlp/extractor/kika.py +++ b/yt_dlp/extractor/kika.py @@ -119,7 +119,7 @@ class KikaIE(InfoExtractor): 'width': ('frameWidth', {int_or_none}), 'height': ('frameHeight', {int_or_none}), # NB: filesize is 0 if unknown, bitrate is -1 if unknown - 'filesize': ('fileSize', {int_or_none}, {lambda x: x or None}), + 'filesize': ('fileSize', {int_or_none}, filter), 'abr': ('bitrateAudio', {int_or_none}, {lambda x: None if x == -1 else x}), 'vbr': ('bitrateVideo', {int_or_none}, {lambda x: None if x == -1 else x}), }), diff --git a/yt_dlp/extractor/laracasts.py b/yt_dlp/extractor/laracasts.py index 4494c4b79a..4a61d6ab14 100644 --- a/yt_dlp/extractor/laracasts.py +++ b/yt_dlp/extractor/laracasts.py @@ -32,7 +32,7 @@ class LaracastsBaseIE(InfoExtractor): VimeoIE, url_transparent=True, **traverse_obj(episode, { 'id': ('id', {int}, {str_or_none}), - 'webpage_url': ('path', {lambda x: urljoin('https://laracasts.com', x)}), + 'webpage_url': ('path', {urljoin('https://laracasts.com')}), 'title': ('title', {clean_html}), 'season_number': ('chapter', {int_or_none}), 'episode_number': ('position', {int_or_none}), @@ -104,7 +104,7 @@ class LaracastsPlaylistIE(LaracastsBaseIE): 'description': ('body', {clean_html}), 'thumbnail': (('large_thumbnail', 'thumbnail'), {url_or_none}, any), 'duration': ('runTime', {parse_duration}), - 'categories': ('taxonomy', 'name', {str}, {lambda x: x and [x]}), + 'categories': ('taxonomy', 'name', {str}, all, filter), 'tags': ('topics', ..., 'name', {str}), 'modified_date': ('lastUpdated', {unified_strdate}), }), diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index 322852dd6f..0445b7cbfc 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -66,7 +66,7 @@ class LBRYBaseIE(InfoExtractor): 'license': ('value', 'license', {str}), 'timestamp': ('timestamp', {int_or_none}), 'release_timestamp': ('value', 'release_time', {int_or_none}), - 'tags': ('value', 'tags', ..., {lambda x: x or None}), + 'tags': ('value', 'tags', ..., filter), 'duration': ('value', stream_type, 'duration', {int_or_none}), 'channel': ('signing_channel', 'value', 'title', {str}), 'channel_id': ('signing_channel', 'claim_id', {str}), diff --git a/yt_dlp/extractor/learningonscreen.py b/yt_dlp/extractor/learningonscreen.py index dcf83144c8..f4b51e66c3 100644 --- a/yt_dlp/extractor/learningonscreen.py +++ b/yt_dlp/extractor/learningonscreen.py @@ -6,13 +6,11 @@ from ..utils import ( ExtractorError, clean_html, extract_attributes, - get_element_by_class, - get_element_html_by_id, join_nonempty, parse_duration, unified_timestamp, ) -from ..utils.traversal import traverse_obj +from ..utils.traversal import find_element, traverse_obj class LearningOnScreenIE(InfoExtractor): @@ -32,28 +30,24 @@ class LearningOnScreenIE(InfoExtractor): def _real_initialize(self): if not self._get_cookies('https://learningonscreen.ac.uk/').get('PHPSESSID-BOB-LIVE'): - self.raise_login_required( - 'Use --cookies for authentication. See ' - ' https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp ' - 'for how to manually pass cookies', method=None) + self.raise_login_required(method='session_cookies') def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) details = traverse_obj(webpage, ( - {functools.partial(get_element_html_by_id, 'programme-details')}, { - 'title': ({functools.partial(re.search, r'

([^<]+)

')}, 1, {clean_html}), + {find_element(id='programme-details', html=True)}, { + 'title': ({find_element(tag='h2')}, {clean_html}), 'timestamp': ( - {functools.partial(get_element_by_class, 'broadcast-date')}, + {find_element(cls='broadcast-date')}, {functools.partial(re.match, r'([^<]+)')}, 1, {unified_timestamp}), 'duration': ( - {functools.partial(get_element_by_class, 'prog-running-time')}, - {clean_html}, {parse_duration}), + {find_element(cls='prog-running-time')}, {clean_html}, {parse_duration}), })) title = details.pop('title', None) or traverse_obj(webpage, ( - {functools.partial(get_element_html_by_id, 'add-to-existing-playlist')}, + {find_element(id='add-to-existing-playlist', html=True)}, {extract_attributes}, 'data-record-title', {clean_html})) entries = self._parse_html5_media_entries( diff --git a/yt_dlp/extractor/listennotes.py b/yt_dlp/extractor/listennotes.py index 61eae95edf..9d68e18301 100644 --- a/yt_dlp/extractor/listennotes.py +++ b/yt_dlp/extractor/listennotes.py @@ -6,12 +6,10 @@ from ..utils import ( extract_attributes, get_element_by_class, get_element_html_by_id, - get_element_text_and_html_by_tag, parse_duration, strip_or_none, - traverse_obj, - try_call, ) +from ..utils.traversal import find_element, traverse_obj class ListenNotesIE(InfoExtractor): @@ -22,14 +20,14 @@ class ListenNotesIE(InfoExtractor): 'info_dict': { 'id': 'KrDgvNb_u1n', 'ext': 'mp3', - 'title': 'md5:32236591a921adf17bbdbf0441b6c0e9', - 'description': 'md5:c581ed197eeddcee55a67cdb547c8cbd', - 'duration': 2148.0, - 'channel': 'Thriving on Overload', + 'title': r're:Tim O’Reilly on noticing things other people .{113}', + 'description': r're:(?s)‘’We shape reality by what we notice and .{27459}', + 'duration': 2215.0, + 'channel': 'Amplifying Cognition', 'channel_id': 'ed84wITivxF', 'episode_id': 'e1312583fa7b4e24acfbb5131050be00', - 'thumbnail': 'https://production.listennotes.com/podcasts/thriving-on-overload-ross-dawson-1wb_KospA3P-ed84wITivxF.300x300.jpg', - 'channel_url': 'https://www.listennotes.com/podcasts/thriving-on-overload-ross-dawson-ed84wITivxF/', + 'thumbnail': 'https://cdn-images-3.listennotes.com/podcasts/amplifying-cognition-ross-dawson-Iemft4Gdr0k-ed84wITivxF.300x300.jpg', + 'channel_url': 'https://www.listennotes.com/podcasts/amplifying-cognition-ross-dawson-ed84wITivxF/', 'cast': ['Tim O’Reilly', 'Cookie Monster', 'Lao Tzu', 'Wallace Steven', 'Eric Raymond', 'Christine Peterson', 'John Maynard Keyne', 'Ross Dawson'], }, }, { @@ -39,13 +37,13 @@ class ListenNotesIE(InfoExtractor): 'id': 'lwEA3154JzG', 'ext': 'mp3', 'title': 'Episode 177: WireGuard with Jason Donenfeld', - 'description': 'md5:24744f36456a3e95f83c1193a3458594', + 'description': r're:(?s)Jason Donenfeld lead developer joins us this hour to discuss WireGuard, .{3169}', 'duration': 3861.0, 'channel': 'Ask Noah Show', 'channel_id': '4DQTzdS5-j7', 'episode_id': '8c8954b95e0b4859ad1eecec8bf6d3a4', 'channel_url': 'https://www.listennotes.com/podcasts/ask-noah-show-noah-j-chelliah-4DQTzdS5-j7/', - 'thumbnail': 'https://production.listennotes.com/podcasts/ask-noah-show-noah-j-chelliah-cfbRUw9Gs3F-4DQTzdS5-j7.300x300.jpg', + 'thumbnail': 'https://cdn-images-3.listennotes.com/podcasts/ask-noah-show-noah-j-chelliah-gD7vG150cxf-4DQTzdS5-j7.300x300.jpg', 'cast': ['noah showlink', 'noah show', 'noah dashboard', 'jason donenfeld'], }, }] @@ -70,7 +68,7 @@ class ListenNotesIE(InfoExtractor): 'id': audio_id, 'url': data['audio'], 'title': (data.get('data-title') - or try_call(lambda: get_element_text_and_html_by_tag('h1', webpage)[0]) + or traverse_obj(webpage, ({find_element(tag='h1')}, {clean_html})) or self._html_search_meta(('og:title', 'title', 'twitter:title'), webpage, 'title')), 'description': (self._clean_description(get_element_by_class('ln-text-p', webpage)) or strip_or_none(description)), diff --git a/yt_dlp/extractor/lsm.py b/yt_dlp/extractor/lsm.py index f5be08f97d..56c06d7458 100644 --- a/yt_dlp/extractor/lsm.py +++ b/yt_dlp/extractor/lsm.py @@ -114,7 +114,7 @@ class LSMLREmbedIE(InfoExtractor): def _real_extract(self, url): query = parse_qs(url) video_id = traverse_obj(query, ( - ('show', 'id'), 0, {int_or_none}, {lambda x: x or None}, {str_or_none}), get_all=False) + ('show', 'id'), 0, {int_or_none}, filter, {str_or_none}), get_all=False) webpage = self._download_webpage(url, video_id) player_data, media_data = self._search_regex( diff --git a/yt_dlp/extractor/magentamusik.py b/yt_dlp/extractor/magentamusik.py index 5bfc0a1545..24c46a1529 100644 --- a/yt_dlp/extractor/magentamusik.py +++ b/yt_dlp/extractor/magentamusik.py @@ -57,6 +57,6 @@ class MagentaMusikIE(InfoExtractor): 'duration': ('runtimeInSeconds', {int_or_none}), 'location': ('countriesOfProduction', {list}, {lambda x: join_nonempty(*x, delim=', ')}), 'release_year': ('yearOfProduction', {int_or_none}), - 'categories': ('mainGenre', {str}, {lambda x: x and [x]}), + 'categories': ('mainGenre', {str}, all, filter), })), } diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py index ae0fb2aed2..d2a22f98f3 100644 --- a/yt_dlp/extractor/mediastream.py +++ b/yt_dlp/extractor/mediastream.py @@ -17,7 +17,7 @@ class MediaStreamBaseIE(InfoExtractor): _BASE_URL_RE = r'https?://mdstrm\.com/(?:embed|live-stream)' def _extract_mediastream_urls(self, webpage): - yield from traverse_obj(list(self._yield_json_ld(webpage, None, fatal=False)), ( + yield from traverse_obj(list(self._yield_json_ld(webpage, None, default={})), ( lambda _, v: v['@type'] == 'VideoObject', ('embedUrl', 'contentUrl'), {lambda x: x if re.match(rf'{self._BASE_URL_RE}/\w+', x) else None})) diff --git a/yt_dlp/extractor/mixch.py b/yt_dlp/extractor/mixch.py index 9b7c7b89b9..4bccc81bdc 100644 --- a/yt_dlp/extractor/mixch.py +++ b/yt_dlp/extractor/mixch.py @@ -12,7 +12,7 @@ from ..utils.traversal import traverse_obj class MixchIE(InfoExtractor): IE_NAME = 'mixch' - _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P\d+)' + _VALID_URL = r'https?://mixch\.tv/u/(?P\d+)' _TESTS = [{ 'url': 'https://mixch.tv/u/16943797/live', @@ -66,7 +66,7 @@ class MixchIE(InfoExtractor): note='Downloading comments', errnote='Failed to download comments'), (..., { 'author': ('name', {str}), 'author_id': ('user_id', {str_or_none}), - 'id': ('message_id', {str}, {lambda x: x or None}), + 'id': ('message_id', {str}, filter), 'text': ('body', {str}), 'timestamp': ('created', {int}), })) @@ -74,7 +74,7 @@ class MixchIE(InfoExtractor): class MixchArchiveIE(InfoExtractor): IE_NAME = 'mixch:archive' - _VALID_URL = r'https?://(?:www\.)?mixch\.tv/archive/(?P\d+)' + _VALID_URL = r'https?://mixch\.tv/archive/(?P\d+)' _TESTS = [{ 'url': 'https://mixch.tv/archive/421', @@ -116,3 +116,56 @@ class MixchArchiveIE(InfoExtractor): 'formats': self._extract_m3u8_formats(info_json['archiveURL'], video_id), 'thumbnail': traverse_obj(info_json, ('thumbnailURL', {url_or_none})), } + + +class MixchMovieIE(InfoExtractor): + IE_NAME = 'mixch:movie' + _VALID_URL = r'https?://mixch\.tv/m/(?P\w+)' + + _TESTS = [{ + 'url': 'https://mixch.tv/m/Ve8KNkJ5', + 'info_dict': { + 'id': 'Ve8KNkJ5', + 'title': '夏☀️\nムービーへのポイントは本イベントに加算されないので配信にてお願い致します🙇🏻\u200d♀️\n#TGCCAMPUS #ミス東大 #ミス東大2024 ', + 'ext': 'mp4', + 'uploader': 'ミス東大No.5 松藤百香🍑💫', + 'uploader_id': '12299174', + 'channel_follower_count': int, + 'view_count': int, + 'like_count': int, + 'comment_count': int, + 'timestamp': 1724070828, + 'uploader_url': 'https://mixch.tv/u/12299174', + 'live_status': 'not_live', + 'upload_date': '20240819', + }, + }, { + 'url': 'https://mixch.tv/m/61DzpIKE', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + data = self._download_json( + f'https://mixch.tv/api-web/movies/{video_id}', video_id) + return { + 'id': video_id, + 'formats': [{ + 'format_id': 'mp4', + 'url': data['movie']['file'], + 'ext': 'mp4', + }], + **traverse_obj(data, { + 'title': ('movie', 'title', {str}), + 'thumbnail': ('movie', 'thumbnailURL', {url_or_none}), + 'uploader': ('ownerInfo', 'name', {str}), + 'uploader_id': ('ownerInfo', 'id', {int}, {str_or_none}), + 'channel_follower_count': ('ownerInfo', 'fan', {int_or_none}), + 'view_count': ('ownerInfo', 'view', {int_or_none}), + 'like_count': ('movie', 'favCount', {int_or_none}), + 'comment_count': ('movie', 'commentCount', {int_or_none}), + 'timestamp': ('movie', 'published', {int_or_none}), + 'uploader_url': ('ownerInfo', 'id', {lambda x: x and f'https://mixch.tv/u/{x}'}, filter), + }), + 'live_status': 'not_live', + } diff --git a/yt_dlp/extractor/monstercat.py b/yt_dlp/extractor/monstercat.py index 930c13e278..f17b91f5a2 100644 --- a/yt_dlp/extractor/monstercat.py +++ b/yt_dlp/extractor/monstercat.py @@ -4,15 +4,11 @@ from .common import InfoExtractor from ..utils import ( clean_html, extract_attributes, - get_element_by_class, - get_element_html_by_class, - get_element_text_and_html_by_tag, int_or_none, strip_or_none, - traverse_obj, - try_call, unified_strdate, ) +from ..utils.traversal import find_element, traverse_obj class MonstercatIE(InfoExtractor): @@ -26,19 +22,21 @@ class MonstercatIE(InfoExtractor): 'thumbnail': 'https://www.monstercat.com/release/742779548009/cover', 'release_date': '20230711', 'album': 'The Secret Language of Trees', - 'album_artist': 'BT', + 'album_artists': ['BT'], }, }] def _extract_tracks(self, table, album_meta): for td in re.findall(r'((?:(?!)[\w\W])+)', table): # regex by chatgpt due to lack of get_elements_by_tag - title = clean_html(try_call( - lambda: get_element_by_class('d-inline-flex flex-column', td).partition(' ]+data-audiopath[^>]+>)', playlist), 1): entry = traverse_obj(extract_attributes(track), { @@ -200,12 +201,12 @@ class NekoHackerIE(InfoExtractor): 'album': 'data-albumtitle', 'duration': ('data-tracktime', {parse_duration}), 'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0), - 'thumbnail': ('data-albumart', {url_or_none}), }) entries.append({ **entry, + 'thumbnail': url_or_none(player_params.get('artwork')), 'track_number': track_number, - 'artist': 'Neko Hacker', + 'artists': ['Neko Hacker'], 'vcodec': 'none', 'acodec': 'mp3' if entry['ext'] == 'mp3' else None, }) diff --git a/yt_dlp/extractor/neteasemusic.py b/yt_dlp/extractor/neteasemusic.py index a759da2147..900b8b2a30 100644 --- a/yt_dlp/extractor/neteasemusic.py +++ b/yt_dlp/extractor/neteasemusic.py @@ -36,10 +36,6 @@ class NetEaseMusicBaseIE(InfoExtractor): _API_BASE = 'http://music.163.com/api/' _GEO_BYPASS = False - @staticmethod - def _kilo_or_none(value): - return int_or_none(value, scale=1000) - def _create_eapi_cipher(self, api_path, query_body, cookies): request_text = json.dumps({**query_body, 'header': cookies}, separators=(',', ':')) @@ -101,7 +97,7 @@ class NetEaseMusicBaseIE(InfoExtractor): 'vcodec': 'none', **traverse_obj(song, { 'ext': ('type', {str}), - 'abr': ('br', {self._kilo_or_none}), + 'abr': ('br', {int_or_none(scale=1000)}), 'filesize': ('size', {int_or_none}), }), }) @@ -282,9 +278,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE): **lyric_data, **traverse_obj(info, { 'title': ('name', {str}), - 'timestamp': ('album', 'publishTime', {self._kilo_or_none}), + 'timestamp': ('album', 'publishTime', {int_or_none(scale=1000)}), 'thumbnail': ('album', 'picUrl', {url_or_none}), - 'duration': ('duration', {self._kilo_or_none}), + 'duration': ('duration', {int_or_none(scale=1000)}), 'album': ('album', 'name', {str}), 'average_rating': ('score', {int_or_none}), }), @@ -440,7 +436,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE): 'tags': ('tags', ..., {str}), 'uploader': ('creator', 'nickname', {str}), 'uploader_id': ('creator', 'userId', {str_or_none}), - 'timestamp': ('updateTime', {self._kilo_or_none}), + 'timestamp': ('updateTime', {int_or_none(scale=1000)}), })) if traverse_obj(info, ('playlist', 'specialType')) == 10: metainfo['title'] = f'{metainfo.get("title")} {strftime_or_none(metainfo.get("timestamp"), "%Y-%m-%d")}' @@ -517,10 +513,10 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE): 'creators': traverse_obj(info, ('artists', ..., 'name')) or [info.get('artistName')], **traverse_obj(info, { 'title': ('name', {str}), - 'description': (('desc', 'briefDesc'), {str}, {lambda x: x or None}), + 'description': (('desc', 'briefDesc'), {str}, filter), 'upload_date': ('publishTime', {unified_strdate}), 'thumbnail': ('cover', {url_or_none}), - 'duration': ('duration', {self._kilo_or_none}), + 'duration': ('duration', {int_or_none(scale=1000)}), 'view_count': ('playCount', {int_or_none}), 'like_count': ('likeCount', {int_or_none}), 'comment_count': ('commentCount', {int_or_none}), @@ -588,7 +584,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE): 'description': ('description', {str}), 'creator': ('dj', 'brand', {str}), 'thumbnail': ('coverUrl', {url_or_none}), - 'timestamp': ('createTime', {self._kilo_or_none}), + 'timestamp': ('createTime', {int_or_none(scale=1000)}), }) if not self._yes_playlist( @@ -598,7 +594,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE): return { 'id': str(info['mainSong']['id']), 'formats': formats, - 'duration': traverse_obj(info, ('mainSong', 'duration', {self._kilo_or_none})), + 'duration': traverse_obj(info, ('mainSong', 'duration', {int_or_none(scale=1000)})), **metainfo, } diff --git a/yt_dlp/extractor/nfl.py b/yt_dlp/extractor/nfl.py index c537c1c47c..59213a44be 100644 --- a/yt_dlp/extractor/nfl.py +++ b/yt_dlp/extractor/nfl.py @@ -11,9 +11,12 @@ from ..utils import ( clean_html, determine_ext, get_element_by_class, - traverse_obj, + int_or_none, + make_archive_id, + url_or_none, urlencode_postdata, ) +from ..utils.traversal import traverse_obj class NFLBaseIE(InfoExtractor): @@ -75,22 +78,15 @@ class NFLBaseIE(InfoExtractor): 'osVersion': '10.0', }, separators=(',', ':')).encode()).decode(), 'networkType': 'other', - 'nflClaimGroupsToAdd': [], - 'nflClaimGroupsToRemove': [], + 'peacockUUID': 'undefined', } _ACCOUNT_INFO = {} - _API_KEY = None + _API_KEY = '3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f' _TOKEN = None _TOKEN_EXPIRY = 0 - def _get_account_info(self, url, slug): - if not self._API_KEY: - webpage = self._download_webpage(url, slug, fatal=False) or '' - self._API_KEY = self._search_regex( - r'window\.gigyaApiKey\s*=\s*["\'](\w+)["\'];', webpage, 'API key', - fatal=False) or '3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f' - + def _get_account_info(self): cookies = self._get_cookies('https://auth-id.nfl.com/') login_token = traverse_obj(cookies, ( (f'glt_{self._API_KEY}', lambda k, _: k.startswith('glt_')), {lambda x: x.value}), get_all=False) @@ -103,7 +99,7 @@ class NFLBaseIE(InfoExtractor): 'or else try using --cookies-from-browser instead', expected=True) account = self._download_json( - 'https://auth-id.nfl.com/accounts.getAccountInfo', slug, + 'https://auth-id.nfl.com/accounts.getAccountInfo', None, note='Downloading account info', data=urlencode_postdata({ 'include': 'profile,data', 'lang': 'en', @@ -111,7 +107,7 @@ class NFLBaseIE(InfoExtractor): 'sdk': 'js_latest', 'login_token': login_token, 'authMode': 'cookie', - 'pageURL': url, + 'pageURL': 'https://www.nfl.com/', 'sdkBuild': traverse_obj(cookies, ( 'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='15170'), 'format': 'json', @@ -126,55 +122,78 @@ class NFLBaseIE(InfoExtractor): if len(self._ACCOUNT_INFO) != 3: raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True) - def _get_auth_token(self, url, slug): + def _get_auth_token(self): if self._TOKEN and self._TOKEN_EXPIRY > int(time.time() + 30): return - if not self._ACCOUNT_INFO: - self._get_account_info(url, slug) - token = self._download_json( 'https://api.nfl.com/identity/v3/token%s' % ( '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''), - slug, headers={'Content-Type': 'application/json'}, note='Downloading access token', + None, headers={'Content-Type': 'application/json'}, note='Downloading access token', data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode()) self._TOKEN = token['accessToken'] self._TOKEN_EXPIRY = token['expiresIn'] self._ACCOUNT_INFO['refreshToken'] = token['refreshToken'] + def _extract_video(self, mcp_id, is_live=False): + self._get_auth_token() + data = self._download_json( + f'https://api.nfl.com/play/v1/asset/{mcp_id}', mcp_id, headers={ + 'Authorization': f'Bearer {self._TOKEN}', + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, data=json.dumps({'init': True, 'live': is_live}, separators=(',', ':')).encode()) + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + data['accessUrl'], mcp_id, 'mp4', m3u8_id='hls') + + return { + 'id': mcp_id, + 'formats': formats, + 'subtitles': subtitles, + 'is_live': is_live, + '_old_archive_ids': [make_archive_id(AnvatoIE, mcp_id)], + **traverse_obj(data, ('metadata', { + 'title': ('event', ('def_title', 'friendlyName'), {str}, any), + 'description': ('event', 'def_description', {str}), + 'duration': ('event', 'duration', {int_or_none}), + 'thumbnails': ('thumbnails', ..., 'url', {'url': {url_or_none}}), + })), + } + def _parse_video_config(self, video_config, display_id): video_config = self._parse_json(video_config, display_id) + is_live = traverse_obj(video_config, ('live', {bool})) or False item = video_config['playlist'][0] - mcp_id = item.get('mcpID') - if mcp_id: - info = self.url_result(f'{self._ANVATO_PREFIX}{mcp_id}', AnvatoIE, mcp_id) + if mcp_id := item.get('mcpID'): + return self._extract_video(mcp_id, is_live=is_live) + + info = {'id': item.get('id') or item['entityId']} + + item_url = item['url'] + ext = determine_ext(item_url) + if ext == 'm3u8': + info['formats'] = self._extract_m3u8_formats(item_url, info['id'], 'mp4') else: - media_id = item.get('id') or item['entityId'] - title = item.get('title') - item_url = item['url'] - info = {'id': media_id} - ext = determine_ext(item_url) - if ext == 'm3u8': - info['formats'] = self._extract_m3u8_formats(item_url, media_id, 'mp4') - else: - info['url'] = item_url - if item.get('audio') is True: - info['vcodec'] = 'none' - is_live = video_config.get('live') is True - thumbnails = None - image_url = item.get(item.get('imageSrc')) or item.get(item.get('posterImage')) - if image_url: - thumbnails = [{ - 'url': image_url, - 'ext': determine_ext(image_url, 'jpg'), - }] - info.update({ - 'title': title, - 'is_live': is_live, - 'description': clean_html(item.get('description')), - 'thumbnails': thumbnails, - }) + info['url'] = item_url + if item.get('audio') is True: + info['vcodec'] = 'none' + + thumbnails = None + if image_url := traverse_obj(item, 'imageSrc', 'posterImage', expected_type=url_or_none): + thumbnails = [{ + 'url': image_url, + 'ext': determine_ext(image_url, 'jpg'), + }] + + info.update({ + **traverse_obj(item, { + 'title': ('title', {str}), + 'description': ('description', {clean_html}), + }), + 'is_live': is_live, + 'thumbnails': thumbnails, + }) return info @@ -188,24 +207,20 @@ class NFLIE(NFLBaseIE): 'ext': 'mp4', 'title': "Baker Mayfield's game-changing plays from 3-TD game Week 14", 'description': 'md5:85e05a3cc163f8c344340f220521136d', - 'upload_date': '20201215', - 'timestamp': 1608009755, - 'thumbnail': r're:^https?://.*\.jpg$', - 'uploader': 'NFL', - 'tags': 'count:6', + 'thumbnail': r're:https?://.+\.jpg', 'duration': 157, - 'categories': 'count:3', + '_old_archive_ids': ['anvato 899441'], }, }, { 'url': 'https://www.chiefs.com/listen/patrick-mahomes-travis-kelce-react-to-win-over-dolphins-the-breakdown', - 'md5': '6886b32c24b463038c760ceb55a34566', + 'md5': '92a517f05bd3eb50fe50244bc621aec8', 'info_dict': { - 'id': 'd87e8790-3e14-11eb-8ceb-ff05c2867f99', + 'id': '8b7c3625-a461-4751-8db4-85f536f2bbd0', 'ext': 'mp3', 'title': 'Patrick Mahomes, Travis Kelce React to Win Over Dolphins | The Breakdown', 'description': 'md5:12ada8ee70e6762658c30e223e095075', + 'thumbnail': 'https://static.clubs.nfl.com/image/private/t_editorial_landscape_12_desktop/v1571153441/chiefs/rfljejccnyhhkpkfq855', }, - 'skip': 'HTTP Error 404: Not Found', }, { 'url': 'https://www.buffalobills.com/video/buffalo-bills-military-recognition-week-14', 'only_matching': True, @@ -236,13 +251,16 @@ class NFLArticleIE(NFLBaseIE): def _real_extract(self, url): display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - entries = [] - for video_config in re.findall(self._VIDEO_CONFIG_REGEX, webpage): - entries.append(self._parse_video_config(video_config, display_id)) + + def entries(): + for video_config in re.findall(self._VIDEO_CONFIG_REGEX, webpage): + yield self._parse_video_config(video_config, display_id) + title = clean_html(get_element_by_class( 'nfl-c-article__title', webpage)) or self._html_search_meta( ['og:title', 'twitter:title'], webpage) - return self.playlist_result(entries, display_id, title) + + return self.playlist_result(entries(), display_id, title) class NFLPlusReplayIE(NFLBaseIE): @@ -307,6 +325,9 @@ class NFLPlusReplayIE(NFLBaseIE): 'all_22': 'All-22', } + def _real_initialize(self): + self._get_account_info() + def _real_extract(self, url): slug, video_id = self._match_valid_url(url).group('slug', 'id') requested_types = self._configuration_arg('type', ['all']) @@ -315,7 +336,7 @@ class NFLPlusReplayIE(NFLBaseIE): requested_types = traverse_obj(self._REPLAY_TYPES, (None, requested_types)) if not video_id: - self._get_auth_token(url, slug) + self._get_auth_token() headers = {'Authorization': f'Bearer {self._TOKEN}'} game_id = self._download_json( f'https://api.nfl.com/football/v2/games/externalId/slug/{slug}', slug, @@ -328,14 +349,13 @@ class NFLPlusReplayIE(NFLBaseIE): 'items', lambda _, v: v['subType'] == requested_types[0], 'mcpPlaybackId'), get_all=False) if video_id: - return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id) + return self._extract_video(video_id) def entries(): for replay in traverse_obj( replays, ('items', lambda _, v: v['mcpPlaybackId'] and v['subType'] in requested_types), ): - video_id = replay['mcpPlaybackId'] - yield self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id) + yield self._extract_video(replay['mcpPlaybackId']) return self.playlist_result(entries(), slug) @@ -362,12 +382,15 @@ class NFLPlusEpisodeIE(NFLBaseIE): 'params': {'skip_download': 'm3u8'}, }] + def _real_initialize(self): + self._get_account_info() + def _real_extract(self, url): slug = self._match_id(url) - self._get_auth_token(url, slug) + self._get_auth_token() video_id = self._download_json( f'https://api.nfl.com/content/v1/videos/episodes/{slug}', slug, headers={ 'Authorization': f'Bearer {self._TOKEN}', })['mcpPlaybackId'] - return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id) + return self._extract_video(video_id) diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py index 961dd0c5e9..29fc1da1e2 100644 --- a/yt_dlp/extractor/niconico.py +++ b/yt_dlp/extractor/niconico.py @@ -371,11 +371,11 @@ class NiconicoIE(InfoExtractor): 'acodec': 'aac', 'vcodec': 'h264', **traverse_obj(audio_quality, ('metadata', { - 'abr': ('bitrate', {functools.partial(float_or_none, scale=1000)}), + 'abr': ('bitrate', {float_or_none(scale=1000)}), 'asr': ('samplingRate', {int_or_none}), })), **traverse_obj(video_quality, ('metadata', { - 'vbr': ('bitrate', {functools.partial(float_or_none, scale=1000)}), + 'vbr': ('bitrate', {float_or_none(scale=1000)}), 'height': ('resolution', 'height', {int_or_none}), 'width': ('resolution', 'width', {int_or_none}), })), @@ -428,7 +428,7 @@ class NiconicoIE(InfoExtractor): **audio_fmt, **traverse_obj(audios, (lambda _, v: audio_fmt['format_id'].startswith(v['id']), { 'format_id': ('id', {str}), - 'abr': ('bitRate', {functools.partial(float_or_none, scale=1000)}), + 'abr': ('bitRate', {float_or_none(scale=1000)}), 'asr': ('samplingRate', {int_or_none}), }), get_all=False), 'acodec': 'aac', diff --git a/yt_dlp/extractor/nubilesporn.py b/yt_dlp/extractor/nubilesporn.py index c2079d8b07..47c7be61d5 100644 --- a/yt_dlp/extractor/nubilesporn.py +++ b/yt_dlp/extractor/nubilesporn.py @@ -10,10 +10,10 @@ from ..utils import ( get_element_html_by_class, get_elements_by_class, int_or_none, - try_call, unified_timestamp, urlencode_postdata, ) +from ..utils.traversal import find_element, find_elements, traverse_obj class NubilesPornIE(InfoExtractor): @@ -70,9 +70,8 @@ class NubilesPornIE(InfoExtractor): url, get_element_by_class('watch-page-video-wrapper', page), video_id)[0] channel_id, channel_name = self._search_regex( - r'/video/website/(?P\d+).+>(?P\w+).com', get_element_html_by_class('site-link', page), + r'/video/website/(?P\d+).+>(?P\w+).com', get_element_html_by_class('site-link', page) or '', 'channel', fatal=False, group=('id', 'name')) or (None, None) - channel_name = re.sub(r'([^A-Z]+)([A-Z]+)', r'\1 \2', channel_name) return { 'id': video_id, @@ -82,14 +81,14 @@ class NubilesPornIE(InfoExtractor): 'thumbnail': media_entries.get('thumbnail'), 'description': clean_html(get_element_html_by_class('content-pane-description', page)), 'timestamp': unified_timestamp(get_element_by_class('date', page)), - 'channel': channel_name, + 'channel': re.sub(r'([^A-Z]+)([A-Z]+)', r'\1 \2', channel_name) if channel_name else None, 'channel_id': channel_id, 'channel_url': format_field(channel_id, None, 'https://members.nubiles-porn.com/video/website/%s'), 'like_count': int_or_none(get_element_by_id('likecount', page)), 'average_rating': float_or_none(get_element_by_class('score', page)), 'age_limit': 18, - 'categories': try_call(lambda: list(map(clean_html, get_elements_by_class('btn', get_element_by_class('categories', page))))), - 'tags': try_call(lambda: list(map(clean_html, get_elements_by_class('btn', get_elements_by_class('tags', page)[1])))), + 'categories': traverse_obj(page, ({find_element(cls='categories')}, {find_elements(cls='btn')}, ..., {clean_html})), + 'tags': traverse_obj(page, ({find_elements(cls='tags')}, 1, {find_elements(cls='btn')}, ..., {clean_html})), 'cast': get_elements_by_class('content-pane-performer', page), 'availability': 'needs_auth', 'series': channel_name, diff --git a/yt_dlp/extractor/nytimes.py b/yt_dlp/extractor/nytimes.py index 5ec3cdd675..9ef57410ac 100644 --- a/yt_dlp/extractor/nytimes.py +++ b/yt_dlp/extractor/nytimes.py @@ -235,7 +235,7 @@ class NYTimesArticleIE(NYTimesBaseIE): details = traverse_obj(block, { 'id': ('sourceId', {str}), 'uploader': ('bylines', ..., 'renderedRepresentation', {str}), - 'duration': (None, (('duration', {lambda x: float_or_none(x, scale=1000)}), ('length', {int_or_none}))), + 'duration': (None, (('duration', {float_or_none(scale=1000)}), ('length', {int_or_none}))), 'timestamp': ('firstPublished', {parse_iso8601}), 'series': ('podcastSeries', {str}), }, get_all=False) diff --git a/yt_dlp/extractor/ondemandkorea.py b/yt_dlp/extractor/ondemandkorea.py index 591b4147eb..1921f3fd8a 100644 --- a/yt_dlp/extractor/ondemandkorea.py +++ b/yt_dlp/extractor/ondemandkorea.py @@ -115,7 +115,7 @@ class OnDemandKoreaIE(InfoExtractor): **traverse_obj(data, { 'thumbnail': ('episode', 'images', 'thumbnail', {url_or_none}), 'release_date': ('episode', 'release_date', {lambda x: x.replace('-', '')}, {unified_strdate}), - 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('duration', {float_or_none(scale=1000)}), 'age_limit': ('age_rating', 'name', {lambda x: x.replace('R', '')}, {parse_age_limit}), 'series': ('episode', {if_series(key='program')}, 'title'), 'series_id': ('episode', {if_series(key='program')}, 'id', {str_or_none}), diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py index 9c37a54d62..12c4a21041 100644 --- a/yt_dlp/extractor/orf.py +++ b/yt_dlp/extractor/orf.py @@ -1,5 +1,4 @@ import base64 -import functools import re from .common import InfoExtractor @@ -192,7 +191,7 @@ class ORFPodcastIE(InfoExtractor): 'ext': ('enclosures', 0, 'type', {mimetype2ext}), 'title': 'title', 'description': ('description', {clean_html}), - 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('duration', {float_or_none(scale=1000)}), 'series': ('podcast', 'title'), })), } @@ -494,7 +493,7 @@ class ORFONIE(InfoExtractor): return traverse_obj(api_json, { 'id': ('id', {int}, {str_or_none}), 'age_limit': ('age_classification', {parse_age_limit}), - 'duration': ('exact_duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('exact_duration', {float_or_none(scale=1000)}), 'title': (('title', 'headline'), {str}), 'description': (('description', 'teaser_text'), {str}), 'media_type': ('video_type', {str}), diff --git a/yt_dlp/extractor/parler.py b/yt_dlp/extractor/parler.py index 9be288a7d0..e5bb3be4ee 100644 --- a/yt_dlp/extractor/parler.py +++ b/yt_dlp/extractor/parler.py @@ -1,5 +1,3 @@ -import functools - from .common import InfoExtractor from .youtube import YoutubeIE from ..utils import ( @@ -83,7 +81,7 @@ class ParlerIE(InfoExtractor): 'timestamp': ('date_created', {unified_timestamp}), 'uploader': ('user', 'name', {strip_or_none}), 'uploader_id': ('user', 'username', {str}), - 'uploader_url': ('user', 'username', {functools.partial(urljoin, 'https://parler.com/')}), + 'uploader_url': ('user', 'username', {urljoin('https://parler.com/')}), 'view_count': ('views', {int_or_none}), 'comment_count': ('total_comments', {int_or_none}), 'repost_count': ('echos', {int_or_none}), diff --git a/yt_dlp/extractor/pornbox.py b/yt_dlp/extractor/pornbox.py index 9b89adbf9d..0996e4d979 100644 --- a/yt_dlp/extractor/pornbox.py +++ b/yt_dlp/extractor/pornbox.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..utils import ( @@ -105,7 +104,7 @@ class PornboxIE(InfoExtractor): get_quality = qualities(['web', 'vga', 'hd', '1080p', '4k', '8k']) metadata['formats'] = traverse_obj(stream_data, ('qualities', lambda _, v: v['src'], { 'url': 'src', - 'vbr': ('bitrate', {functools.partial(int_or_none, scale=1000)}), + 'vbr': ('bitrate', {int_or_none(scale=1000)}), 'format_id': ('quality', {str_or_none}), 'quality': ('quality', {get_quality}), 'width': ('size', {lambda x: int(x[:-1])}), diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py index b0d6475fe4..d5d6ecdfd8 100644 --- a/yt_dlp/extractor/pr0gramm.py +++ b/yt_dlp/extractor/pr0gramm.py @@ -198,6 +198,6 @@ class Pr0grammIE(InfoExtractor): 'dislike_count': ('down', {int}), 'timestamp': ('created', {int}), 'upload_date': ('created', {int}, {dt.date.fromtimestamp}, {lambda x: x.strftime('%Y%m%d')}), - 'thumbnail': ('thumb', {lambda x: urljoin('https://thumb.pr0gramm.com', x)}), + 'thumbnail': ('thumb', {urljoin('https://thumb.pr0gramm.com')}), }), } diff --git a/yt_dlp/extractor/qdance.py b/yt_dlp/extractor/qdance.py index 934ebbfd70..4f71657c3f 100644 --- a/yt_dlp/extractor/qdance.py +++ b/yt_dlp/extractor/qdance.py @@ -140,7 +140,7 @@ class QDanceIE(InfoExtractor): 'description': ('description', {str.strip}), 'display_id': ('slug', {str}), 'thumbnail': ('thumbnail', {url_or_none}), - 'duration': ('durationInSeconds', {int_or_none}, {lambda x: x or None}), + 'duration': ('durationInSeconds', {int_or_none}, filter), 'availability': ('subscription', 'level', {extract_availability}), 'is_live': ('type', {lambda x: x.lower() == 'live'}), 'artist': ('acts', ..., {str}), diff --git a/yt_dlp/extractor/qqmusic.py b/yt_dlp/extractor/qqmusic.py index d0238692f6..fb46e0d124 100644 --- a/yt_dlp/extractor/qqmusic.py +++ b/yt_dlp/extractor/qqmusic.py @@ -211,10 +211,10 @@ class QQMusicIE(QQMusicBaseIE): 'formats': formats, **traverse_obj(info_data, { 'title': ('title', {str}), - 'album': ('album', 'title', {str}, {lambda x: x or None}), + 'album': ('album', 'title', {str}, filter), 'release_date': ('time_public', {lambda x: x.replace('-', '') or None}), 'creators': ('singer', ..., 'name', {str}), - 'alt_title': ('subtitle', {str}, {lambda x: x or None}), + 'alt_title': ('subtitle', {str}, filter), 'duration': ('interval', {int_or_none}), }), **traverse_obj(init_data, ('detail', { diff --git a/yt_dlp/extractor/redge.py b/yt_dlp/extractor/redge.py index 7cb91eea48..5ae09a096b 100644 --- a/yt_dlp/extractor/redge.py +++ b/yt_dlp/extractor/redge.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..networking import HEADRequest @@ -118,7 +117,7 @@ class RedCDNLivxIE(InfoExtractor): time_scale = traverse_obj(ism_doc, ('@TimeScale', {int_or_none})) or 10000000 duration = traverse_obj( - ism_doc, ('@Duration', {functools.partial(float_or_none, scale=time_scale)})) or None + ism_doc, ('@Duration', {float_or_none(scale=time_scale)})) or None live_status = None if traverse_obj(ism_doc, '@IsLive') == 'TRUE': diff --git a/yt_dlp/extractor/rtvslo.py b/yt_dlp/extractor/rtvslo.py index 9c2e6fb6b5..49bebb178a 100644 --- a/yt_dlp/extractor/rtvslo.py +++ b/yt_dlp/extractor/rtvslo.py @@ -187,4 +187,4 @@ class RTVSLOShowIE(InfoExtractor): return self.playlist_from_matches( re.findall(r']*\bhref="(/arhiv/[^"]+)"', webpage), playlist_id, self._html_extract_title(webpage), - getter=lambda x: urljoin('https://365.rtvslo.si', x), ie=RTVSLOIE) + getter=urljoin('https://365.rtvslo.si'), ie=RTVSLOIE) diff --git a/yt_dlp/extractor/rutube.py b/yt_dlp/extractor/rutube.py index 2c416811af..abf9aec727 100644 --- a/yt_dlp/extractor/rutube.py +++ b/yt_dlp/extractor/rutube.py @@ -2,15 +2,18 @@ import itertools from .common import InfoExtractor from ..utils import ( + UnsupportedError, bool_or_none, determine_ext, int_or_none, + js_to_json, parse_qs, - traverse_obj, + str_or_none, try_get, unified_timestamp, url_or_none, ) +from ..utils.traversal import traverse_obj class RutubeBaseIE(InfoExtractor): @@ -19,7 +22,7 @@ class RutubeBaseIE(InfoExtractor): query = {} query['format'] = 'json' return self._download_json( - f'http://rutube.ru/api/video/{video_id}/', + f'https://rutube.ru/api/video/{video_id}/', video_id, 'Downloading video JSON', 'Unable to download video JSON', query=query) @@ -61,18 +64,21 @@ class RutubeBaseIE(InfoExtractor): query = {} query['format'] = 'json' return self._download_json( - f'http://rutube.ru/api/play/options/{video_id}/', + f'https://rutube.ru/api/play/options/{video_id}/', video_id, 'Downloading options JSON', 'Unable to download options JSON', headers=self.geo_verification_headers(), query=query) - def _extract_formats(self, options, video_id): + def _extract_formats_and_subtitles(self, options, video_id): formats = [] + subtitles = {} for format_id, format_url in options['video_balancer'].items(): ext = determine_ext(format_url) if ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( - format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)) + fmts, subs = self._extract_m3u8_formats_and_subtitles( + format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) elif ext == 'f4m': formats.extend(self._extract_f4m_formats( format_url, video_id, f4m_id=format_id, fatal=False)) @@ -82,11 +88,19 @@ class RutubeBaseIE(InfoExtractor): 'format_id': format_id, }) for hls_url in traverse_obj(options, ('live_streams', 'hls', ..., 'url', {url_or_none})): - formats.extend(self._extract_m3u8_formats(hls_url, video_id, ext='mp4', fatal=False)) - return formats + fmts, subs = self._extract_m3u8_formats_and_subtitles( + hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls') + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + for caption in traverse_obj(options, ('captions', lambda _, v: url_or_none(v['file']))): + subtitles.setdefault(caption.get('code') or 'ru', []).append({ + 'url': caption['file'], + 'name': caption.get('langTitle'), + }) + return formats, subtitles - def _download_and_extract_formats(self, video_id, query=None): - return self._extract_formats( + def _download_and_extract_formats_and_subtitles(self, video_id, query=None): + return self._extract_formats_and_subtitles( self._download_api_options(video_id, query=query), video_id) @@ -97,8 +111,8 @@ class RutubeIE(RutubeBaseIE): _EMBED_REGEX = [r']+?src=(["\'])(?P(?:https?:)?//rutube\.ru/(?:play/)?embed/[\da-z]{32}.*?)\1'] _TESTS = [{ - 'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/', - 'md5': 'e33ac625efca66aba86cbec9851f2692', + 'url': 'https://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/', + 'md5': '3d73fdfe5bb81b9aef139e22ef3de26a', 'info_dict': { 'id': '3eac3b4561676c17df9132a9a1e62e3e', 'ext': 'mp4', @@ -111,26 +125,25 @@ class RutubeIE(RutubeBaseIE): 'upload_date': '20131016', 'age_limit': 0, 'view_count': int, - 'thumbnail': 'http://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg', + 'thumbnail': 'https://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg', 'categories': ['Новости и СМИ'], 'chapters': [], }, - 'expected_warnings': ['Unable to download f4m'], }, { - 'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661', + 'url': 'https://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661', 'only_matching': True, }, { - 'url': 'http://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661', + 'url': 'https://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661', 'only_matching': True, }, { - 'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252', + 'url': 'https://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252', 'only_matching': True, }, { 'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_type=source', 'only_matching': True, }, { 'url': 'https://rutube.ru/video/private/884fb55f07a97ab673c7d654553e0f48/?p=x2QojCumHTS3rsKHWXN8Lg', - 'md5': 'd106225f15d625538fe22971158e896f', + 'md5': '4fce7b4fcc7b1bcaa3f45eb1e1ad0dd7', 'info_dict': { 'id': '884fb55f07a97ab673c7d654553e0f48', 'ext': 'mp4', @@ -143,11 +156,10 @@ class RutubeIE(RutubeBaseIE): 'upload_date': '20221210', 'age_limit': 0, 'view_count': int, - 'thumbnail': 'http://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg', + 'thumbnail': 'https://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg', 'categories': ['Видеоигры'], 'chapters': [], }, - 'expected_warnings': ['Unable to download f4m'], }, { 'url': 'https://rutube.ru/video/c65b465ad0c98c89f3b25cb03dcc87c6/', 'info_dict': { @@ -156,17 +168,16 @@ class RutubeIE(RutubeBaseIE): 'chapters': 'count:4', 'categories': ['Бизнес и предпринимательство'], 'description': 'md5:252feac1305257d8c1bab215cedde75d', - 'thumbnail': 'http://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png', + 'thumbnail': 'https://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png', 'duration': 782, 'age_limit': 0, 'uploader_id': '23491359', 'timestamp': 1677153329, 'view_count': int, 'upload_date': '20230223', - 'title': 'Бизнес с нуля: найм сотрудников. Интервью с директором строительной компании', + 'title': 'Бизнес с нуля: найм сотрудников. Интервью с директором строительной компании #1', 'uploader': 'Стас Быков', }, - 'expected_warnings': ['Unable to download f4m'], }, { 'url': 'https://rutube.ru/live/video/c58f502c7bb34a8fcdd976b221fca292/', 'info_dict': { @@ -174,7 +185,7 @@ class RutubeIE(RutubeBaseIE): 'ext': 'mp4', 'categories': ['Телепередачи'], 'description': '', - 'thumbnail': 'http://pic.rutubelist.ru/video/14/19/14190807c0c48b40361aca93ad0867c7.jpg', + 'thumbnail': 'https://pic.rutubelist.ru/video/14/19/14190807c0c48b40361aca93ad0867c7.jpg', 'live_status': 'is_live', 'age_limit': 0, 'uploader_id': '23460655', @@ -184,6 +195,24 @@ class RutubeIE(RutubeBaseIE): 'title': r're:Первый канал. Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', 'uploader': 'Первый канал', }, + }, { + 'url': 'https://rutube.ru/play/embed/03a9cb54bac3376af4c5cb0f18444e01/', + 'info_dict': { + 'id': '03a9cb54bac3376af4c5cb0f18444e01', + 'ext': 'mp4', + 'age_limit': 0, + 'description': '', + 'title': 'Церемония начала торгов акциями ПАО «ЕвроТранс»', + 'chapters': [], + 'upload_date': '20240829', + 'duration': 293, + 'uploader': 'MOEX - Московская биржа', + 'timestamp': 1724946628, + 'thumbnail': 'https://pic.rutubelist.ru/video/2e/24/2e241fddb459baf0fa54acfca44874f4.jpg', + 'view_count': int, + 'uploader_id': '38420507', + 'categories': ['Интервью'], + }, }, { 'url': 'https://rutube.ru/video/5ab908fccfac5bb43ef2b1e4182256b0/', 'only_matching': True, @@ -192,40 +221,46 @@ class RutubeIE(RutubeBaseIE): 'only_matching': True, }] - @classmethod - def suitable(cls, url): - return False if RutubePlaylistIE.suitable(url) else super().suitable(url) - def _real_extract(self, url): video_id = self._match_id(url) query = parse_qs(url) info = self._download_and_extract_info(video_id, query) - info['formats'] = self._download_and_extract_formats(video_id, query) - return info + formats, subtitles = self._download_and_extract_formats_and_subtitles(video_id, query) + return { + **info, + 'formats': formats, + 'subtitles': subtitles, + } class RutubeEmbedIE(RutubeBaseIE): IE_NAME = 'rutube:embed' IE_DESC = 'Rutube embedded videos' - _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P[0-9]+)' + _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P[0-9]+)(?:[?#/]|$)' _TESTS = [{ - 'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=', + 'url': 'https://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=', 'info_dict': { 'id': 'a10e53b86e8f349080f718582ce4c661', 'ext': 'mp4', 'timestamp': 1387830582, 'upload_date': '20131223', 'uploader_id': '297833', - 'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix

восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89', 'uploader': 'subziro89 ILya', 'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89', + 'age_limit': 0, + 'duration': 1395, + 'chapters': [], + 'description': 'md5:a5acea57bbc3ccdc3cacd1f11a014b5b', + 'view_count': int, + 'thumbnail': 'https://pic.rutubelist.ru/video/d3/03/d3031f4670a6e6170d88fb3607948418.jpg', + 'categories': ['Сериалы'], }, 'params': { 'skip_download': True, }, }, { - 'url': 'http://rutube.ru/play/embed/8083783', + 'url': 'https://rutube.ru/play/embed/8083783', 'only_matching': True, }, { # private video @@ -240,11 +275,12 @@ class RutubeEmbedIE(RutubeBaseIE): query = parse_qs(url) options = self._download_api_options(embed_id, query) video_id = options['effective_video'] - formats = self._extract_formats(options, video_id) + formats, subtitles = self._extract_formats_and_subtitles(options, video_id) info = self._download_and_extract_info(video_id, query) info.update({ 'extractor_key': 'Rutube', 'formats': formats, + 'subtitles': subtitles, }) return info @@ -295,14 +331,14 @@ class RutubeTagsIE(RutubePlaylistBaseIE): IE_DESC = 'Rutube tags' _VALID_URL = r'https?://rutube\.ru/tags/video/(?P\d+)' _TESTS = [{ - 'url': 'http://rutube.ru/tags/video/1800/', + 'url': 'https://rutube.ru/tags/video/1800/', 'info_dict': { 'id': '1800', }, 'playlist_mincount': 68, }] - _PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json' + _PAGE_TEMPLATE = 'https://rutube.ru/api/tags/video/%s/?page=%s&format=json' class RutubeMovieIE(RutubePlaylistBaseIE): @@ -310,8 +346,8 @@ class RutubeMovieIE(RutubePlaylistBaseIE): IE_DESC = 'Rutube movies' _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P\d+)' - _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json' - _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json' + _MOVIE_TEMPLATE = 'https://rutube.ru/api/metainfo/tv/%s/?format=json' + _PAGE_TEMPLATE = 'https://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json' def _real_extract(self, url): movie_id = self._match_id(url) @@ -327,62 +363,82 @@ class RutubePersonIE(RutubePlaylistBaseIE): IE_DESC = 'Rutube person videos' _VALID_URL = r'https?://rutube\.ru/video/person/(?P\d+)' _TESTS = [{ - 'url': 'http://rutube.ru/video/person/313878/', + 'url': 'https://rutube.ru/video/person/313878/', 'info_dict': { 'id': '313878', }, - 'playlist_mincount': 37, + 'playlist_mincount': 36, }] - _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json' + _PAGE_TEMPLATE = 'https://rutube.ru/api/video/person/%s/?page=%s&format=json' class RutubePlaylistIE(RutubePlaylistBaseIE): IE_NAME = 'rutube:playlist' IE_DESC = 'Rutube playlists' - _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/[\da-z]{32}/\?.*?\bpl_id=(?P\d+)' + _VALID_URL = r'https?://rutube\.ru/plst/(?P\d+)' _TESTS = [{ - 'url': 'https://rutube.ru/video/cecd58ed7d531fc0f3d795d51cee9026/?pl_id=3097&pl_type=tag', + 'url': 'https://rutube.ru/plst/308547/', 'info_dict': { - 'id': '3097', + 'id': '308547', }, - 'playlist_count': 27, - }, { - 'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source', - 'only_matching': True, + 'playlist_mincount': 22, }] - - _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/%s/%s/?page=%s&format=json' - - @classmethod - def suitable(cls, url): - from ..utils import int_or_none, parse_qs - - if not super().suitable(url): - return False - params = parse_qs(url) - return params.get('pl_type', [None])[0] and int_or_none(params.get('pl_id', [None])[0]) - - def _next_page_url(self, page_num, playlist_id, item_kind): - return self._PAGE_TEMPLATE % (item_kind, playlist_id, page_num) - - def _real_extract(self, url): - qs = parse_qs(url) - playlist_kind = qs['pl_type'][0] - playlist_id = qs['pl_id'][0] - return self._extract_playlist(playlist_id, item_kind=playlist_kind) + _PAGE_TEMPLATE = 'https://rutube.ru/api/playlist/custom/%s/videos?page=%s&format=json' class RutubeChannelIE(RutubePlaylistBaseIE): IE_NAME = 'rutube:channel' IE_DESC = 'Rutube channel' - _VALID_URL = r'https?://rutube\.ru/channel/(?P\d+)/videos' + _VALID_URL = r'https?://rutube\.ru/(?:channel/(?P\d+)|u/(?P\w+))(?:/(?P
videos|shorts|playlists))?' _TESTS = [{ 'url': 'https://rutube.ru/channel/639184/videos/', 'info_dict': { - 'id': '639184', + 'id': '639184_videos', }, - 'playlist_mincount': 133, + 'playlist_mincount': 129, + }, { + 'url': 'https://rutube.ru/channel/25902603/shorts/', + 'info_dict': { + 'id': '25902603_shorts', + }, + 'playlist_mincount': 277, + }, { + 'url': 'https://rutube.ru/channel/25902603/', + 'info_dict': { + 'id': '25902603', + }, + 'playlist_mincount': 406, + }, { + 'url': 'https://rutube.ru/u/rutube/videos/', + 'info_dict': { + 'id': '23704195_videos', + }, + 'playlist_mincount': 113, }] - _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json' + _PAGE_TEMPLATE = 'https://rutube.ru/api/video/person/%s/?page=%s&format=json&origin__type=%s' + + def _next_page_url(self, page_num, playlist_id, section): + origin_type = { + 'videos': 'rtb,rst,ifrm,rspa', + 'shorts': 'rshorts', + None: '', + }.get(section) + return self._PAGE_TEMPLATE % (playlist_id, page_num, origin_type) + + def _real_extract(self, url): + playlist_id, slug, section = self._match_valid_url(url).group('id', 'slug', 'section') + if section == 'playlists': + raise UnsupportedError(url) + if slug: + webpage = self._download_webpage(url, slug) + redux_state = self._search_json( + r'window\.reduxState\s*=', webpage, 'redux state', slug, transform_source=js_to_json) + playlist_id = traverse_obj(redux_state, ( + 'api', 'queries', lambda k, _: k.startswith('channelIdBySlug'), + 'data', 'channel_id', {int}, {str_or_none}, any)) + playlist = self._extract_playlist(playlist_id, section=section) + if section: + playlist['id'] = f'{playlist_id}_{section}' + return playlist diff --git a/yt_dlp/extractor/snapchat.py b/yt_dlp/extractor/snapchat.py index 732677c190..09e5766d4f 100644 --- a/yt_dlp/extractor/snapchat.py +++ b/yt_dlp/extractor/snapchat.py @@ -56,13 +56,13 @@ class SnapchatSpotlightIE(InfoExtractor): **traverse_obj(video_data, ('videoMetadata', { 'title': ('name', {str}), 'description': ('description', {str}), - 'timestamp': ('uploadDateMs', {lambda x: float_or_none(x, 1000)}), + 'timestamp': ('uploadDateMs', {float_or_none(scale=1000)}), 'view_count': ('viewCount', {int_or_none}, {lambda x: None if x == -1 else x}), 'repost_count': ('shareCount', {int_or_none}), 'url': ('contentUrl', {url_or_none}), 'width': ('width', {int_or_none}), 'height': ('height', {int_or_none}), - 'duration': ('durationMs', {lambda x: float_or_none(x, 1000)}), + 'duration': ('durationMs', {float_or_none(scale=1000)}), 'thumbnail': ('thumbnailUrl', {url_or_none}), 'uploader': ('creator', 'personCreator', 'username', {str}), 'uploader_url': ('creator', 'personCreator', 'url', {url_or_none}), diff --git a/yt_dlp/extractor/spreaker.py b/yt_dlp/extractor/spreaker.py index d1df45969b..c64c2fcd2e 100644 --- a/yt_dlp/extractor/spreaker.py +++ b/yt_dlp/extractor/spreaker.py @@ -2,13 +2,16 @@ import itertools from .common import InfoExtractor from ..utils import ( + filter_dict, float_or_none, int_or_none, + parse_qs, str_or_none, try_get, unified_timestamp, url_or_none, ) +from ..utils.traversal import traverse_obj def _extract_episode(data, episode_id=None): @@ -58,15 +61,10 @@ def _extract_episode(data, episode_id=None): class SpreakerIE(InfoExtractor): - _VALID_URL = r'''(?x) - https?:// - api\.spreaker\.com/ - (?: - (?:download/)?episode| - v2/episodes - )/ - (?P\d+) - ''' + _VALID_URL = [ + r'https?://api\.spreaker\.com/(?:(?:download/)?episode|v2/episodes)/(?P\d+)', + r'https?://(?:www\.)?spreaker\.com/episode/[^#?/]*?(?P\d+)/?(?:[?#]|$)', + ] _TESTS = [{ 'url': 'https://api.spreaker.com/episode/12534508', 'info_dict': { @@ -83,7 +81,9 @@ class SpreakerIE(InfoExtractor): 'view_count': int, 'like_count': int, 'comment_count': int, - 'series': 'Success With Music (SWM)', + 'series': 'Success With Music | SWM', + 'thumbnail': 'https://d3wo5wojvuv7l.cloudfront.net/t_square_limited_160/images.spreaker.com/original/777ce4f96b71b0e1b7c09a5e625210e3.jpg', + 'creators': ['SWM'], }, }, { 'url': 'https://api.spreaker.com/download/episode/12534508/swm_ep15_how_to_market_your_music_part_2.mp3', @@ -91,52 +91,75 @@ class SpreakerIE(InfoExtractor): }, { 'url': 'https://api.spreaker.com/v2/episodes/12534508?export=episode_segments', 'only_matching': True, + }, { + 'note': 'episode', + 'url': 'https://www.spreaker.com/episode/grunge-music-origins-the-raw-sound-that-defined-a-generation--60269615', + 'info_dict': { + 'id': '60269615', + 'display_id': 'grunge-music-origins-the-raw-sound-that-', + 'ext': 'mp3', + 'title': 'Grunge Music Origins - The Raw Sound that Defined a Generation', + 'description': str, + 'timestamp': 1717468905, + 'upload_date': '20240604', + 'uploader': 'Katie Brown 2', + 'uploader_id': '17733249', + 'duration': 818.83, + 'view_count': int, + 'like_count': int, + 'comment_count': int, + 'series': '90s Grunge', + 'thumbnail': 'https://d3wo5wojvuv7l.cloudfront.net/t_square_limited_160/images.spreaker.com/original/bb0d4178f7cf57cc8786dedbd9c5d969.jpg', + 'creators': ['Katie Brown 2'], + }, + }, { + 'url': 'https://www.spreaker.com/episode/60269615', + 'only_matching': True, }] def _real_extract(self, url): episode_id = self._match_id(url) data = self._download_json( - f'https://api.spreaker.com/v2/episodes/{episode_id}', - episode_id)['response']['episode'] + f'https://api.spreaker.com/v2/episodes/{episode_id}', episode_id, + query=traverse_obj(parse_qs(url), {'key': ('key', 0)}))['response']['episode'] return _extract_episode(data, episode_id) -class SpreakerPageIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?spreaker\.com/user/[^/]+/(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.spreaker.com/user/9780658/swm-ep15-how-to-market-your-music-part-2', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - episode_id = self._search_regex( - (r'data-episode_id=["\'](?P\d+)', - r'episode_id\s*:\s*(?P\d+)'), webpage, 'episode id') - return self.url_result( - f'https://api.spreaker.com/episode/{episode_id}', - ie=SpreakerIE.ie_key(), video_id=episode_id) - - class SpreakerShowIE(InfoExtractor): - _VALID_URL = r'https?://api\.spreaker\.com/show/(?P\d+)' + _VALID_URL = [ + r'https?://api\.spreaker\.com/show/(?P\d+)', + r'https?://(?:www\.)?spreaker\.com/podcast/[\w-]+--(?P[\d]+)', + r'https?://(?:www\.)?spreaker\.com/show/(?P\d+)/episodes/feed', + ] _TESTS = [{ 'url': 'https://api.spreaker.com/show/4652058', 'info_dict': { 'id': '4652058', }, 'playlist_mincount': 118, + }, { + 'url': 'https://www.spreaker.com/podcast/health-wealth--5918323', + 'info_dict': { + 'id': '5918323', + }, + 'playlist_mincount': 60, + }, { + 'url': 'https://www.spreaker.com/show/5887186/episodes/feed', + 'info_dict': { + 'id': '5887186', + }, + 'playlist_mincount': 290, }] - def _entries(self, show_id): + def _entries(self, show_id, key=None): for page_num in itertools.count(1): episodes = self._download_json( f'https://api.spreaker.com/show/{show_id}/episodes', - show_id, note=f'Downloading JSON page {page_num}', query={ + show_id, note=f'Downloading JSON page {page_num}', query=filter_dict({ 'page': page_num, 'max_per_page': 100, - }) + 'key': key, + })) pager = try_get(episodes, lambda x: x['response']['pager'], dict) if not pager: break @@ -152,21 +175,5 @@ class SpreakerShowIE(InfoExtractor): def _real_extract(self, url): show_id = self._match_id(url) - return self.playlist_result(self._entries(show_id), playlist_id=show_id) - - -class SpreakerShowPageIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?spreaker\.com/show/(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.spreaker.com/show/success-with-music', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - show_id = self._search_regex( - r'show_id\s*:\s*(?P\d+)', webpage, 'show id') - return self.url_result( - f'https://api.spreaker.com/show/{show_id}', - ie=SpreakerShowIE.ie_key(), video_id=show_id) + key = traverse_obj(parse_qs(url), ('key', 0)) + return self.playlist_result(self._entries(show_id, key), playlist_id=show_id) diff --git a/yt_dlp/extractor/tbsjp.py b/yt_dlp/extractor/tbsjp.py index 32f9cfbdec..0d521f1061 100644 --- a/yt_dlp/extractor/tbsjp.py +++ b/yt_dlp/extractor/tbsjp.py @@ -3,14 +3,12 @@ from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, clean_html, - get_element_text_and_html_by_tag, int_or_none, str_or_none, - traverse_obj, - try_call, unified_timestamp, urljoin, ) +from ..utils.traversal import find_element, traverse_obj class TBSJPEpisodeIE(InfoExtractor): @@ -64,7 +62,7 @@ class TBSJPEpisodeIE(InfoExtractor): self._merge_subtitles(subs, target=subtitles) return { - 'title': try_call(lambda: clean_html(get_element_text_and_html_by_tag('h3', webpage)[0])), + 'title': traverse_obj(webpage, ({find_element(tag='h3')}, {clean_html})), 'id': video_id, **traverse_obj(episode, { 'categories': ('keywords', {list}), diff --git a/yt_dlp/extractor/teamcoco.py b/yt_dlp/extractor/teamcoco.py index 3fb899cac5..a94ff9b332 100644 --- a/yt_dlp/extractor/teamcoco.py +++ b/yt_dlp/extractor/teamcoco.py @@ -136,7 +136,7 @@ class TeamcocoIE(TeamcocoBaseIE): 'blocks', lambda _, v: v['name'] in ('meta-tags', 'video-player', 'video-info'), 'props', {dict}))) thumbnail = traverse_obj( - info, (('image', 'poster'), {lambda x: urljoin('https://teamcoco.com/', x)}), get_all=False) + info, (('image', 'poster'), {urljoin('https://teamcoco.com/')}), get_all=False) video_id = traverse_obj(parse_qs(thumbnail), ('id', 0)) or display_id formats, subtitles = self._get_formats_and_subtitles(info, video_id) diff --git a/yt_dlp/extractor/telewebion.py b/yt_dlp/extractor/telewebion.py index b651160240..02a6ea85bc 100644 --- a/yt_dlp/extractor/telewebion.py +++ b/yt_dlp/extractor/telewebion.py @@ -10,10 +10,11 @@ from ..utils.traversal import traverse_obj def _fmt_url(url): - return functools.partial(format_field, template=url, default=None) + return format_field(template=url, default=None) class TelewebionIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?telewebion\.com/episode/(?P(?:0x[a-fA-F\d]+|\d+))' _TESTS = [{ 'url': 'http://www.telewebion.com/episode/0x1b3139c/', diff --git a/yt_dlp/extractor/tencent.py b/yt_dlp/extractor/tencent.py index fc2b07ac27..b281ad1a9f 100644 --- a/yt_dlp/extractor/tencent.py +++ b/yt_dlp/extractor/tencent.py @@ -1,4 +1,3 @@ -import functools import random import re import string @@ -278,7 +277,7 @@ class VQQSeriesIE(VQQBaseIE): webpage)] return self.playlist_from_matches( - episode_paths, series_id, ie=VQQVideoIE, getter=functools.partial(urljoin, url), + episode_paths, series_id, ie=VQQVideoIE, getter=urljoin(url), title=self._get_clean_title(traverse_obj(webpage_metadata, ('coverInfo', 'title')) or self._og_search_title(webpage)), description=(traverse_obj(webpage_metadata, ('coverInfo', 'description')) @@ -328,7 +327,7 @@ class WeTvBaseIE(TencentBaseIE): or re.findall(r']+class="play-video__link"[^>]+href="(?P[^"]+)', webpage)) return self.playlist_from_matches( - episode_paths, series_id, ie=ie, getter=functools.partial(urljoin, url), + episode_paths, series_id, ie=ie, getter=urljoin(url), title=self._get_clean_title(traverse_obj(webpage_metadata, ('coverInfo', 'title')) or self._og_search_title(webpage)), description=(traverse_obj(webpage_metadata, ('coverInfo', 'description')) diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py index 07db583470..cc7bc3b2fc 100644 --- a/yt_dlp/extractor/tenplay.py +++ b/yt_dlp/extractor/tenplay.py @@ -1,4 +1,3 @@ -import functools import itertools from .common import InfoExtractor @@ -161,4 +160,4 @@ class TenPlaySeasonIE(InfoExtractor): return self.playlist_from_matches( self._entries(urljoin(url, episodes_carousel['loadMoreUrl']), playlist_id), playlist_id, traverse_obj(season_info, ('content', 0, 'title', {str})), - getter=functools.partial(urljoin, url)) + getter=urljoin(url)) diff --git a/yt_dlp/extractor/theguardian.py b/yt_dlp/extractor/theguardian.py index a9e4990649..7e8f9fef26 100644 --- a/yt_dlp/extractor/theguardian.py +++ b/yt_dlp/extractor/theguardian.py @@ -131,4 +131,4 @@ class TheGuardianPodcastPlaylistIE(InfoExtractor): return self.playlist_from_matches( self._entries(url, podcast_id), podcast_id, title, description=description, - ie=TheGuardianPodcastIE, getter=lambda x: urljoin('https://www.theguardian.com', x)) + ie=TheGuardianPodcastIE, getter=urljoin('https://www.theguardian.com')) diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py index f7e103fe9f..ba15f08b6d 100644 --- a/yt_dlp/extractor/tiktok.py +++ b/yt_dlp/extractor/tiktok.py @@ -469,7 +469,7 @@ class TikTokBaseIE(InfoExtractor): aweme_detail, aweme_id, traverse_obj(author_info, 'uploader', 'uploader_id', 'channel_id')), 'thumbnails': thumbnails, 'duration': (traverse_obj(video_info, ( - (None, 'download_addr'), 'duration', {functools.partial(int_or_none, scale=1000)}, any)) + (None, 'download_addr'), 'duration', {int_or_none(scale=1000)}, any)) or traverse_obj(music_info, ('duration', {int_or_none}))), 'availability': self._availability( is_private='Private' in labels, @@ -583,7 +583,7 @@ class TikTokBaseIE(InfoExtractor): author_info, ['uploader', 'uploader_id'], self._UPLOADER_URL_FORMAT, default=None), **traverse_obj(aweme_detail, ('music', { 'track': ('title', {str}), - 'album': ('album', {str}, {lambda x: x or None}), + 'album': ('album', {str}, filter), 'artists': ('authorName', {str}, {lambda x: re.split(r'(?:, | & )', x) if x else None}), 'duration': ('duration', {int_or_none}), })), @@ -591,7 +591,7 @@ class TikTokBaseIE(InfoExtractor): 'title': ('desc', {str}), 'description': ('desc', {str}), # audio-only slideshows have a video duration of 0 and an actual audio duration - 'duration': ('video', 'duration', {int_or_none}, {lambda x: x or None}), + 'duration': ('video', 'duration', {int_or_none}, filter), 'timestamp': ('createTime', {int_or_none}), }), **traverse_obj(aweme_detail, ('stats', { @@ -1493,7 +1493,7 @@ class TikTokLiveIE(TikTokBaseIE): sdk_params = traverse_obj(stream, ('main', 'sdk_params', {parse_inner}, { 'vcodec': ('VCodec', {str}), - 'tbr': ('vbitrate', {lambda x: int_or_none(x, 1000)}), + 'tbr': ('vbitrate', {int_or_none(scale=1000)}), 'resolution': ('resolution', {lambda x: re.match(r'(?i)\d+x\d+|\d+p', x).group().lower()}), })) diff --git a/yt_dlp/extractor/tumblr.py b/yt_dlp/extractor/tumblr.py index 7f851bf63b..d6d4368839 100644 --- a/yt_dlp/extractor/tumblr.py +++ b/yt_dlp/extractor/tumblr.py @@ -3,12 +3,13 @@ from ..utils import ( ExtractorError, int_or_none, traverse_obj, + url_or_none, urlencode_postdata, ) class TumblrIE(InfoExtractor): - _VALID_URL = r'https?://(?P[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P[0-9]+)(?:$|[/?#])' + _VALID_URL = r'https?://(?P[^/?#&]+)\.tumblr\.com/(?:post|video|(?P[a-zA-Z\d-]+))/(?P[0-9]+)(?:$|[/?#])' _NETRC_MACHINE = 'tumblr' _LOGIN_URL = 'https://www.tumblr.com/login' _OAUTH_URL = 'https://www.tumblr.com/api/v2/oauth2/token' @@ -66,6 +67,7 @@ class TumblrIE(InfoExtractor): 'age_limit': 0, 'tags': [], }, + 'skip': '404', }, { 'note': 'dashboard only (original post)', 'url': 'https://jujanon.tumblr.com/post/159704441298/my-baby-eating', @@ -98,7 +100,6 @@ class TumblrIE(InfoExtractor): 'like_count': int, 'repost_count': int, 'age_limit': 0, - 'tags': [], }, }, { 'note': 'dashboard only (external)', @@ -109,14 +110,13 @@ class TumblrIE(InfoExtractor): 'title': 'The Blues Remembers Everything the Country Forgot', 'alt_title': 'The Blues Remembers Everything the Country Forgot', 'description': 'md5:1a6b4097e451216835a24c1023707c79', - 'release_date': '20201224', 'creator': 'md5:c2239ba15430e87c3b971ba450773272', 'uploader': 'Moor Mother - Topic', 'upload_date': '20201223', 'uploader_id': 'UCxrMtFBRkFvQJ_vVM4il08w', 'uploader_url': 'http://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w', 'thumbnail': r're:^https?://i.ytimg.com/.*', - 'channel': 'Moor Mother - Topic', + 'channel': 'Moor Mother', 'channel_id': 'UCxrMtFBRkFvQJ_vVM4il08w', 'channel_url': 'https://www.youtube.com/channel/UCxrMtFBRkFvQJ_vVM4il08w', 'channel_follower_count': int, @@ -135,24 +135,10 @@ class TumblrIE(InfoExtractor): 'release_year': 2020, }, 'add_ie': ['Youtube'], - }, { - 'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching', - 'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab', - 'info_dict': { - 'id': 'Wmur', - 'ext': 'mp4', - 'title': 'naked smoking & stretching', - 'upload_date': '20150506', - 'timestamp': 1430931613, - 'age_limit': 18, - 'uploader_id': '1638622', - 'uploader': 'naked-yogi', - }, - # 'add_ie': ['Vidme'], - 'skip': 'dead embedded video host', + 'skip': 'Video Unavailable', }, { 'url': 'https://prozdvoices.tumblr.com/post/673201091169681408/what-recording-voice-acting-sounds-like', - 'md5': 'a0063fc8110e6c9afe44065b4ea68177', + 'md5': 'cb8328a6723c30556cef59e370202918', 'info_dict': { 'id': 'eomhW5MLGWA', 'ext': 'mp4', @@ -160,8 +146,8 @@ class TumblrIE(InfoExtractor): 'description': 'md5:1da3faa22d0e0b1d8b50216c284ee798', 'uploader': 'ProZD', 'upload_date': '20220112', - 'uploader_id': 'ProZD', - 'uploader_url': 'http://www.youtube.com/user/ProZD', + 'uploader_id': '@ProZD', + 'uploader_url': 'https://www.youtube.com/@ProZD', 'thumbnail': r're:^https?://i.ytimg.com/.*', 'channel': 'ProZD', 'channel_id': 'UC6MFZAOHXlKK1FI7V0XQVeA', @@ -176,6 +162,10 @@ class TumblrIE(InfoExtractor): 'live_status': 'not_live', 'playable_in_embed': True, 'availability': 'public', + 'heatmap': 'count:100', + 'channel_is_verified': True, + 'timestamp': 1642014562, + 'comment_count': int, }, 'add_ie': ['Youtube'], }, { @@ -183,16 +173,20 @@ class TumblrIE(InfoExtractor): 'md5': '203e9eb8077e3f45bfaeb4c86c1467b8', 'info_dict': { 'id': '87816359', - 'ext': 'mov', + 'ext': 'mp4', 'title': 'Harold Ramis', - 'description': 'md5:be8e68cbf56ce0785c77f0c6c6dfaf2c', + 'description': 'md5:c99882405fcca0b1d348ad093f8f1672', 'uploader': 'Resolution Productions Group', 'uploader_id': 'resolutionproductions', 'uploader_url': 'https://vimeo.com/resolutionproductions', 'upload_date': '20140227', 'thumbnail': r're:^https?://i.vimeocdn.com/video/.*', - 'timestamp': 1393523719, + 'timestamp': 1393541719, 'duration': 291, + 'comment_count': int, + 'like_count': int, + 'release_timestamp': 1393541719, + 'release_date': '20140227', }, 'add_ie': ['Vimeo'], }, { @@ -214,6 +208,7 @@ class TumblrIE(InfoExtractor): 'view_count': int, }, 'add_ie': ['Vine'], + 'skip': 'Vine is unavailable', }, { 'url': 'https://silami.tumblr.com/post/84250043974/my-bad-river-flows-in-you-impression-on-maschine', 'md5': '3c92d7c3d867f14ccbeefa2119022277', @@ -232,6 +227,140 @@ class TumblrIE(InfoExtractor): 'upload_date': '20140429', }, 'add_ie': ['Instagram'], + }, { + 'note': 'new url scheme', + 'url': 'https://www.tumblr.com/autumnsister/765162750456578048?source=share', + 'info_dict': { + 'id': '765162750456578048', + 'ext': 'mp4', + 'uploader_url': 'https://autumnsister.tumblr.com/', + 'tags': ['autumn', 'food', 'curators on tumblr'], + 'like_count': int, + 'thumbnail': 'https://64.media.tumblr.com/tumblr_sklad89N3x1ygquow_frame1.jpg', + 'title': '🪹', + 'uploader_id': 'autumnsister', + 'repost_count': int, + 'age_limit': 0, + }, + }, { + 'note': 'bandcamp album embed', + 'url': 'https://patricia-taxxon.tumblr.com/post/704473755725004800/patricia-taxxon-agnes-hilda-patricia-taxxon', + 'info_dict': { + 'id': 'agnes-hilda', + 'title': 'Agnes & Hilda', + 'description': 'The inexplicable joy of an artist. Wash paws after listening.', + 'uploader_id': 'patriciataxxon', + }, + 'playlist_count': 8, + }, { + 'note': 'bandcamp track embeds (many)', + 'url': 'https://www.tumblr.com/felixcosm/730460905855467520/if-youre-looking-for-new-music-to-write-or', + 'info_dict': { + 'id': '730460905855467520', + 'uploader_id': 'felixcosm', + 'repost_count': int, + 'tags': 'count:15', + 'description': 'md5:2eb3482a3c6987280cbefb6839068f32', + 'like_count': int, + 'age_limit': 0, + 'title': 'If you\'re looking for new music to write or imagine scenerios to: STOP. This is for you.', + 'uploader_url': 'https://felixcosm.tumblr.com/', + }, + 'playlist_count': 10, + }, { + 'note': 'soundcloud track embed', + 'url': 'https://silverfoxstole.tumblr.com/post/765305403763556352/jamie-robertson-doctor-who-8th-doctor', + 'info_dict': { + 'id': '1218136399', + 'ext': 'opus', + 'comment_count': int, + 'genres': [], + 'repost_count': int, + 'uploader': 'Jamie Robertson', + 'title': 'Doctor Who - 8th doctor - Stranded Theme never released and used.', + 'duration': 46.106, + 'uploader_id': '2731064', + 'thumbnail': 'https://i1.sndcdn.com/artworks-MVgcPm5jN42isC5M-6Dz22w-original.jpg', + 'timestamp': 1645181261, + 'uploader_url': 'https://soundcloud.com/jamierobertson', + 'view_count': int, + 'upload_date': '20220218', + 'description': 'md5:ab924dd9994d0a7d64d6d31bf2af4625', + 'license': 'all-rights-reserved', + 'like_count': int, + }, + }, { + 'note': 'soundcloud set embed', + 'url': 'https://www.tumblr.com/beyourselfchulanmaria/703505323122638848/chu-lan-maria-the-playlist-%E5%BF%83%E7%9A%84%E5%91%BC%E5%96%9A-call-of-the', + 'info_dict': { + 'id': '691222680', + 'title': '心的呼喚 Call of the heart I', + 'description': 'md5:25952a8d178a3aa55e40fcbb646a38c3', + }, + 'playlist_mincount': 19, + }, { + 'note': 'dailymotion video embed', + 'url': 'https://www.tumblr.com/funvibecentral/759390024460632064', + 'info_dict': { + 'id': 'x94cnnk', + 'ext': 'mp4', + 'description': 'Funny dailymotion shorts.\n#funny #fun#comedy #romantic #exciting', + 'uploader': 'FunVibe Central', + 'like_count': int, + 'view_count': int, + 'timestamp': 1724210553, + 'title': 'Woman watching other Woman', + 'tags': [], + 'upload_date': '20240821', + 'age_limit': 0, + 'uploader_id': 'x32m6ye', + 'duration': 20, + 'thumbnail': r're:https://(?:s[12]\.)dmcdn\.net/v/Wtqh01cnxKNXLG1N8/x1080', + }, + }, { + 'note': 'tiktok video embed', + 'url': 'https://fansofcolor.tumblr.com/post/660637918605475840/blockquote-class-tiktok-embed', + 'info_dict': { + 'id': '7000937272010935558', + 'ext': 'mp4', + 'artists': ['Alicia Dreaming'], + 'like_count': int, + 'repost_count': int, + 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', + 'channel_id': 'MS4wLjABAAAAsJohwz_dU4KfAOc61cbGDAZ46-5hg2ANTXVQlRe1ipDhpX08PywR3PPiple1NTAo', + 'uploader': 'aliciadreaming', + 'description': 'huge casting news Greyworm will be #louisdulac #racebending #interviewwiththevampire', + 'title': 'huge casting news Greyworm will be #louisdulac #racebending #interviewwiththevampire', + 'channel_url': 'https://www.tiktok.com/@MS4wLjABAAAAsJohwz_dU4KfAOc61cbGDAZ46-5hg2ANTXVQlRe1ipDhpX08PywR3PPiple1NTAo', + 'uploader_id': '7000478462196990982', + 'uploader_url': 'https://www.tiktok.com/@aliciadreaming', + 'timestamp': 1630032733, + 'channel': 'Alicia Dreaming', + 'track': 'original sound', + 'upload_date': '20210827', + 'view_count': int, + 'comment_count': int, + 'duration': 59, + }, + }, { + 'note': 'tumblr video AND youtube embed', + 'url': 'https://www.tumblr.com/anyaboz/765332564457209856/my-music-video-for-selkie-by-nobodys-wolf-child', + 'info_dict': { + 'id': '765332564457209856', + 'uploader_id': 'anyaboz', + 'repost_count': int, + 'age_limit': 0, + 'uploader_url': 'https://anyaboz.tumblr.com/', + 'description': 'md5:9a129cf6ce9d87a80ffd3c6dedd4d1e6', + 'like_count': int, + 'title': 'md5:b18a2ac9387681d20303e485db85c1b5', + 'tags': ['music video', 'nobodys wolf child', 'selkie', 'Stop Motion Animation', 'stop Motion', 'room guardians', 'Youtube'], + }, + 'playlist_count': 2, + }, { + # twitch_live provider - error when linked account is not live + 'url': 'https://www.tumblr.com/anarcho-skamunist/722224493650722816/hollow-knight-stream-right-now-going-to-fight', + 'only_matching': True, }] _providers = { @@ -239,6 +368,16 @@ class TumblrIE(InfoExtractor): 'vimeo': 'Vimeo', 'vine': 'Vine', 'youtube': 'Youtube', + 'dailymotion': 'Dailymotion', + 'tiktok': 'TikTok', + 'twitch_live': 'TwitchStream', + 'bandcamp': None, + 'soundcloud': None, + } + # known not to be supported + _unsupported_providers = { + # seems like podcasts can't be embedded + 'spotify', } _ACCESS_TOKEN = None @@ -256,23 +395,40 @@ class TumblrIE(InfoExtractor): if not self._ACCESS_TOKEN: return - self._download_json( - self._OAUTH_URL, None, 'Logging in', - data=urlencode_postdata({ - 'password': password, - 'grant_type': 'password', - 'username': username, - }), headers={ - 'Content-Type': 'application/x-www-form-urlencoded', - 'Authorization': f'Bearer {self._ACCESS_TOKEN}', - }, - errnote='Login failed', fatal=False) + data = { + 'password': password, + 'grant_type': 'password', + 'username': username, + } + if self.get_param('twofactor'): + data['tfa_token'] = self.get_param('twofactor') + + def _call_login(): + return self._download_json( + self._OAUTH_URL, None, 'Logging in', + data=urlencode_postdata(data), + headers={ + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': f'Bearer {self._ACCESS_TOKEN}', + }, + errnote='Login failed', fatal=False, + expected_status=lambda s: 400 <= s < 500) + + response = _call_login() + if traverse_obj(response, 'error') == 'tfa_required': + data['tfa_token'] = self._get_tfa_info() + response = _call_login() + if traverse_obj(response, 'error'): + raise ExtractorError( + f'API returned error {": ".join(traverse_obj(response, (("error", "error_description"), {str})))}') def _real_extract(self, url): - blog, video_id = self._match_valid_url(url).groups() + blog_1, blog_2, video_id = self._match_valid_url(url).groups() + blog = blog_2 or blog_1 - url = f'http://{blog}.tumblr.com/post/{video_id}/' - webpage, urlh = self._download_webpage_handle(url, video_id) + url = f'http://{blog}.tumblr.com/post/{video_id}' + webpage, urlh = self._download_webpage_handle( + url, video_id, headers={'User-Agent': 'WhatsApp/2.0'}) # whatsapp ua bypasses problems redirect_url = urlh.url @@ -289,23 +445,69 @@ class TumblrIE(InfoExtractor): self._download_json( f'https://www.tumblr.com/api/v2/blog/{blog}/posts/{video_id}/permalink', video_id, headers={'Authorization': f'Bearer {self._ACCESS_TOKEN}'}, fatal=False), - ('response', 'timeline', 'elements', 0)) or {} - content_json = traverse_obj(post_json, ('trail', 0, 'content'), ('content')) or [] - video_json = next( - (item for item in content_json if item.get('type') == 'video'), {}) - media_json = video_json.get('media') or {} - if api_only and not media_json.get('url') and not video_json.get('url'): - raise ExtractorError('Failed to find video data for dashboard-only post') + ('response', 'timeline', 'elements', 0, {dict})) or {} + content_json = traverse_obj(post_json, ((('trail', 0), None), 'content', ..., {dict})) - if not media_json.get('url') and video_json.get('url'): - # external video host - return self.url_result( - video_json['url'], - self._providers.get(video_json.get('provider'), 'Generic')) + # the url we're extracting from might be an original post or it might be a reblog. + # if it's a reblog, og:description will be the reblogger's comment, not the uploader's. + # content_json is always the op, so if it exists but has no text, there's no description + if content_json: + description = '\n\n'.join( + item.get('text') for item in content_json if item.get('type') == 'text') or None + else: + description = self._og_search_description(webpage, default=None) + uploader_id = traverse_obj(post_json, 'reblogged_root_name', 'blog_name') - video_url = self._og_search_video_url(webpage, default=None) - duration = None + info_dict = { + 'id': video_id, + 'title': post_json.get('summary') or (blog if api_only else self._html_search_regex( + r'(?s)(?P<title>.*?)(?: \| Tumblr)?', webpage, 'title', default=blog)), + 'description': description, + 'uploader_id': uploader_id, + 'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None, + **traverse_obj(post_json, { + 'like_count': ('like_count', {int_or_none}), + 'repost_count': ('reblog_count', {int_or_none}), + 'tags': ('tags', ..., {str}), + }), + 'age_limit': {True: 18, False: 0}.get(post_json.get('is_nsfw')), + } + + # for tumblr's own video hosting + fallback_format = None formats = [] + video_url = self._og_search_video_url(webpage, default=None) + # for external video hosts + entries = [] + ignored_providers = set() + unknown_providers = set() + + for video_json in traverse_obj(content_json, lambda _, v: v['type'] in ('video', 'audio')): + media_json = video_json.get('media') or {} + if api_only and not media_json.get('url') and not video_json.get('url'): + raise ExtractorError('Failed to find video data for dashboard-only post') + provider = video_json.get('provider') + + if provider in ('tumblr', None): + fallback_format = { + 'url': media_json.get('url') or video_url, + 'width': int_or_none( + media_json.get('width') or self._og_search_property('video:width', webpage, default=None)), + 'height': int_or_none( + media_json.get('height') or self._og_search_property('video:height', webpage, default=None)), + } + continue + elif provider in self._unsupported_providers: + ignored_providers.add(provider) + continue + elif provider and provider not in self._providers: + unknown_providers.add(provider) + if video_json.get('url'): + # external video host + entries.append(self.url_result( + video_json['url'], self._providers.get(provider))) + + duration = None # iframes can supply duration and sometimes additional formats, so check for one iframe_url = self._search_regex( @@ -344,44 +546,36 @@ class TumblrIE(InfoExtractor): 'quality': quality, } for quality, (video_url, format_id) in enumerate(sources)] - if not media_json.get('url') and not video_url and not iframe_url: - # external video host (but we weren't able to figure it out from the api) - iframe_url = self._search_regex( - r'src=["\'](https?://safe\.txmblr\.com/svc/embed/inline/[^"\']+)["\']', - webpage, 'embed iframe url', default=None) - return self.url_result(iframe_url or redirect_url, 'Generic') + if not formats and fallback_format: + formats.append(fallback_format) - formats = formats or [{ - 'url': media_json.get('url') or video_url, - 'width': int_or_none( - media_json.get('width') or self._og_search_property('video:width', webpage, default=None)), - 'height': int_or_none( - media_json.get('height') or self._og_search_property('video:height', webpage, default=None)), - }] + if formats: + # tumblr's own video is always above embeds + entries.insert(0, { + **info_dict, + 'formats': formats, + 'duration': duration, + 'thumbnail': (traverse_obj(video_json, ('poster', 0, 'url', {url_or_none})) + or self._og_search_thumbnail(webpage, default=None)), + }) - # the url we're extracting from might be an original post or it might be a reblog. - # if it's a reblog, og:description will be the reblogger's comment, not the uploader's. - # content_json is always the op, so if it exists but has no text, there's no description - if content_json: - description = '\n\n'.join( - item.get('text') for item in content_json if item.get('type') == 'text') or None - else: - description = self._og_search_description(webpage, default=None) - uploader_id = traverse_obj(post_json, 'reblogged_root_name', 'blog_name') + if ignored_providers: + if not entries: + raise ExtractorError(f'None of embed providers are supported: {", ".join(ignored_providers)!s}', video_id=video_id, expected=True) + else: + self.report_warning(f'Skipped embeds from unsupported providers: {", ".join(ignored_providers)!s}', video_id) + if unknown_providers: + self.report_warning(f'Unrecognized providers, please report: {", ".join(unknown_providers)!s}', video_id) + if not entries: + self.raise_no_formats('No video could be found in this post', expected=True, video_id=video_id) + if len(entries) == 1: + return { + **info_dict, + **entries[0], + } return { - 'id': video_id, - 'title': post_json.get('summary') or (blog if api_only else self._html_search_regex( - r'(?s)(?P<title>.*?)(?: \| Tumblr)?', webpage, 'title')), - 'description': description, - 'thumbnail': (traverse_obj(video_json, ('poster', 0, 'url')) - or self._og_search_thumbnail(webpage, default=None)), - 'uploader_id': uploader_id, - 'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None, - 'duration': duration, - 'like_count': post_json.get('like_count'), - 'repost_count': post_json.get('reblog_count'), - 'age_limit': {True: 18, False: 0}.get(post_json.get('is_nsfw')), - 'tags': post_json.get('tags'), - 'formats': formats, + **info_dict, + '_type': 'playlist', + 'entries': entries, } diff --git a/yt_dlp/extractor/tva.py b/yt_dlp/extractor/tva.py index d702640f33..48c4e9cba6 100644 --- a/yt_dlp/extractor/tva.py +++ b/yt_dlp/extractor/tva.py @@ -1,4 +1,3 @@ -import functools import re from .brightcove import BrightcoveNewIE @@ -68,7 +67,7 @@ class TVAIE(InfoExtractor): 'episode': episode, **traverse_obj(entity, { 'description': ('longDescription', {str}), - 'duration': ('durationMillis', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('durationMillis', {float_or_none(scale=1000)}), 'channel': ('knownEntities', 'channel', 'name', {str}), 'series': ('knownEntities', 'videoShow', 'name', {str}), 'season_number': ('slug', {lambda x: re.search(r'/s(?:ai|ea)son-(\d+)/', x)}, 1, {int_or_none}), diff --git a/yt_dlp/extractor/vidyard.py b/yt_dlp/extractor/vidyard.py index 20a54b1618..2f6d1f4c51 100644 --- a/yt_dlp/extractor/vidyard.py +++ b/yt_dlp/extractor/vidyard.py @@ -1,4 +1,3 @@ -import functools import re from .common import InfoExtractor @@ -72,9 +71,9 @@ class VidyardBaseIE(InfoExtractor): 'id': ('facadeUuid', {str}), 'display_id': ('videoId', {int}, {str_or_none}), 'title': ('name', {str}), - 'description': ('description', {str}, {unescapeHTML}, {lambda x: x or None}), + 'description': ('description', {str}, {unescapeHTML}, filter), 'duration': (( - ('milliseconds', {functools.partial(float_or_none, scale=1000)}), + ('milliseconds', {float_or_none(scale=1000)}), ('seconds', {int_or_none})), any), 'thumbnails': ('thumbnailUrls', ('small', 'normal'), {'url': {url_or_none}}), 'tags': ('tags', ..., 'name', {str}), diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py index 33ff574750..9345ca962b 100644 --- a/yt_dlp/extractor/vrt.py +++ b/yt_dlp/extractor/vrt.py @@ -1,4 +1,3 @@ -import functools import json import time import urllib.parse @@ -171,7 +170,7 @@ class VRTIE(VRTBaseIE): **traverse_obj(data, { 'title': ('title', {str}), 'description': ('shortDescription', {str}), - 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('duration', {float_or_none(scale=1000)}), 'thumbnail': ('posterImageUrl', {url_or_none}), }), } diff --git a/yt_dlp/extractor/weibo.py b/yt_dlp/extractor/weibo.py index b5c0e926f8..e632858e56 100644 --- a/yt_dlp/extractor/weibo.py +++ b/yt_dlp/extractor/weibo.py @@ -67,7 +67,7 @@ class WeiboBaseIE(InfoExtractor): 'format': ('quality_desc', {str}), 'format_id': ('label', {str}), 'ext': ('mime', {mimetype2ext}), - 'tbr': ('bitrate', {int_or_none}, {lambda x: x or None}), + 'tbr': ('bitrate', {int_or_none}, filter), 'vcodec': ('video_codecs', {str}), 'fps': ('fps', {int_or_none}), 'width': ('width', {int_or_none}), @@ -107,14 +107,14 @@ class WeiboBaseIE(InfoExtractor): **traverse_obj(video_info, { 'id': (('id', 'id_str', 'mid'), {str_or_none}), 'display_id': ('mblogid', {str_or_none}), - 'title': ('page_info', 'media_info', ('video_title', 'kol_title', 'name'), {str}, {lambda x: x or None}), + 'title': ('page_info', 'media_info', ('video_title', 'kol_title', 'name'), {str}, filter), 'description': ('text_raw', {str}), 'duration': ('page_info', 'media_info', 'duration', {int_or_none}), 'timestamp': ('page_info', 'media_info', 'video_publish_time', {int_or_none}), 'thumbnail': ('page_info', 'page_pic', {url_or_none}), 'uploader': ('user', 'screen_name', {str}), 'uploader_id': ('user', ('id', 'id_str'), {str_or_none}), - 'uploader_url': ('user', 'profile_url', {lambda x: urljoin('https://weibo.com/', x)}), + 'uploader_url': ('user', 'profile_url', {urljoin('https://weibo.com/')}), 'view_count': ('page_info', 'media_info', 'online_users_number', {int_or_none}), 'like_count': ('attitudes_count', {int_or_none}), 'repost_count': ('reposts_count', {int_or_none}), diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py index 6f1a8b95d8..53ad1100d2 100644 --- a/yt_dlp/extractor/weverse.py +++ b/yt_dlp/extractor/weverse.py @@ -159,8 +159,8 @@ class WeverseBaseIE(InfoExtractor): 'creators': ('community', 'communityName', {str}, all), 'channel_id': (('community', 'author'), 'communityId', {str_or_none}), 'duration': ('extension', 'video', 'playTime', {float_or_none}), - 'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}), - 'release_timestamp': ('extension', 'video', 'onAirStartAt', {lambda x: int_or_none(x, 1000)}), + 'timestamp': ('publishedAt', {int_or_none(scale=1000)}), + 'release_timestamp': ('extension', 'video', 'onAirStartAt', {int_or_none(scale=1000)}), 'thumbnail': ('extension', (('mediaInfo', 'thumbnail', 'url'), ('video', 'thumb')), {url_or_none}), 'view_count': ('extension', 'video', 'playCount', {int_or_none}), 'like_count': ('extension', 'video', 'likeCount', {int_or_none}), @@ -469,7 +469,7 @@ class WeverseMomentIE(WeverseBaseIE): 'creator': (('community', 'author'), 'communityName', {str}), 'channel_id': (('community', 'author'), 'communityId', {str_or_none}), 'duration': ('extension', 'moment', 'video', 'uploadInfo', 'playTime', {float_or_none}), - 'timestamp': ('publishedAt', {lambda x: int_or_none(x, 1000)}), + 'timestamp': ('publishedAt', {int_or_none(scale=1000)}), 'thumbnail': ('extension', 'moment', 'video', 'uploadInfo', 'imageUrl', {url_or_none}), 'like_count': ('emotionCount', {int_or_none}), 'comment_count': ('commentCount', {int_or_none}), diff --git a/yt_dlp/extractor/wevidi.py b/yt_dlp/extractor/wevidi.py index 0db52af43f..88b394fa23 100644 --- a/yt_dlp/extractor/wevidi.py +++ b/yt_dlp/extractor/wevidi.py @@ -78,7 +78,7 @@ class WeVidiIE(InfoExtractor): } src_path = f'{wvplayer_props["srcVID"]}/{wvplayer_props["srcUID"]}/{wvplayer_props["srcNAME"]}' - for res in traverse_obj(wvplayer_props, ('resolutions', ..., {int}, {lambda x: x or None})): + for res in traverse_obj(wvplayer_props, ('resolutions', ..., {int}, filter)): format_id = str(-(res // -2) - 1) yield { 'acodec': 'mp4a.40.2', diff --git a/yt_dlp/extractor/xiaohongshu.py b/yt_dlp/extractor/xiaohongshu.py index 00c6ed7c57..1280ca6a9c 100644 --- a/yt_dlp/extractor/xiaohongshu.py +++ b/yt_dlp/extractor/xiaohongshu.py @@ -1,4 +1,3 @@ -import functools from .common import InfoExtractor from ..utils import ( @@ -51,7 +50,7 @@ class XiaoHongShuIE(InfoExtractor): 'tbr': ('avgBitrate', {int_or_none}), 'format': ('qualityType', {str}), 'filesize': ('size', {int_or_none}), - 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), + 'duration': ('duration', {float_or_none(scale=1000)}), }) formats.extend(traverse_obj(info, (('mediaUrl', ('backupUrls', ...)), { diff --git a/yt_dlp/extractor/youporn.py b/yt_dlp/extractor/youporn.py index 4a00dfe9c3..8eb77aa031 100644 --- a/yt_dlp/extractor/youporn.py +++ b/yt_dlp/extractor/youporn.py @@ -247,7 +247,7 @@ class YouPornListBase(InfoExtractor): if not html: return for element in get_elements_html_by_class('video-title', html): - if video_url := traverse_obj(element, ({extract_attributes}, 'href', {lambda x: urljoin(url, x)})): + if video_url := traverse_obj(element, ({extract_attributes}, 'href', {urljoin(url)})): yield self.url_result(video_url) if page_num is not None: diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 88c032cdba..caa99182ae 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3611,7 +3611,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'frameworkUpdates', 'entityBatchUpdate', 'mutations', lambda _, v: v['payload']['macroMarkersListEntity']['markersList']['markerType'] == 'MARKER_TYPE_HEATMAP', 'payload', 'macroMarkersListEntity', 'markersList', 'markers', ..., { - 'start_time': ('startMillis', {functools.partial(float_or_none, scale=1000)}), + 'start_time': ('startMillis', {float_or_none(scale=1000)}), 'end_time': {lambda x: (int(x['startMillis']) + int(x['durationMillis'])) / 1000}, 'value': ('intensityScoreNormalized', {float_or_none}), })) or None @@ -3637,7 +3637,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'author_is_verified': ('author', 'isVerified', {bool}), 'author_url': ('author', 'channelCommand', 'innertubeCommand', ( ('browseEndpoint', 'canonicalBaseUrl'), ('commandMetadata', 'webCommandMetadata', 'url'), - ), {lambda x: urljoin('https://www.youtube.com', x)}), + ), {urljoin('https://www.youtube.com')}), }, get_all=False), 'is_favorited': (None if toolbar_entity_payload is None else toolbar_entity_payload.get('heartState') == 'TOOLBAR_HEART_STATE_HEARTED'), @@ -4304,7 +4304,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): continue tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) - format_duration = traverse_obj(fmt, ('approxDurationMs', {lambda x: float_or_none(x, 1000)})) + format_duration = traverse_obj(fmt, ('approxDurationMs', {float_or_none(scale=1000)})) # Some formats may have much smaller duration than others (possibly damaged during encoding) # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823 # Make sure to avoid false positives with small duration differences. diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py index 4563b7ba07..13ce5de12e 100644 --- a/yt_dlp/extractor/zaiko.py +++ b/yt_dlp/extractor/zaiko.py @@ -109,7 +109,7 @@ class ZaikoIE(ZaikoBaseIE): 'uploader': ('profile', 'name', {str}), 'uploader_id': ('profile', 'id', {str_or_none}), 'release_timestamp': ('stream', 'start', 'timestamp', {int_or_none}), - 'categories': ('event', 'genres', ..., {lambda x: x or None}), + 'categories': ('event', 'genres', ..., filter), }), 'alt_title': traverse_obj(initial_event_info, ('title', {str})), 'thumbnails': [{'url': url, 'id': url_basename(url)} for url in thumbnail_urls if url_or_none(url)], diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 8eb5f2a564..6c6a0b3f9d 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -700,7 +700,8 @@ def create_parser(): selection.add_option( '--break-on-existing', action='store_true', dest='break_on_existing', default=False, - help='Stop the download process when encountering a file that is in the archive') + help='Stop the download process when encountering a file that is in the archive ' + 'supplied with the --download-archive option') selection.add_option( '--no-break-on-existing', action='store_false', dest='break_on_existing', diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index c5396789f7..01ed4ed167 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -5123,6 +5123,7 @@ class _UnsafeExtensionError(Exception): 'rm', 'swf', 'ts', + 'vid', 'vob', 'vp9', @@ -5155,6 +5156,7 @@ class _UnsafeExtensionError(Exception): 'heic', 'ico', 'image', + 'jfif', 'jng', 'jpe', 'jpeg', diff --git a/yt_dlp/version.py b/yt_dlp/version.py index 17d7881845..75dc563679 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,8 +1,8 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2024.10.22' +__version__ = '2024.11.04' -RELEASE_GIT_HEAD = '67adeb7bab00662ba55d473e405b301abb42fe61' +RELEASE_GIT_HEAD = '197d0b03b6a3c8fe4fa5ace630eeffec629bf72c' VARIANT = None @@ -12,4 +12,4 @@ CHANNEL = 'stable' ORIGIN = 'yt-dlp/yt-dlp' -_pkg_version = '2024.10.22' +_pkg_version = '2024.11.04'