Release 2021.12.01

Authored by: j54vc1bk

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -43,7 +43,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
@@ -51,12 +51,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -54,7 +54,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output using one of the example URLs provided above.
+        Provide the complete verbose output **using one of the example URLs provided above**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
@@ -62,12 +62,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a site feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -37,20 +37,20 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
         [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -43,7 +43,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -54,7 +54,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output using one of the example URLs provided above.
+        Provide the complete verbose output **using one of the example URLs provided above**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -37,8 +37,8 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
         [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']

.github/workflows/build.yml

@@ -1,14 +1,11 @@
 name: Build
-
+on: workflow_dispatch
-on:
-  push:
-    branches:
-      - release
 
 jobs:
   build_unix:
     runs-on: ubuntu-latest
     outputs:
+      version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
       ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
       upload_url: ${{ steps.create_release.outputs.upload_url }}
       sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
@@ -26,23 +23,32 @@ jobs:
           python-version: '3.8'
     - name: Install packages
       run: sudo apt-get -y install zip pandoc man
+    - name: Set version suffix
+      id: version_suffix
+      env:
+        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
+      if: "env.PUSH_VERSION_COMMIT == ''"
+      run: echo ::set-output name=version_suffix::$(date -u +"%H%M%S")
     - name: Bump version
       id: bump_version
       run: |
-        python devscripts/update-version.py
+        python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
         make issuetemplates
-    - name: Print version
+    - name: Push to release
-      run: echo "${{ steps.bump_version.outputs.ytdlp_version }}"
+      id: push_release
-    - name: Update master
-      id: push_update
       run: |
-        git config --global user.email "${{ github.event.pusher.email }}"
+        git config --global user.name github-actions
-        git config --global user.name "${{ github.event.pusher.name }}"
+        git config --global user.email github-actions@example.com
         git add -u
-        git commit -m "[version] update" -m ":ci skip all"
+        git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all"
-        git pull --rebase origin ${{ github.event.repository.master_branch }}
+        git push origin --force ${{ github.event.ref }}:release
-        git push origin ${{ github.event.ref }}:${{ github.event.repository.master_branch }}
         echo ::set-output name=head_sha::$(git rev-parse HEAD)
+    - name: Update master
+      id: push_master
+      env:
+        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
+      if: "env.PUSH_VERSION_COMMIT != ''"
+      run: git push origin ${{ github.event.ref }}
     - name: Get Changelog
       id: get_changelog
       run: |
@@ -115,12 +121,12 @@ jobs:
         release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
         commitish: ${{ steps.push_update.outputs.head_sha }}
         body: |
-          ### Changelog:
+          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
-          ${{ env.changelog }}
 
           ---
 
-          ### See [this](https://github.com/yt-dlp/yt-dlp#release-files) for a description of the release files
+          ### Changelog:
+          ${{ env.changelog }}
         draft: false
         prerelease: false
     - name: Upload yt-dlp Unix binary
@@ -155,10 +161,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     # In order to create a universal2 application, the version of python3 in /usr/bin has to be used
+    # Pyinstaller is pinned to 4.5.1 because the builds are failing in 4.6, 4.7
     - name: Install Requirements
       run: |
           brew install coreutils
-          /usr/bin/python3 -m pip install -U --user pip Pyinstaller mutagen pycryptodomex websockets
+          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.5.1 mutagen pycryptodomex websockets
     - name: Bump version
       id: bump_version
       run: /usr/bin/python3 devscripts/update-version.py
@@ -232,7 +239,9 @@ jobs:
           pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
     - name: Bump version
       id: bump_version
-      run: python devscripts/update-version.py
+      env:
+        version_suffix: ${{ needs.build_unix.outputs.version_suffix }}
+      run: python devscripts/update-version.py ${{ env.version_suffix }}
     - name: Build lazy extractors
       id: lazy_extractors
       run: python devscripts/make_lazy_extractors.py
@@ -319,7 +328,9 @@ jobs:
           pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
     - name: Bump version
       id: bump_version
-      run: python devscripts/update-version.py
+      env:
+        version_suffix: ${{ needs.build_unix.outputs.version_suffix }}
+      run: python devscripts/update-version.py ${{ env.version_suffix }}
     - name: Build lazy extractors
       id: lazy_extractors
       run: python devscripts/make_lazy_extractors.py

.gitignore

@@ -6,41 +6,48 @@ cookies
 .netrc
 
 # Downloaded
-*.srt
-*.ttml
-*.sbv
-*.vtt
-*.flv
-*.mp4
-*.m4a
-*.m4v
-*.mp3
 *.3gp
-*.webm
+*.annotations.xml
-*.wav
 *.ape
-*.mkv
+*.aria2
-*.flac
 *.avi
-*.swf
+*.description
-*.part
+*.desktop
-*.part-*
-*.ytdl
 *.dump
+*.flac
+*.flv
 *.frag
 *.frag.urls
-*.aria2
+*.info.json
-*.swp
+*.jpeg
+*.jpg
+*.live_chat.json
+*.m4a
+*.m4v
+*.mhtml
+*.mkv
+*.mov
+*.mp3
+*.mp4
 *.ogg
 *.opus
-*.info.json
+*.part
-*.live_chat.json
+*.part-*
-*.jpg
-*.jpeg
 *.png
+*.sbv
+*.srt
+*.swf
+*.swp
+*.ttml
+*.unknown_video
+*.url
+*.vtt
+*.wav
+*.webloc
+*.webm
 *.webp
-*.annotations.xml
+*.ytdl
-*.description
+.cache/
 
 # Allow config/media files in testdata
 !test/**

CONTRIBUTING.md

@@ -105,7 +105,7 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about yt-dlp?
 
-Some bug reports are completely unrelated to yt-dlp and relate to a different, or even the reporter's own, application. Please make sure that you are actually using yt-dlp. If you are using a UI for yt-dlp, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for yt-dlp fails in some way you believe is related to yt-dlp, by all means, go ahead and report the bug.
+Some bug reports are completely unrelated to yt-dlp and relate to a different, or even the reporter's own, application. Please make sure that you are actually using yt-dlp. If you are using a UI for yt-dlp, report the bug to the maintainer of the actual application providing the UI. In general, if you are unable to provide the verbose log, you should not be opening the issue here.
 
 If the issue is with `youtube-dl` (the upstream fork of yt-dlp) and not with yt-dlp, the issue should be raised in the youtube-dl project.
 
@@ -117,7 +117,7 @@ By sharing an account with anyone, you agree to bear all risks associated with i
 
 While these steps won't necessarily ensure that no misuse of the account takes place, these are still some good practices to follow.
 
-- Look for people with `Member` or `Contributor` tag on their messages.
+- Look for people with `Member` (maintainers of the project) or `Contributor` (people who have previously contributed code) tag on their messages.
 - Change the password before sharing the account to something random (use [this](https://passwordsgenerator.net/) if you don't have a random password generator).
 - Change the password after receiving the account back.
 
@@ -148,7 +148,7 @@ If you want to create a build of yt-dlp yourself, you can follow the instruction
 
 Before you start writing code for implementing a new feature, open an issue explaining your feature request and atleast one use case. This allows the maintainers to decide whether such a feature is desired for the project in the first place, and will provide an avenue to discuss some implementation details. If you open a pull request for a new feature without discussing with us first, do not be surprised when we ask for large changes to the code, or even reject it outright.
 
-The same applies for overarching changes to the architecture, documentation or code style
+The same applies for changes to the documentation, code style, or overarching changes to the architecture
 
 
 ## Adding support for a new site
@@ -209,7 +209,7 @@ After you have ensured this site is distributing its content legally, you can fo
     ```
 1. Add an import in [`yt_dlp/extractor/extractors.py`](yt_dlp/extractor/extractors.py).
 1. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
-1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the purticular test is disabled from running.
+1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L91-L426). Add tests and code for as many as you want.
 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 

CONTRIBUTORS

@@ -129,3 +129,29 @@ Bojidarist
 nixklai
 smplayer-dev
 Zirro
+CrypticSignal
+flashdagger
+fractalf
+frafra
+kaz-us
+ozburo
+rhendric
+sdomi
+selfisekai
+stanoarn
+0xA7404A/Aurora
+4a1e2y5
+aarubui
+chio0hai
+cntrl-s
+Deer-Spangle
+DEvmIb
+Grabien
+j54vc1bk
+mpeter50
+mrpapersonic
+pabs3
+staubichsauger
+xenova
+Yakabuff
+zulaport

Changelog.md

@@ -5,14 +5,205 @@
 
 * Run `make doc`
 * Update Changelog.md and CONTRIBUTORS
-* Change "Merged with ytdl" version in Readme.md if needed
+* Change "Based on ytdl" version in Readme.md if needed
-* Add new/fixed extractors in "new features" section of Readme.md
+* Commit as `Release <version>` and push to master
-* Commit as `Release <version>`
+* Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
-* Push to origin/release using `git push origin master:release`
-    build task will now run
-
 -->
 
+### 2021.12.01
+
+* **Add option `--wait-for-video` to wait for scheduled streams**
+* Add option `--break-per-input` to apply --break-on... to each input URL
+* Add option `--embed-info-json` to embed info.json in mkv
+* Add compat-option `embed-metadata`
+* Allow using a custom format selector through API
+* [AES] Add ECB mode by [nao20010128nao](https://github.com/nao20010128nao)
+* [build] Fix MacOS Build
+* [build] Save Git HEAD at release alongside version info
+* [build] Use `workflow_dispatch` for release
+* [downloader/ffmpeg] Fix for direct videos inside mpd manifests
+* [downloader] Add colors to download progress
+* [EmbedSubtitles] Slightly relax duration check and related cleanup
+* [ExtractAudio] Fix conversion to `wav` and `vorbis`
+* [ExtractAudio] Support `alac`
+* [extractor] Extract `average_rating` from JSON-LD
+* [FixupM3u8] Fixup MPEG-TS in MP4 container
+* [generic] Support mpd manifests without extension by [shirt](https://github.com/shirt-dev)
+* [hls] Better FairPlay DRM detection by [nyuszika7h](https://github.com/nyuszika7h)
+* [jsinterp] Fix splice to handle float (for youtube js player f1ca6900)
+* [utils] Allow alignment in `render_table` and add tests
+* [utils] Fix `PagedList`
+* [utils] Fix error when copying `LazyList`
+* Clarify video/audio-only formats in -F
+* Ensure directory exists when checking formats
+* Ensure path for link files exists by [Zirro](https://github.com/Zirro)
+* Ensure same config file is not loaded multiple times
+* Fix 'postprocessor_hooks`
+* Fix `--break-on-archive` when pre-checking
+* Fix `--check-formats` for `mhtml`
+* Fix `--load-info-json` of playlists with failed entries
+* Fix `--trim-filename` when filename has `.`
+* Fix bug in parsing `--add-header`
+* Fix error in `report_unplayable_conflict` by [shirt](https://github.com/shirt-dev)
+* Fix writing playlist infojson with `--no-clean-infojson`
+* Validate --get-bypass-country
+* [blogger] Add extractor by [pabs3](https://github.com/pabs3)
+* [breitbart] Add extractor by [Grabien](https://github.com/Grabien)
+* [CableAV] Add extractor by [j54vc1bk](https://github.com/j54vc1bk)
+* [CanalAlpha] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [CozyTV] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [CPTwentyFour] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [DiscoveryPlus] Add `DiscoveryPlusItalyShowIE` by [Ashish0804](https://github.com/Ashish0804)
+* [ESPNCricInfo] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [LinkedIn] Add extractor by [u-spec-png](https://github.com/u-spec-png)
+* [mixch] Add extractor by [nao20010128nao](https://github.com/nao20010128nao)
+* [nebula] Add `NebulaCollectionIE` and rewrite extractor by [hheimbuerger](https://github.com/hheimbuerger)
+* [OneFootball] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [peer.tv] Add extractor by [u-spec-png](https://github.com/u-spec-png)
+* [radiozet] Add extractor by [0xA7404A](https://github.com/0xA7404A) (Aurora)
+* [redgifs] Add extractor by [chio0hai](https://github.com/chio0hai)
+* [RedGifs] Add Search and User extractors by [Deer-Spangle](https://github.com/Deer-Spangle)
+* [rtrfm] Add extractor by [pabs3](https://github.com/pabs3)
+* [Streamff] Add extractor by [cntrl-s](https://github.com/cntrl-s)
+* [Stripchat] Add extractor by [zulaport](https://github.com/zulaport)
+* [Aljazeera] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [AmazonStoreIE] Fix regex to not match vdp urls by [Ashish0804](https://github.com/Ashish0804)
+* [ARDBetaMediathek] Handle new URLs
+* [bbc] Get all available formats by [nyuszika7h](https://github.com/nyuszika7h)
+* [Bilibili] Fix title extraction by [u-spec-png](https://github.com/u-spec-png)
+* [CBC Gem] Fix for shows that don't have all seasons by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
+* [curiositystream] Add more metadata
+* [CuriosityStream] Fix series
+* [DiscoveryPlus] Rewrite extractors by [Ashish0804](https://github.com/Ashish0804), [pukkandan](https://github.com/pukkandan)
+* [HotStar] Set language field from tags by [Ashish0804](https://github.com/Ashish0804)
+* [instagram, cleanup] Refactor extractors
+* [Instagram] Display more login errors by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [itv] Fix extractor by [staubichsauger](https://github.com/staubichsauger), [pukkandan](https://github.com/pukkandan)
+* [mediaklikk] Expand valid URL
+* [MTV] Improve mgid extraction by [Sipherdrakon](https://github.com/Sipherdrakon), [kikuyan](https://github.com/kikuyan)
+* [nexx] Better error message for unsupported format
+* [NovaEmbed] Fix extractor by [pukkandan](https://github.com/pukkandan), [std-move](https://github.com/std-move)
+* [PatreonUser] Do not capture RSS URLs
+* [Reddit] Add support for 1080p videos by [xenova](https://github.com/xenova)
+* [RoosterTeethSeries] Fix for multiple pages by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [sbs] Fix for movies and livestreams
+* [Senate.gov] Add SenateGovIE and fix SenateISVPIE by [Grabien](https://github.com/Grabien), [pukkandan](https://github.com/pukkandan)
+* [soundcloud:search] Fix pagination
+* [tiktok:user] Set `webpage_url` correctly
+* [Tokentube] Fix description by [u-spec-png](https://github.com/u-spec-png)
+* [trovo] Fix extractor by [nyuszika7h](https://github.com/nyuszika7h)
+* [tv2] Expand valid URL
+* [Tvplayhome] Fix extractor by [pukkandan](https://github.com/pukkandan), [18928172992817182](https://github.com/18928172992817182)
+* [Twitch:vod] Add chapters by [mpeter50](https://github.com/mpeter50)
+* [twitch:vod] Extract live status by [DEvmIb](https://github.com/DEvmIb)
+* [VidLii] Add 720p support by [mrpapersonic](https://github.com/mrpapersonic)
+* [vimeo] Add fallback for config URL
+* [vimeo] Sort http formats higher
+* [WDR] Expand valid URL
+* [willow] Add extractor by [aarubui](https://github.com/aarubui)
+* [xvideos] Detect embed URLs by [4a1e2y5](https://github.com/4a1e2y5)
+* [xvideos] Fix extractor by [Yakabuff](https://github.com/Yakabuff)
+* [youtube, cleanup] Reorganize Tab and Search extractor inheritances
+* [youtube:search_url] Add playlist/channel support
+* [youtube] Add `default` player client by [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Add storyboard formats
+* [youtube] Decrypt n-sig for URLs with `ratebypass`
+* [youtube] Minor improvement to format sorting
+* [cleanup] Add deprecation warnings
+* [cleanup] Minor cleanup
+* [cleanup] Misc cleanup
+* [cleanup] Refactor `JSInterpreter._seperate`
+* [Cleanup] Remove some unnecessary groups in regexes by [Ashish0804](https://github.com/Ashish0804)
+
+
+### 2021.11.10.1
+
+* Temporarily disable MacOS Build
+
+### 2021.11.10
+
+* [youtube] **Fix throttling by decrypting n-sig**
+* Merging extractors from [haruhi-dl](https://git.sakamoto.pl/laudom/haruhi-dl) by [selfisekai](https://github.com/selfisekai)
+    * [extractor] Add `_search_nextjs_data`
+    * [tvp] Fix extractors
+    * [tvp] Add TVPStreamIE
+    * [wppilot] Add extractors
+    * [polskieradio] Add extractors
+    * [radiokapital] Add extractors
+    * [polsatgo] Add extractor by [selfisekai](https://github.com/selfisekai), [sdomi](https://github.com/sdomi)
+* Separate `--check-all-formats` from `--check-formats`
+* Approximate filesize from bitrate
+* Don't create console in `windows_enable_vt_mode`
+* Fix bug in `--load-infojson` of playlists
+* [minicurses] Add colors to `-F` and standardize color-printing code
+* [outtmpl] Add type `link` for internet shortcut files
+* [outtmpl] Add alternate forms for `q` and `j`
+* [outtmpl] Do not traverse `None`
+* [fragment] Fix progress display in fragmented downloads
+* [downloader/ffmpeg] Fix vtt download with ffmpeg
+* [ffmpeg] Detect presence of setts and libavformat version
+* [ExtractAudio] Rescale `--audio-quality` correctly by [CrypticSignal](https://github.com/CrypticSignal), [pukkandan](https://github.com/pukkandan)
+* [ExtractAudio] Use `libfdk_aac` if available by [CrypticSignal](https://github.com/CrypticSignal)
+* [FormatSort] `eac3` is better than `ac3`
+* [FormatSort] Fix some fields' defaults
+* [generic] Detect more json_ld
+* [generic] parse jwplayer with only the json URL
+* [extractor] Add keyword automatically to SearchIE descriptions
+* [extractor] Fix some errors being converted to `ExtractorError`
+* [utils] Add `join_nonempty`
+* [utils] Add `jwt_decode_hs256` by [Ashish0804](https://github.com/Ashish0804)
+* [utils] Create `DownloadCancelled` exception
+* [utils] Parse `vp09` as vp9
+* [utils] Sanitize URL when determining protocol
+* [test/download] Fallback test to `bv`
+* [docs] Minor documentation improvements
+* [cleanup] Improvements to error and debug messages
+* [cleanup] Minor fixes and cleanup
+* [3speak] Add extractors by [Ashish0804](https://github.com/Ashish0804)
+* [AmazonStore] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [Gab] Add extractor by [u-spec-png](https://github.com/u-spec-png)
+* [mediaset] Add playlist support by [nixxo](https://github.com/nixxo)
+* [MLSScoccer] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [N1] Add support for nova.rs by [u-spec-png](https://github.com/u-spec-png)
+* [PlanetMarathi] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [RaiplayRadio] Add extractors by [frafra](https://github.com/frafra)
+* [roosterteeth] Add series extractor
+* [sky] Add `SkyNewsStoryIE` by [ajj8](https://github.com/ajj8)
+* [youtube] Fix sorting for some videos
+* [youtube] Populate `thumbnail` with the best "known" thumbnail
+* [youtube] Refactor itag processing
+* [youtube] Remove unnecessary no-playlist warning
+* [youtube:tab] Add Invidious list for playlists/channels by [rhendric](https://github.com/rhendric)
+* [Bilibili:comments] Fix infinite loop by [u-spec-png](https://github.com/u-spec-png)
+* [ceskatelevize] Fix extractor by [flashdagger](https://github.com/flashdagger)
+* [Coub] Fix media format identification by [wlritchi](https://github.com/wlritchi)
+* [crunchyroll] Add extractor-args `language` and `hardsub`
+* [DiscoveryPlus] Allow language codes in URL
+* [imdb] Fix thumbnail by [ozburo](https://github.com/ozburo)
+* [instagram] Add IOS URL support by [u-spec-png](https://github.com/u-spec-png)
+* [instagram] Improve login code by [u-spec-png](https://github.com/u-spec-png)
+* [Instagram] Improve metadata extraction by [u-spec-png](https://github.com/u-spec-png)
+* [iPrima] Fix extractor by [stanoarn](https://github.com/stanoarn)
+* [itv] Add support for ITV News by [ajj8](https://github.com/ajj8)
+* [la7] Fix extractor by [nixxo](https://github.com/nixxo)
+* [linkedin] Don't login multiple times
+* [mtv] Fix some videos by [Sipherdrakon](https://github.com/Sipherdrakon)
+* [Newgrounds] Fix description by [u-spec-png](https://github.com/u-spec-png)
+* [Nrk] Minor fixes by [fractalf](https://github.com/fractalf)
+* [Olympics] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [piksel] Fix sorting
+* [twitter] Do not sort by codec
+* [viewlift] Add cookie-based login and series support by [Ashish0804](https://github.com/Ashish0804), [pukkandan](https://github.com/pukkandan)
+* [vimeo] Detect source extension and misc cleanup by [flashdagger](https://github.com/flashdagger)
+* [vimeo] Fix ondemand videos and direct URLs with hash
+* [vk] Fix login and add subtitles by [kaz-us](https://github.com/kaz-us)
+* [VLive] Add upload_date and thumbnail by [Ashish0804](https://github.com/Ashish0804)
+* [VRT] Fix login by [pgaig](https://github.com/pgaig)
+* [Vupload] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [wakanim] Add support for MPD manifests by [nyuszika7h](https://github.com/nyuszika7h)
+* [wakanim] Detect geo-restriction by [nyuszika7h](https://github.com/nyuszika7h)
+* [ZenYandex] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+
 
 ### 2021.10.22
 
@@ -61,7 +252,7 @@
 * [AdobePass] Fix RCN MSO by [jfogelman](https://github.com/jfogelman)
 * [CBC] Fix Gem livestream by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
 * [CBC] Support CBC Gem member content by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
-* [crunchyroll] Add season to flat-playlist Closes #1319
+* [crunchyroll] Add season to flat-playlist
 * [crunchyroll] Add support for `beta.crunchyroll` URLs and fix series URLs with language code
 * [EUScreen] Add Extractor by [Ashish0804](https://github.com/Ashish0804)
 * [Gronkh] Add extractor by [Ashish0804](https://github.com/Ashish0804)
@@ -1283,9 +1474,8 @@
 * Cleaned up the fork for public use
 
 
-**PS**: All uncredited changes above this point are authored by [pukkandan](https://github.com/pukkandan)
+**Note**: All uncredited changes above this point are authored by [pukkandan](https://github.com/pukkandan)
 
-### Unreleased changes in [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc)
 * Updated to youtube-dl release 2020.11.26 by [pukkandan](https://github.com/pukkandan)
 * Youtube improvements by [pukkandan](https://github.com/pukkandan)
     * Implemented all Youtube Feeds (ytfav, ytwatchlater, ytsubs, ythistory, ytrec) and SearchURL
@@ -1308,8 +1498,110 @@
 * [spreaker] fix SpreakerShowIE test URL by [pukkandan](https://github.com/pukkandan)
 * [Vlive] Fix playlist handling when downloading a channel by [kyuyeunk](https://github.com/kyuyeunk)
 * [tmz] Fix extractor by [diegorodriguezv](https://github.com/diegorodriguezv)
+* [ITV] BTCC URL update by [WolfganP](https://github.com/WolfganP)
 * [generic] Detect embedded bitchute videos by [pukkandan](https://github.com/pukkandan)
 * [generic] Extract embedded youtube and twitter videos by [diegorodriguezv](https://github.com/diegorodriguezv)
 * [ffmpeg] Ensure all streams are copied by [pukkandan](https://github.com/pukkandan)
 * [embedthumbnail] Fix for os.rename error by [pukkandan](https://github.com/pukkandan)
 * make_win.bat: don't use UPX to pack vcruntime140.dll by [jbruchon](https://github.com/jbruchon)
+
+
+### Changelog of [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc) till release 2020.11.11-3
+
+**Note**: This was constructed from the merge commit messages and may not be entirely accurate
+
+* [bandcamp] fix failing test. remove subclass hack by [insaneracist](https://github.com/insaneracist)
+* [bandcamp] restore album downloads by [insaneracist](https://github.com/insaneracist)
+* [francetv] fix extractor by [Surkal](https://github.com/Surkal)
+* [gdcvault] fix extractor by [blackjack4494](https://github.com/blackjack4494)
+* [hotstar] Move to API v1 by [theincognito-inc](https://github.com/theincognito-inc)
+* [hrfernsehen] add extractor by [blocktrron](https://github.com/blocktrron)
+* [kakao] new apis by [blackjack4494](https://github.com/blackjack4494)
+* [la7] fix missing protocol by [nixxo](https://github.com/nixxo)
+* [mailru] removed escaped braces, use urljoin, added tests by [nixxo](https://github.com/nixxo)
+* [MTV/Nick] universal mgid extractor + fix nick.de feed by [blackjack4494](https://github.com/blackjack4494)
+* [mtv] Fix a missing match_id by [nixxo](https://github.com/nixxo)
+* [Mtv] updated extractor logic & more by [blackjack4494](https://github.com/blackjack4494)
+* [ndr] support Daserste ndr by [blackjack4494](https://github.com/blackjack4494)
+* [Netzkino] Only use video id to find metadata by [TobiX](https://github.com/TobiX)
+* [newgrounds] fix: video download by [insaneracist](https://github.com/insaneracist)
+* [nitter] Add new extractor by [B0pol](https://github.com/B0pol)
+* [soundcloud] Resolve audio/x-wav by [tfvlrue](https://github.com/tfvlrue)
+* [soundcloud] sets pattern and tests by [blackjack4494](https://github.com/blackjack4494)
+* [SouthparkDE/MTV] another mgid extraction (mtv_base) feed url updated by [blackjack4494](https://github.com/blackjack4494)
+* [StoryFire] Add new extractor by [sgstair](https://github.com/sgstair)
+* [twitch] by [geauxlo](https://github.com/geauxlo)
+* [videa] Adapt to updates by [adrianheine](https://github.com/adrianheine)
+* [Viki] subtitles, formats by [blackjack4494](https://github.com/blackjack4494)
+* [vlive] fix extractor for revamped website by [exwm](https://github.com/exwm)
+* [xtube] fix extractor by [insaneracist](https://github.com/insaneracist)
+* [youtube] Convert subs when download is skipped by [blackjack4494](https://github.com/blackjack4494)
+* [youtube] Fix age gate detection by [random-nick](https://github.com/random-nick)
+* [youtube] fix yt-only playback when age restricted/gated - requires cookies by [blackjack4494](https://github.com/blackjack4494)
+* [youtube] fix: extract artist metadata from ytInitialData by [insaneracist](https://github.com/insaneracist)
+* [youtube] fix: extract mix playlist ids from ytInitialData by [insaneracist](https://github.com/insaneracist)
+* [youtube] fix: mix playlist title by [insaneracist](https://github.com/insaneracist)
+* [youtube] fix: Youtube Music playlists by [insaneracist](https://github.com/insaneracist)
+* [Youtube] Fixed problem with new youtube player by [peet1993](https://github.com/peet1993)
+* [zoom] Fix url parsing for url's containing /share/ and dots by [Romern](https://github.com/Romern)
+* [zoom] new extractor by [insaneracist](https://github.com/insaneracist)
+* abc by [adrianheine](https://github.com/adrianheine)
+* Added Comcast_SSO fix by [merval](https://github.com/merval)
+* Added DRM logic to brightcove by [merval](https://github.com/merval)
+* Added regex for ABC.com site. by [kucksdorfs](https://github.com/kucksdorfs)
+* alura by [hugohaa](https://github.com/hugohaa)
+* Arbitrary merges by [fstirlitz](https://github.com/fstirlitz)
+* ard.py_add_playlist_support by [martin54](https://github.com/martin54)
+* Bugfix/youtube/chapters fix extractor by [gschizas](https://github.com/gschizas)
+* bugfix_youtube_like_extraction by [RedpointsBots](https://github.com/RedpointsBots)
+* Create build workflow by [blackjack4494](https://github.com/blackjack4494)
+* deezer by [LucBerge](https://github.com/LucBerge)
+* Detect embedded bitchute videos by [pukkandan](https://github.com/pukkandan)
+* Don't install tests by [l29ah](https://github.com/l29ah)
+* Don't try to embed/convert json subtitles generated by [youtube](https://github.com/youtube) livechat by [pukkandan](https://github.com/pukkandan)
+* Doodstream by [sxvghd](https://github.com/sxvghd)
+* duboku by [lkho](https://github.com/lkho)
+* elonet by [tpikonen](https://github.com/tpikonen)
+* ext/remuxe-video by [Zocker1999NET](https://github.com/Zocker1999NET)
+* fall-back to the old way to fetch subtitles, if needed by [RobinD42](https://github.com/RobinD42)
+* feature_subscriber_count by [RedpointsBots](https://github.com/RedpointsBots)
+* Fix external downloader when there is no http_header by [pukkandan](https://github.com/pukkandan)
+* Fix issue triggered by [tubeup](https://github.com/tubeup) by [nsapa](https://github.com/nsapa)
+* Fix YoutubePlaylistsIE by [ZenulAbidin](https://github.com/ZenulAbidin)
+* fix-mitele' by [DjMoren](https://github.com/DjMoren)
+* fix/google-drive-cookie-issue by [legraphista](https://github.com/legraphista)
+* fix_tiktok by [mervel-mervel](https://github.com/mervel-mervel)
+* Fixed problem with JS player URL by [peet1993](https://github.com/peet1993)
+* fixYTSearch by [xarantolus](https://github.com/xarantolus)
+* FliegendeWurst-3sat-zdf-merger-bugfix-feature
+* gilou-bandcamp_update
+* implement ThisVid extractor by [rigstot](https://github.com/rigstot)
+* JensTimmerman-patch-1 by [JensTimmerman](https://github.com/JensTimmerman)
+* Keep download archive in memory for better performance by [jbruchon](https://github.com/jbruchon)
+* la7-fix by [iamleot](https://github.com/iamleot)
+* magenta by [adrianheine](https://github.com/adrianheine)
+* Merge 26564 from [adrianheine](https://github.com/adrianheine)
+* Merge code from [ddland](https://github.com/ddland)
+* Merge code from [nixxo](https://github.com/nixxo)
+* Merge code from [ssaqua](https://github.com/ssaqua)
+* Merge code from [zubearc](https://github.com/zubearc)
+* mkvthumbnail by [MrDoritos](https://github.com/MrDoritos)
+* myvideo_ge by [fonkap](https://github.com/fonkap)
+* naver by [SeonjaeHyeon](https://github.com/SeonjaeHyeon)
+* ondemandkorea by [julien-hadleyjack](https://github.com/julien-hadleyjack)
+* rai-update by [iamleot](https://github.com/iamleot)
+* RFC: youtube: Polymer UI and JSON endpoints for playlists by [wlritchi](https://github.com/wlritchi)
+* rutv by [adrianheine](https://github.com/adrianheine)
+* Sc extractor web auth by [blackjack4494](https://github.com/blackjack4494)
+* Switch from binary search tree to Python sets by [jbruchon](https://github.com/jbruchon)
+* tiktok by [skyme5](https://github.com/skyme5)
+* tvnow by [TinyToweringTree](https://github.com/TinyToweringTree)
+* twitch-fix by [lel-amri](https://github.com/lel-amri)
+* Twitter shortener by [blackjack4494](https://github.com/blackjack4494)
+* Update README.md by [JensTimmerman](https://github.com/JensTimmerman)
+* Update to reflect website changes. by [amigatomte](https://github.com/amigatomte)
+* use webarchive to fix a dead link in README by [B0pol](https://github.com/B0pol)
+* Viki the second by [blackjack4494](https://github.com/blackjack4494)
+* wdr-subtitles by [mrtnmtth](https://github.com/mrtnmtth)
+* Webpfix by [alexmerkel](https://github.com/alexmerkel)
+* Youtube live chat by [siikamiika](https://github.com/siikamiika)

Makefile

@@ -15,9 +15,11 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites com
 clean-test:
 	rm -rf *.3gp *.annotations.xml *.ape *.avi *.description *.dump *.flac *.flv *.frag *.frag.aria2 *.frag.urls \
 	*.info.json *.jpeg *.jpg *.live_chat.json *.m4a *.m4v *.mkv *.mp3 *.mp4 *.ogg *.opus *.part* *.png *.sbv *.srt \
-	*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.ytdl test/testdata/player-*.js
+	*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.mhtml *.mov *.unknown_video *.desktop *.url *.webloc *.ytdl \
+	test/testdata/player-*.js tmp/
 clean-dist:
-	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
+	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
+	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
 clean-cache:
 	find . -name "*.pyc" -o -name "*.class" -delete
 
@@ -31,7 +33,6 @@ DESTDIR ?= .
 BINDIR ?= $(PREFIX)/bin
 MANDIR ?= $(PREFIX)/man
 SHAREDIR ?= $(PREFIX)/share
-# make_supportedsites.py doesnot work correctly in python2
 PYTHON ?= /usr/bin/env python3
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local

README.md

@@ -61,26 +61,24 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
     * [Opening an Issue](CONTRIBUTING.md#opening-an-issue)
     * [Developer Instructions](CONTRIBUTING.md#developer-instructions)
 * [MORE](#more)
-</div>
 
 
 # NEW FEATURES
-The major new features from the latest release of [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc) are:
+
+* Based on **youtube-dl 2021.06.06 [commit/379f52a](https://github.com/ytdl-org/youtube-dl/commit/379f52a4954013767219d25099cce9e0f9401961)** and **youtube-dlc 2020.11.11-3 [commit/98e248f](https://github.com/blackjack4494/yt-dlc/commit/98e248faa49e69d795abc60f7cdefcf91e2612aa)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
 * **[Format Sorting](#sorting-formats)**: The default format sorting options have been changed so that higher resolution and better codecs will be now preferred instead of simply using larger bitrate. Furthermore, you can now specify the sort order using `-S`. This allows for much easier format selection than what is possible by simply using `--format` ([examples](#format-selection-examples))
 
-* **Merged with youtube-dl [commit/379f52a](https://github.com/ytdl-org/youtube-dl/commit/379f52a4954013767219d25099cce9e0f9401961)**: (v2021.06.06) You get all the latest features and patches of [youtube-dl](https://github.com/ytdl-org/youtube-dl) in addition to all the features of [youtube-dlc](https://github.com/blackjack4494/yt-dlc)
-
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that the NicoNico improvements are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
 * **Youtube improvements**:
     * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`) and private playlists supports downloading multiple pages of content
     * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works
     * Mixes supports downloading multiple pages of content
-    * Most (but not all) age-gated content can be downloaded without cookies
+    * Some (but not all) age-gated content can be downloaded without cookies
-    * Partial workaround for throttling issue
+    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326)
     * Redirect channel's home URL automatically to `/video` to preserve the old behaviour
     * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
     * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
@@ -93,9 +91,9 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats
 
-* **New extractors**: AnimeLab, Philo MSO, Spectrum MSO, SlingTV MSO, Cablevision MSO, RCN MSO, Rcs, Gedi, bitwave.tv, mildom, audius, zee5, mtv.it, wimtv, pluto.tv, niconico users, discoveryplus.in, mediathek, NFHSNetwork, nebula, ukcolumn, whowatch, MxplayerShow, parlview (au), YoutubeWebArchive, fancode, Saitosan, ShemarooMe, telemundo, VootSeries, SonyLIVSeries, HotstarSeries, VidioPremier, VidioLive, RCTIPlus, TBS Live, douyin, pornflip, ParamountPlusSeries, ScienceChannel, Utreon, OpenRec, BandcampMusic, blackboardcollaborate, eroprofile albums, mirrativ, BannedVideo, bilibili categories, Epicon, filmmodu, GabTV, HungamaAlbum, ManotoTV, Niconico search, Patreon User, peloton, ProjectVeritas, radiko, StarTV, tiktok user, Tokentube, voicy, TV2HuSeries, biliintl, 17live, NewgroundsUser, peertube channel/playlist, ZenYandex, CAM4, CGTN, damtomo, gotostage, Koo, Mediaite, Mediaklikk, MuseScore, nzherald, Olympics replay, radlive, SovietsCloset, Streamanity, Theta, Chingari, ciscowebex, Gettr, GoPro, N1, Theta, Veo, Vupload, NovaPlay, SkyNewsAU, EUScreen, Gronkh, microsoftstream, on24, trovo channels
+* **New and fixed extractors**: Many new extractors have been added and a lot of exisiting ones have been fixed. See the [changelog](Changelog.md) or the [list of supported sites](supportedsites.md)
 
-* **Fixed/improved extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, akamai, ina, rumble, tennistv, amcnetworks, la7 podcasts, linuxacadamy, nitter, twitcasting, viu, crackle, curiositystream, mediasite, rmcdecouverte, sonyliv, tubi, tenplay, patreon, videa, yahoo, BravoTV, crunchyroll, RTP, viki, Hotstar, vidio, vimeo, mediaset, Mxplayer, nbcolympics, ParamountPlus, Newgrounds, SAML Verizon login, Hungama, afreecatv, aljazeera, ATV, bitchute, camtube, CDA, eroprofile, facebook, HearThisAtIE, iwara, kakao, Motherless, Nova, peertube, pornhub, reddit, tiktok, TV2, TV2Hu, tv5mondeplus, VH1, Viafree, XHamster, 9Now, AnimalPlanet, Arte, CBC, Chingari, comedycentral, DIYNetwork, niconico, dw, funimation, globo, HiDive, NDR, Nuvid, Oreilly, pbs, plutotv, reddit, redtube, soundcloud, SpankBang, VrtNU, bbc, Bilibili, LinkedInLearning, parliamentlive, PolskieRadio, Streamable, vidme, francetv, 7plus, tagesschau
+* **New MSOs**: Philo, Spectrum, SlingTV, Cablevision, RCN
 
 * **Subtitle extraction from manifests**: Subtitles can be extracted from streaming media manifests. See [commit/be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details
 
@@ -105,35 +103,30 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata`
 
-* **Other new options**: `--print`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
+* **Other new options**: Many new options have been added such as `--print`, `--wait-for-video`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
 
-* **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection) etc
+* **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio etc
 
-* **Plugin extractors**: Extractors can be loaded from an external file. See [plugins](#plugins) for details
+* **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details
 
 * **Self-updater**: The releases can be updated using `yt-dlp -U`
 
-
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
-
-**PS**: Some of these changes are already in youtube-dlc, but are still unreleased. See [this](Changelog.md#unreleased-changes-in-blackjack4494yt-dlc) for details
-
-If you are coming from [youtube-dl](https://github.com/ytdl-org/youtube-dl), the amount of changes are very large. Compare [options](#options) and [supported sites](supportedsites.md) with youtube-dl's to get an idea of the massive number of features/patches [youtube-dlc](https://github.com/blackjack4494/yt-dlc) has accumulated.
-
 ### Differences in default behavior
 
-Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc.
+Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
 
-* The options `--id`, `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
+* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
 * `avconv` is not supported as as an alternative to `ffmpeg`
-* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s.%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
+* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
 * The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
 * The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be prefered. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
 * Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both
 * `--ignore-errors` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead
 * When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files
-* `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-infojson`. Use `--compat-options no-attach-info-json` to revert this
+* `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-info-json`. Use `--no-embed-info-json` or `--compat-options no-attach-info-json` to revert this
+* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](https://github.com/yt-dlp/yt-dlp#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
 * `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior
 * The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this
 * All *experiences* of a funimation episode are considered as a single video. This behavior breaks existing archives. Use `--compat-options seperate-video-versions` to extract information from only the default player
@@ -143,7 +136,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
 * Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
-* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the seperate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this.
+* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the seperate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
 
 For ease of use, a few more compat options are available:
 * `--compat-options all`: Use all compat options
@@ -152,17 +145,14 @@ For ease of use, a few more compat options are available:
 
 
 # INSTALLATION
-yt-dlp is not platform specific. So it should work on your Unix box, on Windows or on macOS
 
 You can install yt-dlp using one of the following methods:
-* Download [the binary](#release-files) from the [latest release](https://github.com/yt-dlp/yt-dlp/releases/latest)
-* With Homebrew, `brew install yt-dlp/taps/yt-dlp`
-* Use [PyPI package](https://pypi.org/project/yt-dlp): `python3 -m pip install --upgrade yt-dlp`
-* Install master branch: `python3 -m pip3 install -U https://github.com/yt-dlp/yt-dlp/archive/master.zip`
 
-Note that on some systems, you may need to use `py` or `python` instead of `python3`
+### Using the release binary
 
-UNIX users (Linux, macOS, BSD) can also install the [latest release](https://github.com/yt-dlp/yt-dlp/releases/latest) one of the following ways:
+You can simply download the [correct binary file](#release-files) for your OS: **[[Windows](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)] [[UNIX-like](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)]**
+
+In UNIX-like OSes (MacOS, Linux, BSD), you can also install the same in one of the following ways:
 
 ```
 sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
@@ -179,35 +169,60 @@ sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o
 sudo chmod a+rx /usr/local/bin/yt-dlp
 ```
 
-macOS or Linux users that are using Homebrew (formerly known as Linuxbrew for Linux users) can also install it by:
+PS: The manpages, shell completion files etc. are available in [yt-dlp.tar.gz](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+
+### With [PIP](https://pypi.org/project/pip)
+
+You can install the [PyPI package](https://pypi.org/project/yt-dlp) with:
+```
+python3 -m pip install -U yt-dlp
+```
+
+You can install without any of the optional dependencies using:
+```
+python3 -m pip install --no-deps -U yt-dlp
+```
+
+If you want to be on the cutting edge, you can also install the master branch with:
+```
+python3 -m pip install --force-reinstall https://github.com/yt-dlp/yt-dlp/archive/master.zip
+```
+
+Note that on some systems, you may need to use `py` or `python` instead of `python3`
+
+### With [Homebrew](https://brew.sh)
+
+macOS or Linux users that are using Homebrew can also install it by:
 
 ```
 brew install yt-dlp/taps/yt-dlp
 ```
 
-### UPDATE
+## UPDATE
-You can use `yt-dlp -U` to update if you are using the provided release.
+You can use `yt-dlp -U` to update if you are [using the provided release](#using-the-release-binary)
-If you are using `pip`, simply re-run the same command that was used to install the program.
-If you have installed using Homebrew, run `brew upgrade yt-dlp/taps/yt-dlp`
 
-### RELEASE FILES
+If you [installed with pip](#with-pip), simply re-run the same command that was used to install the program
+
+If you [installed using Homebrew](#with-homebrew), run `brew upgrade yt-dlp/taps/yt-dlp`
+
+## RELEASE FILES
 
 #### Recommended
 
 File|Description
 :---|:---
-[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform independant binary. Needs Python (Recommended for **UNIX-like systems**)
+[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform-independant binary. Needs Python (recommended for **UNIX-like systems**)
-[yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows standalone x64 binary (Recommended for **Windows**)
+[yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows (Win7 SP1+) standalone x64 binary (recommended for **Windows**)
 
 #### Alternatives
 
 File|Description
 :---|:---
-[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS standalone executable
+[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS (10.15+) standalone executable
-[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows standalone x86 (32bit) binary
+[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Vista SP2+) standalone x86 (32-bit) binary
-[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`, needs VC++14
+[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`, needs VC++14
-[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged windows executable (No auto-update)
+[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
-[yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS executable (No auto-update)
+[yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
 
 #### Misc
 
@@ -217,7 +232,7 @@ File|Description
 [SHA2-512SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-512SUMS)|GNU-style SHA512 sums
 [SHA2-256SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS)|GNU-style SHA256 sums
 
-### DEPENDENCIES
+## DEPENDENCIES
 Python versions 3.6+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
 
 <!-- Python 3.5+ uses VC++14 and it is already embedded in the binary created
@@ -227,25 +242,25 @@ On windows, [Microsoft Visual C++ 2010 SP1 Redistributable Package (x86)](https:
 
 While all the other dependancies are optional, `ffmpeg` and `ffprobe` are highly recommended
 * [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging seperate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. Licence [depends on the build](https://www.ffmpeg.org/legal.html)
-* [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licenced under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
+* [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
-* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licenced under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
+* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
-* [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licenced under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
+* [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licensed under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
-* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licenced under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
+* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licensed under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
-* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licenced under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
+* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
-* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licenced under [GPLv2+](http://rtmpdump.mplayerhq.hu)
+* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
-* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licenced under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
+* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
-* [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licenced under [BSD3](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
+* [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD3](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
-* [**sponskrub**](https://github.com/faissaloo/SponSkrub) - For using the now **deprecated** [sponskrub options](#sponskrub-options). Licenced under [GPLv3+](https://github.com/faissaloo/SponSkrub/blob/master/LICENCE.md)
+* [**sponskrub**](https://github.com/faissaloo/SponSkrub) - For using the now **deprecated** [sponskrub options](#sponskrub-options). Licensed under [GPLv3+](https://github.com/faissaloo/SponSkrub/blob/master/LICENCE.md)
 * Any external downloader that you want to use with `--downloader`
 
 To use or redistribute the dependencies, you must agree to their respective licensing terms.
 
-The windows releases are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
+The Windows and MacOS standalone release binaries are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
 
 **Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependancy, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds/wiki/Latest#latest-autobuilds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specifc issues solved by these builds
 
 
-### COMPILE
+## COMPILE
 
 **For Windows**:
 To build the Windows executable, you must have pyinstaller (and optionally mutagen, pycryptodomex, websockets). Once you have all the necessary dependencies installed, (optionally) build lazy extractors using `devscripts/make_lazy_extractors.py`, and then just run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the python used to build it.
@@ -262,6 +277,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
 
 **Note**: In either platform, `devscripts/update-version.py` can be used to automatically update the version number
 
+You can also fork the project on github and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a release
+
 # USAGE AND OPTIONS
 
     yt-dlp [OPTIONS] [--] URL [URL...]
@@ -276,7 +293,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      sure that you have sufficient permissions
                                      (run with sudo if needed)
     -i, --ignore-errors              Ignore download and postprocessing errors.
-                                     The download will be considered successfull
+                                     The download will be considered successful
                                      even if the postprocessing fails
     --no-abort-on-error              Continue with next video on download
                                      errors; e.g. to skip unavailable videos in
@@ -312,6 +329,10 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them
     --no-flat-playlist               Extract the videos of a playlist
+    --wait-for-video MIN[-MAX]       Wait for scheduled streams to become
+                                     available. Pass the minimum number of
+                                     seconds (or range) to wait between retries
+    --no-wait-for-video              Do not wait for scheduled streams (default)
     --mark-watched                   Mark videos watched (even with --simulate).
                                      Currently only supported for YouTube
     --no-mark-watched                Do not mark videos watched (default)
@@ -361,12 +382,11 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      specify range: "--playlist-items
                                      1-3,7,10-13", it will download the videos
                                      at index 1, 2, 3, 7, 10, 11, 12 and 13
-    --max-downloads NUMBER           Abort after downloading NUMBER files
     --min-filesize SIZE              Do not download any videos smaller than
                                      SIZE (e.g. 50k or 44.6m)
     --max-filesize SIZE              Do not download any videos larger than SIZE
                                      (e.g. 50k or 44.6m)
-    --date DATE                      Download only videos uploaded in this date.
+    --date DATE                      Download only videos uploaded on this date.
                                      The date can be "YYYYMMDD" or in the format
                                      "(now|today)[+-][0-9](day|week|month|year)(s)?"
     --datebefore DATE                Download only videos uploaded on or before
@@ -403,13 +423,18 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --download-archive FILE          Download only videos not listed in the
                                      archive file. Record the IDs of all
                                      downloaded videos in it
+    --no-download-archive            Do not use archive file (default)
+    --max-downloads NUMBER           Abort after downloading NUMBER files
     --break-on-existing              Stop the download process when encountering
                                      a file that is in the archive
     --break-on-reject                Stop the download process when encountering
                                      a file that has been filtered out
+    --break-per-input                Make --break-on-existing and --break-on-
+                                     reject act only on the current input URL
+    --no-break-per-input             --break-on-existing and --break-on-reject
+                                     terminates the entire download queue
     --skip-playlist-after-errors N   Number of allowed failures until the rest
                                      of the playlist is skipped
-    --no-download-archive            Do not use archive file (default)
 
 ## Download Options:
     -N, --concurrent-fragments N     Number of fragments of a dash/hlsnative
@@ -510,9 +535,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      filenames
     --no-restrict-filenames          Allow Unicode characters, "&" and spaces in
                                      filenames (default)
-    --windows-filenames              Force filenames to be windows compatible
+    --windows-filenames              Force filenames to be Windows-compatible
-    --no-windows-filenames           Make filenames windows compatible only if
+    --no-windows-filenames           Make filenames Windows-compatible only if
-                                     using windows (default)
+                                     using Windows (default)
     --trim-filenames LENGTH          Limit the filename length (excluding
                                      extension) to the specified number of
                                      characters
@@ -560,8 +585,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --load-info-json FILE            JSON file containing the video information
                                      (created with the "--write-info-json"
                                      option)
-    --cookies FILE                   File to read cookies from and dump cookie
+    --cookies FILE                   Netscape formatted file to read cookies
-                                     jar in
+                                     from and dump cookie jar in
     --no-cookies                     Do not read/dump cookies from/to file
                                      (default)
     --cookies-from-browser BROWSER[:PROFILE]
@@ -608,9 +633,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      anything to disk
     --no-simulate                    Download the video even if printing/listing
                                      options are used
-    --ignore-no-formats-error        Ignore "No video formats" error. Usefull
+    --ignore-no-formats-error        Ignore "No video formats" error. Useful for
-                                     for extracting metadata even if the videos
+                                     extracting metadata even if the videos are
-                                     are not actually available for download
+                                     not actually available for download
                                      (experimental)
     --no-ignore-no-formats-error     Throw error when no downloadable video
                                      formats are found (default)
@@ -644,7 +669,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      "postprocess:",  or "postprocess-title:".
                                      The video's fields are accessible under the
                                      "info" key and the progress attributes are
-                                     accessible under "progress" key. Eg:
+                                     accessible under "progress" key. E.g.:
                                      --console-title --progress-template
                                      "download-title:%(info.id)s-%(progress.eta)s"
     -v, --verbose                    Print various debugging information
@@ -657,7 +682,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
-    --no-check-certificate           Suppress HTTPS certificate validation
+    --no-check-certificates          Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve
                                      information about the video (Currently
                                      supported only for YouTube)
@@ -706,10 +731,12 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      containers irrespective of quality
     --no-prefer-free-formats         Don't give any special preference to free
                                      containers (default)
-    --check-formats                  Check that the formats selected are
+    --check-formats                  Check that the selected formats are
                                      actually downloadable
-    --no-check-formats               Do not check that the formats selected are
+    --check-all-formats              Check all formats for whether they are
                                      actually downloadable
+    --no-check-formats               Do not check that the formats are actually
+                                     downloadable
     -F, --list-formats               List available formats of each video.
                                      Simulate unless --no-simulate is used
     --merge-output-format FORMAT     If a merge is required (e.g.
@@ -731,7 +758,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      "ass/srt/best"
     --sub-langs LANGS                Languages of the subtitles to download (can
                                      be regex) or "all" separated by commas.
-                                     (Eg: --sub-langs en.*,ja) You can prefix
+                                     (Eg: --sub-langs "en.*,ja") You can prefix
                                      the language code with a "-" to exempt it
                                      from the requested languages. (Eg: --sub-
                                      langs all,-live_chat) Use --list-subs for a
@@ -763,9 +790,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --audio-format FORMAT            Specify audio format to convert the audio
                                      to when -x is used. Currently supported
                                      formats are: best (default) or one of
-                                     best|aac|flac|mp3|m4a|opus|vorbis|wav
+                                     best|aac|flac|mp3|m4a|opus|vorbis|wav|alac
     --audio-quality QUALITY          Specify ffmpeg audio quality, insert a
-                                     value between 0 (better) and 9 (worse) for
+                                     value between 0 (best) and 10 (worst) for
                                      VBR or a specific bitrate like 128K
                                      (default 5)
     --remux-video FORMAT             Remux the video into another container if
@@ -814,15 +841,20 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --no-embed-subs                  Do not embed subtitles (default)
     --embed-thumbnail                Embed thumbnail in the video as cover art
     --no-embed-thumbnail             Do not embed thumbnail (default)
-    --embed-metadata                 Embed metadata to the video file. Also adds
+    --embed-metadata                 Embed metadata to the video file. Also
-                                     chapters to file unless --no-add-chapters
+                                     embeds chapters/infojson if present unless
-                                     is used (Alias: --add-metadata)
+                                     --no-embed-chapters/--no-embed-info-json
+                                     are used (Alias: --add-metadata)
     --no-embed-metadata              Do not add metadata to file (default)
                                      (Alias: --no-add-metadata)
     --embed-chapters                 Add chapter markers to the video file
                                      (Alias: --add-chapters)
     --no-embed-chapters              Do not add chapter markers (default)
                                      (Alias: --no-add-chapters)
+    --embed-info-json                Embed the infojson as an attachment to
+                                     mkv/mka video files
+    --no-embed-info-json             Do not embed the infojson as an attachment
+                                     to the video file
     --parse-metadata FROM:TO         Parse additional metadata like title/artist
                                      from other fields; see "MODIFYING METADATA"
                                      for details
@@ -966,7 +998,7 @@ You can configure yt-dlp by placing any supported command line option to a confi
     * `~/yt-dlp.conf`
     * `~/yt-dlp.conf.txt`
 
-    `%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `~` points to %HOME% if present, `%USERPROFILE%` (generally `C:\Users\<user name>`) or `%HOMEDRIVE%%HOMEPATH%`.
+    `%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `%APPDATA%` generally points to (`C:\Users\<user name>\AppData\Roaming`) and `~` points to `%HOME%` if present, `%USERPROFILE%` (generally `C:\Users\<user name>`), or `%HOMEDRIVE%%HOMEPATH%`
 1. **System Configuration**: `/etc/yt-dlp.conf`
 
 For example, with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
@@ -988,7 +1020,7 @@ For example, with the following configuration file yt-dlp will always extract th
 
 Note that options in configuration file are just the same options aka switches used in regular command line calls; thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
 
-You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of user and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
+You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of home, user, and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
 
 ### Authentication with `.netrc` file
 
@@ -1018,7 +1050,7 @@ The `-o` option is used to indicate a template for the output file names while `
 
 The simplest usage of `-o` is not to set any template arguments when downloading a single file, like in `yt-dlp -o funny_video.flv "https://some/video"` (hard-coding file extension like this is _not_ recommended and could break some post-processing).
 
-It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
+It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
 
 The field names themselves (the part inside the parenthesis) can also have some special formatting:
 1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
@@ -1026,7 +1058,7 @@ The field names themselves (the part inside the parenthesis) can also have some
 1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
 1. **Alternatives**: Alternate fields can be specified seperated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
 1. **Default**: A literal default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
-1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son, a comma seperated **l**ist (alternate form flag `#` makes it new line `\n` seperated) and a string **q**uoted for the terminal, respectively
+1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma seperated **l**ist (flag `#` for `\n` newline-seperated) and a string **q**uoted for the terminal (flag `#` to split a list into different arguments), respectively
 1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
 
 To summarize, the general syntax for a field is:
@@ -1034,7 +1066,7 @@ To summarize, the general syntax for a field is:
 %(name[.keys][addition][>strf][,alternate][|default])[flags][width][.precision][length]type
 ```
 
-Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'`  will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
+Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'`  will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
 
 The available fields are:
 
@@ -1159,7 +1191,7 @@ Each aforementioned sequence when referenced in an output template will be repla
 
 Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default).
 
-**Tip**: Look at the `-j` output to identify which fields are available for the purticular URL
+**Tip**: Look at the `-j` output to identify which fields are available for the particular URL
 
 For numeric sequences you can use [numeric related formatting](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting), for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
 
@@ -1169,7 +1201,7 @@ To use percent literals in an output template use `%%`. To output to stdout use
 
 The current default template is `%(title)s [%(id)s].%(ext)s`.
 
-In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
+In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title.
 
 #### Output template and Windows batch files
 
@@ -1180,11 +1212,14 @@ If you are using an output template inside a Windows batch file then you must es
 Note that on Windows you need to use double quotes instead of single.
 
 ```bash
+$ yt-dlp --get-filename -o 'test video.%(ext)s' BaW_jenozKc
+test video.webm    # Literal name with correct extension
+
 $ yt-dlp --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
-youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
+youtube-dl test video ''_ä↭𝕐.webm    # All kinds of weird characters
 
 $ yt-dlp --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames
-youtube-dl_test_video_.mp4          # A simple file name
+youtube-dl_test_video_.webm    # Restricted file name
 
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
 $ yt-dlp -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
@@ -1222,19 +1257,19 @@ You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`,
 
 You can also use special names to select particular edge case formats:
 
- - `all`: Select all formats
+ - `all`: Select **all formats** separately
- - `mergeall`: Select and merge all formats (Must be used with `--audio-multistreams`, `--video-multistreams` or both)
+ - `mergeall`: Select and **merge all formats** (Must be used with `--audio-multistreams`, `--video-multistreams` or both)
- - `b*`, `best*`: Select the best quality format irrespective of whether it contains video or audio
+ - `b*`, `best*`: Select the best quality format that **contains either** a video or an audio
- - `w*`, `worst*`: Select the worst quality format irrespective of whether it contains video or audio
+ - `b`, `best`: Select the best quality format that **contains both** video and audio. Equivalent to `best*[vcodec!=none][acodec!=none]`
- - `b`, `best`: Select the best quality format that contains both video and audio. Equivalent to `best*[vcodec!=none][acodec!=none]`
+ - `bv`, `bestvideo`: Select the best quality **video-only** format. Equivalent to `best*[acodec=none]`
+ - `bv*`, `bestvideo*`: Select the best quality format that **contains video**. It may also contain audio. Equivalent to `best*[vcodec!=none]`
+ - `ba`, `bestaudio`: Select the best quality **audio-only** format. Equivalent to `best*[vcodec=none]`
+ - `ba*`, `bestaudio*`: Select the best quality format that **contains audio**. It may also contain video. Equivalent to `best*[acodec!=none]`
+ - `w*`, `worst*`: Select the worst quality format that contains either a video or an audio
  - `w`, `worst`: Select the worst quality format that contains both video and audio. Equivalent to `worst*[vcodec!=none][acodec!=none]`
- - `bv`, `bestvideo`: Select the best quality video-only format. Equivalent to `best*[acodec=none]`
  - `wv`, `worstvideo`: Select the worst quality video-only format. Equivalent to `worst*[acodec=none]`
- - `bv*`, `bestvideo*`: Select the best quality format that contains video. It may also contain audio. Equivalent to `best*[vcodec!=none]`
  - `wv*`, `worstvideo*`: Select the worst quality format that contains video. It may also contain audio. Equivalent to `worst*[vcodec!=none]`
- - `ba`, `bestaudio`: Select the best quality audio-only format. Equivalent to `best*[vcodec=none]`
  - `wa`, `worstaudio`: Select the worst quality audio-only format. Equivalent to `worst*[vcodec=none]`
- - `ba*`, `bestaudio*`: Select the best quality format that contains audio. It may also contain video. Equivalent to `best*[acodec!=none]`
  - `wa*`, `worstaudio*`: Select the worst quality format that contains audio. It may also contain video. Equivalent to `worst*[acodec!=none]`
 
 For example, to download the worst quality video-only format you can use `-f worstvideo`. It is however recommended not to use `worst` and related options. When your format selector is `worst`, the format which is worst in all respects is selected. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-f best -S +size,+br,+res,+fps` instead of `-f worst`. See [sorting formats](#sorting-formats) for more details.
@@ -1298,12 +1333,12 @@ The available fields are:
  - `source`: Preference of the source as given by the extractor
  - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > other > `mms`/`rtsp` > unknown > `f4f`/`f4m`)
  - `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other > unknown)
- - `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `ac3` > `dts` > other > unknown)
+ - `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other > unknown)
  - `codec`: Equivalent to `vcodec,acodec`
  - `vext`: Video Extension (`mp4` > `webm` > `flv` > other > unknown). If `--prefer-free-formats` is used, `webm` is prefered.
  - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other > unknown). If `--prefer-free-formats` is used, the order changes to `opus` > `ogg` > `webm` > `m4a` > `mp3` > `aac`.
  - `ext`: Equivalent to `vext,aext`
- - `filesize`: Exact filesize, if know in advance. This will be unavailable for mu38 and DASH formats.
+ - `filesize`: Exact filesize, if known in advance
  - `fs_approx`: Approximate filesize calculated from the manifests
  - `size`: Exact filesize if available, otherwise approximate filesize
  - `height`: Height of video
@@ -1455,7 +1490,7 @@ $ yt-dlp -S '+res:480,codec,br'
 
 # MODIFYING METADATA
 
-The metadata obtained the the extractors can be modified by using `--parse-metadata` and `--replace-in-metadata`
+The metadata obtained by the extractors can be modified by using `--parse-metadata` and `--replace-in-metadata`
 
 `--replace-in-metadata FIELDS REGEX REPLACE` is used to replace text in any metadata field using [python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax). [Backreferences](https://docs.python.org/3/library/re.html?highlight=backreferences#re.sub) can be used in the replace string for advanced use.
 
@@ -1506,6 +1541,9 @@ $ yt-dlp --parse-metadata '%(series)s S%(season_number)02dE%(episode_number)02d:
 # Set "comment" field in video metadata using description instead of webpage_url
 $ yt-dlp --parse-metadata 'description:(?s)(?P<meta_comment>.+)' --add-metadata
 
+# Remove "formats" field from the infojson by setting it to an empty string
+$ yt-dlp --parse-metadata ':(?P<formats>)' -j
+
 # Replace all spaces and "_" in title and uploader with a `-`
 $ yt-dlp --replace-in-metadata 'title,uploader' '[ _]' '-'
 
@@ -1513,27 +1551,32 @@ $ yt-dlp --replace-in-metadata 'title,uploader' '[ _]' '-'
 
 # EXTRACTOR ARGUMENTS
 
-Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) seperated string of `ARG=VAL1,VAL2`. Eg: `--extractor-args "youtube:player_client=android_agegate,web;include_live_dash" --extractor-args "funimation:version=uncut"`
+Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. Eg: `--extractor-args "youtube:player-client=android_agegate,web;include_live_dash" --extractor-args "funimation:version=uncut"`
 
 The following extractors use this feature:
-* **youtube**
-    * `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
-    * `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients
-    * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
-    * `include_live_dash`: Include live dash formats (These formats don't download properly)
-    * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side).
-    * `max_comments`: Maximum amount of comments to download (default all).
-    * `max_comment_depth`: Maximum depth for nested comments. YouTube supports depths 1 or 2 (default).
-* **youtubetab**
-  (YouTube playlists, channels, feeds, etc.)
-   * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
 
-* **funimation**
+#### youtube
-    * `language`: Languages to extract. Eg: `funimation:language=english,japanese`
+* `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
-    * `version`: The video version to extract - `uncut` or `simulcast`
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients, and `default` for the default clients.
+* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
+* `include_live_dash`: Include live dash formats (These formats don't download properly)
+* `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
+* `max_comments`: Maximum amount of comments to download (default all)
+* `max_comment_depth`: Maximum depth for nested comments. YouTube supports depths 1 or 2 (default)
 
-* **vikiChannel**
+#### youtubetab (YouTube playlists, channels, feeds, etc.)
-    * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
+* `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
+
+#### funimation
+* `language`: Languages to extract. Eg: `funimation:language=english,japanese`
+* `version`: The video version to extract - `uncut` or `simulcast`
+
+#### crunchyroll
+* `language`: Languages to extract. Eg: `crunchyroll:language=jaJp`
+* `hardsub`: Which hard-sub versions to extract. Eg: `crunchyroll:hardsub=None,enUS`
+
+#### vikichannel
+* `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
 
 NOTE: These options may be changed/removed in the future without concern for backward compatibility
 
@@ -1561,22 +1604,20 @@ Your program should avoid parsing the normal stdout since they may change in fut
 From a Python program, you can embed yt-dlp in a more powerful fashion, like this:
 
 ```python
-import yt_dlp
+from yt_dlp import YoutubeDL
 
-ydl_opts = {}
+ydl_opts = {'format': 'bestaudio'}
-with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+with YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L154-L452).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L162).
 
-Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), converts the video to an mp3 file, implements a custom postprocessor and prints the final info_dict as json:
+Here's a more complete example demonstrating various functionality:
 
 ```python
 import json
-
 import yt_dlp
-from yt_dlp.postprocessor.common import PostProcessor
 
 
 class MyLogger:
@@ -1598,36 +1639,75 @@ class MyLogger:
         print(msg)
 
 
-class MyCustomPP(PostProcessor):
+# ℹ️ See the docstring of yt_dlp.postprocessor.common.PostProcessor
+class MyCustomPP(yt_dlp.postprocessor.PostProcessor):
+    # ℹ️ See docstring of yt_dlp.postprocessor.common.PostProcessor.run
     def run(self, info):
         self.to_screen('Doing stuff')
         return [], info
 
 
+# ℹ️ See "progress_hooks" in the docstring of yt_dlp.YoutubeDL
 def my_hook(d):
     if d['status'] == 'finished':
         print('Done downloading, now converting ...')
 
 
+def format_selector(ctx):
+    """ Select the best video and the best audio that won't result in an mkv.
+    This is just an example and does not handle all cases """
+
+    # formats are already sorted worst to best
+    formats = ctx.get('formats')[::-1]
+
+    # acodec='none' means there is no audio
+    best_video = next(f for f in formats
+                      if f['vcodec'] != 'none' and f['acodec'] == 'none')
+
+    # find compatible audio extension
+    audio_ext = {'mp4': 'm4a', 'webm': 'webm'}[best_video['ext']]
+    # vcodec='none' means there is no video
+    best_audio = next(f for f in formats if (
+        f['acodec'] != 'none' and f['vcodec'] == 'none' and f['ext'] == audio_ext))
+
+    yield {
+        # These are the minimum required fields for a merged format
+        'format_id': f'{best_video["format_id"]}+{best_audio["format_id"]}',
+        'ext': best_video['ext'],
+        'requested_formats': [best_video, best_audio],
+        # Must be + seperated list of protocols
+        'protocol': f'{best_video["protocol"]}+{best_audio["protocol"]}'
+    }
+
+
+# ℹ️ See docstring of yt_dlp.YoutubeDL for a description of the options
 ydl_opts = {
-    'format': 'bestaudio/best',
+    'format': format_selector,
     'postprocessors': [{
-        'key': 'FFmpegExtractAudio',
+        # Embed metadata in video using ffmpeg.
-        'preferredcodec': 'mp3',
+        # ℹ️ See yt_dlp.postprocessor.FFmpegMetadataPP for the arguments it accepts
-        'preferredquality': '192',
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True,
     }],
     'logger': MyLogger(),
     'progress_hooks': [my_hook],
 }
 
+
+# Add custom headers
+yt_dlp.utils.std_headers.update({'Referer': 'https://www.google.com'})
+
+# ℹ️ See the public functions in yt_dlp.YoutubeDL for for other available functions.
+# Eg: "ydl.download", "ydl.download_with_info_file"
 with yt_dlp.YoutubeDL(ydl_opts) as ydl:
     ydl.add_post_processor(MyCustomPP())
     info = ydl.extract_info('https://www.youtube.com/watch?v=BaW_jenozKc')
+
+    # ℹ️ ydl.sanitize_info makes the info json-serializable
     print(json.dumps(ydl.sanitize_info(info)))
 ```
 
-See the public functions in [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py) for other available functions. Eg: `ydl.download`, `ydl.download_with_info_file`
-
 **Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the example above
 
 
@@ -1702,7 +1782,7 @@ These are aliases that are no longer documented for various reasons
     --yes-overwrites                 --force-overwrites
 
 #### Sponskrub Options
-Support for [SponSkrub](https://github.com/faissaloo/SponSkrub) has been deprecated in favor of `--sponsorblock`
+Support for [SponSkrub](https://github.com/faissaloo/SponSkrub) has been deprecated in favor of the `--sponsorblock` options
 
     --sponskrub                      --sponsorblock-mark all
     --no-sponskrub                   --no-sponsorblock

devscripts/make_supportedsites.py

@@ -29,6 +29,9 @@ def main():
                 continue
             if ie_desc is not None:
                 ie_md += ': {0}'.format(ie.IE_DESC)
+            search_key = getattr(ie, 'SEARCH_KEY', None)
+            if search_key is not None:
+                ie_md += f'; "{ie.SEARCH_KEY}:" prefix'
             if not ie.working():
                 ie_md += ' (Currently broken)'
             yield ie_md

devscripts/update-version.py

@@ -1,33 +1,42 @@
 #!/usr/bin/env python3
-from __future__ import unicode_literals
-
 from datetime import datetime
-# import urllib.request
+import sys
+import subprocess
 
-# response = urllib.request.urlopen('https://blackjack4494.github.io/youtube-dlc/update/LATEST_VERSION')
-# old_version = response.read().decode('utf-8')
 
-exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec'))
+with open('yt_dlp/version.py', 'rt') as f:
+    exec(compile(f.read(), 'yt_dlp/version.py', 'exec'))
 old_version = locals()['__version__']
 
-old_version_list = old_version.split(".", 4)
+old_version_list = old_version.split('.')
 
 old_ver = '.'.join(old_version_list[:3])
 old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
 
 ver = datetime.utcnow().strftime("%Y.%m.%d")
-rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
+
+rev = (sys.argv[1:] or [''])[0]  # Use first argument, if present as revision number
+if not rev:
+    rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
 
 VERSION = '.'.join((ver, rev)) if rev else ver
-# VERSION_LIST = [(int(v) for v in ver.split(".") + [rev or 0])]
+
+try:
+    sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
+    GIT_HEAD = sp.communicate()[0].decode().strip() or None
+except Exception:
+    GIT_HEAD = None
+
+VERSION_FILE = f'''
+# Autogenerated by devscripts/update-version.py
+
+__version__ = {VERSION!r}
+
+RELEASE_GIT_HEAD = {GIT_HEAD!r}
+'''.lstrip()
+
+with open('yt_dlp/version.py', 'wt') as f:
+    f.write(VERSION_FILE)
 
 print('::set-output name=ytdlp_version::' + VERSION)
-
+print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')
-file_version_py = open('yt_dlp/version.py', 'rt')
-data = file_version_py.read()
-data = data.replace(old_version, VERSION)
-file_version_py.close()
-
-file_version_py = open('yt_dlp/version.py', 'wt')
-file_version_py.write(data)
-file_version_py.close()

setup.py

@@ -16,7 +16,7 @@ from distutils.spawn import spawn
 exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec'))
 
 
-DESCRIPTION = 'Command-line program to download videos from YouTube.com and many other other video platforms.'
+DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',

supportedsites.md

@@ -48,6 +48,7 @@
  - **Alura**
  - **AluraCourse**
  - **Amara**
+ - **AmazonStore**
  - **AMCNetworks**
  - **AmericasTestKitchen**
  - **AmericasTestKitchenSeason**
@@ -127,7 +128,7 @@
  - **BilibiliAudioAlbum**
  - **BilibiliChannel**
  - **BiliBiliPlayer**
- - **BiliBiliSearch**: Bilibili video search, "bilisearch" keyword
+ - **BiliBiliSearch**: Bilibili video search; "bilisearch:" prefix
  - **BiliIntl**
  - **BiliIntlSeries**
  - **BioBioChileTV**
@@ -140,6 +141,7 @@
  - **BlackboardCollaborate**
  - **BleacherReport**
  - **BleacherReportCMS**
+ - **blogger.com**
  - **Bloomberg**
  - **BokeCC**
  - **BongaCams**
@@ -149,6 +151,7 @@
  - **BR**: Bayerischer Rundfunk
  - **BravoTV**
  - **Break**
+ - **BreitBart**
  - **brightcove:legacy**
  - **brightcove:new**
  - **BRMediathek**: Bayerischer Rundfunk Mediathek
@@ -157,11 +160,13 @@
  - **BusinessInsider**
  - **BuzzFeed**
  - **BYUtv**
+ - **CableAV**
  - **CAM4**
  - **Camdemy**
  - **CamdemyFolder**
  - **CamModels**
  - **CamWithHer**
+ - **CanalAlpha**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
  - **Canvas**
@@ -184,7 +189,6 @@
  - **CCTV**: 央视网
  - **CDA**
  - **CeskaTelevize**
- - **CeskaTelevizePorady**
  - **CGTN**
  - **channel9**: Channel 9
  - **CharlieRose**
@@ -222,6 +226,8 @@
  - **CONtv**
  - **Corus**
  - **Coub**
+ - **CozyTV**
+ - **cp24**
  - **Cracked**
  - **Crackle**
  - **CrooksAndLiars**
@@ -236,7 +242,8 @@
  - **cu.ntv.co.jp**: Nippon Television Network
  - **CultureUnplugged**
  - **curiositystream**
- - **curiositystream:collection**
+ - **curiositystream:collections**
+ - **curiositystream:series**
  - **CWTV**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
@@ -266,6 +273,7 @@
  - **DiscoveryPlus**
  - **DiscoveryPlusIndia**
  - **DiscoveryPlusIndiaShow**
+ - **DiscoveryPlusItalyShow**
  - **DiscoveryVR**
  - **Disney**
  - **DIYNetwork**
@@ -315,6 +323,7 @@
  - **Escapist**
  - **ESPN**
  - **ESPNArticle**
+ - **ESPNCricInfo**
  - **EsriVideo**
  - **Europa**
  - **EUScreen**
@@ -366,6 +375,7 @@
  - **Funk**
  - **Fusion**
  - **Fux**
+ - **Gab**
  - **GabTV**
  - **Gaia**
  - **GameInformer**
@@ -447,11 +457,13 @@
  - **IndavideoEmbed**
  - **InfoQ**
  - **Instagram**
- - **instagram:tag**: Instagram hashtag search
+ - **instagram:tag**: Instagram hashtag search URLs
  - **instagram:user**: Instagram user profile
+ - **InstagramIOS**: IOS instagram:// URL
  - **Internazionale**
  - **InternetVideoArchive**
  - **IPrima**
+ - **IPrimaCNN**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ITTF**
@@ -521,6 +533,7 @@
  - **LineLive**
  - **LineLiveChannel**
  - **LineTV**
+ - **LinkedIn**
  - **linkedin:learning**
  - **linkedin:learning:course**
  - **LinuxAcademy**
@@ -560,6 +573,7 @@
  - **MediaKlikk**
  - **Medialaan**
  - **Mediaset**
+ - **MediasetShow**
  - **Mediasite**
  - **MediasiteCatalog**
  - **MediasiteNamedCatalog**
@@ -587,11 +601,13 @@
  - **mirrativ**
  - **mirrativ:user**
  - **MiTele**: mitele.es
+ - **mixch**
  - **mixcloud**
  - **mixcloud:playlist**
  - **mixcloud:user**
  - **MLB**
  - **MLBVideo**
+ - **MLSSoccer**
  - **Mnet**
  - **MNetTV**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -658,6 +674,7 @@
  - **ndr:embed:base**
  - **NDTV**
  - **Nebula**
+ - **nebula:collection**
  - **NerdCubedFeed**
  - **netease:album**: 网易云音乐 - 专辑
  - **netease:djradio**: 网易云音乐 - 电台
@@ -691,8 +708,8 @@
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **NiconicoUser**
- - **nicovideo:search**: Nico video searches
+ - **nicovideo:search**: Nico video search; "nicosearch:" prefix
- - **nicovideo:search:date**: Nico video searches, newest first
+ - **nicovideo:search:date**: Nico video search, newest first; "nicosearchdate:" prefix
  - **nicovideo:search_url**: Nico video search URLs
  - **Nintendo**
  - **Nitter**
@@ -741,6 +758,7 @@
  - **OlympicsReplay**
  - **on24**: ON24
  - **OnDemandKorea**
+ - **OneFootball**
  - **onet.pl**
  - **onet.tv**
  - **onet.tv:channel**
@@ -783,6 +801,7 @@
  - **PatreonUser**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **PearVideo**
+ - **peer.tv**
  - **PeerTube**
  - **PeerTube:Playlist**
  - **peloton**
@@ -801,6 +820,7 @@
  - **Pinterest**
  - **PinterestCollection**
  - **Pladform**
+ - **PlanetMarathi**
  - **Platzi**
  - **PlatziCourse**
  - **play.fm**
@@ -817,7 +837,12 @@
  - **podomatic**
  - **Pokemon**
  - **PokemonWatch**
+ - **PolsatGo**
  - **PolskieRadio**
+ - **polskieradio:kierowcow**
+ - **polskieradio:player**
+ - **polskieradio:podcast**
+ - **polskieradio:podcast:list**
  - **PolskieRadioCategory**
  - **Popcorntimes**
  - **PopcornTV**
@@ -860,6 +885,9 @@
  - **radiocanada:audiovideo**
  - **radiofrance**
  - **RadioJavan**
+ - **radiokapital**
+ - **radiokapital:show**
+ - **RadioZetPodcast**
  - **radlive**
  - **radlive:channel**
  - **radlive:season**
@@ -867,6 +895,8 @@
  - **RaiPlay**
  - **RaiPlayLive**
  - **RaiPlayPlaylist**
+ - **RaiPlayRadio**
+ - **RaiPlayRadioPlaylist**
  - **RayWenderlich**
  - **RayWenderlichCourse**
  - **RBMARadio**
@@ -882,7 +912,9 @@
  - **RedBullTV**
  - **RedBullTVRrnContent**
  - **Reddit**
- - **RedditR**
+ - **RedGifs**
+ - **RedGifsSearch**: Redgifs search
+ - **RedGifsUser**: Redgifs user
  - **RedTube**
  - **RegioTV**
  - **RENTV**
@@ -894,6 +926,7 @@
  - **RMCDecouverte**
  - **RockstarGames**
  - **RoosterTeeth**
+ - **RoosterTeethSeries**
  - **RottenTomatoes**
  - **Roxwel**
  - **Rozhlas**
@@ -905,6 +938,7 @@
  - **rtl2:you**
  - **rtl2:you:series**
  - **RTP**
+ - **RTRFM**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:infantil**: RTVE infantil
@@ -936,7 +970,7 @@
  - **SBS**: sbs.com.au
  - **schooltv**
  - **ScienceChannel**
- - **screen.yahoo:search**: Yahoo screen search
+ - **screen.yahoo:search**: Yahoo screen search; "yvsearch:" prefix
  - **Screencast**
  - **ScreencastOMatic**
  - **ScrippsNetworks**
@@ -944,6 +978,7 @@
  - **SCTE**
  - **SCTECourse**
  - **Seeker**
+ - **SenateGov**
  - **SenateISVP**
  - **SendtoNews**
  - **Servus**
@@ -961,6 +996,7 @@
  - **Sina**
  - **sky.it**
  - **sky:news**
+ - **sky:news:story**
  - **sky:sports**
  - **sky:sports:news**
  - **skyacademy.it**
@@ -977,7 +1013,7 @@
  - **SonyLIVSeries**
  - **soundcloud**
  - **soundcloud:playlist**
- - **soundcloud:search**: Soundcloud search, "scsearch" keyword
+ - **soundcloud:search**: Soundcloud search; "scsearch:" prefix
  - **soundcloud:set**
  - **soundcloud:trackstation**
  - **soundcloud:user**
@@ -1021,8 +1057,10 @@
  - **Streamanity**
  - **streamcloud.eu**
  - **StreamCZ**
+ - **StreamFF**
  - **StreetVoice**
  - **StretchInternet**
+ - **Stripchat**
  - **stv:player**
  - **SunPorno**
  - **sverigesradio:episode**
@@ -1079,6 +1117,8 @@
  - **ThisAmericanLife**
  - **ThisAV**
  - **ThisOldHouse**
+ - **ThreeSpeak**
+ - **ThreeSpeakUser**
  - **TikTok**
  - **tiktok:user**
  - **tinypic**: tinypic.com videos
@@ -1095,8 +1135,8 @@
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
  - **Trovo**
- - **TrovoChannelClip**: All Clips of a trovo.live channel, "trovoclip" keyword
+ - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
- - **TrovoChannelVod**: All VODs of a trovo.live channel, "trovovod" keyword
+ - **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
  - **TrovoVod**
  - **TruNews**
  - **TruTV**
@@ -1142,6 +1182,7 @@
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
+ - **tvp:stream**
  - **TVPlayer**
  - **TVPlayHome**
  - **Tweakers**
@@ -1201,7 +1242,7 @@
  - **Viddler**
  - **Videa**
  - **video.arnes.si**: Arnes Video
- - **video.google:search**: Google Video search (Currently broken)
+ - **video.google:search**: Google Video search; "gvsearch:" prefix (Currently broken)
  - **video.sky.it**
  - **video.sky.it:live**
  - **VideoDetective**
@@ -1291,11 +1332,14 @@
  - **WeiboMobile**
  - **WeiqiTV**: WQTV
  - **whowatch**
+ - **Willow**
  - **WimTV**
  - **Wistia**
  - **WistiaPlaylist**
  - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **WorldStarHipHop**
+ - **wppilot**
+ - **wppilot:channels**
  - **WSJ**: Wall Street Journal
  - **WSJArticle**
  - **WWE**
@@ -1343,19 +1387,19 @@
  - **YouPorn**
  - **YourPorn**
  - **YourUpload**
- - **youtube**: YouTube.com
+ - **youtube**: YouTube
- - **youtube:favorites**: YouTube.com liked videos, ":ytfav" for short (requires authentication)
+ - **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)
- - **youtube:history**: Youtube watch history, ":ythis" for short (requires authentication)
+ - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies)
- - **youtube:playlist**: YouTube.com playlists
+ - **youtube:playlist**: YouTube playlists
- - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
+ - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword
- - **youtube:search**: YouTube.com searches, "ytsearch" keyword
+ - **youtube:search**: YouTube search; "ytsearch:" prefix
- - **youtube:search:date**: YouTube.com searches, newest videos first, "ytsearchdate" keyword
+ - **youtube:search:date**: YouTube search, newest videos first; "ytsearchdate:" prefix
- - **youtube:search_url**: YouTube.com search URLs
+ - **youtube:search_url**: YouTube search URLs with sorting and filter support
- - **youtube:subscriptions**: YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)
+ - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)
- - **youtube:tab**: YouTube.com tab
+ - **youtube:tab**: YouTube Tabs
- - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **youtube:watchlater**: Youtube watch later list; ":ytwatchlater" keyword (requires cookies)
  - **YoutubeYtBe**: youtu.be
- - **YoutubeYtUser**: YouTube.com user videos, URL or "ytuser" keyword
+ - **YoutubeYtUser**: YouTube user videos; "ytuser:" prefix
  - **Zapiks**
  - **Zattoo**
  - **ZattooLive**

test/parameters.json

@@ -9,7 +9,7 @@
     "forcetitle": false,
     "forceurl": false,
     "force_write_download_archive": false,
-    "format": "best",
+    "format": "b/bv",
     "ignoreerrors": false,
     "listformats": null,
     "logtostderr": false,

test/test_YoutubeDL.py

@@ -137,7 +137,7 @@ class TestFormatSelection(unittest.TestCase):
         test('webm/mp4', '47')
         test('3gp/40/mp4', '35')
         test('example-with-dashes', 'example-with-dashes')
-        test('all', '35', 'example-with-dashes', '45', '47', '2')  # Order doesn't actually matter for this
+        test('all', '2', '47', '45', 'example-with-dashes', '35')
         test('mergeall', '2+47+45+example-with-dashes+35', multi=True)
 
     def test_format_selection_audio(self):
@@ -520,7 +520,7 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'format': 'all[width>=400][width<=600]'})
         ydl.process_ie_result(info_dict)
         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
-        self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
+        self.assertEqual(downloaded_ids, ['D', 'C', 'B'])
 
         ydl = YDL({'format': 'best[height<40]'})
         try:
@@ -656,7 +656,7 @@ class TestYoutubeDL(unittest.TestCase):
         'playlist_autonumber': 2,
         '_last_playlist_index': 100,
         'n_entries': 10,
-        'formats': [{'id': 'id1'}, {'id': 'id2'}, {'id': 'id3'}]
+        'formats': [{'id': 'id 1'}, {'id': 'id 2'}, {'id': 'id 3'}]
     }
 
     def test_prepare_outtmpl_and_filename(self):
@@ -737,6 +737,7 @@ class TestYoutubeDL(unittest.TestCase):
         test(NA_TEST_OUTTMPL, 'NA-NA-def-1234.mp4')
         test(NA_TEST_OUTTMPL, 'none-none-def-1234.mp4', outtmpl_na_placeholder='none')
         test(NA_TEST_OUTTMPL, '--def-1234.mp4', outtmpl_na_placeholder='')
+        test('%(non_existent.0)s', 'NA')
 
         # String formatting
         FMT_TEST_OUTTMPL = '%%(height)%s.%%(ext)s'
@@ -762,14 +763,15 @@ class TestYoutubeDL(unittest.TestCase):
         test('a%(width|)d', 'a', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
-        sanitize = lambda x: x.replace(':', ' -').replace('"', "'")
+        sanitize = lambda x: x.replace(':', ' -').replace('"', "'").replace('\n', ' ')
 
         # Custom type casting
-        test('%(formats.:.id)l', 'id1, id2, id3')
+        test('%(formats.:.id)l', 'id 1, id 2, id 3')
-        test('%(formats.:.id)#l', ('id1\nid2\nid3', 'id1 id2 id3'))
+        test('%(formats.:.id)#l', ('id 1\nid 2\nid 3', 'id 1 id 2 id 3'))
         test('%(ext)l', 'mp4')
-        test('%(formats.:.id) 15l', '  id1, id2, id3')
+        test('%(formats.:.id) 18l', '  id 1, id 2, id 3')
         test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
+        test('%(formats)#j', (json.dumps(FORMATS, indent=4), sanitize(json.dumps(FORMATS, indent=4))))
         test('%(title5).3B', 'á')
         test('%(title5)U', 'áéí 𝐀')
         test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
@@ -777,8 +779,12 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
         if compat_os_name == 'nt':
             test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
+            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))
+            test('%(formats.0.id)#q', ('"id 1"', "'id 1'"))
         else:
             test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'"))
+            test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'")
+            test('%(formats.0.id)#q', "'id 1'")
 
         # Internal formatting
         test('%(timestamp-1000>%H-%M-%S)s', '11-43-20')

test/test_aes.py

@@ -10,6 +10,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from yt_dlp.aes import (
     aes_decrypt,
     aes_encrypt,
+    aes_ecb_encrypt,
+    aes_ecb_decrypt,
     aes_cbc_decrypt,
     aes_cbc_decrypt_bytes,
     aes_cbc_encrypt,
@@ -17,7 +19,8 @@ from yt_dlp.aes import (
     aes_ctr_encrypt,
     aes_gcm_decrypt_and_verify,
     aes_gcm_decrypt_and_verify_bytes,
-    aes_decrypt_text
+    aes_decrypt_text,
+    BLOCK_SIZE_BYTES,
 )
 from yt_dlp.compat import compat_pycrypto_AES
 from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
@@ -94,6 +97,19 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 
+    def test_ecb_encrypt(self):
+        data = bytes_to_intlist(self.secret_msg)
+        data += [0x08] * (BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES)
+        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key, self.iv))
+        self.assertEqual(
+            encrypted,
+            b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
+
+    def test_ecb_decrypt(self):
+        data = bytes_to_intlist(b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
+        decrypted = intlist_to_bytes(aes_ecb_decrypt(data, self.key, self.iv))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_all_urls.py

@@ -38,7 +38,6 @@ class TestAllURLsMatching(unittest.TestCase):
         assertTab('https://www.youtube.com/AsapSCIENCE')
         assertTab('https://www.youtube.com/embedded')
         assertTab('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
-        assertTab('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
         assertTab('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
         assertTab('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')  # 668
         self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))

test/test_jsinterp.py

@@ -112,6 +112,71 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

@@ -1222,12 +1222,49 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
     def test_render_table(self):
         self.assertEqual(
             render_table(
-                ['a', 'bcd'],
+                ['a', 'empty', 'bcd'],
-                [[123, 4], [9999, 51]]),
+                [[123, '', 4], [9999, '', 51]]),
+            'a    empty bcd\n'
+            '123        4\n'
+            '9999       51')
+
+        self.assertEqual(
+            render_table(
+                ['a', 'empty', 'bcd'],
+                [[123, '', 4], [9999, '', 51]],
+                hide_empty=True),
             'a    bcd\n'
             '123  4\n'
             '9999 51')
 
+        self.assertEqual(
+            render_table(
+                ['\ta', 'bcd'],
+                [['1\t23', 4], ['\t9999', 51]]),
+            '   a bcd\n'
+            '1 23 4\n'
+            '9999 51')
+
+        self.assertEqual(
+            render_table(
+                ['a', 'bcd'],
+                [[123, 4], [9999, 51]],
+                delim='-'),
+            'a    bcd\n'
+            '--------\n'
+            '123  4\n'
+            '9999 51')
+
+        self.assertEqual(
+            render_table(
+                ['a', 'bcd'],
+                [[123, 4], [9999, 51]],
+                delim='-', extra_gap=2),
+            'a      bcd\n'
+            '----------\n'
+            '123    4\n'
+            '9999   51')
+
     def test_match_str(self):
         # Unary
         self.assertFalse(match_str('xy', {'x': 1200}))
@@ -1620,9 +1657,9 @@ Line 1
         self.assertEqual(repr(LazyList(it)), repr(it))
         self.assertEqual(str(LazyList(it)), str(it))
 
-        self.assertEqual(list(LazyList(it).reverse()), it[::-1])
+        self.assertEqual(list(LazyList(it, reverse=True)), it[::-1])
-        self.assertEqual(list(LazyList(it).reverse()[1:3:7]), it[::-1][1:3:7])
+        self.assertEqual(list(reversed(LazyList(it))[::-1]), it)
-        self.assertEqual(list(LazyList(it).reverse()[::-1]), it)
+        self.assertEqual(list(reversed(LazyList(it))[1:3:7]), it[::-1][1:3:7])
 
     def test_LazyList_laziness(self):
 
@@ -1635,13 +1672,13 @@ Line 1
         test(ll, 5, 5, range(6))
         test(ll, -3, 7, range(10))
 
-        ll = LazyList(range(10)).reverse()
+        ll = LazyList(range(10), reverse=True)
         test(ll, -1, 0, range(1))
         test(ll, 3, 6, range(10))
 
         ll = LazyList(itertools.count())
         test(ll, 10, 10, range(11))
-        ll.reverse()
+        ll = reversed(ll)
         test(ll, -15, 14, range(15))
 
 

test/test_youtube_lists.py

@@ -26,29 +26,31 @@ class TestYoutubeLists(unittest.TestCase):
     def test_youtube_playlist_noplaylist(self):
         dl = FakeYDL()
         dl.params['noplaylist'] = True
-        ie = YoutubePlaylistIE(dl)
+        ie = YoutubeTabIE(dl)
         result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
         self.assertEqual(result['_type'], 'url')
-        self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
+        self.assertEqual(YoutubeIE.extract_id(result['url']), 'FXxLjLQi3Fg')
 
     def test_youtube_course(self):
+        print('Skipping: Course URLs no longer exists')
+        return
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)
         # TODO find a > 100 (paginating?) videos course
         result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
         entries = list(result['entries'])
-        self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
+        self.assertEqual(YoutubeIE.extract_id(entries[0]['url']), 'j9WZyLZCBzs')
         self.assertEqual(len(entries), 25)
-        self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
+        self.assertEqual(YoutubeIE.extract_id(entries[-1]['url']), 'rYefUsYuEp0')
 
     def test_youtube_mix(self):
         dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
+        ie = YoutubeTabIE(dl)
-        result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
+        result = ie.extract('https://www.youtube.com/watch?v=tyITL_exICo&list=RDCLAK5uy_kLWIr9gv1XLlPbaDS965-Db4TrBoUTxQ8')
-        entries = result['entries']
+        entries = list(result['entries'])
         self.assertTrue(len(entries) >= 50)
         original_video = entries[0]
-        self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
+        self.assertEqual(original_video['id'], 'tyITL_exICo')
 
     def test_youtube_toptracks(self):
         print('Skipping: The playlist page gives error 500')
@@ -68,10 +70,10 @@ class TestYoutubeLists(unittest.TestCase):
         entries = list(result['entries'])
         self.assertTrue(len(entries) == 1)
         video = entries[0]
-        self.assertEqual(video['_type'], 'url_transparent')
+        self.assertEqual(video['_type'], 'url')
         self.assertEqual(video['ie_key'], 'Youtube')
         self.assertEqual(video['id'], 'BaW_jenozKc')
-        self.assertEqual(video['url'], 'BaW_jenozKc')
+        self.assertEqual(video['url'], 'https://www.youtube.com/watch?v=BaW_jenozKc')
         self.assertEqual(video['title'], 'youtube-dl test video "\'/\\ä↭𝕐')
         self.assertEqual(video['duration'], 10)
         self.assertEqual(video['uploader'], 'Philipp Hagemeister')

test/test_youtube_signature.py

@@ -14,9 +14,10 @@ import string
 
 from test.helper import FakeYDL, is_download_test
 from yt_dlp.extractor import YoutubeIE
+from yt_dlp.jsinterp import JSInterpreter
 from yt_dlp.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@ _TESTS = [
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 @is_download_test
 class TestPlayerInfo(unittest.TestCase):
@@ -97,35 +117,49 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
+def t_factory(name, sig_func, url_pattern):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
+    def make_tfunc(url, sig_input, expected_sig):
-    assert m, '%r should follow URL format' % url
+        m = url_pattern.match(url)
-    test_id = m.group(1)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
 
-    def test_func(self):
+        def test_func(self):
-        basename = 'player-%s.js' % test_id
+            basename = f'player-{name}-{test_id}.js'
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+            fn = os.path.join(self.TESTDATA_DIR, basename)
 
-        if not os.path.exists(fn):
+            if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
 
-        ydl = FakeYDL()
+        test_func.__name__ = f'test_{name}_js_{test_id}'
-        ie = YoutubeIE(ydl)
+        setattr(TestSignature, test_func.__name__, test_func)
-        with io.open(fn, encoding='utf-8') as testf:
+    return make_tfunc
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
-
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
 
-for test_spec in _TESTS:
+def signature(jscode, sig_input):
-    make_tfunc(*test_spec)
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
+
+
+def n_sig(jscode, sig_input):
+    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
+
+
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
+
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':

yt_dlp/__init__.py

@@ -25,16 +25,17 @@ from .cookies import SUPPORTED_BROWSERS
 from .utils import (
     DateRange,
     decodeOption,
+    DownloadCancelled,
     DownloadError,
     error_to_compat_str,
-    ExistingVideoReached,
     expand_path,
+    GeoUtils,
+    float_or_none,
+    int_or_none,
     match_filter_func,
-    MaxDownloadsReached,
     parse_duration,
     preferredencoding,
     read_batch_urls,
-    RejectedVideoReached,
     render_table,
     SameFileError,
     setproctitle,
@@ -71,7 +72,7 @@ def _real_main(argv=None):
     setproctitle('yt-dlp')
 
     parser, opts, args = parseOpts(argv)
-    warnings = []
+    warnings, deprecation_warnings = [], []
 
     # Set user agent
     if opts.user_agent is not None:
@@ -94,6 +95,7 @@ def _real_main(argv=None):
     if opts.batchfile is not None:
         try:
             if opts.batchfile == '-':
+                write_string('Reading URLs from stdin:\n')
                 batchfd = sys.stdin
             else:
                 batchfd = io.open(
@@ -122,10 +124,10 @@ def _real_main(argv=None):
             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
             if desc is False:
                 continue
-            if hasattr(ie, 'SEARCH_KEY'):
+            if getattr(ie, 'SEARCH_KEY', None) is not None:
                 _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                 _COUNTS = ('', '5', '10', 'all')
-                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+                desc += f'; "{ie.SEARCH_KEY}:" prefix (Example: "{ie.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(_SEARCHES)}")'
             write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
     if opts.ap_list_mso:
@@ -193,7 +195,15 @@ def _real_main(argv=None):
     if opts.overwrites:  # --yes-overwrites implies --no-continue
         opts.continue_dl = False
     if opts.concurrent_fragment_downloads <= 0:
-        raise ValueError('Concurrent fragments must be positive')
+        parser.error('Concurrent fragments must be positive')
+    if opts.wait_for_video is not None:
+        mobj = re.match(r'(?P<min>\d+)(?:-(?P<max>\d+))?$', opts.wait_for_video)
+        if not mobj:
+            parser.error('Invalid time range to wait')
+        min_wait, max_wait = map(int_or_none, mobj.group('min', 'max'))
+        if max_wait is not None and max_wait < min_wait:
+            parser.error('Invalid time range to wait')
+        opts.wait_for_video = (min_wait, max_wait)
 
     def parse_retries(retries, name=''):
         if retries in ('inf', 'infinite'):
@@ -221,15 +231,17 @@ def _real_main(argv=None):
             parser.error('invalid http chunk size specified')
         opts.http_chunk_size = numeric_chunksize
     if opts.playliststart <= 0:
-        raise ValueError('Playlist start must be positive')
+        raise parser.error('Playlist start must be positive')
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
-        raise ValueError('Playlist end must be greater than playlist start')
+        raise parser.error('Playlist end must be greater than playlist start')
     if opts.extractaudio:
+        opts.audioformat = opts.audioformat.lower()
         if opts.audioformat not in ['best'] + list(FFmpegExtractAudioPP.SUPPORTED_EXTS):
             parser.error('invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
-        if not opts.audioquality.isdigit():
+        audioquality = int_or_none(float_or_none(opts.audioquality))  # int_or_none prevents inf, nan
+        if audioquality is None or audioquality < 0:
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
         opts.recodevideo = opts.recodevideo.replace(' ', '')
@@ -245,12 +257,17 @@ def _real_main(argv=None):
     if opts.convertthumbnails is not None:
         if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
             parser.error('invalid thumbnail format specified')
-
     if opts.cookiesfrombrowser is not None:
         opts.cookiesfrombrowser = [
             part.strip() or None for part in opts.cookiesfrombrowser.split(':', 1)]
         if opts.cookiesfrombrowser[0].lower() not in SUPPORTED_BROWSERS:
             parser.error('unsupported browser specified for cookies')
+    geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
+    if geo_bypass_code is not None:
+        try:
+            GeoUtils.random_ipv4(geo_bypass_code)
+        except Exception:
+            parser.error('unsupported geo-bypass country or ip-block')
 
     if opts.date is not None:
         date = DateRange.day(opts.date)
@@ -286,6 +303,11 @@ def _real_main(argv=None):
     set_default_compat('abort-on-error', 'ignoreerrors', 'only_download')
     set_default_compat('no-playlist-metafiles', 'allow_playlist_files')
     set_default_compat('no-clean-infojson', 'clean_infojson')
+    if 'no-attach-info-json' in compat_opts:
+        if opts.embed_infojson:
+            _unused_compat_opt('no-attach-info-json')
+        else:
+            opts.embed_infojson = False
     if 'format-sort' in compat_opts:
         opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default)
     _video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False)
@@ -369,8 +391,6 @@ def _real_main(argv=None):
         opts.sponsorblock_remove = set()
     sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove
 
-    if (opts.addmetadata or opts.sponsorblock_mark) and opts.addchapters is None:
-        opts.addchapters = True
     opts.remove_chapters = opts.remove_chapters or []
 
     if (opts.remove_chapters or sponsorblock_query) and opts.sponskrub is not False:
@@ -391,40 +411,32 @@ def _real_main(argv=None):
         opts.remuxvideo = False
 
     if opts.allow_unplayable_formats:
-        if opts.extractaudio:
+        def report_unplayable_conflict(opt_name, arg, default=False, allowed=None):
-            report_conflict('--allow-unplayable-formats', '--extract-audio')
+            val = getattr(opts, opt_name)
-            opts.extractaudio = False
+            if (not allowed and val) or (allowed and not allowed(val)):
-        if opts.remuxvideo:
+                report_conflict('--allow-unplayable-formats', arg)
-            report_conflict('--allow-unplayable-formats', '--remux-video')
+                setattr(opts, opt_name, default)
-            opts.remuxvideo = False
+
-        if opts.recodevideo:
+        report_unplayable_conflict('extractaudio', '--extract-audio')
-            report_conflict('--allow-unplayable-formats', '--recode-video')
+        report_unplayable_conflict('remuxvideo', '--remux-video')
-            opts.recodevideo = False
+        report_unplayable_conflict('recodevideo', '--recode-video')
-        if opts.addmetadata:
+        report_unplayable_conflict('addmetadata', '--embed-metadata')
-            report_conflict('--allow-unplayable-formats', '--add-metadata')
+        report_unplayable_conflict('addchapters', '--embed-chapters')
-            opts.addmetadata = False
+        report_unplayable_conflict('embed_infojson', '--embed-info-json')
-        if opts.embedsubtitles:
+        opts.embed_infojson = False
-            report_conflict('--allow-unplayable-formats', '--embed-subs')
+        report_unplayable_conflict('embedsubtitles', '--embed-subs')
-            opts.embedsubtitles = False
+        report_unplayable_conflict('embedthumbnail', '--embed-thumbnail')
-        if opts.embedthumbnail:
+        report_unplayable_conflict('xattrs', '--xattrs')
-            report_conflict('--allow-unplayable-formats', '--embed-thumbnail')
+        report_unplayable_conflict('fixup', '--fixup', default='never', allowed=lambda x: x in (None, 'never', 'ignore'))
-            opts.embedthumbnail = False
-        if opts.xattrs:
-            report_conflict('--allow-unplayable-formats', '--xattrs')
-            opts.xattrs = False
-        if opts.fixup and opts.fixup.lower() not in ('never', 'ignore'):
-            report_conflict('--allow-unplayable-formats', '--fixup')
         opts.fixup = 'never'
-        if opts.remove_chapters:
+        report_unplayable_conflict('remove_chapters', '--remove-chapters', default=[])
-            report_conflict('--allow-unplayable-formats', '--remove-chapters')
+        report_unplayable_conflict('sponsorblock_remove', '--sponsorblock-remove', default=set())
-            opts.remove_chapters = []
+        report_unplayable_conflict('sponskrub', '--sponskrub', default=set())
-        if opts.sponsorblock_remove:
-            report_conflict('--allow-unplayable-formats', '--sponsorblock-remove')
-            opts.sponsorblock_remove = set()
-        if opts.sponskrub:
-            report_conflict('--allow-unplayable-formats', '--sponskrub')
         opts.sponskrub = False
 
+    if (opts.addmetadata or opts.sponsorblock_mark) and opts.addchapters is None:
+        opts.addchapters = True
+
     # PostProcessors
     postprocessors = list(opts.add_postprocessors)
     if sponsorblock_query:
@@ -522,13 +534,16 @@ def _real_main(argv=None):
     # By default ffmpeg preserves metadata applicable for both
     # source and target containers. From this point the container won't change,
     # so metadata can be added here.
-    if opts.addmetadata or opts.addchapters:
+    if opts.addmetadata or opts.addchapters or opts.embed_infojson:
+        if opts.embed_infojson is None:
+            opts.embed_infojson = 'if_exists'
         postprocessors.append({
             'key': 'FFmpegMetadata',
             'add_chapters': opts.addchapters,
             'add_metadata': opts.addmetadata,
+            'add_infojson': opts.embed_infojson,
         })
-    # Note: Deprecated
+    # Deprecated
     # This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment
     # but must be below EmbedSubtitle and FFmpegMetadata
     # See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29
@@ -541,6 +556,7 @@ def _real_main(argv=None):
             'cut': opts.sponskrub_cut,
             'force': opts.sponskrub_force,
             'ignoreerror': opts.sponskrub is None,
+            '_from_cli': True,
         })
     if opts.embedthumbnail:
         already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
@@ -580,6 +596,19 @@ def _real_main(argv=None):
         opts.postprocessor_args.setdefault('sponskrub', [])
         opts.postprocessor_args['default'] = opts.postprocessor_args['default-compat']
 
+    def report_deprecation(val, old, new=None):
+        if not val:
+            return
+        deprecation_warnings.append(
+            f'{old} is deprecated and may be removed in a future version. Use {new} instead' if new
+            else f'{old} is deprecated and may not work as expected')
+
+    report_deprecation(opts.sponskrub, '--sponskrub', '--sponsorblock-mark or --sponsorblock-remove')
+    report_deprecation(not opts.prefer_ffmpeg, '--prefer-avconv', 'ffmpeg')
+    report_deprecation(opts.include_ads, '--include-ads')
+    # report_deprecation(opts.call_home, '--call-home')  # We may re-implement this in future
+    # report_deprecation(opts.writeannotations, '--write-annotations')  # It's just that no website has it
+
     final_ext = (
         opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS
         else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS
@@ -699,6 +728,7 @@ def _real_main(argv=None):
         'download_archive': download_archive_fn,
         'break_on_existing': opts.break_on_existing,
         'break_on_reject': opts.break_on_reject,
+        'break_per_url': opts.break_per_url,
         'skip_playlist_after_errors': opts.skip_playlist_after_errors,
         'cookiefile': opts.cookiefile,
         'cookiesfrombrowser': opts.cookiesfrombrowser,
@@ -717,6 +747,7 @@ def _real_main(argv=None):
         'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
         'encoding': opts.encoding,
         'extract_flat': opts.extract_flat,
+        'wait_for_video': opts.wait_for_video,
         'mark_watched': opts.mark_watched,
         'merge_output_format': opts.merge_output_format,
         'final_ext': final_ext,
@@ -746,11 +777,12 @@ def _real_main(argv=None):
         'geo_bypass_country': opts.geo_bypass_country,
         'geo_bypass_ip_block': opts.geo_bypass_ip_block,
         '_warnings': warnings,
+        '_deprecation_warnings': deprecation_warnings,
         'compat_opts': compat_opts,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
-        actual_use = len(all_urls) or opts.load_info_filename
+        actual_use = all_urls or opts.load_info_filename
 
         # Remove cache dir
         if opts.rm_cachedir:
@@ -779,7 +811,7 @@ def _real_main(argv=None):
                 retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
             else:
                 retcode = ydl.download(all_urls)
-        except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached):
+        except DownloadCancelled:
             ydl.to_screen('Aborting remaining downloads')
             retcode = 101
 
@@ -791,15 +823,15 @@ def main(argv=None):
         _real_main(argv)
     except DownloadError:
         sys.exit(1)
-    except SameFileError:
+    except SameFileError as e:
-        sys.exit('ERROR: fixed output name but more than one file to download')
+        sys.exit(f'ERROR: {e}')
     except KeyboardInterrupt:
         sys.exit('\nERROR: Interrupted by user')
-    except BrokenPipeError:
+    except BrokenPipeError as e:
         # https://docs.python.org/3/library/signal.html#note-on-sigpipe
         devnull = os.open(os.devnull, os.O_WRONLY)
         os.dup2(devnull, sys.stdout.fileno())
-        sys.exit(r'\nERROR: {err}')
+        sys.exit(f'\nERROR: {e}')
 
 
 __all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

yt_dlp/aes.py

@@ -28,6 +28,48 @@ else:
 BLOCK_SIZE_BYTES = 16
 
 
+def aes_ecb_encrypt(data, key, iv=None):
+    """
+    Encrypt with aes in ECB mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          Unused for this mode
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        encrypted_data += aes_encrypt(block, expanded_key)
+    encrypted_data = encrypted_data[:len(data)]
+
+    return encrypted_data
+
+
+def aes_ecb_decrypt(data, key, iv=None):
+    """
+    Decrypt with aes in ECB mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          Unused for this mode
+    @returns {int[]}           decrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        encrypted_data += aes_decrypt(block, expanded_key)
+    encrypted_data = encrypted_data[:len(data)]
+
+    return encrypted_data
+
+
 def aes_ctr_decrypt(data, key, iv):
     """
     Decrypt with aes in counter mode

yt_dlp/compat.py

@@ -19,6 +19,7 @@ import shlex
 import shutil
 import socket
 import struct
+import subprocess
 import sys
 import tokenize
 import urllib
@@ -162,7 +163,9 @@ except ImportError:
 def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
     if compat_os_name != 'nt':
         return
-    os.system('')
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+    subprocess.Popen('', shell=True, startupinfo=startupinfo)
 
 
 #  Deprecated

yt_dlp/cookies.py

@@ -117,7 +117,7 @@ def _extract_firefox_cookies(profile, logger):
         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 
-    with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)
@@ -236,7 +236,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
 
     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
 
-    with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)

yt_dlp/downloader/__init__.py

@@ -41,6 +41,7 @@ from .external import (
 
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
+    'rtmpe': RtmpFD,
     'rtmp_ffmpeg': FFmpegFD,
     'm3u8_native': HlsFD,
     'm3u8': FFmpegFD,

yt_dlp/downloader/common.py

@@ -93,6 +93,8 @@ class FileDownloader(object):
     def format_percent(percent):
         if percent is None:
             return '---.-%'
+        elif percent == 100:
+            return '100%'
         return '%6s' % ('%3.1f%%' % percent)
 
     @staticmethod
@@ -247,11 +249,29 @@ class FileDownloader(object):
             self._multiline = BreaklineStatusPrinter(self.ydl._screen_file, lines)
         else:
             self._multiline = MultilinePrinter(self.ydl._screen_file, lines, not self.params.get('quiet'))
+        self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
 
     def _finish_multiline_status(self):
         self._multiline.end()
 
-    def _report_progress_status(self, s):
+    _progress_styles = {
+        'downloaded_bytes': 'light blue',
+        'percent': 'light blue',
+        'eta': 'yellow',
+        'speed': 'green',
+        'elapsed': 'bold white',
+        'total_bytes': '',
+        'total_bytes_estimate': '',
+    }
+
+    def _report_progress_status(self, s, default_template):
+        for name, style in self._progress_styles.items():
+            name = f'_{name}_str'
+            if name not in s:
+                continue
+            s[name] = self._format_progress(s[name], style)
+        s['_default_template'] = default_template % s
+
         progress_dict = s.copy()
         progress_dict.pop('info_dict')
         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
@@ -264,6 +284,10 @@ class FileDownloader(object):
             progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
             progress_dict))
 
+    def _format_progress(self, *args, **kwargs):
+        return self.ydl._format_text(
+            self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
+
     def report_progress(self, s):
         if s['status'] == 'finished':
             if self.params.get('noprogress'):
@@ -276,8 +300,7 @@ class FileDownloader(object):
                 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                 msg_template += ' in %(_elapsed_str)s'
             s['_percent_str'] = self.format_percent(100)
-            s['_default_template'] = msg_template % s
+            self._report_progress_status(s, msg_template)
-            self._report_progress_status(s)
             return
 
         if s['status'] != 'downloading':
@@ -286,7 +309,7 @@ class FileDownloader(object):
         if s.get('eta') is not None:
             s['_eta_str'] = self.format_eta(s['eta'])
         else:
-            s['_eta_str'] = 'Unknown ETA'
+            s['_eta_str'] = 'Unknown'
 
         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
@@ -318,9 +341,12 @@ class FileDownloader(object):
                 else:
                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
             else:
-                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+                msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
-        s['_default_template'] = msg_template % s
+        if s.get('fragment_index') and s.get('fragment_count'):
-        self._report_progress_status(s)
+            msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
+        elif s.get('fragment_index'):
+            msg_template += ' (frag %(fragment_index)s)'
+        self._report_progress_status(s, msg_template)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""

yt_dlp/downloader/external.py

@@ -21,7 +21,6 @@ from ..utils import (
     encodeArgument,
     handle_youtubedl_headers,
     check_executable,
-    is_outdated_version,
     Popen,
     sanitize_open,
 )
@@ -444,8 +443,7 @@ class FFmpegFD(ExternalFD):
         if info_dict.get('requested_formats') or protocol == 'http_dash_segments':
             for (i, fmt) in enumerate(info_dict.get('requested_formats') or [info_dict]):
                 stream_number = fmt.get('manifest_stream_number', 0)
-                a_or_v = 'a' if fmt.get('acodec') != 'none' else 'v'
+                args.extend(['-map', f'{i}:{stream_number}'])
-                args.extend(['-map', f'{i}:{a_or_v}:{stream_number}'])
 
         if self.params.get('test', False):
             args += ['-fs', compat_str(self._TEST_FILE_SIZE)]
@@ -459,7 +457,7 @@ class FFmpegFD(ExternalFD):
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4']
-                if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2', False)) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
+                if (ffpp.basename == 'ffmpeg' and ffpp._features.get('needs_adtstoasc')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
                     args += ['-bsf:a', 'aac_adtstoasc']
         elif protocol == 'rtmp':
             args += ['-f', 'flv']

yt_dlp/downloader/fragment.py

@@ -31,6 +31,10 @@ class HttpQuietDownloader(HttpFD):
     def to_screen(self, *args, **kargs):
         pass
 
+    def report_retry(self, err, count, retries):
+        super().to_screen(
+            f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
+
 
 class FragmentFD(FileDownloader):
     """
@@ -44,6 +48,7 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
                         finished
+    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
     _no_ytdl_file:      Don't use .ytdl file
 
     For each incomplete fragment download yt-dlp keeps on disk a special
@@ -167,7 +172,7 @@ class FragmentFD(FileDownloader):
             self.ydl,
             {
                 'continuedl': True,
-                'quiet': True,
+                'quiet': self.params.get('quiet'),
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
@@ -237,6 +242,7 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
+            'fragment_started': start,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -267,6 +273,9 @@ class FragmentFD(FileDownloader):
                 ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_total_bytes)
+                ctx['fragment_started'] = time.time()
                 ctx['prev_frag_downloaded_bytes'] = 0
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
@@ -275,8 +284,8 @@ class FragmentFD(FileDownloader):
                     state['eta'] = self.calc_eta(
                         start, time_now, estimated_size - resume_len,
                         state['downloaded_bytes'] - resume_len)
-                state['speed'] = s.get('speed') or ctx.get('speed')
+                ctx['speed'] = state['speed'] = self.calc_speed(
-                ctx['speed'] = state['speed']
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes)
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state, info_dict)
 

yt_dlp/downloader/hls.py

@@ -77,6 +77,15 @@ class HlsFD(FragmentFD):
                 message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
                            'Decryption will be performed natively, but will be extremely slow')
         if not can_download:
+            has_drm = re.search('|'.join([
+                r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
+                r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
+            ]), s)
+            if has_drm and not self.params.get('allow_unplayable_formats'):
+                self.report_error(
+                    'This video is DRM protected; Try selecting another format with --format or '
+                    'add --check-formats to automatically fallback to the next best format')
+                return False
             message = message or 'Unsupported features have been detected'
             fd = FFmpegFD(self.ydl, self.params)
             self.report_warning(f'{message}; extraction will be delegated to {fd.get_basename()}')

yt_dlp/downloader/mhtml.py

@@ -114,8 +114,8 @@ body > figure > img {
         fragment_base_url = info_dict.get('fragment_base_url')
         fragments = info_dict['fragments'][:1] if self.params.get(
             'test', False) else info_dict['fragments']
-        title = info_dict['title']
+        title = info_dict.get('title', info_dict['format_id'])
-        origin = info_dict['webpage_url']
+        origin = info_dict.get('webpage_url', info_dict['url'])
 
         ctx = {
             'filename': filename,

yt_dlp/extractor/adobetv.py

@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     ISO639Utils,
+    join_nonempty,
     OnDemandPagedList,
     parse_duration,
     str_or_none,
@@ -263,7 +264,7 @@ class AdobeTVVideoIE(AdobeTVBaseIE):
                 continue
             formats.append({
                 'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000),
-                'format_id': '-'.join(filter(None, [source.get('format'), source.get('label')])),
+                'format_id': join_nonempty(source.get('format'), source.get('label')),
                 'height': int_or_none(source.get('height') or None),
                 'tbr': int_or_none(source.get('bitrate') or None),
                 'width': int_or_none(source.get('width') or None),

yt_dlp/extractor/aljazeera.py

@@ -1,55 +1,86 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import json
 
 from .common import InfoExtractor
+from ..utils import (
+    try_get,
+)
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?P<type>program/[^/]+|(?:feature|video)s)/\d{4}/\d{1,2}/\d{1,2}/(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://(?P<base>\w+\.aljazeera\.\w+)/(?P<type>programs?/[^/]+|(?:feature|video|new)s)?/\d{4}/\d{1,2}/\d{1,2}/(?P<id>[^/?&#]+)'
 
     _TESTS = [{
-        'url': 'https://www.aljazeera.com/program/episode/2014/9/19/deliverance',
+        'url': 'https://balkans.aljazeera.net/videos/2021/11/6/pojedini-domovi-u-sarajevu-jos-pod-vodom-mjestanima-se-dostavlja-hrana',
         'info_dict': {
-            'id': '3792260579001',
+            'id': '6280641530001',
             'ext': 'mp4',
-            'title': 'The Slum - Episode 1: Deliverance',
+            'title': 'Pojedini domovi u Sarajevu još pod vodom, mještanima se dostavlja hrana',
-            'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
+            'timestamp': 1636219149,
-            'uploader_id': '665003303001',
+            'description': 'U sarajevskim naseljima Rajlovac i Reljevo stambeni objekti, ali i industrijska postrojenja i dalje su pod vodom.',
-            'timestamp': 1411116829,
+            'upload_date': '20211106',
-            'upload_date': '20140919',
+        }
+    }, {
+        'url': 'https://balkans.aljazeera.net/videos/2021/11/6/djokovic-usao-u-finale-mastersa-u-parizu',
+        'info_dict': {
+            'id': '6280654936001',
+            'ext': 'mp4',
+            'title': 'Đoković ušao u finale Mastersa u Parizu',
+            'timestamp': 1636221686,
+            'description': 'Novak Đoković je u polufinalu Mastersa u Parizu nakon preokreta pobijedio Poljaka Huberta Hurkacza.',
+            'upload_date': '20211106',
         },
-        'add_ie': ['BrightcoveNew'],
-        'skip': 'Not accessible from Travis CI server',
-    }, {
-        'url': 'https://www.aljazeera.com/videos/2017/5/11/sierra-leone-709-carat-diamond-to-be-auctioned-off',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.aljazeera.com/features/2017/8/21/transforming-pakistans-buses-into-art',
-        'only_matching': True,
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+    BRIGHTCOVE_URL_RE = r'https?://players.brightcove.net/(?P<account>\d+)/(?P<player_id>[a-zA-Z0-9]+)_(?P<embed>[^/]+)/index.html\?videoId=(?P<id>\d+)'
 
     def _real_extract(self, url):
-        post_type, name = self._match_valid_url(url).groups()
+        base, post_type, id = self._match_valid_url(url).groups()
+        wp = {
+            'balkans.aljazeera.net': 'ajb',
+            'chinese.aljazeera.net': 'chinese',
+            'mubasher.aljazeera.net': 'ajm',
+        }.get(base) or 'aje'
         post_type = {
             'features': 'post',
             'program': 'episode',
+            'programs': 'episode',
             'videos': 'video',
+            'news': 'news',
         }[post_type.split('/')[0]]
         video = self._download_json(
-            'https://www.aljazeera.com/graphql', name, query={
+            f'https://{base}/graphql', id, query={
+                'wp-site': wp,
                 'operationName': 'ArchipelagoSingleArticleQuery',
                 'variables': json.dumps({
-                    'name': name,
+                    'name': id,
                     'postType': post_type,
                 }),
             }, headers={
-                'wp-site': 'aje',
+                'wp-site': wp,
-            })['data']['article']['video']
+            })
-        video_id = video['id']
+        video = try_get(video, lambda x: x['data']['article']['video']) or {}
-        account_id = video.get('accountId') or '665003303001'
+        video_id = video.get('id')
-        player_id = video.get('playerId') or 'BkeSH5BDb'
+        account = video.get('accountId') or '911432371001'
-        return self.url_result(
+        player_id = video.get('playerId') or 'csvTfAlKW'
-            self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id),
+        embed = 'default'
-            'BrightcoveNew', video_id)
+
+        if video_id is None:
+            webpage = self._download_webpage(url, id)
+
+            account, player_id, embed, video_id = self._search_regex(self.BRIGHTCOVE_URL_RE, webpage, 'video id',
+                                                                     group=(1, 2, 3, 4), default=(None, None, None, None))
+
+            if video_id is None:
+                return {
+                    '_type': 'url_transparent',
+                    'url': url,
+                    'ie_key': 'Generic'
+                }
+
+        return {
+            '_type': 'url_transparent',
+            'url': f'https://players.brightcove.net/{account}/{player_id}_{embed}/index.html?videoId={video_id}',
+            'ie_key': 'BrightcoveNew'
+        }

yt_dlp/extractor/amazon.py

@@ -0,0 +1,53 @@
+# coding: utf-8
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class AmazonStoreIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?amazon\.(?:[a-z]{2,3})(?:\.[a-z]{2})?/(?:[^/]+/)?(?:dp|gp/product)/(?P<id>[^/&#$?]+)'
+
+    _TESTS = [{
+        'url': 'https://www.amazon.co.uk/dp/B098XNCHLD/',
+        'info_dict': {
+            'id': 'B098XNCHLD',
+            'title': 'md5:5f3194dbf75a8dcfc83079bd63a2abed',
+        },
+        'playlist_mincount': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': 'A1F83G8C2ARO7P',
+                'ext': 'mp4',
+                'title': 'mcdodo usb c cable 100W 5a',
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+        }]
+    }, {
+        'url': 'https://www.amazon.in/Sony-WH-1000XM4-Cancelling-Headphones-Bluetooth/dp/B0863TXGM3',
+        'info_dict': {
+            'id': 'B0863TXGM3',
+            'title': 'md5:b0bde4881d3cfd40d63af19f7898b8ff',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'https://www.amazon.com/dp/B0845NXCXF/',
+        'info_dict': {
+            'id': 'B0845NXCXF',
+            'title': 'md5:2145cd4e3c7782f1ee73649a3cff1171',
+        },
+        'playlist-mincount': 1,
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        data_json = self._parse_json(self._html_search_regex(r'var\s?obj\s?=\s?jQuery\.parseJSON\(\'(.*)\'\)', webpage, 'data'), id)
+        entries = [{
+            'id': video['marketPlaceID'],
+            'url': video['url'],
+            'title': video.get('title'),
+            'thumbnail': video.get('thumbUrl') or video.get('thumb'),
+            'duration': video.get('durationSeconds'),
+            'height': int_or_none(video.get('videoHeight')),
+            'width': int_or_none(video.get('videoWidth')),
+        } for video in (data_json.get('videos') or []) if video.get('isVideo') and video.get('url')]
+        return self.playlist_result(entries, playlist_id=id, playlist_title=data_json['title'])

yt_dlp/extractor/animeondemand.py

@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     extract_attributes,
     ExtractorError,
+    join_nonempty,
     url_or_none,
     urlencode_postdata,
     urljoin,
@@ -140,15 +141,8 @@ class AnimeOnDemandIE(InfoExtractor):
                     kind = self._search_regex(
                         r'videomaterialurl/\d+/([^/]+)/',
                         playlist_url, 'media kind', default=None)
-                    format_id_list = []
+                    format_id = join_nonempty(lang, kind) if lang or kind else str(num)
-                    if lang:
+                    format_note = join_nonempty(kind, lang_note, delim=', ')
-                        format_id_list.append(lang)
-                    if kind:
-                        format_id_list.append(kind)
-                    if not format_id_list and num is not None:
-                        format_id_list.append(compat_str(num))
-                    format_id = '-'.join(format_id_list)
-                    format_note = ', '.join(filter(None, (kind, lang_note)))
                     item_id_list = []
                     if format_id:
                         item_id_list.append(format_id)
@@ -195,12 +189,10 @@ class AnimeOnDemandIE(InfoExtractor):
                         if not file_:
                             continue
                         ext = determine_ext(file_)
-                        format_id_list = [lang, kind]
+                        format_id = join_nonempty(
-                        if ext == 'm3u8':
+                            lang, kind,
-                            format_id_list.append('hls')
+                            'hls' if ext == 'm3u8' else None,
-                        elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            'dash' if source.get('type') == 'video/dash' or ext == 'mpd' else None)
-                            format_id_list.append('dash')
-                        format_id = '-'.join(filter(None, format_id_list))
                         if ext == 'm3u8':
                             file_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',

yt_dlp/extractor/anvato.py

@@ -16,6 +16,7 @@ from ..utils import (
     determine_ext,
     intlist_to_bytes,
     int_or_none,
+    join_nonempty,
     strip_jsonp,
     unescapeHTML,
     unsmuggle_url,
@@ -303,13 +304,13 @@ class AnvatoIE(InfoExtractor):
             tbr = int_or_none(published_url.get('kbps'))
             a_format = {
                 'url': video_url,
-                'format_id': ('-'.join(filter(None, ['http', published_url.get('cdn_name')]))).lower(),
+                'format_id': join_nonempty('http', published_url.get('cdn_name')).lower(),
-                'tbr': tbr if tbr != 0 else None,
+                'tbr': tbr or None,
             }
 
             if media_format == 'm3u8' and tbr is not None:
                 a_format.update({
-                    'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
+                    'format_id': join_nonempty('hls', tbr),
                     'ext': 'mp4',
                 })
             elif media_format == 'm3u8-variant' or ext == 'm3u8':

yt_dlp/extractor/ard.py

@@ -388,7 +388,13 @@ class ARDIE(InfoExtractor):
 
 
 class ARDBetaMediathekIE(ARDMediathekBaseIE):
-    _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?P<mode>player|live|video|sendung|sammlung)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'''(?x)https://
+        (?:(?:beta|www)\.)?ardmediathek\.de/
+        (?:(?P<client>[^/]+)/)?
+        (?:player|live|video|(?P<playlist>sendung|sammlung))/
+        (?:(?P<display_id>[^?#]+)/)?
+        (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)'''
+
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
         'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
@@ -403,6 +409,18 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'upload_date': '20200805',
             'ext': 'mp4',
         },
+        'skip': 'Error',
+    }, {
+        'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
+        'md5': 'f1837e563323b8a642a8ddeff0131f51',
+        'info_dict': {
+            'id': '10049223',
+            'ext': 'mp4',
+            'title': 'tagesschau, 20:00 Uhr',
+            'timestamp': 1636398000,
+            'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
+            'upload_date': '20211108',
+        },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
         'only_matching': True,
@@ -426,6 +444,12 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         # playlist of type 'sammlung'
         'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
+        'only_matching': True,
     }]
 
     def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
@@ -525,20 +549,12 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         return self.playlist_result(entries, playlist_title=display_id)
 
     def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
+        video_id, display_id, playlist_type, client = self._match_valid_url(url).group(
-        video_id = mobj.group('video_id')
+            'id', 'display_id', 'playlist', 'client')
-        display_id = mobj.group('display_id')
+        display_id, client = display_id or video_id, client or 'ard'
-        if display_id:
-            display_id = display_id.rstrip('/')
-        if not display_id:
-            display_id = video_id
 
-        if mobj.group('mode') in ('sendung', 'sammlung'):
+        if playlist_type:
-            # this is a playlist-URL
+            return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
-            return self._ARD_extract_playlist(
-                url, video_id, display_id,
-                mobj.group('client'),
-                mobj.group('mode'))
 
         player_page = self._download_json(
             'https://api.ardmediathek.de/public-gateway',
@@ -574,7 +590,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
       }
     }
   }
-}''' % (mobj.group('client'), video_id),
+}''' % (client, video_id),
             }).encode(), headers={
                 'Content-Type': 'application/json'
             })['data']['playerPage']

yt_dlp/extractor/atresplayer.py

@@ -24,9 +24,6 @@ class AtresPlayerIE(InfoExtractor):
                 'description': 'md5:7634cdcb4d50d5381bedf93efb537fbc',
                 'duration': 3413,
             },
-            'params': {
-                'format': 'bestvideo',
-            },
             'skip': 'This video is only available for registered users'
         },
         {

yt_dlp/extractor/bandaichannel.py

@@ -21,7 +21,6 @@ class BandaiChannelIE(BrightcoveNewIE):
             'duration': 1387.733,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }]

yt_dlp/extractor/bbc.py

@@ -451,9 +451,10 @@ class BBCCoUkIE(InfoExtractor):
             playlist = self._download_json(
                 'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
                 playlist_id, 'Downloading playlist JSON')
+            formats = []
+            subtitles = {}
 
-            version = playlist.get('defaultAvailableVersion')
+            for version in playlist.get('allAvailableVersions', []):
-            if version:
                 smp_config = version['smpConfig']
                 title = smp_config['title']
                 description = smp_config['summary']
@@ -463,8 +464,17 @@ class BBCCoUkIE(InfoExtractor):
                         continue
                     programme_id = item.get('vpid')
                     duration = int_or_none(item.get('duration'))
-                    formats, subtitles = self._download_media_selector(programme_id)
+                    version_formats, version_subtitles = self._download_media_selector(programme_id)
-                return programme_id, title, description, duration, formats, subtitles
+                    types = version['types']
+                    for f in version_formats:
+                        f['format_note'] = ', '.join(types)
+                        if any('AudioDescribed' in x for x in types):
+                            f['language_preference'] = -10
+                    formats += version_formats
+                    for tag, subformats in (version_subtitles or {}).items():
+                        subtitles.setdefault(tag, []).extend(subformats)
+
+            return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
             if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
                 raise

yt_dlp/extractor/bilibili.py

@@ -346,7 +346,8 @@ class BiliBiliIE(InfoExtractor):
     def _extract_anthology_entries(self, bv_id, video_id, webpage):
         title = self._html_search_regex(
             (r'<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
-             r'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
+             r'(?s)<h1[^>]*>(?P<title>.+?)</h1>',
+             r'<title>(?P<title>.+?)</title>'), webpage, 'title',
             group='title')
         json_data = self._download_json(
             f'https://api.bilibili.com/x/player/pagelist?bvid={bv_id}&jsonp=jsonp',
@@ -376,8 +377,10 @@ class BiliBiliIE(InfoExtractor):
             replies = traverse_obj(
                 self._download_json(
                     f'https://api.bilibili.com/x/v2/reply?pn={idx}&oid={video_id}&type=1&jsonp=jsonp&sort=2&_=1567227301685',
-                    video_id, note=f'Extracting comments from page {idx}'),
+                    video_id, note=f'Extracting comments from page {idx}', fatal=False),
-                ('data', 'replies')) or []
+                ('data', 'replies'))
+            if not replies:
+                return
             for children in map(self._get_all_children, replies):
                 yield from children
 
@@ -566,7 +569,7 @@ class BilibiliCategoryIE(InfoExtractor):
 
 
 class BiliBiliSearchIE(SearchInfoExtractor):
-    IE_DESC = 'Bilibili video search, "bilisearch" keyword'
+    IE_DESC = 'Bilibili video search'
     _MAX_RESULTS = 100000
     _SEARCH_KEY = 'bilisearch'
 

yt_dlp/extractor/blogger.py

@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from ..utils import (
+    mimetype2ext,
+    parse_duration,
+    parse_qs,
+    str_or_none,
+    traverse_obj,
+)
+from .common import InfoExtractor
+
+
+class BloggerIE(InfoExtractor):
+    IE_NAME = 'blogger.com'
+    _VALID_URL = r'https?://(?:www\.)?blogger\.com/video\.g\?token=(?P<id>.+)'
+    _VALID_EMBED = r'''<iframe[^>]+src=["']((?:https?:)?//(?:www\.)?blogger\.com/video\.g\?token=[^"']+)["']'''
+    _TESTS = [{
+        'url': 'https://www.blogger.com/video.g?token=AD6v5dzEe9hfcARr5Hlq1WTkYy6t-fXH3BBahVhGvVHe5szdEUBEloSEDSTA8-b111089KbfWuBvTN7fnbxMtymsHhXAXwVvyzHH4Qch2cfLQdGxKQrrEuFpC1amSl_9GuLWODjPgw',
+        'md5': 'f1bc19b6ea1b0fd1d81e84ca9ec467ac',
+        'info_dict': {
+            'id': 'BLOGGER-video-3c740e3a49197e16-796',
+            'title': 'BLOGGER-video-3c740e3a49197e16-796',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*',
+            'duration': 76.068,
+        }
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(BloggerIE._VALID_EMBED, webpage)
+
+    def _real_extract(self, url):
+        token_id = self._match_id(url)
+        webpage = self._download_webpage(url, token_id)
+        data_json = self._search_regex(r'var\s+VIDEO_CONFIG\s*=\s*(\{.*)', webpage, 'JSON data')
+        data = self._parse_json(data_json.encode('utf-8').decode('unicode_escape'), token_id)
+        streams = data['streams']
+        formats = [{
+            'ext': mimetype2ext(traverse_obj(parse_qs(stream['play_url']), ('mime', 0))),
+            'url': stream['play_url'],
+            'format_id': str_or_none(stream.get('format_id')),
+        } for stream in streams]
+
+        return {
+            'id': data.get('iframe_id', token_id),
+            'title': data.get('iframe_id', token_id),
+            'formats': formats,
+            'thumbnail': data.get('thumbnail'),
+            'duration': parse_duration(traverse_obj(parse_qs(streams[0]['play_url']), ('dur', 0))),
+        }

yt_dlp/extractor/breitbart.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class BreitBartIE(InfoExtractor):
+    _VALID_URL = r'https?:\/\/(?:www\.)breitbart.com/videos/v/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.breitbart.com/videos/v/5cOz1yup/?pl=Ij6NDOji',
+        'md5': '0aa6d1d6e183ac5ca09207fe49f17ade',
+        'info_dict': {
+            'id': '5cOz1yup',
+            'ext': 'mp4',
+            'title': 'Watch \u2013 Clyburn: Statues in Congress Have to Go Because they Are Honoring Slavery',
+            'description': 'md5:bac35eb0256d1cb17f517f54c79404d5',
+            'thumbnail': 'https://cdn.jwplayer.com/thumbs/5cOz1yup-1920.jpg',
+            'age_limit': 0,
+        }
+    }, {
+        'url': 'https://www.breitbart.com/videos/v/eaiZjVOn/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        formats = self._extract_m3u8_formats(f'https://cdn.jwplayer.com/manifests/{video_id}.m3u8', video_id, ext='mp4')
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': self._og_search_title(
+                webpage, default=None) or self._html_search_regex(
+                r'(?s)<title>(.*?)</title>', webpage, 'video title'),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'age_limit': self._rta_search(webpage),
+            'formats': formats
+        }

yt_dlp/extractor/cableav.py

@@ -0,0 +1,34 @@
+# coding: utf-8
+from .common import InfoExtractor
+
+
+class CableAVIE(InfoExtractor):
+    _VALID_URL = r'https://cableav\.tv/(?P<id>[a-zA-Z0-9]+)'
+    _TESTS = [{
+        'url': 'https://cableav.tv/lS4iR9lWjN8/',
+        'md5': '7e3fe5e49d61c4233b7f5b0f69b15e18',
+        'info_dict': {
+            'id': 'lS4iR9lWjN8',
+            'ext': 'mp4',
+            'title': '國產麻豆AV 叮叮映畫 DDF001 情欲小說家 - CableAV',
+            'description': '國產AV 480p, 720p 国产麻豆AV 叮叮映画 DDF001 情欲小说家',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._og_search_video_url(webpage, secure=False)
+
+        formats = self._extract_m3u8_formats(video_url, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+        }

yt_dlp/extractor/canalalpha.py

@@ -0,0 +1,98 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    dict_get,
+    try_get,
+    unified_strdate,
+)
+
+
+class CanalAlphaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?canalalpha\.ch/play/[^/]+/[^/]+/(?P<id>\d+)/?.*'
+
+    _TESTS = [{
+        'url': 'https://www.canalalpha.ch/play/le-journal/episode/24520/jeudi-28-octobre-2021',
+        'info_dict': {
+            'id': '24520',
+            'ext': 'mp4',
+            'title': 'Jeudi 28 octobre 2021',
+            'description': 'md5:d30c6c3e53f8ad40d405379601973b30',
+            'thumbnail': 'https://static.canalalpha.ch/poster/journal/journal_20211028.jpg',
+            'upload_date': '20211028',
+            'duration': 1125,
+        },
+        'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.canalalpha.ch/play/le-journal/topic/24512/la-poste-fait-de-neuchatel-un-pole-cryptographique',
+        'info_dict': {
+            'id': '24512',
+            'ext': 'mp4',
+            'title': 'La Poste fait de Neuchâtel un pôle cryptographique',
+            'description': 'md5:4ba63ae78a0974d1a53d6703b6e1dedf',
+            'thumbnail': 'https://static.canalalpha.ch/poster/news/news_39712.jpg',
+            'upload_date': '20211028',
+            'duration': 138,
+        },
+        'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.canalalpha.ch/play/eureka/episode/24484/ces-innovations-qui-veulent-rendre-lagriculture-plus-durable',
+        'info_dict': {
+            'id': '24484',
+            'ext': 'mp4',
+            'title': 'Ces innovations qui veulent rendre l’agriculture plus durable',
+            'description': 'md5:3de3f151180684621e85be7c10e4e613',
+            'thumbnail': 'https://static.canalalpha.ch/poster/magazine/magazine_10236.jpg',
+            'upload_date': '20211026',
+            'duration': 360,
+        },
+        'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.canalalpha.ch/play/avec-le-temps/episode/23516/redonner-de-leclat-grace-au-polissage',
+        'info_dict': {
+            'id': '23516',
+            'ext': 'mp4',
+            'title': 'Redonner de l\'éclat grâce au polissage',
+            'description': 'md5:0d8fbcda1a5a4d6f6daa3165402177e1',
+            'thumbnail': 'https://static.canalalpha.ch/poster/magazine/magazine_9990.png',
+            'upload_date': '20210726',
+            'duration': 360,
+        },
+        'params': {'skip_download': True}
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        data_json = self._parse_json(self._search_regex(
+            r'window\.__SERVER_STATE__\s?=\s?({(?:(?!};)[^"]|"([^"]|\\")*")+})\s?;',
+            webpage, 'data_json'), id)['1']['data']['data']
+        manifests = try_get(data_json, lambda x: x['video']['manifests'], expected_type=dict) or {}
+        subtitles = {}
+        formats = [{
+            'url': video['$url'],
+            'ext': 'mp4',
+            'width': try_get(video, lambda x: x['res']['width'], expected_type=int),
+            'height': try_get(video, lambda x: x['res']['height'], expected_type=int),
+        } for video in try_get(data_json, lambda x: x['video']['mp4'], expected_type=list) or [] if video.get('$url')]
+        if manifests.get('hls'):
+            m3u8_frmts, m3u8_subs = self._parse_m3u8_formats_and_subtitles(manifests['hls'], id)
+            formats.extend(m3u8_frmts)
+            subtitles = self._merge_subtitles(subtitles, m3u8_subs)
+        if manifests.get('dash'):
+            dash_frmts, dash_subs = self._parse_mpd_formats_and_subtitles(manifests['dash'], id)
+            formats.extend(dash_frmts)
+            subtitles = self._merge_subtitles(subtitles, dash_subs)
+        self._sort_formats(formats)
+        return {
+            'id': id,
+            'title': data_json.get('title').strip(),
+            'description': clean_html(dict_get(data_json, ('longDesc', 'shortDesc'))),
+            'thumbnail': data_json.get('poster'),
+            'upload_date': unified_strdate(dict_get(data_json, ('webPublishAt', 'featuredAt', 'diffusionDate'))),
+            'duration': try_get(data_json, lambda x: x['video']['duration'], expected_type=int),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/canvas.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import json
 
 
 from .common import InfoExtractor
@@ -41,9 +42,9 @@ class CanvasIE(InfoExtractor):
     _GEO_BYPASS = False
     _HLS_ENTRY_PROTOCOLS_MAP = {
         'HLS': 'm3u8_native',
-        'HLS_AES': 'm3u8',
+        'HLS_AES': 'm3u8_native',
     }
-    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1'
+    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
 
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
@@ -59,16 +60,21 @@ class CanvasIE(InfoExtractor):
 
         # New API endpoint
         if not data:
+            vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
+                                             video_id, note='refreshtoken: Retrieve vrtnutoken',
+                                             errnote='refreshtoken failed')['vrtnutoken']
             headers = self.geo_verification_headers()
-            headers.update({'Content-Type': 'application/json'})
+            headers.update({'Content-Type': 'application/json; charset=utf-8'})
-            token = self._download_json(
+            vrtPlayerToken = self._download_json(
                 '%s/tokens' % self._REST_API_BASE, video_id,
-                'Downloading token', data=b'', headers=headers)['vrtPlayerToken']
+                'Downloading token', headers=headers, data=json.dumps({
+                    'identityToken': vrtnutoken
+                }).encode('utf-8'))['vrtPlayerToken']
             data = self._download_json(
                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
                 video_id, 'Downloading video JSON', query={
-                    'vrtPlayerToken': token,
+                    'vrtPlayerToken': vrtPlayerToken,
-                    'client': '%s@PROD' % site_id,
+                    'client': 'null',
                 }, expected_status=400)
             if not data.get('title'):
                 code = data.get('code')
@@ -264,7 +270,7 @@ class VrtNUIE(GigyaBaseIE):
         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
     }]
     _NETRC_MACHINE = 'vrtnu'
-    _APIKEY = '3_qhEcPa5JGFROVwu5SWKqJ4mVOIkwlFNMSKwzPDAh8QZOtHqu6L4nD5Q7lk0eXOOG'
+    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
     _CONTEXT_ID = 'R3595707040'
 
     def _real_initialize(self):
@@ -275,16 +281,13 @@ class VrtNUIE(GigyaBaseIE):
         if username is None:
             return
 
-        auth_info = self._download_json(
+        auth_info = self._gigya_login({
-            'https://accounts.vrt.be/accounts.login', None,
+            'APIKey': self._APIKEY,
-            note='Login data', errnote='Could not get Login data',
+            'targetEnv': 'jssdk',
-            headers={}, data=urlencode_postdata({
+            'loginID': username,
-                'loginID': username,
+            'password': password,
-                'password': password,
+            'authMode': 'cookie',
-                'sessionExpiration': '-2',
+        })
-                'APIKey': self._APIKEY,
-                'targetEnv': 'jssdk',
-            }))
 
         if auth_info.get('errorDetails'):
             raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
@@ -301,14 +304,15 @@ class VrtNUIE(GigyaBaseIE):
                     'UID': auth_info['UID'],
                     'UIDSignature': auth_info['UIDSignature'],
                     'signatureTimestamp': auth_info['signatureTimestamp'],
-                    'client_id': 'vrtnu-site',
                     '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
                 }
 
                 self._request_webpage(
                     'https://login.vrt.be/perform_login',
-                    None, note='Requesting a token', errnote='Could not get a token',
+                    None, note='Performing login', errnote='perform login failed',
-                    headers={}, data=urlencode_postdata(post_data))
+                    headers={}, query={
+                        'client_id': 'vrtnu-site'
+                    }, data=urlencode_postdata(post_data))
 
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:

yt_dlp/extractor/cbc.py

@@ -390,7 +390,8 @@ class CBCGemPlaylistIE(InfoExtractor):
         show = match.group('show')
         show_info = self._download_json(self._API_BASE + show, season_id)
         season = int(match.group('season'))
-        season_info = try_get(show_info, lambda x: x['seasons'][season - 1])
+
+        season_info = next((s for s in show_info['seasons'] if s.get('season') == season), None)
 
         if season_info is None:
             raise ExtractorError(f'Couldn\'t find season {season} of {show}')

yt_dlp/extractor/ceskatelevize.py

@@ -20,22 +20,8 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
-        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
-        'info_dict': {
-            'id': '61924494877246241',
-            'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Život v Grónsku',
-            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3350,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
         'info_dict': {
             'id': '61924494877028507',
@@ -66,12 +52,58 @@ class CeskaTelevizeIE(InfoExtractor):
     }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
         'only_matching': True,
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Hlavní město Kolumbie v doprovodu queer umělců. Vroucí svět plný vášně, sebevědomí, ale i násilí a bolesti. Připravil Peter Serge Butko',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494877311053',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 11.9,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # iframe embed
+        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-
+        parsed_url = compat_urllib_parse_urlparse(url)
         webpage = self._download_webpage(url, playlist_id)
+        site_name = self._og_search_property('site_name', webpage, fatal=False, default=None)
+        playlist_title = self._og_search_title(webpage, default=None)
+        if site_name and playlist_title:
+            playlist_title = playlist_title.replace(f' — {site_name}', '', 1)
+        playlist_description = self._og_search_description(webpage, default=None)
+        if playlist_description:
+            playlist_description = playlist_description.replace('\xa0', ' ')
+
+        if parsed_url.path.startswith('/porady/'):
+            refer_url = update_url_query(unescapeHTML(self._search_regex(
+                (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+                 r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
+                webpage, 'iframe player url', group='url')), query={'autoStart': 'true'})
+            webpage = self._download_webpage(refer_url, playlist_id)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
@@ -100,7 +132,7 @@ class CeskaTelevizeIE(InfoExtractor):
         data = {
             'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
-            'requestUrl': compat_urllib_parse_urlparse(url).path,
+            'requestUrl': parsed_url.path,
             'requestSource': 'iVysilani',
         }
 
@@ -108,7 +140,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         for user_agent in (None, USER_AGENTS['Safari']):
             req = sanitized_Request(
-                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
@@ -130,9 +162,6 @@ class CeskaTelevizeIE(InfoExtractor):
             req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
             req.add_header('Referer', url)
 
-            playlist_title = self._og_search_title(webpage, default=None)
-            playlist_description = self._og_search_description(webpage, default=None)
-
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:
                 continue
@@ -237,54 +266,3 @@ class CeskaTelevizeIE(InfoExtractor):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
-
-
-class CeskaTelevizePoradyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
-    _TESTS = [{
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # iframe embed
-        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        data_url = update_url_query(unescapeHTML(self._search_regex(
-            (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
-            webpage, 'iframe player url', group='url')), query={
-                'autoStart': 'true',
-        })
-
-        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())

yt_dlp/extractor/chingari.py

@@ -67,7 +67,7 @@ class ChingariBaseIE(InfoExtractor):
 
 
 class ChingariIE(ChingariBaseIE):
-    _VALID_URL = r'(?:https?://)(?:www\.)?chingari\.io/share/post\?id=(?P<id>[^&/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?chingari\.io/share/post\?id=(?P<id>[^&/#?]+)'
     _TESTS = [{
         'url': 'https://chingari.io/share/post?id=612f8f4ce1dc57090e8a7beb',
         'info_dict': {
@@ -102,7 +102,7 @@ class ChingariIE(ChingariBaseIE):
 
 
 class ChingariUserIE(ChingariBaseIE):
-    _VALID_URL = r'(?:https?://)(?:www\.)?chingari\.io/(?!share/post)(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?chingari\.io/(?!share/post)(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'https://chingari.io/dada1023',
         'playlist_mincount': 3,

yt_dlp/extractor/common.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import collections
 import datetime
 import hashlib
 import itertools
@@ -54,6 +55,7 @@ from ..utils import (
     GeoRestrictedError,
     GeoUtils,
     int_or_none,
+    join_nonempty,
     js_to_json,
     JSON_LD_RE,
     mimetype2ext,
@@ -74,6 +76,7 @@ from ..utils import (
     strip_or_none,
     traverse_obj,
     unescapeHTML,
+    UnsupportedError,
     unified_strdate,
     unified_timestamp,
     update_Request,
@@ -340,6 +343,7 @@ class InfoExtractor(object):
     series, programme or podcast:
 
     series:         Title of the series or programme the video episode belongs to.
+    series_id:      Id of the series or programme the video episode belongs to, as a unicode string.
     season:         Title of the season the video episode belongs to.
     season_number:  Number of the season the video episode belongs to, as an integer.
     season_id:      Id of the season the video episode belongs to, as a unicode string.
@@ -440,11 +444,11 @@ class InfoExtractor(object):
     _WORKING = True
 
     _LOGIN_HINTS = {
-        'any': 'Use --cookies, --username and --password or --netrc to provide account credentials',
+        'any': 'Use --cookies, --username and --password, or --netrc to provide account credentials',
         'cookies': (
             'Use --cookies-from-browser or --cookies for the authentication. '
             'See  https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl  for how to manually pass cookies'),
-        'password': 'Use --username and --password or --netrc to provide account credentials',
+        'password': 'Use --username and --password, or --netrc to provide account credentials',
     }
 
     def __init__(self, downloader=None):
@@ -604,10 +608,19 @@ class InfoExtractor(object):
                     if self.__maybe_fake_ip_and_retry(e.countries):
                         continue
                     raise
+        except UnsupportedError:
+            raise
         except ExtractorError as e:
-            video_id = e.video_id or self.get_temp_id(url)
+            kwargs = {
-            raise ExtractorError(
+                'video_id': e.video_id or self.get_temp_id(url),
-                e.msg, video_id=video_id, ie=self.IE_NAME, tb=e.traceback, expected=e.expected, cause=e.cause)
+                'ie': self.IE_NAME,
+                'tb': e.traceback,
+                'expected': e.expected,
+                'cause': e.cause
+            }
+            if hasattr(e, 'countries'):
+                kwargs['countries'] = e.countries
+            raise type(e)(e.msg, **kwargs)
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
@@ -1066,7 +1079,8 @@ class InfoExtractor(object):
     def raise_login_required(
             self, msg='This video is only available for registered users',
             metadata_available=False, method='any'):
-        if metadata_available and self.get_param('ignore_no_formats_error'):
+        if metadata_available and (
+                self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')):
             self.report_warning(msg)
         if method is not None:
             msg = '%s. %s' % (msg, self._LOGIN_HINTS[method])
@@ -1075,13 +1089,15 @@ class InfoExtractor(object):
     def raise_geo_restricted(
             self, msg='This video is not available from your location due to geo restriction',
             countries=None, metadata_available=False):
-        if metadata_available and self.get_param('ignore_no_formats_error'):
+        if metadata_available and (
+                self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')):
             self.report_warning(msg)
         else:
             raise GeoRestrictedError(msg, countries=countries)
 
     def raise_no_formats(self, msg, expected=False, video_id=None):
-        if expected and self.get_param('ignore_no_formats_error'):
+        if expected and (
+                self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')):
             self.report_warning(msg, video_id)
         elif isinstance(msg, ExtractorError):
             raise msg
@@ -1139,7 +1155,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        _name = self._downloader._color_text(name, 'blue')
+        _name = self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
 
         if mobj:
             if group is None:
@@ -1439,6 +1455,9 @@ class InfoExtractor(object):
                 item_type = e.get('@type')
                 if expected_type is not None and expected_type != item_type:
                     continue
+                rating = traverse_obj(e, ('aggregateRating', 'ratingValue'), expected_type=float_or_none)
+                if rating is not None:
+                    info['average_rating'] = rating
                 if item_type in ('TVEpisode', 'Episode'):
                     episode_name = unescapeHTML(e.get('name'))
                     info.update({
@@ -1485,6 +1504,13 @@ class InfoExtractor(object):
                     break
         return dict((k, v) for k, v in info.items() if v is not None)
 
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
     @staticmethod
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
@@ -1512,20 +1538,20 @@ class InfoExtractor(object):
 
         default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality',
                    'res', 'fps', 'hdr:12', 'codec:vp9.2', 'size', 'br', 'asr',
-                   'proto', 'ext', 'hasaud', 'source', 'format_id')  # These must not be aliases
+                   'proto', 'ext', 'hasaud', 'source', 'id')  # These must not be aliases
         ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr',
                         'height', 'width', 'proto', 'vext', 'abr', 'aext',
-                        'fps', 'fs_approx', 'source', 'format_id')
+                        'fps', 'fs_approx', 'source', 'id')
 
         settings = {
             'vcodec': {'type': 'ordered', 'regex': True,
                        'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
             'acodec': {'type': 'ordered', 'regex': True,
-                       'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e?a?c-?3', 'dts', '', None, 'none']},
+                       'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
             'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
                     'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
             'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
-                      'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.+', '.*dash', 'ws|websocket', '', 'mms|rtsp', 'none', 'f4']},
+                      'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.*', '.*dash', 'websocket_frag', 'rtmpe?', '', 'mms|rtsp', 'ws|websocket', 'f4']},
             'vext': {'type': 'ordered', 'field': 'video_ext',
                      'order': ('mp4', 'webm', 'flv', '', 'none'),
                      'order_free': ('webm', 'mp4', 'flv', '', 'none')},
@@ -1539,8 +1565,8 @@ class InfoExtractor(object):
             'ie_pref': {'priority': True, 'type': 'extractor'},
             'hasvid': {'priority': True, 'field': 'vcodec', 'type': 'boolean', 'not_in_list': ('none',)},
             'hasaud': {'field': 'acodec', 'type': 'boolean', 'not_in_list': ('none',)},
-            'lang': {'convert': 'ignore', 'field': 'language_preference'},
+            'lang': {'convert': 'float', 'field': 'language_preference', 'default': -1},
-            'quality': {'convert': 'float_none', 'default': -1},
+            'quality': {'convert': 'float', 'default': -1},
             'filesize': {'convert': 'bytes'},
             'fs_approx': {'convert': 'bytes', 'field': 'filesize_approx'},
             'id': {'convert': 'string', 'field': 'format_id'},
@@ -1551,7 +1577,7 @@ class InfoExtractor(object):
             'vbr': {'convert': 'float_none'},
             'abr': {'convert': 'float_none'},
             'asr': {'convert': 'float_none'},
-            'source': {'convert': 'ignore', 'field': 'source_preference'},
+            'source': {'convert': 'float', 'field': 'source_preference', 'default': -1},
 
             'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
             'br': {'type': 'combined', 'field': ('tbr', 'vbr', 'abr'), 'same_limit': True},
@@ -1560,7 +1586,7 @@ class InfoExtractor(object):
             'res': {'type': 'multiple', 'field': ('height', 'width'),
                     'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
 
-            # Most of these exist only for compatibility reasons
+            # Deprecated
             'dimension': {'type': 'alias', 'field': 'res'},
             'resolution': {'type': 'alias', 'field': 'res'},
             'extension': {'type': 'alias', 'field': 'ext'},
@@ -1569,7 +1595,7 @@ class InfoExtractor(object):
             'video_bitrate': {'type': 'alias', 'field': 'vbr'},
             'audio_bitrate': {'type': 'alias', 'field': 'abr'},
             'framerate': {'type': 'alias', 'field': 'fps'},
-            'language_preference': {'type': 'alias', 'field': 'lang'},  # not named as 'language' because such a field exists
+            'language_preference': {'type': 'alias', 'field': 'lang'},
             'protocol': {'type': 'alias', 'field': 'proto'},
             'source_preference': {'type': 'alias', 'field': 'source'},
             'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
@@ -1589,10 +1615,20 @@ class InfoExtractor(object):
             'format_id': {'type': 'alias', 'field': 'id'},
         }
 
-        _order = []
+        def __init__(self, ie, field_preference):
+            self._order = []
+            self.ydl = ie._downloader
+            self.evaluate_params(self.ydl.params, field_preference)
+            if ie.get_param('verbose'):
+                self.print_verbose_info(self.ydl.write_debug)
 
         def _get_field_setting(self, field, key):
             if field not in self.settings:
+                if key in ('forced', 'priority'):
+                    return False
+                self.ydl.deprecation_warning(
+                    f'Using arbitrary fields ({field}) for format sorting is deprecated '
+                    'and may be removed in a future version')
                 self.settings[field] = {}
             propObj = self.settings[field]
             if key not in propObj:
@@ -1675,7 +1711,10 @@ class InfoExtractor(object):
                 if field is None:
                     continue
                 if self._get_field_setting(field, 'type') == 'alias':
-                    field = self._get_field_setting(field, 'field')
+                    alias, field = field, self._get_field_setting(field, 'field')
+                    self.ydl.deprecation_warning(
+                        f'Format sorting alias {alias} is deprecated '
+                        f'and may be removed in a future version. Please use {field} instead')
                 reverse = match.group('reverse') is not None
                 closest = match.group('separator') == '~'
                 limit_text = match.group('limit')
@@ -1779,10 +1818,7 @@ class InfoExtractor(object):
     def _sort_formats(self, formats, field_preference=[]):
         if not formats:
             return
-        format_sort = self.FormatSort()  # params and to_screen are taken from the downloader
+        format_sort = self.FormatSort(self, field_preference)
-        format_sort.evaluate_params(self._downloader.params, field_preference)
-        if self.get_param('verbose', False):
-            format_sort.print_verbose_info(self._downloader.write_debug)
         formats.sort(key=lambda f: format_sort.calculate_preference(f))
 
     def _check_formats(self, formats, video_id):
@@ -1901,7 +1937,7 @@ class InfoExtractor(object):
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             width = int_or_none(media_el.attrib.get('width'))
             height = int_or_none(media_el.attrib.get('height'))
-            format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
+            format_id = join_nonempty(f4m_id, tbr or i)
             # If <bootstrapInfo> is present, the specified f4m is a
             # stream-level manifest, and only set-level manifests may refer to
             # external resources.  See section 11.4 and section 4 of F4M spec
@@ -1963,7 +1999,7 @@ class InfoExtractor(object):
 
     def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, quality=None, m3u8_id=None):
         return {
-            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
+            'format_id': join_nonempty(m3u8_id, 'meta'),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
@@ -2016,10 +2052,10 @@ class InfoExtractor(object):
             video_id=None):
         formats, subtitles = [], {}
 
-        if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
+        has_drm = re.search('|'.join([
-            return formats, subtitles
+            r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
-
+            r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
-        has_drm = re.search(r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://', m3u8_doc)
+        ]), m3u8_doc)
 
         def format_url(url):
             return url if re.match(r'^https?://', url) else compat_urlparse.urljoin(m3u8_url, url)
@@ -2058,7 +2094,7 @@ class InfoExtractor(object):
 
         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             formats = [{
-                'format_id': '-'.join(map(str, filter(None, [m3u8_id, idx]))),
+                'format_id': join_nonempty(m3u8_id, idx),
                 'format_index': idx,
                 'url': m3u8_url,
                 'ext': ext,
@@ -2107,7 +2143,7 @@ class InfoExtractor(object):
             if media_url:
                 manifest_url = format_url(media_url)
                 formats.extend({
-                    'format_id': '-'.join(map(str, filter(None, (m3u8_id, group_id, name, idx)))),
+                    'format_id': join_nonempty(m3u8_id, group_id, name, idx),
                     'format_note': name,
                     'format_index': idx,
                     'url': manifest_url,
@@ -2164,9 +2200,9 @@ class InfoExtractor(object):
                     # format_id intact.
                     if not live:
                         stream_name = build_stream_name()
-                        format_id[1] = stream_name if stream_name else '%d' % (tbr if tbr else len(formats))
+                        format_id[1] = stream_name or '%d' % (tbr or len(formats))
                     f = {
-                        'format_id': '-'.join(map(str, filter(None, format_id))),
+                        'format_id': join_nonempty(*format_id),
                         'format_index': idx,
                         'url': manifest_url,
                         'manifest_url': m3u8_url,
@@ -2630,7 +2666,7 @@ class InfoExtractor(object):
 
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats, subtitles = [], {}
-        stream_numbers = {'audio': 0, 'video': 0}
+        stream_numbers = collections.defaultdict(int)
         for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
@@ -2696,10 +2732,8 @@ class InfoExtractor(object):
                             'format_note': 'DASH %s' % content_type,
                             'filesize': filesize,
                             'container': mimetype2ext(mime_type) + '_dash',
-                            'manifest_stream_number': stream_numbers[content_type]
                         }
                         f.update(parse_codecs(codecs))
-                        stream_numbers[content_type] += 1
                     elif content_type == 'text':
                         f = {
                             'ext': mimetype2ext(mime_type),
@@ -2866,7 +2900,9 @@ class InfoExtractor(object):
                     else:
                         # Assuming direct URL to unfragmented media.
                         f['url'] = base_url
-                    if content_type in ('video', 'audio') or mime_type == 'image/jpeg':
+                    if content_type in ('video', 'audio', 'image/jpeg'):
+                        f['manifest_stream_number'] = stream_numbers[f['url']]
+                        stream_numbers[f['url']] += 1
                         formats.append(f)
                     elif content_type == 'text':
                         subtitles.setdefault(lang or 'und', []).append(f)
@@ -2955,13 +2991,6 @@ class InfoExtractor(object):
                         })
                         fragment_ctx['time'] += fragment_ctx['duration']
 
-                format_id = []
-                if ism_id:
-                    format_id.append(ism_id)
-                if stream_name:
-                    format_id.append(stream_name)
-                format_id.append(compat_str(tbr))
-
                 if stream_type == 'text':
                     subtitles.setdefault(stream_language, []).append({
                         'ext': 'ismt',
@@ -2980,7 +3009,7 @@ class InfoExtractor(object):
                     })
                 elif stream_type in ('video', 'audio'):
                     formats.append({
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty(ism_id, stream_name, tbr),
                         'url': ism_url,
                         'manifest_url': ism_url,
                         'ext': 'ismv' if stream_type == 'video' else 'isma',
@@ -3620,9 +3649,11 @@ class SearchInfoExtractor(InfoExtractor):
     """
     Base class for paged search queries extractors.
     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
-    Instances should define _SEARCH_KEY and _MAX_RESULTS.
+    Instances should define _SEARCH_KEY and optionally _MAX_RESULTS
     """
 
+    _MAX_RESULTS = float('inf')
+
     @classmethod
     def _make_valid_url(cls):
         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY

yt_dlp/extractor/corus.py

@@ -55,7 +55,6 @@ class CorusIE(ThePlatformFeedIE):
             'timestamp': 1486392197,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
         'expected_warnings': ['Failed to parse JSON'],

yt_dlp/extractor/coub.py

@@ -57,7 +57,7 @@ class CoubIE(InfoExtractor):
 
         file_versions = coub['file_versions']
 
-        QUALITIES = ('low', 'med', 'high')
+        QUALITIES = ('low', 'med', 'high', 'higher')
 
         MOBILE = 'mobile'
         IPHONE = 'iphone'
@@ -86,6 +86,7 @@ class CoubIE(InfoExtractor):
                     'format_id': '%s-%s-%s' % (HTML5, kind, quality),
                     'filesize': int_or_none(item.get('size')),
                     'vcodec': 'none' if kind == 'audio' else None,
+                    'acodec': 'none' if kind == 'video' else None,
                     'quality': quality_key(quality),
                     'source_preference': preference_key(HTML5),
                 })

yt_dlp/extractor/cozytv.py

@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class CozyTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cozy\.tv/(?P<uploader>[^/]+)/replays/(?P<id>[^/$#&?]+)'
+
+    _TESTS = [{
+        'url': 'https://cozy.tv/beardson/replays/2021-11-19_1',
+        'info_dict': {
+            'id': 'beardson-2021-11-19_1',
+            'ext': 'mp4',
+            'title': 'pokemon pt2',
+            'uploader': 'beardson',
+            'upload_date': '20211119',
+            'was_live': True,
+            'duration': 7981,
+        },
+        'params': {'skip_download': True}
+    }]
+
+    def _real_extract(self, url):
+        uploader, date = self._match_valid_url(url).groups()
+        id = f'{uploader}-{date}'
+        data_json = self._download_json(f'https://api.cozy.tv/cache/{uploader}/replay/{date}', id)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            f'https://cozycdn.foxtrotstream.xyz/replays/{uploader}/{date}/index.m3u8', id, ext='mp4')
+        return {
+            'id': id,
+            'title': data_json.get('title'),
+            'uploader': data_json.get('user') or uploader,
+            'upload_date': unified_strdate(data_json.get('date')),
+            'was_live': True,
+            'duration': data_json.get('duration'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/crunchyroll.py

@@ -27,6 +27,7 @@ from ..utils import (
     int_or_none,
     lowercase_escape,
     merge_dicts,
+    qualities,
     remove_end,
     sanitized_Request,
     try_get,
@@ -478,19 +479,24 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             [r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', r'<div>\s*Publisher:\s*<span>\s*(.+?)\s*</span>\s*</div>'],
             webpage, 'video_uploader', default=False)
 
+        requested_languages = self._configuration_arg('language')
+        requested_hardsubs = [('' if val == 'none' else val) for val in self._configuration_arg('hardsub')]
+        language_preference = qualities((requested_languages or [language or ''])[::-1])
+        hardsub_preference = qualities((requested_hardsubs or ['', language or ''])[::-1])
+
         formats = []
         for stream in media.get('streams', []):
-            audio_lang = stream.get('audio_lang')
+            audio_lang = stream.get('audio_lang') or ''
-            hardsub_lang = stream.get('hardsub_lang')
+            hardsub_lang = stream.get('hardsub_lang') or ''
+            if (requested_languages and audio_lang.lower() not in requested_languages
+                    or requested_hardsubs and hardsub_lang.lower() not in requested_hardsubs):
+                continue
             vrv_formats = self._extract_vrv_formats(
                 stream.get('url'), video_id, stream.get('format'),
                 audio_lang, hardsub_lang)
             for f in vrv_formats:
-                f['language_preference'] = 1 if audio_lang == language else 0
+                f['language_preference'] = language_preference(audio_lang)
-                f['quality'] = (
+                f['quality'] = hardsub_preference(hardsub_lang)
-                    1 if not hardsub_lang
-                    else 0 if hardsub_lang == language
-                    else -1)
             formats.extend(vrv_formats)
         if not formats:
             available_fmts = []

yt_dlp/extractor/cspan.py

@@ -18,7 +18,7 @@ from ..utils import (
     str_to_int,
     unescapeHTML,
 )
-from .senateisvp import SenateISVPIE
+from .senategov import SenateISVPIE
 from .ustream import UstreamIE
 
 

yt_dlp/extractor/curiositystream.py

@@ -15,7 +15,6 @@ from ..utils import (
 class CuriosityStreamBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'curiositystream'
     _auth_token = None
-    _API_BASE_URL = 'https://api.curiositystream.com/v1/'
 
     def _handle_errors(self, result):
         error = result.get('error', {}).get('message')
@@ -39,38 +38,44 @@ class CuriosityStreamBaseIE(InfoExtractor):
         if email is None:
             return
         result = self._download_json(
-            self._API_BASE_URL + 'login', None, data=urlencode_postdata({
+            'https://api.curiositystream.com/v1/login', None,
+            note='Logging in', data=urlencode_postdata({
                 'email': email,
                 'password': password,
             }))
         self._handle_errors(result)
-        self._auth_token = result['message']['auth_token']
+        CuriosityStreamBaseIE._auth_token = result['message']['auth_token']
 
 
 class CuriosityStreamIE(CuriosityStreamBaseIE):
     IE_NAME = 'curiositystream'
     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://app.curiositystream.com/video/2',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',
             'title': 'How Did You Develop The Internet?',
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
+            'channel': 'Curiosity Stream',
+            'categories': ['Technology', 'Interview'],
+            'average_rating': 96.79,
+            'series_id': '2',
         },
         'params': {
-            'format': 'bestvideo',
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
+
+    _API_BASE_URL = 'https://api.curiositystream.com/v1/media/'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         formats = []
         for encoding_format in ('m3u8', 'mpd'):
-            media = self._call_api('media/' + video_id, video_id, query={
+            media = self._call_api(video_id, video_id, query={
                 'encodingsNew': 'true',
                 'encodingsFormat': encoding_format,
             })
@@ -140,12 +145,33 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
             'duration': int_or_none(media.get('duration')),
             'tags': media.get('tags'),
             'subtitles': subtitles,
+            'channel': media.get('producer'),
+            'categories': [media.get('primary_category'), media.get('type')],
+            'average_rating': media.get('rating_percentage'),
+            'series_id': str(media.get('collection_id') or '') or None,
         }
 
 
-class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
+class CuriosityStreamCollectionBaseIE(CuriosityStreamBaseIE):
-    IE_NAME = 'curiositystream:collection'
+
-    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:collections?|series)/(?P<id>\d+)'
+    def _real_extract(self, url):
+        collection_id = self._match_id(url)
+        collection = self._call_api(collection_id, collection_id)
+        entries = []
+        for media in collection.get('media', []):
+            media_id = compat_str(media.get('id'))
+            media_type, ie = ('series', CuriosityStreamSeriesIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
+            entries.append(self.url_result(
+                'https://curiositystream.com/%s/%s' % (media_type, media_id),
+                ie=ie.ie_key(), video_id=media_id))
+        return self.playlist_result(
+            entries, collection_id,
+            collection.get('title'), collection.get('description'))
+
+
+class CuriosityStreamCollectionsIE(CuriosityStreamCollectionBaseIE):
+    IE_NAME = 'curiositystream:collections'
+    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/collections/(?P<id>\d+)'
     _API_BASE_URL = 'https://api.curiositystream.com/v2/collections/'
     _TESTS = [{
         'url': 'https://curiositystream.com/collections/86',
@@ -156,7 +182,17 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
         },
         'playlist_mincount': 7,
     }, {
-        'url': 'https://app.curiositystream.com/collection/2',
+        'url': 'https://curiositystream.com/collections/36',
+        'only_matching': True,
+    }]
+
+
+class CuriosityStreamSeriesIE(CuriosityStreamCollectionBaseIE):
+    IE_NAME = 'curiositystream:series'
+    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:series|collection)/(?P<id>\d+)'
+    _API_BASE_URL = 'https://api.curiositystream.com/v2/series/'
+    _TESTS = [{
+        'url': 'https://curiositystream.com/series/2',
         'info_dict': {
             'id': '2',
             'title': 'Curious Minds: The Internet',
@@ -164,23 +200,6 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
         },
         'playlist_mincount': 16,
     }, {
-        'url': 'https://curiositystream.com/series/2',
+        'url': 'https://curiositystream.com/collection/2',
-        'only_matching': True,
-    }, {
-        'url': 'https://curiositystream.com/collections/36',
         'only_matching': True,
     }]
-
-    def _real_extract(self, url):
-        collection_id = self._match_id(url)
-        collection = self._call_api(collection_id, collection_id)
-        entries = []
-        for media in collection.get('media', []):
-            media_id = compat_str(media.get('id'))
-            media_type, ie = ('series', CuriosityStreamCollectionIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
-            entries.append(self.url_result(
-                'https://curiositystream.com/%s/%s' % (media_type, media_id),
-                ie=ie.ie_key(), video_id=media_id))
-        return self.playlist_result(
-            entries, collection_id,
-            collection.get('title'), collection.get('description'))

yt_dlp/extractor/discoverynetworks.py

@@ -1,42 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-
-from .dplay import DPlayIE
-
-
-class DiscoveryNetworksDeIE(DPlayIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de|dplay\.co\.uk)/(?:programme|show|sendungen)/(?P<programme>[^/]+)/(?:video/)?(?P<alternate_id>[^/]+)'
-
-    _TESTS = [{
-        'url': 'https://www.tlc.de/programme/breaking-amish/video/die-welt-da-drauen/DCB331270001100',
-        'info_dict': {
-            'id': '78867',
-            'ext': 'mp4',
-            'title': 'Die Welt da draußen',
-            'description': 'md5:61033c12b73286e409d99a41742ef608',
-            'timestamp': 1554069600,
-            'upload_date': '20190331',
-        },
-        'params': {
-            'format': 'bestvideo',
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.dmax.de/programme/dmax-highlights/video/tuning-star-sidney-hoffmann-exklusiv-bei-dmax/191023082312316',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.dplay.co.uk/show/ghost-adventures/video/hotel-leger-103620/EHD_280313B',
-        'only_matching': True,
-    }, {
-        'url': 'https://tlc.de/sendungen/breaking-amish/die-welt-da-drauen/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        domain, programme, alternate_id = self._match_valid_url(url).groups()
-        country = 'GB' if domain == 'dplay.co.uk' else 'DE'
-        realm = 'questuk' if country == 'GB' else domain.replace('.', '')
-        return self._get_disco_api_info(
-            url, '%s/%s' % (programme, alternate_id),
-            'sonic-eu1-prod.disco-api.com', realm, country)

yt_dlp/extractor/discoveryplusindia.py

@@ -1,98 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from ..compat import compat_str
-from ..utils import try_get
-from .common import InfoExtractor
-from .dplay import DPlayIE
-
-
-class DiscoveryPlusIndiaIE(DPlayIE):
-    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.in/videos?' + DPlayIE._PATH_REGEX
-    _TESTS = [{
-        'url': 'https://www.discoveryplus.in/videos/how-do-they-do-it/fugu-and-more?seasonId=8&type=EPISODE',
-        'info_dict': {
-            'id': '27104',
-            'ext': 'mp4',
-            'display_id': 'how-do-they-do-it/fugu-and-more',
-            'title': 'Fugu and More',
-            'description': 'The Japanese catch, prepare and eat the deadliest fish on the planet.',
-            'duration': 1319,
-            'timestamp': 1582309800,
-            'upload_date': '20200221',
-            'series': 'How Do They Do It?',
-            'season_number': 8,
-            'episode_number': 2,
-            'creator': 'Discovery Channel',
-        },
-        'params': {
-            'format': 'bestvideo',
-            'skip_download': True,
-        },
-        'skip': 'Cookies (not necessarily logged in) are needed'
-    }]
-
-    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
-        headers['x-disco-params'] = 'realm=%s' % realm
-        headers['x-disco-client'] = 'WEB:UNKNOWN:dplus-india:17.0.0'
-
-    def _download_video_playback_info(self, disco_base, video_id, headers):
-        return self._download_json(
-            disco_base + 'playback/v3/videoPlaybackInfo',
-            video_id, headers=headers, data=json.dumps({
-                'deviceInfo': {
-                    'adBlocker': False,
-                },
-                'videoId': video_id,
-            }).encode('utf-8'))['data']['attributes']['streaming']
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        return self._get_disco_api_info(
-            url, display_id, 'ap2-prod-direct.discoveryplus.in', 'dplusindia', 'in')
-
-
-class DiscoveryPlusIndiaShowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.in/show/(?P<show_name>[^/]+)/?(?:[?#]|$)'
-    _TESTS = [{
-        'url': 'https://www.discoveryplus.in/show/how-do-they-do-it',
-        'playlist_mincount': 140,
-        'info_dict': {
-            'id': 'how-do-they-do-it',
-        },
-    }]
-
-    def _entries(self, show_name):
-        headers = {
-            'x-disco-client': 'WEB:UNKNOWN:dplus-india:prod',
-            'x-disco-params': 'realm=dplusindia',
-            'referer': 'https://www.discoveryplus.in/',
-        }
-        show_url = 'https://ap2-prod-direct.discoveryplus.in/cms/routes/show/{}?include=default'.format(show_name)
-        show_json = self._download_json(show_url,
-                                        video_id=show_name,
-                                        headers=headers)['included'][4]['attributes']['component']
-        show_id = show_json['mandatoryParams'].split('=')[-1]
-        season_url = 'https://ap2-prod-direct.discoveryplus.in/content/videos?sort=episodeNumber&filter[seasonNumber]={}&filter[show.id]={}&page[size]=100&page[number]={}'
-        for season in show_json['filters'][0]['options']:
-            season_id = season['id']
-            total_pages, page_num = 1, 0
-            while page_num < total_pages:
-                season_json = self._download_json(season_url.format(season_id, show_id, compat_str(page_num + 1)),
-                                                  video_id=show_id, headers=headers,
-                                                  note='Downloading JSON metadata%s' % (' page %d' % page_num if page_num else ''))
-                if page_num == 0:
-                    total_pages = try_get(season_json, lambda x: x['meta']['totalPages'], int) or 1
-                episodes_json = season_json['data']
-                for episode in episodes_json:
-                    video_id = episode['attributes']['path']
-                    yield self.url_result(
-                        'https://discoveryplus.in/videos/%s' % video_id,
-                        ie=DiscoveryPlusIndiaIE.ie_key(), video_id=video_id)
-                page_num += 1
-
-    def _real_extract(self, url):
-        show_name = self._match_valid_url(url).group('show_name')
-        return self.playlist_result(self._entries(show_name), playlist_id=show_name)

yt_dlp/extractor/disney.py

@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
-    compat_str,
     determine_ext,
+    join_nonempty,
     update_url_query,
 )
 
@@ -119,18 +119,13 @@ class DisneyIE(InfoExtractor):
                         continue
                     formats.append(f)
                 continue
-            format_id = []
-            if flavor_format:
-                format_id.append(flavor_format)
-            if tbr:
-                format_id.append(compat_str(tbr))
             ext = determine_ext(flavor_url)
             if flavor_format == 'applehttp' or ext == 'm3u8':
                 ext = 'mp4'
             width = int_or_none(flavor.get('width'))
             height = int_or_none(flavor.get('height'))
             formats.append({
-                'format_id': '-'.join(format_id),
+                'format_id': join_nonempty(flavor_format, tbr),
                 'url': flavor_url,
                 'width': width,
                 'height': height,

yt_dlp/extractor/dplay.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+import uuid
 
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
@@ -11,12 +12,172 @@ from ..utils import (
     float_or_none,
     int_or_none,
     strip_or_none,
+    try_get,
     unified_timestamp,
 )
 
 
-class DPlayIE(InfoExtractor):
+class DPlayBaseIE(InfoExtractor):
     _PATH_REGEX = r'/(?P<id>[^/]+/[^/?#]+)'
+    _auth_token_cache = {}
+
+    def _get_auth(self, disco_base, display_id, realm, needs_device_id=True):
+        key = (disco_base, realm)
+        st = self._get_cookies(disco_base).get('st')
+        token = (st and st.value) or self._auth_token_cache.get(key)
+
+        if not token:
+            query = {'realm': realm}
+            if needs_device_id:
+                query['deviceId'] = uuid.uuid4().hex
+            token = self._download_json(
+                disco_base + 'token', display_id, 'Downloading token',
+                query=query)['data']['attributes']['token']
+
+            # Save cache only if cookies are not being set
+            if not self._get_cookies(disco_base).get('st'):
+                self._auth_token_cache[key] = token
+
+        return f'Bearer {token}'
+
+    def _process_errors(self, e, geo_countries):
+        info = self._parse_json(e.cause.read().decode('utf-8'), None)
+        error = info['errors'][0]
+        error_code = error.get('code')
+        if error_code == 'access.denied.geoblocked':
+            self.raise_geo_restricted(countries=geo_countries)
+        elif error_code in ('access.denied.missingpackage', 'invalid.token'):
+            raise ExtractorError(
+                'This video is only available for registered users. You may want to use --cookies.', expected=True)
+        raise ExtractorError(info['errors'][0]['detail'], expected=True)
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['Authorization'] = self._get_auth(disco_base, display_id, realm, False)
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        streaming = self._download_json(
+            disco_base + 'playback/videoPlaybackInfo/' + video_id,
+            video_id, headers=headers)['data']['attributes']['streaming']
+        streaming_list = []
+        for format_id, format_dict in streaming.items():
+            streaming_list.append({
+                'type': format_id,
+                'url': format_dict.get('url'),
+            })
+        return streaming_list
+
+    def _get_disco_api_info(self, url, display_id, disco_host, realm, country, domain=''):
+        geo_countries = [country.upper()]
+        self._initialize_geo_bypass({
+            'countries': geo_countries,
+        })
+        disco_base = 'https://%s/' % disco_host
+        headers = {
+            'Referer': url,
+        }
+        self._update_disco_api_headers(headers, disco_base, display_id, realm)
+        try:
+            video = self._download_json(
+                disco_base + 'content/videos/' + display_id, display_id,
+                headers=headers, query={
+                    'fields[channel]': 'name',
+                    'fields[image]': 'height,src,width',
+                    'fields[show]': 'name',
+                    'fields[tag]': 'name',
+                    'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
+                    'include': 'images,primaryChannel,show,tags'
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                self._process_errors(e, geo_countries)
+            raise
+        video_id = video['data']['id']
+        info = video['data']['attributes']
+        title = info['name'].strip()
+        formats = []
+        subtitles = {}
+        try:
+            streaming = self._download_video_playback_info(
+                disco_base, video_id, headers)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                self._process_errors(e, geo_countries)
+            raise
+        for format_dict in streaming:
+            if not isinstance(format_dict, dict):
+                continue
+            format_url = format_dict.get('url')
+            if not format_url:
+                continue
+            format_id = format_dict.get('type')
+            ext = determine_ext(format_url)
+            if format_id == 'dash' or ext == 'mpd':
+                dash_fmts, dash_subs = self._extract_mpd_formats_and_subtitles(
+                    format_url, display_id, mpd_id='dash', fatal=False)
+                formats.extend(dash_fmts)
+                subtitles = self._merge_subtitles(subtitles, dash_subs)
+            elif format_id == 'hls' or ext == 'm3u8':
+                m3u8_fmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
+                    format_url, display_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls',
+                    fatal=False)
+                formats.extend(m3u8_fmts)
+                subtitles = self._merge_subtitles(subtitles, m3u8_subs)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        creator = series = None
+        tags = []
+        thumbnails = []
+        included = video.get('included') or []
+        if isinstance(included, list):
+            for e in included:
+                attributes = e.get('attributes')
+                if not attributes:
+                    continue
+                e_type = e.get('type')
+                if e_type == 'channel':
+                    creator = attributes.get('name')
+                elif e_type == 'image':
+                    src = attributes.get('src')
+                    if src:
+                        thumbnails.append({
+                            'url': src,
+                            'width': int_or_none(attributes.get('width')),
+                            'height': int_or_none(attributes.get('height')),
+                        })
+                if e_type == 'show':
+                    series = attributes.get('name')
+                elif e_type == 'tag':
+                    name = attributes.get('name')
+                    if name:
+                        tags.append(name)
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': strip_or_none(info.get('description')),
+            'duration': float_or_none(info.get('videoDuration'), 1000),
+            'timestamp': unified_timestamp(info.get('publishStart')),
+            'series': series,
+            'season_number': int_or_none(info.get('seasonNumber')),
+            'episode_number': int_or_none(info.get('episodeNumber')),
+            'creator': creator,
+            'tags': tags,
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
+            'http_headers': {
+                'referer': domain,
+            },
+        }
+
+
+class DPlayIE(DPlayBaseIE):
     _VALID_URL = r'''(?x)https?://
         (?P<domain>
             (?:www\.)?(?P<host>d
@@ -26,7 +187,7 @@ class DPlayIE(InfoExtractor):
                 )
             )|
             (?P<subdomain_country>es|it)\.dplay\.com
-        )/[^/]+''' + _PATH_REGEX
+        )/[^/]+''' + DPlayBaseIE._PATH_REGEX
 
     _TESTS = [{
         # non geo restricted, via secure api, unsigned download hls URL
@@ -46,7 +207,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 1,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -67,7 +227,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 1,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -87,7 +246,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 7,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
         'skip': 'Available for Premium users',
@@ -153,138 +311,6 @@ class DPlayIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _process_errors(self, e, geo_countries):
-        info = self._parse_json(e.cause.read().decode('utf-8'), None)
-        error = info['errors'][0]
-        error_code = error.get('code')
-        if error_code == 'access.denied.geoblocked':
-            self.raise_geo_restricted(countries=geo_countries)
-        elif error_code in ('access.denied.missingpackage', 'invalid.token'):
-            raise ExtractorError(
-                'This video is only available for registered users. You may want to use --cookies.', expected=True)
-        raise ExtractorError(info['errors'][0]['detail'], expected=True)
-
-    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
-        headers['Authorization'] = 'Bearer ' + self._download_json(
-            disco_base + 'token', display_id, 'Downloading token',
-            query={
-                'realm': realm,
-            })['data']['attributes']['token']
-
-    def _download_video_playback_info(self, disco_base, video_id, headers):
-        streaming = self._download_json(
-            disco_base + 'playback/videoPlaybackInfo/' + video_id,
-            video_id, headers=headers)['data']['attributes']['streaming']
-        streaming_list = []
-        for format_id, format_dict in streaming.items():
-            streaming_list.append({
-                'type': format_id,
-                'url': format_dict.get('url'),
-            })
-        return streaming_list
-
-    def _get_disco_api_info(self, url, display_id, disco_host, realm, country):
-        geo_countries = [country.upper()]
-        self._initialize_geo_bypass({
-            'countries': geo_countries,
-        })
-        disco_base = 'https://%s/' % disco_host
-        headers = {
-            'Referer': url,
-        }
-        self._update_disco_api_headers(headers, disco_base, display_id, realm)
-        try:
-            video = self._download_json(
-                disco_base + 'content/videos/' + display_id, display_id,
-                headers=headers, query={
-                    'fields[channel]': 'name',
-                    'fields[image]': 'height,src,width',
-                    'fields[show]': 'name',
-                    'fields[tag]': 'name',
-                    'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
-                    'include': 'images,primaryChannel,show,tags'
-                })
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
-                self._process_errors(e, geo_countries)
-            raise
-        video_id = video['data']['id']
-        info = video['data']['attributes']
-        title = info['name'].strip()
-        formats = []
-        try:
-            streaming = self._download_video_playback_info(
-                disco_base, video_id, headers)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self._process_errors(e, geo_countries)
-            raise
-        for format_dict in streaming:
-            if not isinstance(format_dict, dict):
-                continue
-            format_url = format_dict.get('url')
-            if not format_url:
-                continue
-            format_id = format_dict.get('type')
-            ext = determine_ext(format_url)
-            if format_id == 'dash' or ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, display_id, mpd_id='dash', fatal=False))
-            elif format_id == 'hls' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, display_id, 'mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls',
-                    fatal=False))
-            else:
-                formats.append({
-                    'url': format_url,
-                    'format_id': format_id,
-                })
-        self._sort_formats(formats)
-
-        creator = series = None
-        tags = []
-        thumbnails = []
-        included = video.get('included') or []
-        if isinstance(included, list):
-            for e in included:
-                attributes = e.get('attributes')
-                if not attributes:
-                    continue
-                e_type = e.get('type')
-                if e_type == 'channel':
-                    creator = attributes.get('name')
-                elif e_type == 'image':
-                    src = attributes.get('src')
-                    if src:
-                        thumbnails.append({
-                            'url': src,
-                            'width': int_or_none(attributes.get('width')),
-                            'height': int_or_none(attributes.get('height')),
-                        })
-                if e_type == 'show':
-                    series = attributes.get('name')
-                elif e_type == 'tag':
-                    name = attributes.get('name')
-                    if name:
-                        tags.append(name)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': strip_or_none(info.get('description')),
-            'duration': float_or_none(info.get('videoDuration'), 1000),
-            'timestamp': unified_timestamp(info.get('publishStart')),
-            'series': series,
-            'season_number': int_or_none(info.get('seasonNumber')),
-            'episode_number': int_or_none(info.get('episodeNumber')),
-            'creator': creator,
-            'tags': tags,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
-
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         display_id = mobj.group('id')
@@ -292,11 +318,11 @@ class DPlayIE(InfoExtractor):
         country = mobj.group('country') or mobj.group('subdomain_country') or mobj.group('plus_country')
         host = 'disco-api.' + domain if domain[0] == 'd' else 'eu2-prod.disco-api.com'
         return self._get_disco_api_info(
-            url, display_id, host, 'dplay' + country, country)
+            url, display_id, host, 'dplay' + country, country, domain)
 
 
-class HGTVDeIE(DPlayIE):
+class HGTVDeIE(DPlayBaseIE):
-    _VALID_URL = r'https?://de\.hgtv\.com/sendungen' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://de\.hgtv\.com/sendungen' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://de.hgtv.com/sendungen/tiny-house-klein-aber-oho/wer-braucht-schon-eine-toilette/',
         'info_dict': {
@@ -313,9 +339,6 @@ class HGTVDeIE(DPlayIE):
             'season_number': 3,
             'episode_number': 3,
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }]
 
     def _real_extract(self, url):
@@ -324,8 +347,8 @@ class HGTVDeIE(DPlayIE):
             url, display_id, 'eu1-prod.disco-api.com', 'hgtv', 'de')
 
 
-class DiscoveryPlusIE(DPlayIE):
+class DiscoveryPlusIE(DPlayBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/(?:\w{2}/)?video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
         'info_dict': {
@@ -343,6 +366,9 @@ class DiscoveryPlusIE(DPlayIE):
             'episode_number': 1,
         },
         'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://discoveryplus.com/ca/video/bering-sea-gold-discovery-ca/goldslingers',
+        'only_matching': True,
     }]
 
     _PRODUCT = 'dplus_us'
@@ -372,7 +398,7 @@ class DiscoveryPlusIE(DPlayIE):
 
 
 class ScienceChannelIE(DiscoveryPlusIE):
-    _VALID_URL = r'https?://(?:www\.)?sciencechannel\.com/video' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:www\.)?sciencechannel\.com/video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.sciencechannel.com/video/strangest-things-science-atve-us/nazi-mystery-machine',
         'info_dict': {
@@ -392,7 +418,7 @@ class ScienceChannelIE(DiscoveryPlusIE):
 
 
 class DIYNetworkIE(DiscoveryPlusIE):
-    _VALID_URL = r'https?://(?:watch\.)?diynetwork\.com/video' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:watch\.)?diynetwork\.com/video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://watch.diynetwork.com/video/pool-kings-diy-network/bringing-beach-life-to-texas',
         'info_dict': {
@@ -412,7 +438,7 @@ class DIYNetworkIE(DiscoveryPlusIE):
 
 
 class AnimalPlanetIE(DiscoveryPlusIE):
-    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/video' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.animalplanet.com/video/north-woods-law-animal-planet/squirrel-showdown',
         'info_dict': {
@@ -429,3 +455,159 @@ class AnimalPlanetIE(DiscoveryPlusIE):
 
     _PRODUCT = 'apl'
     _API_URL = 'us1-prod-direct.animalplanet.com'
+
+
+class DiscoveryPlusIndiaIE(DPlayBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.in/videos?' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.discoveryplus.in/videos/how-do-they-do-it/fugu-and-more?seasonId=8&type=EPISODE',
+        'info_dict': {
+            'id': '27104',
+            'ext': 'mp4',
+            'display_id': 'how-do-they-do-it/fugu-and-more',
+            'title': 'Fugu and More',
+            'description': 'The Japanese catch, prepare and eat the deadliest fish on the planet.',
+            'duration': 1319,
+            'timestamp': 1582309800,
+            'upload_date': '20200221',
+            'series': 'How Do They Do It?',
+            'season_number': 8,
+            'episode_number': 2,
+            'creator': 'Discovery Channel',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers.update({
+            'x-disco-params': 'realm=%s' % realm,
+            'x-disco-client': 'WEB:UNKNOWN:dplus-india:17.0.0',
+            'Authorization': self._get_auth(disco_base, display_id, realm),
+        })
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        return self._download_json(
+            disco_base + 'playback/v3/videoPlaybackInfo',
+            video_id, headers=headers, data=json.dumps({
+                'deviceInfo': {
+                    'adBlocker': False,
+                },
+                'videoId': video_id,
+            }).encode('utf-8'))['data']['attributes']['streaming']
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._get_disco_api_info(
+            url, display_id, 'ap2-prod-direct.discoveryplus.in', 'dplusindia', 'in', 'https://www.discoveryplus.in/')
+
+
+class DiscoveryNetworksDeIE(DPlayBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de|dplay\.co\.uk)/(?:programme|show|sendungen)/(?P<programme>[^/]+)/(?:video/)?(?P<alternate_id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'https://www.tlc.de/programme/breaking-amish/video/die-welt-da-drauen/DCB331270001100',
+        'info_dict': {
+            'id': '78867',
+            'ext': 'mp4',
+            'title': 'Die Welt da draußen',
+            'description': 'md5:61033c12b73286e409d99a41742ef608',
+            'timestamp': 1554069600,
+            'upload_date': '20190331',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.dmax.de/programme/dmax-highlights/video/tuning-star-sidney-hoffmann-exklusiv-bei-dmax/191023082312316',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dplay.co.uk/show/ghost-adventures/video/hotel-leger-103620/EHD_280313B',
+        'only_matching': True,
+    }, {
+        'url': 'https://tlc.de/sendungen/breaking-amish/die-welt-da-drauen/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        domain, programme, alternate_id = self._match_valid_url(url).groups()
+        country = 'GB' if domain == 'dplay.co.uk' else 'DE'
+        realm = 'questuk' if country == 'GB' else domain.replace('.', '')
+        return self._get_disco_api_info(
+            url, '%s/%s' % (programme, alternate_id),
+            'sonic-eu1-prod.disco-api.com', realm, country)
+
+
+class DiscoveryPlusShowBaseIE(DPlayBaseIE):
+
+    def _entries(self, show_name):
+        headers = {
+            'x-disco-client': self._X_CLIENT,
+            'x-disco-params': f'realm={self._REALM}',
+            'referer': self._DOMAIN,
+            'Authentication': self._get_auth(self._BASE_API, None, self._REALM),
+        }
+        show_json = self._download_json(
+            f'{self._BASE_API}cms/routes/{self._SHOW_STR}/{show_name}?include=default',
+            video_id=show_name, headers=headers)['included'][self._INDEX]['attributes']['component']
+        show_id = show_json['mandatoryParams'].split('=')[-1]
+        season_url = self._BASE_API + 'content/videos?sort=episodeNumber&filter[seasonNumber]={}&filter[show.id]={}&page[size]=100&page[number]={}'
+        for season in show_json['filters'][0]['options']:
+            season_id = season['id']
+            total_pages, page_num = 1, 0
+            while page_num < total_pages:
+                season_json = self._download_json(
+                    season_url.format(season_id, show_id, str(page_num + 1)), show_name, headers=headers,
+                    note='Downloading season %s JSON metadata%s' % (season_id, ' page %d' % page_num if page_num else ''))
+                if page_num == 0:
+                    total_pages = try_get(season_json, lambda x: x['meta']['totalPages'], int) or 1
+                episodes_json = season_json['data']
+                for episode in episodes_json:
+                    video_id = episode['attributes']['path']
+                    yield self.url_result(
+                        '%svideos/%s' % (self._DOMAIN, video_id),
+                        ie=self._VIDEO_IE.ie_key(), video_id=video_id)
+                page_num += 1
+
+    def _real_extract(self, url):
+        show_name = self._match_valid_url(url).group('show_name')
+        return self.playlist_result(self._entries(show_name), playlist_id=show_name)
+
+
+class DiscoveryPlusItalyShowIE(DiscoveryPlusShowBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.it/programmi/(?P<show_name>[^/]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.discoveryplus.it/programmi/deal-with-it-stai-al-gioco',
+        'playlist_mincount': 168,
+        'info_dict': {
+            'id': 'deal-with-it-stai-al-gioco',
+        },
+    }]
+
+    _BASE_API = 'https://disco-api.discoveryplus.it/'
+    _DOMAIN = 'https://www.discoveryplus.it/'
+    _X_CLIENT = 'WEB:UNKNOWN:dplay-client:2.6.0'
+    _REALM = 'dplayit'
+    _SHOW_STR = 'programmi'
+    _INDEX = 1
+    _VIDEO_IE = DPlayIE
+
+
+class DiscoveryPlusIndiaShowIE(DiscoveryPlusShowBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.in/show/(?P<show_name>[^/]+)/?(?:[?#]|$)'
+    _TESTS = [{
+        'url': 'https://www.discoveryplus.in/show/how-do-they-do-it',
+        'playlist_mincount': 140,
+        'info_dict': {
+            'id': 'how-do-they-do-it',
+        },
+    }]
+
+    _BASE_API = 'https://ap2-prod-direct.discoveryplus.in/'
+    _DOMAIN = 'https://www.discoveryplus.in/'
+    _X_CLIENT = 'WEB:UNKNOWN:dplus-india:prod'
+    _REALM = 'dplusindia'
+    _SHOW_STR = 'show'
+    _INDEX = 4
+    _VIDEO_IE = DiscoveryPlusIndiaIE

yt_dlp/extractor/dvtv.py

@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    join_nonempty,
     js_to_json,
     mimetype2ext,
     try_get,
@@ -139,13 +140,9 @@ class DVTVIE(InfoExtractor):
                     label = video.get('label')
                     height = self._search_regex(
                         r'^(\d+)[pP]', label or '', 'height', default=None)
-                    format_id = ['http']
-                    for f in (ext, label):
-                        if f:
-                            format_id.append(f)
                     formats.append({
                         'url': video_url,
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty('http', ext, label),
                         'height': int_or_none(height),
                     })
         self._sort_formats(formats)

yt_dlp/extractor/egghead.py

@@ -86,7 +86,6 @@ class EggheadLessonIE(EggheadBaseIE):
         },
         'params': {
             'skip_download': True,
-            'format': 'bestvideo',
         },
     }, {
         'url': 'https://egghead.io/api/v1/lessons/react-add-redux-to-a-react-application',

yt_dlp/extractor/epicon.py

@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class EpiconIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?epicon\.in/(?:documentaries|movies|tv-shows/[^/?#]+/[^/?#]+)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?epicon\.in/(?:documentaries|movies|tv-shows/[^/?#]+/[^/?#]+)/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.epicon.in/documentaries/air-battle-of-srinagar',
         'info_dict': {
@@ -84,7 +84,7 @@ class EpiconIE(InfoExtractor):
 
 
 class EpiconSeriesIE(InfoExtractor):
-    _VALID_URL = r'(?!.*season)(?:https?://)(?:www\.)?epicon\.in/tv-shows/(?P<id>[^/?#]+)'
+    _VALID_URL = r'(?!.*season)https?://(?:www\.)?epicon\.in/tv-shows/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.epicon.in/tv-shows/1-of-something',
         'playlist_mincount': 5,

yt_dlp/extractor/espn.py

@@ -7,7 +7,9 @@ from .once import OnceIE
 from ..compat import compat_str
 from ..utils import (
     determine_ext,
+    dict_get,
     int_or_none,
+    unified_strdate,
     unified_timestamp,
 )
 
@@ -236,3 +238,44 @@ class FiveThirtyEightIE(InfoExtractor):
             webpage, 'embed url')
 
         return self.url_result(embed_url, 'AbcNewsVideo')
+
+
+class ESPNCricInfoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/video/[^#$&?/]+-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.espncricinfo.com/video/finch-chasing-comes-with-risks-despite-world-cup-trend-1289135',
+        'info_dict': {
+            'id': '1289135',
+            'ext': 'mp4',
+            'title': 'Finch: Chasing comes with \'risks\' despite World Cup trend',
+            'description': 'md5:ea32373303e25efbb146efdfc8a37829',
+            'upload_date': '20211113',
+            'duration': 96,
+        },
+        'params': {'skip_download': True}
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        data_json = self._download_json(f'https://hs-consumer-api.espncricinfo.com/v1/pages/video/video-details?videoId={id}', id)['video']
+        formats, subtitles = [], {}
+        for item in data_json.get('playbacks') or []:
+            if item.get('type') == 'HLS' and item.get('url'):
+                m3u8_frmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(item['url'], id)
+                formats.extend(m3u8_frmts)
+                subtitles = self._merge_subtitles(subtitles, m3u8_subs)
+            elif item.get('type') == 'AUDIO' and item.get('url'):
+                formats.append({
+                    'url': item['url'],
+                    'vcodec': 'none',
+                })
+        self._sort_formats(formats)
+        return {
+            'id': id,
+            'title': data_json.get('title'),
+            'description': data_json.get('summary'),
+            'upload_date': unified_strdate(dict_get(data_json, ('publishedAt', 'recordedAt'))),
+            'duration': data_json.get('duration'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/euscreen.py

@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class EUScreenIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?euscreen\.eu/item.html\?id=(?P<id>[^&?$/]+)'
+    _VALID_URL = r'https?://(?:www\.)?euscreen\.eu/item.html\?id=(?P<id>[^&?$/]+)'
 
     _TESTS = [{
         'url': 'https://euscreen.eu/item.html?id=EUS_0EBCBF356BFC4E12A014023BA41BD98C',

yt_dlp/extractor/extractors.py

@@ -50,6 +50,7 @@ from .animelab import (
     AnimeLabIE,
     AnimeLabShowsIE,
 )
+from .amazon import AmazonStoreIE
 from .americastestkitchen import (
     AmericasTestKitchenIE,
     AmericasTestKitchenSeasonIE,
@@ -165,6 +166,7 @@ from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
 )
+from .blogger import BloggerIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE
@@ -177,6 +179,7 @@ from .br import (
 )
 from .bravotv import BravoTVIE
 from .breakcom import BreakIE
+from .breitbart import BreitBartIE
 from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
@@ -185,6 +188,7 @@ from .businessinsider import BusinessInsiderIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .cableav import CableAVIE
 from .cam4 import CAM4IE
 from .camdemy import (
     CamdemyIE,
@@ -192,6 +196,7 @@ from .camdemy import (
 )
 from .cammodels import CamModelsIE
 from .camwithher import CamWithHerIE
+from .canalalpha import CanalAlphaIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import (
@@ -235,10 +240,7 @@ from .ccc import (
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import (
+from .ceskatelevize import CeskaTelevizeIE
-    CeskaTelevizeIE,
-    CeskaTelevizePoradyIE,
-)
 from .cgtn import CGTNIE
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
@@ -293,6 +295,7 @@ from .commonprotocols import (
 from .condenast import CondeNastIE
 from .contv import CONtvIE
 from .corus import CorusIE
+from .cozytv import CozyTVIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
 from .crooksandliars import CrooksAndLiarsIE
@@ -309,7 +312,8 @@ from .ctvnews import CTVNewsIE
 from .cultureunplugged import CultureUnpluggedIE
 from .curiositystream import (
     CuriosityStreamIE,
-    CuriosityStreamCollectionIE,
+    CuriosityStreamCollectionsIE,
+    CuriosityStreamSeriesIE,
 )
 from .cwtv import CWTVIE
 from .dailymail import DailyMailIE
@@ -338,10 +342,6 @@ from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE
-from .discoveryplusindia import (
-    DiscoveryPlusIndiaIE,
-    DiscoveryPlusIndiaShowIE,
-)
 from .dotsub import DotsubIE
 from .douyutv import (
     DouyuShowIE,
@@ -353,7 +353,11 @@ from .dplay import (
     HGTVDeIE,
     ScienceChannelIE,
     DIYNetworkIE,
-    AnimalPlanetIE
+    AnimalPlanetIE,
+    DiscoveryPlusIndiaIE,
+    DiscoveryNetworksDeIE,
+    DiscoveryPlusItalyShowIE,
+    DiscoveryPlusIndiaShowIE,
 )
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
@@ -375,7 +379,6 @@ from .discoverygo import (
     DiscoveryGoIE,
     DiscoveryGoPlaylistIE,
 )
-from .discoverynetworks import DiscoveryNetworksDeIE
 from .discoveryvr import DiscoveryVRIE
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
@@ -419,6 +422,7 @@ from .espn import (
     ESPNIE,
     ESPNArticleIE,
     FiveThirtyEightIE,
+    ESPNCricInfoIE,
 )
 from .esri import EsriVideoIE
 from .europa import EuropaIE
@@ -495,7 +499,10 @@ from .funimation import (
 )
 from .funk import FunkIE
 from .fusion import FusionIE
-from .gab import GabTVIE
+from .gab import (
+    GabTVIE,
+    GabIE,
+)
 from .gaia import GaiaIE
 from .gameinformer import GameInformerIE
 from .gamespot import GameSpotIE
@@ -591,12 +598,16 @@ from .indavideo import IndavideoEmbedIE
 from .infoq import InfoQIE
 from .instagram import (
     InstagramIE,
+    InstagramIOSIE,
     InstagramUserIE,
     InstagramTagIE,
 )
 from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE
-from .iprima import IPrimaIE
+from .iprima import (
+    IPrimaIE,
+    IPrimaCNNIE
+)
 from .iqiyi import IqiyiIE
 from .ir90tv import Ir90TvIE
 from .itv import (
@@ -696,6 +707,7 @@ from .line import (
     LineLiveChannelIE,
 )
 from .linkedin import (
+    LinkedInIE,
     LinkedInLearningIE,
     LinkedInLearningCourseIE,
 )
@@ -744,7 +756,10 @@ from .mdr import MDRIE
 from .medaltv import MedalTVIE
 from .mediaite import MediaiteIE
 from .mediaklikk import MediaKlikkIE
-from .mediaset import MediasetIE
+from .mediaset import (
+    MediasetIE,
+    MediasetShowIE,
+)
 from .mediasite import (
     MediasiteIE,
     MediasiteCatalogIE,
@@ -784,6 +799,7 @@ from .mirrativ import (
 )
 from .mit import TechTVMITIE, OCWMITIE
 from .mitele import MiTeleIE
+from .mixch import MixchIE
 from .mixcloud import (
     MixcloudIE,
     MixcloudUserIE,
@@ -793,6 +809,7 @@ from .mlb import (
     MLBIE,
     MLBVideoIE,
 )
+from .mlssoccer import MLSSoccerIE
 from .mnet import MnetIE
 from .moevideo import MoeVideoIE
 from .mofosex import (
@@ -835,7 +852,10 @@ from .myvi import (
 )
 from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
-from .n1 import N1InfoIIE, N1InfoAssetIE
+from .n1 import (
+    N1InfoAssetIE,
+    N1InfoIIE,
+)
 from .nationalgeographic import (
     NationalGeographicVideoIE,
     NationalGeographicTVIE,
@@ -869,7 +889,10 @@ from .ndr import (
     NJoyEmbedIE,
 )
 from .ndtv import NDTVIE
-from .nebula import NebulaIE
+from .nebula import (
+    NebulaIE,
+    NebulaCollectionIE,
+)
 from .nerdcubed import NerdCubedFeedIE
 from .netzkino import NetzkinoIE
 from .neteasemusic import (
@@ -923,7 +946,10 @@ from .niconico import (
     NicovideoSearchIE,
     NicovideoSearchURLIE,
 )
-from .ninecninemedia import NineCNineMediaIE
+from .ninecninemedia import (
+    NineCNineMediaIE,
+    CPTwentyFourIE,
+)
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
@@ -987,6 +1013,7 @@ from .oktoberfesttv import OktoberfestTVIE
 from .olympics import OlympicsReplayIE
 from .on24 import On24IE
 from .ondemandkorea import OnDemandKoreaIE
+from .onefootball import OneFootballIE
 from .onet import (
     OnetIE,
     OnetChannelIE,
@@ -1047,6 +1074,7 @@ from .peertube import (
     PeerTubeIE,
     PeerTubePlaylistIE,
 )
+from .peertv import PeerTVIE
 from .peloton import (
     PelotonIE,
     PelotonLiveIE
@@ -1071,6 +1099,7 @@ from .pinterest import (
     PinterestCollectionIE,
 )
 from .pladform import PladformIE
+from .planetmarathi import PlanetMarathiIE
 from .platzi import (
     PlatziIE,
     PlatziCourseIE,
@@ -1092,9 +1121,14 @@ from .pokemon import (
     PokemonIE,
     PokemonWatchIE,
 )
+from .polsatgo import PolsatGoIE
 from .polskieradio import (
     PolskieRadioIE,
     PolskieRadioCategoryIE,
+    PolskieRadioPlayerIE,
+    PolskieRadioPodcastIE,
+    PolskieRadioPodcastListIE,
+    PolskieRadioRadioKierowcowIE,
 )
 from .popcorntimes import PopcorntimesIE
 from .popcorntv import PopcornTVIE
@@ -1141,6 +1175,11 @@ from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
+from .radiozet import RadioZetPodcastIE
+from .radiokapital import (
+    RadioKapitalIE,
+    RadioKapitalShowIE,
+)
 from .radlive import (
     RadLiveIE,
     RadLiveChannelIE,
@@ -1151,6 +1190,8 @@ from .rai import (
     RaiPlayLiveIE,
     RaiPlayPlaylistIE,
     RaiIE,
+    RaiPlayRadioIE,
+    RaiPlayRadioPlaylistIE,
 )
 from .raywenderlich import (
     RayWenderlichIE,
@@ -1174,9 +1215,11 @@ from .redbulltv import (
     RedBullTVRrnContentIE,
     RedBullIE,
 )
-from .reddit import (
+from .reddit import RedditIE
-    RedditIE,
+from .redgifs import (
-    RedditRIE,
+    RedGifsIE,
+    RedGifsSearchIE,
+    RedGifsUserIE,
 )
 from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
@@ -1191,7 +1234,7 @@ from .rice import RICEIE
 from .rmcdecouverte import RMCDecouverteIE
 from .ro220 import Ro220IE
 from .rockstargames import RockstarGamesIE
-from .roosterteeth import RoosterTeethIE
+from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rozhlas import RozhlasIE
@@ -1204,6 +1247,7 @@ from .rtl2 import (
     RTL2YouSeriesIE,
 )
 from .rtp import RTPIE
+from .rtrfm import RTRFMIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
 from .rtvnh import RTVNHIE
@@ -1245,7 +1289,7 @@ from .scte import (
     SCTECourseIE,
 )
 from .seeker import SeekerIE
-from .senateisvp import SenateISVPIE
+from .senategov import SenateISVPIE, SenateGovIE
 from .sendtonews import SendtoNewsIE
 from .servus import ServusIE
 from .sevenplus import SevenPlusIE
@@ -1289,6 +1333,7 @@ from .skynewsarabia import (
 from .skynewsau import SkyNewsAUIE
 from .sky import (
     SkyNewsIE,
+    SkyNewsStoryIE,
     SkySportsIE,
     SkySportsNewsIE,
 )
@@ -1371,8 +1416,10 @@ from .streamable import StreamableIE
 from .streamanity import StreamanityIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streamff import StreamFFIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
+from .stripchat import StripchatIE
 from .stv import STVPlayerIE
 from .sunporno import SunPornoIE
 from .sverigesradio import (
@@ -1442,6 +1489,10 @@ from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
+from .threespeak import (
+    ThreeSpeakIE,
+    ThreeSpeakUserIE,
+)
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,
@@ -1542,6 +1593,7 @@ from .tvnow import (
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
+    TVPStreamIE,
     TVPWebsiteIE,
 )
 from .tvplay import (
@@ -1752,6 +1804,7 @@ from .weibo import (
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .willow import WillowIE
 from .wimtv import WimTVIE
 from .whowatch import WhoWatchIE
 from .wistia import (
@@ -1759,6 +1812,10 @@ from .wistia import (
     WistiaPlaylistIE,
 )
 from .worldstarhiphop import WorldStarHipHopIE
+from .wppilot import (
+    WPPilotIE,
+    WPPilotChannelsIE,
+)
 from .wsj import (
     WSJIE,
     WSJArticleIE,

yt_dlp/extractor/fancode.py

@@ -21,7 +21,6 @@ class FancodeVodIE(InfoExtractor):
         'url': 'https://fancode.com/video/15043/match-preview-pbks-vs-mi',
         'params': {
             'skip_download': True,
-            'format': 'bestvideo'
         },
         'info_dict': {
             'id': '6249806281001',

yt_dlp/extractor/francetv.py

@@ -185,7 +185,7 @@ class FranceTVIE(InfoExtractor):
                 'vcodec': 'none',
                 'ext': 'mhtml',
                 'protocol': 'mhtml',
-                'url': 'about:dummy',
+                'url': 'about:invalid',
                 'fragments': [{
                     'path': sheet,
                     # XXX: not entirely accurate; each spritesheet seems to be

yt_dlp/extractor/funimation.py

@@ -10,6 +10,7 @@ from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     int_or_none,
+    join_nonempty,
     js_to_json,
     orderedSet,
     qualities,
@@ -275,7 +276,7 @@ class FunimationIE(FunimationBaseIE):
     def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name):
         if isinstance(episode, str):
             webpage = self._download_webpage(
-                f'https://www.funimation.com/player/{experience_id}', display_id,
+                f'https://www.funimation.com/player/{experience_id}/', display_id,
                 fatal=False, note=f'Downloading player webpage for {format_name}')
             episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False)
 
@@ -288,10 +289,11 @@ class FunimationIE(FunimationBaseIE):
                     sub_type = sub_type if sub_type != 'FULL' else None
                     current_sub = {
                         'url': text_track['src'],
-                        'name': ' '.join(filter(None, (version, text_track.get('label'), sub_type)))
+                        'name': join_nonempty(version, text_track.get('label'), sub_type, delim=' ')
                     }
-                    lang = '_'.join(filter(None, (
+                    lang = join_nonempty(text_track.get('language', 'und'),
-                        text_track.get('language', 'und'), version if version != 'Simulcast' else None, sub_type)))
+                                         version if version != 'Simulcast' else None,
+                                         sub_type, delim='_')
                     if current_sub not in subtitles.get(lang, []):
                         subtitles.setdefault(lang, []).append(current_sub)
         return subtitles

yt_dlp/extractor/gab.py

@@ -6,12 +6,16 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    int_or_none,
+    parse_codecs,
+    parse_duration,
     str_to_int,
+    unified_timestamp
 )
 
 
 class GabTVIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)tv.gab.com/channel/[^/]+/view/(?P<id>[a-z0-9-]+)'
+    _VALID_URL = r'https?://tv\.gab\.com/channel/[^/]+/view/(?P<id>[a-z0-9-]+)'
     _TESTS = [{
         'url': 'https://tv.gab.com/channel/wurzelroot/view/why-was-america-in-afghanistan-61217eacea5665de450d0488',
         'info_dict': {
@@ -32,8 +36,10 @@ class GabTVIE(InfoExtractor):
         channel_name = self._search_regex(r'data-channel-name=\"(?P<channel_id>[^\"]+)', webpage, 'channel_name')
         title = self._search_regex(r'data-episode-title=\"(?P<channel_id>[^\"]+)', webpage, 'title')
         view_key = self._search_regex(r'data-view-key=\"(?P<channel_id>[^\"]+)', webpage, 'view_key')
-        description = clean_html(self._html_search_regex(self._meta_regex('description'), webpage, 'description', group='content')) or None
+        description = clean_html(
-        available_resolutions = re.findall(r'<a\ data-episode-id=\"%s\"\ data-resolution=\"(?P<resolution>[^\"]+)' % id, webpage)
+            self._html_search_regex(self._meta_regex('description'), webpage, 'description', group='content')) or None
+        available_resolutions = re.findall(r'<a\ data-episode-id=\"%s\"\ data-resolution=\"(?P<resolution>[^\"]+)' % id,
+                                           webpage)
 
         formats = []
         for resolution in available_resolutions:
@@ -62,3 +68,80 @@ class GabTVIE(InfoExtractor):
             'uploader_id': channel_id,
             'thumbnail': f'https://tv.gab.com/image/{id}',
         }
+
+
+class GabIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gab\.com/[^/]+/posts/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://gab.com/SomeBitchIKnow/posts/107163961867310434',
+        'md5': '8ca34fb00f1e1033b5c5988d79ec531d',
+        'info_dict': {
+            'id': '107163961867310434-0',
+            'ext': 'mp4',
+            'title': 'L on Gab',
+            'uploader_id': '946600',
+            'uploader': 'SomeBitchIKnow',
+            'description': 'md5:204055fafd5e1a519f5d6db953567ca3',
+            'timestamp': 1635192289,
+            'upload_date': '20211025',
+        }
+    }, {
+        'url': 'https://gab.com/TheLonelyProud/posts/107045884469287653',
+        'md5': 'f9cefcfdff6418e392611a828d47839d',
+        'info_dict': {
+            'id': '107045884469287653-0',
+            'ext': 'mp4',
+            'title': 'Jody Sadowski on Gab',
+            'uploader_id': '1390705',
+            'timestamp': 1633390571,
+            'upload_date': '20211004',
+            'uploader': 'TheLonelyProud',
+        }
+    }]
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+        json_data = self._download_json(f'https://gab.com/api/v1/statuses/{post_id}', post_id)
+
+        entries = []
+        for idx, media in enumerate(json_data['media_attachments']):
+            if media.get('type') not in ('video', 'gifv'):
+                continue
+            metadata = media['meta']
+            format_metadata = {
+                'acodec': parse_codecs(metadata.get('audio_encode')).get('acodec'),
+                'asr': int_or_none((metadata.get('audio_bitrate') or '').split(' ')[0]),
+                'fps': metadata.get('fps'),
+            }
+
+            formats = [{
+                'url': url,
+                'width': f.get('width'),
+                'height': f.get('height'),
+                'tbr': int_or_none(f.get('bitrate'), scale=1000),
+                **format_metadata,
+            } for url, f in ((media.get('url'), metadata.get('original') or {}),
+                             (media.get('source_mp4'), metadata.get('playable') or {})) if url]
+
+            self._sort_formats(formats)
+
+            author = json_data.get('account') or {}
+            entries.append({
+                'id': f'{post_id}-{idx}',
+                'title': f'{json_data["account"]["display_name"]} on Gab',
+                'timestamp': unified_timestamp(json_data.get('created_at')),
+                'formats': formats,
+                'description': clean_html(json_data.get('content')),
+                'duration': metadata.get('duration') or parse_duration(metadata.get('length')),
+                'like_count': json_data.get('favourites_count'),
+                'comment_count': json_data.get('replies_count'),
+                'repost_count': json_data.get('reblogs_count'),
+                'uploader': author.get('username'),
+                'uploader_id': author.get('id'),
+                'uploader_url': author.get('url'),
+            })
+
+        if len(entries) > 1:
+            return self.playlist_result(entries, post_id)
+
+        return entries[0]

yt_dlp/extractor/generic.py

@@ -56,7 +56,7 @@ from .sportbox import SportBoxIE
 from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
-from .senateisvp import SenateISVPIE
+from .senategov import SenateISVPIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
@@ -135,6 +135,8 @@ from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
 from .simplecast import SimplecastIE
 from .wimtv import WimTVIE
+from .tvp import TVPEmbedIE
+from .blogger import BloggerIE
 
 
 class GenericIE(InfoExtractor):
@@ -359,9 +361,6 @@ class GenericIE(InfoExtractor):
                 'formats': 'mincount:9',
                 'upload_date': '20130904',
             },
-            'params': {
-                'format': 'bestvideo',
-            },
         },
         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
         {
@@ -1188,6 +1187,21 @@ class GenericIE(InfoExtractor):
             },
             'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
         },
+        # jwplayer with only the json URL
+        {
+            'url': 'https://www.hollywoodreporter.com/news/general-news/dunkirk-team-reveals-what-christopher-nolan-said-oscar-win-meet-your-oscar-winner-1092454',
+            'info_dict': {
+                'id': 'TljWkvWH',
+                'ext': 'mp4',
+                'upload_date': '20180306',
+                'title': 'md5:91eb1862f6526415214f62c00b453936',
+                'description': 'md5:73048ae50ae953da10549d1d2fe9b3aa',
+                'timestamp': 1520367225,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Complex jwplayer
         {
             'url': 'http://www.indiedb.com/games/king-machine/videos',
@@ -2160,6 +2174,17 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # blogger embed
+            'url': 'https://blog.tomeuvizoso.net/2019/01/a-panfrost-milestone.html',
+            'md5': 'f1bc19b6ea1b0fd1d81e84ca9ec467ac',
+            'info_dict': {
+                'id': 'BLOGGER-video-3c740e3a49197e16-796',
+                'ext': 'mp4',
+                'title': 'Blogger',
+                'thumbnail': r're:^https?://.*',
+            },
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2319,12 +2344,43 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
             }
         },
+        {
+            # Reddit-hosted video that will redirect and be processed by RedditIE
+            # Redirects to https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/
+            'url': 'https://v.redd.it/zv89llsvexdz',
+            'md5': '87f5f02f6c1582654146f830f21f8662',
+            'info_dict': {
+                'id': 'zv89llsvexdz',
+                'ext': 'mp4',
+                'timestamp': 1501941939.0,
+                'title': 'That small heart attack.',
+                'upload_date': '20170805',
+                'uploader': 'Antw87'
+            }
+        },
+        {
+            # 1080p Reddit-hosted video that will redirect and be processed by RedditIE
+            'url': 'https://v.redd.it/33hgok7dfbz71/',
+            'md5': '7a1d587940242c9bb3bd6eb320b39258',
+            'info_dict': {
+                'id': '33hgok7dfbz71',
+                'ext': 'mp4',
+                'title': "The game Didn't want me to Knife that Guy I guess",
+                'uploader': 'paraf1ve',
+                'timestamp': 1636788683.0,
+                'upload_date': '20211113'
+            }
+        }
+        #
     ]
 
     def report_following_redirect(self, new_url):
         """Report information extraction."""
         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
 
+    def report_detected(self, name):
+        self._downloader.write_debug(f'Identified a {name}')
+
     def _extract_rss(self, url, video_id, doc):
         playlist_title = doc.find('./channel/title').text
         playlist_desc_el = doc.find('./channel/description')
@@ -2540,10 +2596,13 @@ class GenericIE(InfoExtractor):
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
+            self.report_detected('direct video link')
             format_id = compat_str(m.group('format_id'))
             subtitles = {}
             if format_id.endswith('mpegurl'):
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
+            elif format_id.endswith('mpd') or format_id.endswith('dash+xml'):
+                formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id)
             elif format_id == 'f4m':
                 formats = self._extract_f4m_formats(url, video_id)
             else:
@@ -2580,6 +2639,7 @@ class GenericIE(InfoExtractor):
 
         # Is it an M3U playlist?
         if first_bytes.startswith(b'#EXTM3U'):
+            self.report_detected('M3U playlist')
             info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
             self._sort_formats(info_dict['formats'])
             return info_dict
@@ -2610,16 +2670,20 @@ class GenericIE(InfoExtractor):
             except compat_xml_parse_error:
                 doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
+                self.report_detected('RSS feed')
                 return self._extract_rss(url, video_id, doc)
             elif doc.tag == 'SmoothStreamingMedia':
                 info_dict['formats'], info_dict['subtitles'] = self._parse_ism_formats_and_subtitles(doc, url)
+                self.report_detected('ISM manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                 smil = self._parse_smil(doc, url, video_id)
+                self.report_detected('SMIL file')
                 self._sort_formats(smil['formats'])
                 return smil
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
+                self.report_detected('XSPF playlist')
                 return self.playlist_result(
                     self._parse_xspf(
                         doc, video_id, xspf_url=url,
@@ -2630,10 +2694,12 @@ class GenericIE(InfoExtractor):
                     doc,
                     mpd_base_url=full_response.geturl().rpartition('/')[0],
                     mpd_url=url)
+                self.report_detected('DASH manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
+                self.report_detected('F4M manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
         except compat_xml_parse_error:
@@ -2642,6 +2708,7 @@ class GenericIE(InfoExtractor):
         # Is it a Camtasia project?
         camtasia_res = self._extract_camtasia(url, video_id, webpage)
         if camtasia_res is not None:
+            self.report_detected('Camtasia video')
             return camtasia_res
 
         # Sometimes embedded video player is hidden behind percent encoding
@@ -2692,6 +2759,8 @@ class GenericIE(InfoExtractor):
             'age_limit': age_limit,
         })
 
+        self._downloader.write_debug('Looking for video embeds')
+
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
@@ -3189,6 +3258,11 @@ class GenericIE(InfoExtractor):
         if onionstudios_url:
             return self.url_result(onionstudios_url)
 
+        # Look for Blogger embeds
+        blogger_urls = BloggerIE._extract_urls(webpage)
+        if blogger_urls:
+            return self.playlist_from_matches(blogger_urls, video_id, video_title, ie=BloggerIE.ie_key())
+
         # Look for ViewLift embeds
         viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
         if viewlift_url:
@@ -3482,9 +3556,14 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 rumble_urls, video_id, video_title, ie=RumbleEmbedIE.ie_key())
 
+        tvp_urls = TVPEmbedIE._extract_urls(webpage)
+        if tvp_urls:
+            return self.playlist_from_matches(tvp_urls, video_id, video_title, ie=TVPEmbedIE.ie_key())
+
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
+            self.report_detected('HTML5 media')
             if len(entries) == 1:
                 entries[0].update({
                     'id': video_id,
@@ -3503,9 +3582,18 @@ class GenericIE(InfoExtractor):
         jwplayer_data = self._find_jwplayer_data(
             webpage, video_id, transform_source=js_to_json)
         if jwplayer_data:
+            if isinstance(jwplayer_data.get('playlist'), str):
+                self.report_detected('JW Player playlist')
+                return {
+                    **info_dict,
+                    '_type': 'url',
+                    'ie_key': JWPlatformIE.ie_key(),
+                    'url': jwplayer_data['playlist'],
+                }
             try:
                 info = self._parse_jwplayer_data(
                     jwplayer_data, video_id, require_title=False, base_url=url)
+                self.report_detected('JW Player data')
                 return merge_dicts(info, info_dict)
             except ExtractorError:
                 # See https://github.com/ytdl-org/youtube-dl/pull/16735
@@ -3555,15 +3643,16 @@ class GenericIE(InfoExtractor):
                         },
                     })
             if formats or subtitles:
+                self.report_detected('video.js embed')
                 self._sort_formats(formats)
                 info_dict['formats'] = formats
                 info_dict['subtitles'] = subtitles
                 return info_dict
 
         # Looking for http://schema.org/VideoObject
-        json_ld = self._search_json_ld(
+        json_ld = self._search_json_ld(webpage, video_id, default={})
-            webpage, video_id, default={}, expected_type='VideoObject')
         if json_ld.get('url'):
+            self.report_detected('JSON LD')
             return merge_dicts(json_ld, info_dict)
 
         def check_video(vurl):
@@ -3580,7 +3669,9 @@ class GenericIE(InfoExtractor):
 
         # Start with something easy: JW Player in SWFObject
         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
-        if not found:
+        if found:
+            self.report_detected('JW Player in SFWObject')
+        else:
             # Look for gorilla-vid style embedding
             found = filter_video(re.findall(r'''(?sx)
                 (?:
@@ -3590,10 +3681,13 @@ class GenericIE(InfoExtractor):
                 )
                 .*?
                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
+            if found:
+                self.report_detected('JW Player embed')
         if not found:
             # Look for generic KVS player
             found = re.search(r'<script [^>]*?src="https://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
             if found:
+                self.report_detected('KWS Player')
                 if found.group('maj_ver') not in ['4', '5']:
                     self.report_warning('Untested major version (%s) in player engine--Download may fail.' % found.group('ver'))
                 flashvars = re.search(r'(?ms)<script.*?>.*?var\s+flashvars\s*=\s*(\{.*?\});.*?</script>', webpage)
@@ -3639,10 +3733,14 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the search a little bit
             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
+            if found:
+                self.report_detected('video file')
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = filter_video(re.findall(
                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
+            if found:
+                self.report_detected('JW Player JS loader')
         if not found:
             # Flow player
             found = filter_video(re.findall(r'''(?xs)
@@ -3651,10 +3749,14 @@ class GenericIE(InfoExtractor):
                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
+            if found:
+                self.report_detected('Flow Player')
         if not found:
             # Cinerama player
             found = re.findall(
                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
+            if found:
+                self.report_detected('Cinerama player')
         if not found:
             # Try to find twitter cards info
             # twitter:player:stream should be checked before twitter:player since
@@ -3662,6 +3764,8 @@ class GenericIE(InfoExtractor):
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             found = filter_video(re.findall(
                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
+            if found:
+                self.report_detected('Twitter card')
         if not found:
             # We look for Open Graph info:
             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
@@ -3669,6 +3773,8 @@ class GenericIE(InfoExtractor):
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 found = filter_video(re.findall(r'<meta.*?property="og:(?:video|audio)".*?content="(.*?)"', webpage))
+                if found:
+                    self.report_detected('Open Graph video info')
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
@@ -3700,6 +3806,7 @@ class GenericIE(InfoExtractor):
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
             if embed_url and embed_url != url:
+                self.report_detected('twitter:player iframe')
                 return self.url_result(embed_url)
 
         if not found:

yt_dlp/extractor/gronkh.py

@@ -6,7 +6,7 @@ from ..utils import unified_strdate
 
 
 class GronkhIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?gronkh\.tv/stream/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?gronkh\.tv/stream/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'https://gronkh.tv/stream/536',

yt_dlp/extractor/hotstar.py

@@ -230,6 +230,11 @@ class HotStarIE(HotStarBaseIE):
             if tags and 'encryption:plain' not in tags:
                 for f in current_formats:
                     f['has_drm'] = True
+            if tags and 'language' in tags:
+                lang = re.search(r'language:(?P<lang>[a-z]+)', tags).group('lang')
+                for f in current_formats:
+                    if not f.get('langauge'):
+                        f['language'] = lang
             formats.extend(current_formats)
             subs = self._merge_subtitles(subs, current_subs)
         if not formats and geo_restricted:
@@ -291,7 +296,7 @@ class HotStarPlaylistIE(HotStarBaseIE):
 
 class HotStarSeriesIE(HotStarBaseIE):
     IE_NAME = 'hotstar:series'
-    _VALID_URL = r'(?P<url>(?:https?://)(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/(?P<id>\d+))'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/(?P<id>\d+))'
     _TESTS = [{
         'url': 'https://www.hotstar.com/in/tv/radhakrishn/1260000646',
         'info_dict': {

yt_dlp/extractor/imdb.py

@@ -111,7 +111,7 @@ class ImdbIE(InfoExtractor):
             'formats': formats,
             'description': info.get('videoDescription'),
             'thumbnail': url_or_none(try_get(
-                video_metadata, lambda x: x['videoSlate']['source'])),
+                info, lambda x: x['videoSlate']['source'])),
             'duration': parse_duration(info.get('videoRuntime')),
         }
 

yt_dlp/extractor/instagram.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+# coding: utf-8
 
 import itertools
 import hashlib
@@ -8,7 +8,6 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
     compat_HTTPError,
 )
 from ..utils import (
@@ -18,16 +17,160 @@ from ..utils import (
     int_or_none,
     lowercase_escape,
     std_headers,
-    try_get,
+    traverse_obj,
     url_or_none,
-    variadic,
     urlencode_postdata,
 )
 
 
-class InstagramIE(InfoExtractor):
+class InstagramBaseIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
     _NETRC_MACHINE = 'instagram'
+    _IS_LOGGED_IN = False
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None or self._IS_LOGGED_IN:
+            return
+
+        login_webpage = self._download_webpage(
+            'https://www.instagram.com/accounts/login/', None,
+            note='Downloading login webpage', errnote='Failed to download login webpage')
+
+        shared_data = self._parse_json(
+            self._search_regex(
+                r'window\._sharedData\s*=\s*({.+?});',
+                login_webpage, 'shared data', default='{}'),
+            None)
+
+        login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
+            'Accept': '*/*',
+            'X-IG-App-ID': '936619743392459',
+            'X-ASBD-ID': '198387',
+            'X-IG-WWW-Claim': '0',
+            'X-Requested-With': 'XMLHttpRequest',
+            'X-CSRFToken': shared_data['config']['csrf_token'],
+            'X-Instagram-AJAX': shared_data['rollout_hash'],
+            'Referer': 'https://www.instagram.com/',
+        }, data=urlencode_postdata({
+            'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
+            'username': username,
+            'queryParams': '{}',
+            'optIntoOneTap': 'false',
+            'stopDeletionNonce': '',
+            'trustedDeviceRecords': '{}',
+        }))
+
+        if not login.get('authenticated'):
+            if login.get('message'):
+                raise ExtractorError(f'Unable to login: {login["message"]}')
+            elif login.get('user'):
+                raise ExtractorError('Unable to login: Sorry, your password was incorrect. Please double-check your password.', expected=True)
+            elif login.get('user') is False:
+                raise ExtractorError('Unable to login: The username you entered doesn\'t belong to an account. Please check your username and try again.', expected=True)
+            raise ExtractorError('Unable to login')
+        InstagramBaseIE._IS_LOGGED_IN = True
+
+    def _real_initialize(self):
+        self._login()
+
+    def _get_count(self, media, kind, *keys):
+        return traverse_obj(
+            media, (kind, 'count'), *((f'edge_media_{key}', 'count') for key in keys),
+            expected_type=int_or_none)
+
+    def _get_dimension(self, name, media, webpage=None):
+        return (
+            traverse_obj(media, ('dimensions', name), expected_type=int_or_none)
+            or int_or_none(self._html_search_meta(
+                (f'og:video:{name}', f'video:{name}'), webpage or '', default=None)))
+
+    def _extract_nodes(self, nodes, is_direct=False):
+        for idx, node in enumerate(nodes, start=1):
+            if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
+                continue
+
+            video_id = node.get('shortcode')
+
+            if is_direct:
+                info = {
+                    'id': video_id or node['id'],
+                    'url': node.get('video_url'),
+                    'width': self._get_dimension('width', node),
+                    'height': self._get_dimension('height', node),
+                    'http_headers': {
+                        'Referer': 'https://www.instagram.com/',
+                    }
+                }
+            elif not video_id:
+                continue
+            else:
+                info = {
+                    '_type': 'url',
+                    'ie_key': 'Instagram',
+                    'id': video_id,
+                    'url': f'https://instagram.com/p/{video_id}',
+                }
+
+            yield {
+                **info,
+                'title': node.get('title') or (f'Video {idx}' if is_direct else None),
+                'description': traverse_obj(
+                    node, ('edge_media_to_caption', 'edges', 0, 'node', 'text'), expected_type=str),
+                'thumbnail': traverse_obj(
+                    node, 'display_url', 'thumbnail_src', 'display_src', expected_type=url_or_none),
+                'duration': float_or_none(node.get('video_duration')),
+                'timestamp': int_or_none(node.get('taken_at_timestamp')),
+                'view_count': int_or_none(node.get('video_view_count')),
+                'comment_count': self._get_count(node, 'comments', 'preview_comment', 'to_comment', 'to_parent_comment'),
+                'like_count': self._get_count(node, 'likes', 'preview_like'),
+            }
+
+
+class InstagramIOSIE(InfoExtractor):
+    IE_DESC = 'IOS instagram:// URL'
+    _VALID_URL = r'instagram://media\?id=(?P<id>[\d_]+)'
+    _TESTS = [{
+        'url': 'instagram://media?id=482584233761418119',
+        'md5': '0d2da106a9d2631273e192b372806516',
+        'info_dict': {
+            'id': 'aye83DjauH',
+            'ext': 'mp4',
+            'title': 'Video by naomipq',
+            'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 0,
+            'timestamp': 1371748545,
+            'upload_date': '20130620',
+            'uploader_id': 'naomipq',
+            'uploader': 'B E A U T Y  F O R  A S H E S',
+            'like_count': int,
+            'comment_count': int,
+            'comments': list,
+        },
+        'add_ie': ['Instagram']
+    }]
+
+    def _get_id(self, id):
+        """Source: https://stackoverflow.com/questions/24437823/getting-instagram-post-url-from-media-id"""
+        chrs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+        media_id = int(id.split('_')[0])
+        shortened_id = ''
+        while media_id > 0:
+            r = media_id % 64
+            media_id = (media_id - r) // 64
+            shortened_id = chrs[r] + shortened_id
+        return shortened_id
+
+    def _real_extract(self, url):
+        return {
+            '_type': 'url_transparent',
+            'url': f'http://instagram.com/tv/{self._get_id(self._match_id(url))}/',
+            'ie_key': 'Instagram',
+        }
+
+
+class InstagramIE(InstagramBaseIE):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
@@ -143,69 +286,23 @@ class InstagramIE(InfoExtractor):
         if mobj:
             return mobj.group('link')
 
-    def _login(self):
-        username, password = self._get_login_info()
-
-        login_webpage = self._download_webpage(
-            'https://www.instagram.com/accounts/login/', None,
-            note='Downloading login webpage', errnote='Failed to download login webpage')
-
-        shared_data = self._parse_json(
-            self._search_regex(
-                r'window\._sharedData\s*=\s*({.+?});',
-                login_webpage, 'shared data', default='{}'),
-            None)
-
-        login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
-            'Accept': '*/*',
-            'X-IG-App-ID': '936619743392459',
-            'X-ASBD-ID': '198387',
-            'X-IG-WWW-Claim': '0',
-            'X-Requested-With': 'XMLHttpRequest',
-            'X-CSRFToken': shared_data['config']['csrf_token'],
-            'X-Instagram-AJAX': shared_data['rollout_hash'],
-            'Referer': 'https://www.instagram.com/',
-        }, data=urlencode_postdata({
-            'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
-            'username': username,
-            'queryParams': '{}',
-            'optIntoOneTap': 'false',
-            'stopDeletionNonce': '',
-            'trustedDeviceRecords': '{}',
-        }))
-
-        if not login.get('authenticated'):
-            if login.get('message'):
-                raise ExtractorError(f'Unable to login: {login["message"]}')
-            raise ExtractorError('Unable to login')
-
-    def _real_initialize(self):
-        self._login()
-
     def _real_extract(self, url):
-        mobj = self._match_valid_url(url)
+        video_id, url = self._match_valid_url(url).group('id', 'url')
-        video_id = mobj.group('id')
-        url = mobj.group('url')
-
         webpage, urlh = self._download_webpage_handle(url, video_id)
-        if 'www.instagram.com/accounts/login' in urlh.geturl().rstrip('/'):
+        if 'www.instagram.com/accounts/login' in urlh.geturl():
             self.raise_login_required('You need to log in to access this content')
 
-        (media, video_url, description, thumbnail, timestamp, uploader,
-         uploader_id, like_count, comment_count, comments, height,
-         width) = [None] * 12
-
         shared_data = self._parse_json(
             self._search_regex(
                 r'window\._sharedData\s*=\s*({.+?});',
                 webpage, 'shared data', default='{}'),
             video_id, fatal=False)
-        if shared_data:
+        media = traverse_obj(
-            media = try_get(
+            shared_data,
-                shared_data,
+            ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
-                (lambda x: x['entry_data']['PostPage'][0]['graphql']['shortcode_media'],
+            ('entry_data', 'PostPage', 0, 'media'),
-                 lambda x: x['entry_data']['PostPage'][0]['media']),
+            expected_type=dict)
-                dict)
+
         # _sharedData.entry_data.PostPage is empty when authenticated (see
         # https://github.com/ytdl-org/youtube-dl/pull/22880)
         if not media:
@@ -214,123 +311,78 @@ class InstagramIE(InfoExtractor):
                     r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*({.+?})\s*\)\s*;',
                     webpage, 'additional data', default='{}'),
                 video_id, fatal=False)
-            if additional_data:
+            media = traverse_obj(additional_data, ('graphql', 'shortcode_media'), expected_type=dict) or {}
-                media = try_get(
-                    additional_data, lambda x: x['graphql']['shortcode_media'],
-                    dict)
-        if media:
-            video_url = media.get('video_url')
-            height = int_or_none(media.get('dimensions', {}).get('height'))
-            width = int_or_none(media.get('dimensions', {}).get('width'))
-            description = try_get(
-                media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
-                compat_str) or media.get('caption')
-            title = media.get('title')
-            thumbnail = media.get('display_src') or media.get('display_url')
-            duration = float_or_none(media.get('video_duration'))
-            timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
-            uploader = media.get('owner', {}).get('full_name')
-            uploader_id = media.get('owner', {}).get('username')
 
-            def get_count(keys, kind):
+        uploader_id = traverse_obj(media, ('owner', 'username')) or self._search_regex(
-                for key in variadic(keys):
+            r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', webpage, 'uploader id', fatal=False)
-                    count = int_or_none(try_get(
-                        media, (lambda x: x['edge_media_%s' % key]['count'],
-                                lambda x: x['%ss' % kind]['count'])))
-                    if count is not None:
-                        return count
-
-            like_count = get_count('preview_like', 'like')
-            comment_count = get_count(
-                ('preview_comment', 'to_comment', 'to_parent_comment'), 'comment')
-
-            comments = []
-            for comment in try_get(media, lambda x: x['edge_media_to_parent_comment']['edges']):
-                comment_dict = comment.get('node', {})
-                comment_text = comment_dict.get('text')
-                if comment_text:
-                    comments.append({
-                        'author': try_get(comment_dict, lambda x: x['owner']['username']),
-                        'author_id': try_get(comment_dict, lambda x: x['owner']['id']),
-                        'id': comment_dict.get('id'),
-                        'text': comment_text,
-                        'timestamp': int_or_none(comment_dict.get('created_at')),
-                    })
-            if not video_url:
-                edges = try_get(
-                    media, lambda x: x['edge_sidecar_to_children']['edges'],
-                    list) or []
-                if edges:
-                    entries = []
-                    for edge_num, edge in enumerate(edges, start=1):
-                        node = try_get(edge, lambda x: x['node'], dict)
-                        if not node:
-                            continue
-                        node_video_url = url_or_none(node.get('video_url'))
-                        if not node_video_url:
-                            continue
-                        entries.append({
-                            'id': node.get('shortcode') or node['id'],
-                            'title': node.get('title') or 'Video %d' % edge_num,
-                            'url': node_video_url,
-                            'thumbnail': node.get('display_url'),
-                            'duration': float_or_none(node.get('video_duration')),
-                            'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])),
-                            'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])),
-                            'view_count': int_or_none(node.get('video_view_count')),
-                        })
-                    return self.playlist_result(
-                        entries, video_id,
-                        'Post by %s' % uploader_id if uploader_id else None,
-                        description)
-
-        if not video_url:
-            video_url = self._og_search_video_url(webpage, secure=False)
-
-        formats = [{
-            'url': video_url,
-            'width': width,
-            'height': height,
-        }]
-
-        if not uploader_id:
-            uploader_id = self._search_regex(
-                r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
-                webpage, 'uploader id', fatal=False)
 
+        description = (
+            traverse_obj(media, ('edge_media_to_caption', 'edges', 0, 'node', 'text'), expected_type=str)
+            or media.get('caption'))
         if not description:
             description = self._search_regex(
                 r'"caption"\s*:\s*"(.+?)"', webpage, 'description', default=None)
             if description is not None:
                 description = lowercase_escape(description)
 
-        if not thumbnail:
+        video_url = media.get('video_url')
-            thumbnail = self._og_search_thumbnail(webpage)
+        if not video_url:
+            nodes = traverse_obj(media, ('edge_sidecar_to_children', 'edges', ..., 'node'), expected_type=dict) or []
+            if nodes:
+                return self.playlist_result(
+                    self._extract_nodes(nodes, True), video_id,
+                    'Post by %s' % uploader_id if uploader_id else None, description)
+
+            video_url = self._og_search_video_url(webpage, secure=False)
+
+        formats = [{
+            'url': video_url,
+            'width': self._get_dimension('width', media, webpage),
+            'height': self._get_dimension('height', media, webpage),
+        }]
+        dash = traverse_obj(media, ('dash_info', 'video_dash_manifest'))
+        if dash:
+            formats.extend(self._parse_mpd_formats(self._parse_xml(dash, video_id), mpd_id='dash'))
+        self._sort_formats(formats)
+
+        comments = [{
+            'author': traverse_obj(comment_dict, ('node', 'owner', 'username')),
+            'author_id': traverse_obj(comment_dict, ('node', 'owner', 'id')),
+            'id': traverse_obj(comment_dict, ('node', 'id')),
+            'text': traverse_obj(comment_dict, ('node', 'text')),
+            'timestamp': traverse_obj(comment_dict, ('node', 'created_at'), expected_type=int_or_none),
+        } for comment_dict in traverse_obj(media, ('edge_media_to_parent_comment', 'edges'))]
+
+        display_resources = (
+            media.get('display_resources')
+            or [{'src': media.get(key)} for key in ('display_src', 'display_url')]
+            or [{'src': self._og_search_thumbnail(webpage)}])
+        thumbnails = [{
+            'url': thumbnail['src'],
+            'width': thumbnail.get('config_width'),
+            'height': thumbnail.get('config_height'),
+        } for thumbnail in display_resources if thumbnail.get('src')]
 
         return {
             'id': video_id,
             'formats': formats,
-            'ext': 'mp4',
+            'title': media.get('title') or 'Video by %s' % uploader_id,
-            'title': title or 'Video by %s' % uploader_id,
             'description': description,
-            'duration': duration,
+            'duration': float_or_none(media.get('video_duration')),
-            'thumbnail': thumbnail,
+            'timestamp': traverse_obj(media, 'taken_at_timestamp', 'date', expected_type=int_or_none),
-            'timestamp': timestamp,
             'uploader_id': uploader_id,
-            'uploader': uploader,
+            'uploader': traverse_obj(media, ('owner', 'full_name')),
-            'like_count': like_count,
+            'like_count': self._get_count(media, 'likes', 'preview_like'),
-            'comment_count': comment_count,
+            'comment_count': self._get_count(media, 'comments', 'preview_comment', 'to_comment', 'to_parent_comment'),
             'comments': comments,
+            'thumbnails': thumbnails,
             'http_headers': {
                 'Referer': 'https://www.instagram.com/',
             }
         }
 
 
-class InstagramPlaylistIE(InfoExtractor):
+class InstagramPlaylistBaseIE(InstagramBaseIE):
-    # A superclass for handling any kind of query based on GraphQL which
-    # results in a playlist.
-
     _gis_tmpl = None  # used to cache GIS request type
 
     def _parse_graphql(self, webpage, item_id):
@@ -342,10 +394,6 @@ class InstagramPlaylistIE(InfoExtractor):
 
     def _extract_graphql(self, data, url):
         # Parses GraphQL queries containing videos and generates a playlist.
-        def get_count(suffix):
-            return int_or_none(try_get(
-                node, lambda x: x['edge_media_' + suffix]['count']))
-
         uploader_id = self._match_id(url)
         csrf_token = data['config']['csrf_token']
         rhx_gis = data.get('rhx_gis') or '3c7ca9dcefcf966d11dacf1f151335e8'
@@ -394,55 +442,14 @@ class InstagramPlaylistIE(InfoExtractor):
                             continue
                     raise
 
-            edges = media.get('edges')
+            nodes = traverse_obj(media, ('edges', ..., 'node'), expected_type=dict) or []
-            if not edges or not isinstance(edges, list):
+            if not nodes:
                 break
+            yield from self._extract_nodes(nodes)
 
-            for edge in edges:
+            has_next_page = traverse_obj(media, ('page_info', 'has_next_page'))
-                node = edge.get('node')
+            cursor = traverse_obj(media, ('page_info', 'end_cursor'), expected_type=str)
-                if not node or not isinstance(node, dict):
+            if not has_next_page or not cursor:
-                    continue
-                if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
-                    continue
-                video_id = node.get('shortcode')
-                if not video_id:
-                    continue
-
-                info = self.url_result(
-                    'https://instagram.com/p/%s/' % video_id,
-                    ie=InstagramIE.ie_key(), video_id=video_id)
-
-                description = try_get(
-                    node, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
-                    compat_str)
-                thumbnail = node.get('thumbnail_src') or node.get('display_src')
-                timestamp = int_or_none(node.get('taken_at_timestamp'))
-
-                comment_count = get_count('to_comment')
-                like_count = get_count('preview_like')
-                view_count = int_or_none(node.get('video_view_count'))
-
-                info.update({
-                    'description': description,
-                    'thumbnail': thumbnail,
-                    'timestamp': timestamp,
-                    'comment_count': comment_count,
-                    'like_count': like_count,
-                    'view_count': view_count,
-                })
-
-                yield info
-
-            page_info = media.get('page_info')
-            if not page_info or not isinstance(page_info, dict):
-                break
-
-            has_next_page = page_info.get('has_next_page')
-            if not has_next_page:
-                break
-
-            cursor = page_info.get('end_cursor')
-            if not cursor or not isinstance(cursor, compat_str):
                 break
 
     def _real_extract(self, url):
@@ -456,11 +463,11 @@ class InstagramPlaylistIE(InfoExtractor):
             self._extract_graphql(data, url), user_or_tag, user_or_tag)
 
 
-class InstagramUserIE(InstagramPlaylistIE):
+class InstagramUserIE(InstagramPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/porsche',
         'info_dict': {
             'id': 'porsche',
@@ -472,7 +479,7 @@ class InstagramUserIE(InstagramPlaylistIE):
             'skip_download': True,
             'playlistend': 5,
         }
-    }
+    }]
 
     _QUERY_HASH = '42323d64886122307be10013ad2dcc44',
 
@@ -490,11 +497,11 @@ class InstagramUserIE(InstagramPlaylistIE):
         }
 
 
-class InstagramTagIE(InstagramPlaylistIE):
+class InstagramTagIE(InstagramPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)'
-    IE_DESC = 'Instagram hashtag search'
+    IE_DESC = 'Instagram hashtag search URLs'
     IE_NAME = 'instagram:tag'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/explore/tags/lolcats',
         'info_dict': {
             'id': 'lolcats',
@@ -506,7 +513,7 @@ class InstagramTagIE(InstagramPlaylistIE):
             'skip_download': True,
             'playlistend': 50,
         }
-    }
+    }]
 
     _QUERY_HASH = 'f92f56d47dc7a55b606908374b43a314',
 

yt_dlp/extractor/internazionale.py

@@ -20,9 +20,6 @@ class InternazionaleIE(InfoExtractor):
             'upload_date': '20150219',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'https://www.internazionale.it/video/2018/08/29/telefono-stare-con-noi-stessi',
         'md5': '9db8663704cab73eb972d1cee0082c79',
@@ -36,9 +33,6 @@ class InternazionaleIE(InfoExtractor):
             'upload_date': '20180829',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/iprima.py

@@ -8,12 +8,19 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     js_to_json,
+    urlencode_postdata,
+    ExtractorError,
+    parse_qs
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
+    _NETRC_MACHINE = 'iprima'
+    _LOGIN_URL = 'https://auth.iprima.cz/oauth2/login'
+    _TOKEN_URL = 'https://auth.iprima.cz/oauth2/token'
+    access_token = None
 
     _TESTS = [{
         'url': 'https://prima.iprima.cz/particka/92-epizoda',
@@ -22,16 +29,8 @@ class IPrimaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Partička (92)',
             'description': 'md5:859d53beae4609e6dd7796413f1b6cac',
-        },
+            'upload_date': '20201103',
-        'params': {
+            'timestamp': 1604437480,
-            'skip_download': True,  # m3u8 download
-        },
-    }, {
-        'url': 'https://cnn.iprima.cz/videa/70-epizoda',
-        'info_dict': {
-            'id': 'p681554',
-            'ext': 'mp4',
-            'title': 'HLAVNÍ ZPRÁVY 3.5.2020',
         },
         'params': {
             'skip_download': True,  # m3u8 download
@@ -44,11 +43,9 @@ class IPrimaIE(InfoExtractor):
         'url': 'http://play.iprima.cz/closer-nove-pripady/closer-nove-pripady-iv-1',
         'only_matching': True,
     }, {
-        # iframe api.play-backend.iprima.cz
         'url': 'https://prima.iprima.cz/my-little-pony/mapa-znameni-2-2',
         'only_matching': True,
     }, {
-        # iframe prima.iprima.cz
         'url': 'https://prima.iprima.cz/porady/jak-se-stavi-sen/rodina-rathousova-praha',
         'only_matching': True,
     }, {
@@ -66,9 +63,127 @@ class IPrimaIE(InfoExtractor):
     }, {
         'url': 'https://love.iprima.cz/laska-az-za-hrob/slib-dany-bratrovi',
         'only_matching': True,
-    }, {
+    }]
-        'url': 'https://autosalon.iprima.cz/motorsport/7-epizoda-1',
+
-        'only_matching': True,
+    def _login(self):
+        username, password = self._get_login_info()
+
+        if username is None or password is None:
+            self.raise_login_required('Login is required to access any iPrima content', method='password')
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, note='Downloading login page',
+            errnote='Downloading login page failed')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            '_email': username,
+            '_password': password})
+
+        _, login_handle = self._download_webpage_handle(
+            self._LOGIN_URL, None, data=urlencode_postdata(login_form),
+            note='Logging in')
+
+        code = parse_qs(login_handle.geturl()).get('code')[0]
+        if not code:
+            raise ExtractorError('Login failed', expected=True)
+
+        token_request_data = {
+            'scope': 'openid+email+profile+phone+address+offline_access',
+            'client_id': 'prima_sso',
+            'grant_type': 'authorization_code',
+            'code': code,
+            'redirect_uri': 'https://auth.iprima.cz/sso/auth-check'}
+
+        token_data = self._download_json(
+            self._TOKEN_URL, None,
+            note='Downloading token', errnote='Downloading token failed',
+            data=urlencode_postdata(token_request_data))
+
+        self.access_token = token_data.get('access_token')
+        if self.access_token is None:
+            raise ExtractorError('Getting token failed', expected=True)
+
+    def _raise_access_error(self, error_code):
+        if error_code == 'PLAY_GEOIP_DENIED':
+            self.raise_geo_restricted(countries=['CZ'], metadata_available=True)
+        elif error_code is not None:
+            self.raise_no_formats('Access to stream infos forbidden', expected=True)
+
+    def _real_initialize(self):
+        if not self.access_token:
+            self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title'],
+            webpage, 'title', default=None)
+
+        video_id = self._search_regex((
+            r'productId\s*=\s*([\'"])(?P<id>p\d+)\1',
+            r'pproduct_id\s*=\s*([\'"])(?P<id>p\d+)\1'),
+            webpage, 'real id', group='id')
+
+        metadata = self._download_json(
+            f'https://api.play-backend.iprima.cz/api/v1//products/id-{video_id}/play',
+            video_id, note='Getting manifest URLs', errnote='Failed to get manifest URLs',
+            headers={'X-OTT-Access-Token': self.access_token},
+            expected_status=403)
+
+        self._raise_access_error(metadata.get('errorCode'))
+
+        stream_infos = metadata.get('streamInfos')
+        formats = []
+        if stream_infos is None:
+            self.raise_no_formats('Reading stream infos failed', expected=True)
+        else:
+            for manifest in stream_infos:
+                manifest_type = manifest.get('type')
+                manifest_url = manifest.get('url')
+                ext = determine_ext(manifest_url)
+                if manifest_type == 'HLS' or ext == 'm3u8':
+                    formats += self._extract_m3u8_formats(
+                        manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False)
+                elif manifest_type == 'DASH' or ext == 'mpd':
+                    formats += self._extract_mpd_formats(
+                        manifest_url, video_id, mpd_id='dash', fatal=False)
+            self._sort_formats(formats)
+
+        final_result = self._search_json_ld(webpage, video_id) or {}
+        final_result.update({
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._html_search_meta(
+                ['thumbnail', 'og:image', 'twitter:image'],
+                webpage, 'thumbnail', default=None),
+            'formats': formats,
+            'description': self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'],
+                webpage, 'description', default=None)})
+
+        return final_result
+
+
+class IPrimaCNNIE(InfoExtractor):
+    _VALID_URL = r'https?://cnn\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _GEO_BYPASS = False
+
+    _TESTS = [{
+        'url': 'https://cnn.iprima.cz/porady/strunc/24072020-koronaviru-mam-plne-zuby-strasit-druhou-vlnou-je-absurdni-rika-senatorka-dernerova',
+        'info_dict': {
+            'id': 'p716177',
+            'ext': 'mp4',
+            'title': 'md5:277c6b1ed0577e51b40ddd35602ff43e',
+        },
+        'params': {
+            'skip_download': 'm3u8'
+        }
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/itv.py

@@ -117,7 +117,7 @@ class ITVIE(InfoExtractor):
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_subs, featureset_subs = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
+             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
              if try_get(featureset, lambda x: x[2]) == 'outband-webvtt'),
             (None, None))
 
@@ -146,8 +146,8 @@ class ITVIE(InfoExtractor):
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_video, featureset_video = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
+             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
-             if try_get(featureset, lambda x: x[:2]) == ['hls', 'aes']),
+             if set(try_get(featureset, lambda x: x[:2]) or []) == {'aes', 'hls'}),
             (None, None))
         if not platform_tag_video or not featureset_video:
             raise ExtractorError('No downloads available', expected=True, video_id=video_id)
@@ -220,16 +220,23 @@ class ITVIE(InfoExtractor):
 
 
 class ITVBTCCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:news|btcc)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action',
         'info_dict': {
             'id': 'btcc-2019-brands-hatch-gp-race-action',
             'title': 'BTCC 2019: Brands Hatch GP race action',
         },
         'playlist_count': 12,
-    }
+    }, {
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
+        'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+        'info_dict': {
+            'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+            'title': 'md5:6ef054dd9f069330db3dcc66cb772d32'
+        },
+        'playlist_count': 4
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -240,15 +247,15 @@ class ITVBTCCIE(InfoExtractor):
             '(?s)<script[^>]+id=[\'"]__NEXT_DATA__[^>]*>([^<]+)</script>', webpage, 'json_map'), playlist_id),
             lambda x: x['props']['pageProps']['article']['body']['content']) or []
 
-        # Discard empty objects
+        entries = []
-        video_ids = []
         for video in json_map:
-            if video['data'].get('id'):
+            if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')):
-                video_ids.append(video['data']['id'])
+                continue
-
+            video_id = video['data']['id']
-        entries = [
+            account_id = video['data']['accountId']
-            self.url_result(
+            player_id = video['data']['playerId']
-                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
+            entries.append(self.url_result(
+                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id), {
                     # ITV does not like some GB IP ranges, so here are some
                     # IP blocks it accepts
                     'geo_ip_blocks': [
@@ -256,8 +263,7 @@ class ITVBTCCIE(InfoExtractor):
                     ],
                     'referrer': url,
                 }),
-                ie=BrightcoveNewIE.ie_key(), video_id=video_id)
+                ie=BrightcoveNewIE.ie_key(), video_id=video_id))
-            for video_id in video_ids]
 
         title = self._og_search_title(webpage, fatal=False)
 

yt_dlp/extractor/kinopoisk.py

@@ -23,9 +23,6 @@ class KinoPoiskIE(InfoExtractor):
             'duration': 4533,
             'age_limit': 12,
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'https://www.kinopoisk.ru/film/81041',
         'only_matching': True,

yt_dlp/extractor/koo.py

@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class KooIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?kooapp\.com/koo/[^/]+/(?P<id>[^/&#$?]+)'
+    _VALID_URL = r'https?://(?:www\.)?kooapp\.com/koo/[^/]+/(?P<id>[^/&#$?]+)'
     _TESTS = [{  # Test for video in the comments
         'url': 'https://www.kooapp.com/koo/ytdlpTestAccount/946c4189-bc2d-4524-b95b-43f641e2adde',
         'info_dict': {

yt_dlp/extractor/la7.py

@@ -7,8 +7,9 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     float_or_none,
+    HEADRequest,
+    int_or_none,
     parse_duration,
-    smuggle_url,
     unified_strdate,
 )
 
@@ -25,19 +26,38 @@ class LA7IE(InfoExtractor):
         'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
         'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
         'info_dict': {
-            'id': '0_42j6wd36',
+            'id': 'inccool8-02-10-2015-163722',
             'ext': 'mp4',
             'title': 'Inc.Cool8',
             'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
             'thumbnail': 're:^https?://.*',
-            'uploader_id': 'kdla7pillole@iltrovatore.it',
-            'timestamp': 1443814869,
             'upload_date': '20151002',
         },
     }, {
         'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
         'only_matching': True,
     }]
+    _HOST = 'https://awsvodpkg.iltrovatore.it'
+
+    def _generate_mp4_url(self, quality, m3u8_formats):
+        for f in m3u8_formats:
+            if f['vcodec'] != 'none' and quality in f['url']:
+                http_url = '%s%s.mp4' % (self._HOST, quality)
+
+                urlh = self._request_webpage(
+                    HEADRequest(http_url), quality,
+                    note='Check filesize', fatal=False)
+                if urlh:
+                    http_f = f.copy()
+                    del http_f['manifest_url']
+                    http_f.update({
+                        'format_id': http_f['format_id'].replace('hls-', 'https-'),
+                        'url': http_url,
+                        'protocol': 'https',
+                        'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
+                    })
+                    return http_f
+                return None
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -46,22 +66,30 @@ class LA7IE(InfoExtractor):
             url = '%s//%s' % (self.http_scheme(), url)
 
         webpage = self._download_webpage(url, video_id)
+        video_path = self._search_regex(r'(/content/.*?).mp4', webpage, 'video_path')
 
-        player_data = self._search_regex(
+        formats = self._extract_mpd_formats(
-            [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
+            f'{self._HOST}/local/dash/,{video_path}.mp4.urlset/manifest.mpd',
-            webpage, 'player data')
+            video_id, mpd_id='dash', fatal=False)
-        vid = self._search_regex(r'vid\s*:\s*"(.+?)",', player_data, 'vid')
+        m3u8_formats = self._extract_m3u8_formats(
+            f'{self._HOST}/local/hls/,{video_path}.mp4.urlset/master.m3u8',
+            video_id, 'mp4', m3u8_id='hls', fatal=False)
+        formats.extend(m3u8_formats)
+
+        for q in filter(None, video_path.split(',')):
+            http_f = self._generate_mp4_url(q, m3u8_formats)
+            if http_f:
+                formats.append(http_f)
+
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'url': smuggle_url('kaltura:103:%s' % vid, {
-                'service_url': 'http://nkdam.iltrovatore.it',
-            }),
             'id': video_id,
             'title': self._og_search_title(webpage, default=None),
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
-            'ie_key': 'Kaltura',
+            'formats': formats,
+            'upload_date': unified_strdate(self._search_regex(r'datetime="(.+?)"', webpage, 'upload_date', fatal=False))
         }
 
 

yt_dlp/extractor/lego.py

@@ -8,6 +8,7 @@ from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
+    join_nonempty,
     qualities,
 )
 
@@ -102,12 +103,8 @@ class LEGOIE(InfoExtractor):
                     m3u8_id=video_source_format, fatal=False))
             else:
                 video_source_quality = video_source.get('Quality')
-                format_id = []
-                for v in (video_source_format, video_source_quality):
-                    if v:
-                        format_id.append(v)
                 f = {
-                    'format_id': '-'.join(format_id),
+                    'format_id': join_nonempty(video_source_format, video_source_quality),
                     'quality': q(video_source_quality),
                     'url': video_source_url,
                 }

yt_dlp/extractor/linkedin.py

@@ -6,50 +6,28 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    extract_attributes,
     ExtractorError,
     float_or_none,
+    get_element_by_class,
     int_or_none,
     srt_subtitles_timecode,
+    strip_or_none,
+    mimetype2ext,
     try_get,
     urlencode_postdata,
     urljoin,
 )
 
 
-class LinkedInLearningBaseIE(InfoExtractor):
+class LinkedInBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'linkedin'
-    _LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
+    _logged_in = False
-
-    def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
-        query = {
-            'courseSlug': course_slug,
-            'fields': fields,
-            'q': 'slugs',
-        }
-        sub = ''
-        if video_slug:
-            query.update({
-                'videoSlug': video_slug,
-                'resolution': '_%s' % resolution,
-            })
-            sub = ' %dp' % resolution
-        api_url = 'https://www.linkedin.com/learning-api/detailedCourses'
-        return self._download_json(
-            api_url, video_slug, 'Downloading%s JSON metadata' % sub, headers={
-                'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
-            }, query=query)['elements'][0]
-
-    def _get_urn_id(self, video_data):
-        urn = video_data.get('urn')
-        if urn:
-            mobj = re.search(r'urn:li:lyndaCourse:\d+,(\d+)', urn)
-            if mobj:
-                return mobj.group(1)
-
-    def _get_video_id(self, video_data, course_slug, video_slug):
-        return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
 
     def _real_initialize(self):
+        if self._logged_in:
+            return
         email, password = self._get_login_info()
         if email is None:
             return
@@ -72,6 +50,84 @@ class LinkedInLearningBaseIE(InfoExtractor):
             login_submit_page, 'error', default=None)
         if error:
             raise ExtractorError(error, expected=True)
+        LinkedInBaseIE._logged_in = True
+
+
+class LinkedInLearningBaseIE(LinkedInBaseIE):
+    _LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
+
+    def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
+        query = {
+            'courseSlug': course_slug,
+            'fields': fields,
+            'q': 'slugs',
+        }
+        sub = ''
+        if video_slug:
+            query.update({
+                'videoSlug': video_slug,
+                'resolution': '_%s' % resolution,
+            })
+            sub = ' %dp' % resolution
+        api_url = 'https://www.linkedin.com/learning-api/detailedCourses'
+        if not self._get_cookies(api_url).get('JSESSIONID'):
+            self.raise_login_required()
+        return self._download_json(
+            api_url, video_slug, 'Downloading%s JSON metadata' % sub, headers={
+                'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
+            }, query=query)['elements'][0]
+
+    def _get_urn_id(self, video_data):
+        urn = video_data.get('urn')
+        if urn:
+            mobj = re.search(r'urn:li:lyndaCourse:\d+,(\d+)', urn)
+            if mobj:
+                return mobj.group(1)
+
+    def _get_video_id(self, video_data, course_slug, video_slug):
+        return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
+
+
+class LinkedInIE(LinkedInBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/.+?(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.linkedin.com/posts/mishalkhawaja_sendinblueviews-toronto-digitalmarketing-ugcPost-6850898786781339649-mM20',
+        'info_dict': {
+            'id': '6850898786781339649',
+            'ext': 'mp4',
+            'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing',
+            'description': 'md5:be125430bab1c574f16aeb186a4d5b19',
+            'creator': 'Mishal K.'
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        description = clean_html(get_element_by_class('share-update-card__update-text', webpage))
+        like_count = int_or_none(get_element_by_class('social-counts-reactions__social-counts-numRections', webpage))
+        creator = strip_or_none(clean_html(get_element_by_class('comment__actor-name', webpage)))
+
+        sources = self._parse_json(extract_attributes(self._search_regex(r'(<video[^>]+>)', webpage, 'video'))['data-sources'], video_id)
+        formats = [{
+            'url': source['src'],
+            'ext': mimetype2ext(source.get('type')),
+            'tbr': float_or_none(source.get('data-bitrate'), scale=1000),
+        } for source in sources]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'like_count': like_count,
+            'creator': creator,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': description,
+        }
 
 
 class LinkedInLearningIE(LinkedInLearningBaseIE):
@@ -102,7 +158,6 @@ class LinkedInLearningIE(LinkedInLearningBaseIE):
     def _real_extract(self, url):
         course_slug, video_slug = self._match_valid_url(url).groups()
 
-        video_data = None
         formats = []
         for width, height in ((640, 360), (960, 540), (1280, 720)):
             video_data = self._call_api(

yt_dlp/extractor/mdr.py

@@ -2,13 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
+from ..compat import compat_urlparse
-    compat_str,
-    compat_urlparse,
-)
 from ..utils import (
     determine_ext,
     int_or_none,
+    join_nonempty,
     parse_duration,
     parse_iso8601,
     url_or_none,
@@ -148,13 +146,9 @@ class MDRIE(InfoExtractor):
                     abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
                     filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))
 
-                    format_id = [media_type]
-                    if vbr or abr:
-                        format_id.append(compat_str(vbr or abr))
-
                     f = {
                         'url': video_url,
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty(media_type, vbr or abr),
                         'filesize': filesize,
                         'abr': abr,
                         'vbr': vbr,

yt_dlp/extractor/mediaklikk.py

@@ -12,8 +12,8 @@ from ..compat import (
 
 
 class MediaKlikkIE(InfoExtractor):
-    _VALID_URL = r'''(?x)^https?:\/\/(?:www\.)?
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
-                        (?:mediaklikk|m4sport|hirado|petofilive)\.hu\/.*?videok?\/
+                        (?:mediaklikk|m4sport|hirado|petofilive)\.hu/.*?(?:videok?|cikk)/
                         (?:(?P<year>[0-9]{4})/(?P<month>[0-9]{1,2})/(?P<day>[0-9]{1,2})/)?
                         (?P<id>[^/#?_]+)'''
 

yt_dlp/extractor/mediaset.py

@@ -1,13 +1,17 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import re
 
 from .theplatform import ThePlatformBaseIE
 from ..utils import (
     ExtractorError,
     int_or_none,
+    OnDemandPagedList,
     parse_qs,
+    try_get,
+    urljoin,
     update_url_query,
 )
 
@@ -212,3 +216,81 @@ class MediasetIE(ThePlatformBaseIE):
             'subtitles': subtitles,
         })
         return info
+
+
+class MediasetShowIE(MediasetIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        https?://
+                            (?:(?:www|static3)\.)?mediasetplay\.mediaset\.it/
+                            (?:
+                                (?:fiction|programmi-tv|serie-tv)/(?:.+?/)?
+                                    (?:[a-z]+)_SE(?P<id>\d{12})
+                                    (?:,ST(?P<st>\d{12}))?
+                                    (?:,sb(?P<sb>\d{9}))?$
+                            )
+                    )
+                    '''
+    _TESTS = [{
+        # TV Show webpage (with a single playlist)
+        'url': 'https://www.mediasetplay.mediaset.it/serie-tv/fireforce/episodi_SE000000001556',
+        'info_dict': {
+            'id': '000000001556',
+            'title': 'Fire Force',
+        },
+        'playlist_count': 1,
+    }, {
+        # TV Show webpage (with multiple playlists)
+        'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061,ST000000002763',
+        'info_dict': {
+            'id': '000000002763',
+            'title': 'Le Iene',
+        },
+        'playlist_count': 7,
+    }, {
+        # TV Show specific playlist (single page)
+        'url': 'https://www.mediasetplay.mediaset.it/serie-tv/fireforce/episodi_SE000000001556,ST000000002738,sb100013107',
+        'info_dict': {
+            'id': '100013107',
+            'title': 'Episodi',
+        },
+        'playlist_count': 4,
+    }, {
+        # TV Show specific playlist (with multiple pages)
+        'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/iservizi_SE000000000061,ST000000002763,sb100013375',
+        'info_dict': {
+            'id': '100013375',
+            'title': 'I servizi',
+        },
+        'playlist_count': 53,
+    }]
+
+    _BY_SUBBRAND = 'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2?byCustomValue={subBrandId}{%s}&sort=:publishInfo_lastPublished|desc,tvSeasonEpisodeNumber|desc&range=%d-%d'
+    _PAGE_SIZE = 25
+
+    def _fetch_page(self, sb, page):
+        lower_limit = page * self._PAGE_SIZE + 1
+        upper_limit = lower_limit + self._PAGE_SIZE - 1
+        content = self._download_json(
+            self._BY_SUBBRAND % (sb, lower_limit, upper_limit), sb)
+        for entry in content.get('entries') or []:
+            yield self.url_result(
+                'mediaset:' + entry['guid'],
+                playlist_title=entry['mediasetprogram$subBrandDescription'])
+
+    def _real_extract(self, url):
+        playlist_id, st, sb = self._match_valid_url(url).group('id', 'st', 'sb')
+        if not sb:
+            page = self._download_webpage(url, playlist_id)
+            entries = [self.url_result(urljoin('https://www.mediasetplay.mediaset.it', url))
+                       for url in re.findall(r'href="([^<>=]+SE\d{12},ST\d{12},sb\d{9})">[^<]+<', page)]
+            title = (self._html_search_regex(r'(?s)<h1[^>]*>(.+?)</h1>', page, 'title', default=None)
+                     or self._og_search_title(page))
+            return self.playlist_result(entries, st or playlist_id, title)
+
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, sb),
+            self._PAGE_SIZE)
+        title = try_get(entries, lambda x: x[0]['playlist_title'])
+
+        return self.playlist_result(entries, sb, title)

yt_dlp/extractor/mixch.py

@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    traverse_obj,
+)
+
+
+class MixchIE(InfoExtractor):
+    IE_NAME = 'mixch'
+    _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P<id>\d+)'
+
+    TESTS = [{
+        'url': 'https://mixch.tv/u/16236849/live',
+        'skip': 'don\'t know if this live persists',
+        'info_dict': {
+            'id': '16236849',
+            'title': '24配信シェア⭕️投票🙏💦',
+            'comment_count': 13145,
+            'view_count': 28348,
+            'timestamp': 1636189377,
+            'uploader': '🦥伊咲👶🏻#フレアワ',
+            'uploader_id': '16236849',
+        }
+    }, {
+        'url': 'https://mixch.tv/u/16137876/live',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://mixch.tv/u/{video_id}/live', video_id)
+
+        initial_js_state = self._parse_json(self._search_regex(
+            r'(?m)^\s*window\.__INITIAL_JS_STATE__\s*=\s*(\{.+?\});\s*$', webpage, 'initial JS state'), video_id)
+        if not initial_js_state.get('liveInfo'):
+            raise ExtractorError('Livestream has ended.', expected=True)
+
+        return {
+            'id': video_id,
+            'title': traverse_obj(initial_js_state, ('liveInfo', 'title')),
+            'comment_count': traverse_obj(initial_js_state, ('liveInfo', 'comments')),
+            'view_count': traverse_obj(initial_js_state, ('liveInfo', 'visitor')),
+            'timestamp': traverse_obj(initial_js_state, ('liveInfo', 'created')),
+            'uploader': traverse_obj(initial_js_state, ('broadcasterInfo', 'name')),
+            'uploader_id': video_id,
+            'formats': [{
+                'format_id': 'hls',
+                'url': traverse_obj(initial_js_state, ('liveInfo', 'hls')) or 'https://d1hd0ww6piyb43.cloudfront.net/hls/torte_%s.m3u8' % video_id,
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+            'is_live': True,
+        }

yt_dlp/extractor/mlssoccer.py

@@ -0,0 +1,117 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MLSSoccerIE(InfoExtractor):
+    _VALID_DOMAINS = r'(?:(?:cfmontreal|intermiamicf|lagalaxy|lafc|houstondynamofc|dcunited|atlutd|mlssoccer|fcdallas|columbuscrew|coloradorapids|fccincinnati|chicagofirefc|austinfc|nashvillesc|whitecapsfc|sportingkc|soundersfc|sjearthquakes|rsl|timbers|philadelphiaunion|orlandocitysc|newyorkredbulls|nycfc)\.com|(?:torontofc)\.ca|(?:revolutionsoccer)\.net)'
+    _VALID_URL = r'https?://(?:www\.)?%s/video/#?(?P<id>[^/&$#?]+)' % _VALID_DOMAINS
+
+    _TESTS = [{
+        'url': 'https://www.mlssoccer.com/video/the-octagon-can-alphonso-davies-lead-canada-to-first-world-cup-since-1986#the-octagon-can-alphonso-davies-lead-canada-to-first-world-cup-since-1986',
+        'info_dict': {
+            'id': '6276033198001',
+            'ext': 'mp4',
+            'title': 'The Octagon | Can Alphonso Davies lead Canada to first World Cup since 1986?',
+            'description': 'md5:f0a883ee33592a0221798f451a98be8f',
+            'thumbnail': 'https://cf-images.us-east-1.prod.boltdns.net/v1/static/5530036772001/1bbc44f6-c63c-4981-82fa-46b0c1f891e0/5c1ca44a-a033-4e98-b531-ff24c4947608/160x90/match/image.jpg',
+            'duration': 350.165,
+            'timestamp': 1633627291,
+            'uploader_id': '5530036772001',
+            'tags': ['club/canada'],
+            'is_live': False,
+            'upload_date': '20211007',
+            'filesize_approx': 255193528.83200002
+        },
+        'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.whitecapsfc.com/video/highlights-san-jose-earthquakes-vs-vancouver-whitecaps-fc-october-23-2021#highlights-san-jose-earthquakes-vs-vancouver-whitecaps-fc-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.torontofc.ca/video/highlights-toronto-fc-vs-cf-montreal-october-23-2021-x6733#highlights-toronto-fc-vs-cf-montreal-october-23-2021-x6733',
+        'only_matching': True
+    }, {
+        'url': 'https://www.sportingkc.com/video/post-match-press-conference-john-pulskamp-oct-27-2021#post-match-press-conference-john-pulskamp-oct-27-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.soundersfc.com/video/highlights-seattle-sounders-fc-vs-sporting-kansas-city-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.sjearthquakes.com/video/#highlights-austin-fc-vs-san-jose-earthquakes-june-19-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.rsl.com/video/2021-u-of-u-health-mic-d-up-vs-colorado-10-16-21#2021-u-of-u-health-mic-d-up-vs-colorado-10-16-21',
+        'only_matching': True
+    }, {
+        'url': 'https://www.timbers.com/video/highlights-d-chara-asprilla-with-goals-in-portland-timbers-2-0-win-over-san-jose#highlights-d-chara-asprilla-with-goals-in-portland-timbers-2-0-win-over-san-jose',
+        'only_matching': True
+    }, {
+        'url': 'https://www.philadelphiaunion.com/video/highlights-torvphi',
+        'only_matching': True
+    }, {
+        'url': 'https://www.orlandocitysc.com/video/highlight-columbus-crew-vs-orlando-city-sc',
+        'only_matching': True
+    }, {
+        'url': 'https://www.newyorkredbulls.com/video/all-access-matchday-double-derby-week#all-access-matchday-double-derby-week',
+        'only_matching': True
+    }, {
+        'url': 'https://www.nycfc.com/video/highlights-nycfc-1-0-chicago-fire-fc#highlights-nycfc-1-0-chicago-fire-fc',
+        'only_matching': True
+    }, {
+        'url': 'https://www.revolutionsoccer.net/video/two-minute-highlights-revs-1-rapids-0-october-27-2021#two-minute-highlights-revs-1-rapids-0-october-27-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.nashvillesc.com/video/goal-c-j-sapong-nashville-sc-92nd-minute',
+        'only_matching': True
+    }, {
+        'url': 'https://www.cfmontreal.com/video/faits-saillants-tor-v-mtl#faits-saillants-orl-v-mtl-x5645',
+        'only_matching': True
+    }, {
+        'url': 'https://www.intermiamicf.com/video/all-access-victory-vs-nashville-sc-by-ukg#all-access-victory-vs-nashville-sc-by-ukg',
+        'only_matching': True
+    }, {
+        'url': 'https://www.lagalaxy.com/video/#moment-of-the-month-presented-by-san-manuel-casino-rayan-raveloson-scores-his-se',
+        'only_matching': True
+    }, {
+        'url': 'https://www.lafc.com/video/breaking-down-lafc-s-final-6-matches-of-the-2021-mls-regular-season#breaking-down-lafc-s-final-6-matches-of-the-2021-mls-regular-season',
+        'only_matching': True
+    }, {
+        'url': 'https://www.houstondynamofc.com/video/postgame-press-conference-michael-nelson-presented-by-coushatta-casino-res-x9660#postgame-press-conference-michael-nelson-presented-by-coushatta-casino-res-x9660',
+        'only_matching': True
+    }, {
+        'url': 'https://www.dcunited.com/video/tony-alfaro-my-family-pushed-me-to-believe-everything-was-possible',
+        'only_matching': True
+    }, {
+        'url': 'https://www.fcdallas.com/video/highlights-fc-dallas-vs-minnesota-united-fc-october-02-2021#highlights-fc-dallas-vs-minnesota-united-fc-october-02-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.columbuscrew.com/video/match-rewind-columbus-crew-vs-new-york-red-bulls-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.coloradorapids.com/video/postgame-reaction-robin-fraser-october-27#postgame-reaction-robin-fraser-october-27',
+        'only_matching': True
+    }, {
+        'url': 'https://www.fccincinnati.com/video/#keeping-cincy-chill-presented-by-coors-lite',
+        'only_matching': True
+    }, {
+        'url': 'https://www.chicagofirefc.com/video/all-access-fire-score-dramatic-road-win-in-cincy#all-access-fire-score-dramatic-road-win-in-cincy',
+        'only_matching': True
+    }, {
+        'url': 'https://www.austinfc.com/video/highlights-colorado-rapids-vs-austin-fc-september-29-2021#highlights-colorado-rapids-vs-austin-fc-september-29-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.atlutd.com/video/goal-josef-martinez-scores-in-the-73rd-minute#goal-josef-martinez-scores-in-the-73rd-minute',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        data_json = self._parse_json(self._html_search_regex(r'data-options\=\"([^\"]+)\"', webpage, 'json'), id)['videoList'][0]
+        return {
+            'id': id,
+            '_type': 'url',
+            'url': 'https://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (data_json['accountId'], data_json['videoId']),
+            'ie_key': 'BrightcoveNew',
+        }

yt_dlp/extractor/mtv.py

@@ -15,6 +15,7 @@ from ..utils import (
     float_or_none,
     HEADRequest,
     int_or_none,
+    join_nonempty,
     RegexNotFoundError,
     sanitized_Request,
     strip_or_none,
@@ -99,9 +100,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
                     formats.extend([{
                         'ext': 'flv' if rtmp_video_url.startswith('rtmp') else ext,
                         'url': rtmp_video_url,
-                        'format_id': '-'.join(filter(None, [
+                        'format_id': join_nonempty(
                             'rtmp' if rtmp_video_url.startswith('rtmp') else None,
-                            rendition.get('bitrate')])),
+                            rendition.get('bitrate')),
                         'width': int(rendition.get('width')),
                         'height': int(rendition.get('height')),
                     }])
@@ -311,11 +312,17 @@ class MTVServicesInfoExtractor(InfoExtractor):
             main_container = self._extract_child_with_type(data, 'MainContainer')
             ab_testing = self._extract_child_with_type(main_container, 'ABTesting')
             video_player = self._extract_child_with_type(ab_testing or main_container, 'VideoPlayer')
-            mgid = video_player['props']['media']['video']['config']['uri']
+            if video_player:
+                mgid = try_get(video_player, lambda x: x['props']['media']['video']['config']['uri'])
+            else:
+                flex_wrapper = self._extract_child_with_type(ab_testing or main_container, 'FlexWrapper')
+                auth_suite_wrapper = self._extract_child_with_type(flex_wrapper, 'AuthSuiteWrapper')
+                player = self._extract_child_with_type(auth_suite_wrapper or flex_wrapper, 'Player')
+                if player:
+                    mgid = try_get(player, lambda x: x['props']['videoDetail']['mgid'])
 
         if not mgid:
-            mgid = self._search_regex(
+            raise ExtractorError('Could not extract mgid')
-                r'"media":{"video":{"config":{"uri":"(mgid:.*?)"', webpage, 'mgid', default=None)
 
         return mgid
 

yt_dlp/extractor/musescore.py

@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class MuseScoreIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?musescore\.com/(?:user/\d+|[^/]+)(?:/scores)?/(?P<id>[^#&?]+)'
+    _VALID_URL = r'https?://(?:www\.)?musescore\.com/(?:user/\d+|[^/]+)(?:/scores)?/(?P<id>[^#&?]+)'
     _TESTS = [{
         'url': 'https://musescore.com/user/73797/scores/142975',
         'info_dict': {
@@ -13,7 +13,7 @@ class MuseScoreIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'WA Mozart Marche Turque (Turkish March fingered)',
             'description': 'md5:7ede08230e4eaabd67a4a98bb54d07be',
-            'thumbnail': r're:(?:https?://)(?:www\.)?musescore\.com/.*\.png[^$]+',
+            'thumbnail': r're:https?://(?:www\.)?musescore\.com/.*\.png[^$]+',
             'uploader': 'PapyPiano',
             'creator': 'Wolfgang Amadeus Mozart',
         }
@@ -24,7 +24,7 @@ class MuseScoreIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Sweet Child O\' Mine  – Guns N\' Roses sweet child',
             'description': 'md5:4dca71191c14abc312a0a4192492eace',
-            'thumbnail': r're:(?:https?://)(?:www\.)?musescore\.com/.*\.png[^$]+',
+            'thumbnail': r're:https?://(?:www\.)?musescore\.com/.*\.png[^$]+',
             'uploader': 'roxbelviolin',
             'creator': 'Guns N´Roses Arr. Roxbel Violin',
         }
@@ -35,7 +35,7 @@ class MuseScoreIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Für Elise – Beethoven',
             'description': 'md5:49515a3556d5ecaf9fa4b2514064ac34',
-            'thumbnail': r're:(?:https?://)(?:www\.)?musescore\.com/.*\.png[^$]+',
+            'thumbnail': r're:https?://(?:www\.)?musescore\.com/.*\.png[^$]+',
             'uploader': 'ClassicMan',
             'creator': 'Ludwig van Beethoven (1770–1827)',
         }

yt_dlp/extractor/mxplayer.py

@@ -180,7 +180,7 @@ class MxplayerIE(InfoExtractor):
 
 
 class MxplayerShowIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?mxplayer\.in/show/(?P<display_id>[-\w]+)-(?P<id>\w+)/?(?:$|[#?])'
+    _VALID_URL = r'https?://(?:www\.)?mxplayer\.in/show/(?P<display_id>[-\w]+)-(?P<id>\w+)/?(?:$|[#?])'
     _TESTS = [{
         'url': 'https://www.mxplayer.in/show/watch-chakravartin-ashoka-samrat-series-online-a8f44e3cc0814b5601d17772cedf5417',
         'playlist_mincount': 440,

yt_dlp/extractor/n1.py

@@ -3,8 +3,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .youtube import YoutubeIE
-from .reddit import RedditRIE
 from .common import InfoExtractor
 from ..utils import (
     unified_timestamp,
@@ -40,7 +38,7 @@ class N1InfoAssetIE(InfoExtractor):
 
 class N1InfoIIE(InfoExtractor):
     IE_NAME = 'N1Info:article'
-    _VALID_URL = r'https?://(?:(?:ba|rs|hr)\.)?n1info\.(?:com|si)/(?:[^/]+/){1,2}(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?:(?:ba|rs|hr)\.)?n1info\.(?:com|si)|nova\.rs)/(?:[^/]+/){1,2}(?P<id>[^/]+)'
     _TESTS = [{
         # Youtube embedded
         'url': 'https://rs.n1info.com/sport-klub/tenis/kako-je-djokovic-propustio-istorijsku-priliku-video/',
@@ -90,9 +88,17 @@ class N1InfoIIE(InfoExtractor):
             'uploader': 'YouLotWhatDontStop',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
+    }, {
+        'url': 'https://nova.rs/vesti/politika/zaklina-tatalovic-ani-brnabic-pricate-lazi-video/',
+        'info_dict': {
+            'id': 'tnjganabrnabicizaklinatatalovic100danavladegp-novas-worldwide',
+            'ext': 'mp4',
+            'title': 'Žaklina Tatalović Ani Brnabić: Pričate laži (VIDEO)',
+            'upload_date': '20211102',
+            'timestamp': 1635861677,
+        },
     }, {
         'url': 'https://hr.n1info.com/vijesti/pravobraniteljica-o-ubojstvu-u-zagrebu-radi-se-o-doista-nezapamcenoj-situaciji/',
         'only_matching': True,
@@ -116,16 +122,16 @@ class N1InfoIIE(InfoExtractor):
                 'title': title,
                 'thumbnail': video_data.get('data-thumbnail'),
                 'timestamp': timestamp,
-                'ie_key': N1InfoAssetIE.ie_key()})
+                'ie_key': 'N1InfoAsset'})
 
         embedded_videos = re.findall(r'(<iframe[^>]+>)', webpage)
         for embedded_video in embedded_videos:
             video_data = extract_attributes(embedded_video)
-            url = video_data.get('src')
+            url = video_data.get('src') or ''
             if url.startswith('https://www.youtube.com'):
-                entries.append(self.url_result(url, ie=YoutubeIE.ie_key()))
+                entries.append(self.url_result(url, ie='Youtube'))
             elif url.startswith('https://www.redditmedia.com'):
-                entries.append(self.url_result(url, ie=RedditRIE.ie_key()))
+                entries.append(self.url_result(url, ie='RedditR'))
 
         return {
             '_type': 'playlist',

yt_dlp/extractor/naver.py

@@ -40,6 +40,7 @@ class NaverBaseIE(InfoExtractor):
                 formats.append({
                     'format_id': '%s_%s' % (stream.get('type') or stream_type, dict_get(encoding_option, ('name', 'id'))),
                     'url': stream_url,
+                    'ext': 'mp4',
                     'width': int_or_none(encoding_option.get('width')),
                     'height': int_or_none(encoding_option.get('height')),
                     'vbr': int_or_none(bitrate.get('video')),
@@ -174,7 +175,7 @@ class NaverLiveIE(InfoExtractor):
         'url': 'https://tv.naver.com/l/52010',
         'info_dict': {
             'id': '52010',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': '[LIVE] 뉴스특보 : "수도권 거리두기, 2주간 2단계로 조정"',
             'description': 'md5:df7f0c237a5ed5e786ce5c91efbeaab3',
             'channel_id': 'NTV-ytnnews24-0',
@@ -184,7 +185,7 @@ class NaverLiveIE(InfoExtractor):
         'url': 'https://tv.naver.com/l/51549',
         'info_dict': {
             'id': '51549',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': '연합뉴스TV - 코로나19 뉴스특보',
             'description': 'md5:c655e82091bc21e413f549c0eaccc481',
             'channel_id': 'NTV-yonhapnewstv-0',
@@ -233,7 +234,7 @@ class NaverLiveIE(InfoExtractor):
                 continue
 
             formats.extend(self._extract_m3u8_formats(
-                quality.get('url'), video_id, 'm3u8',
+                quality.get('url'), video_id, 'mp4',
                 m3u8_id=quality.get('qualityId'), live=True
             ))
         self._sort_formats(formats)