[version] update

Created by: pukkandan

:ci skip all

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -1,4 +1,4 @@
-name: Broken site support
+name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.12.25**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2022.03.08**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -44,19 +44,19 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.25 (exe)
+        [debug] yt-dlp version 2022.03.08 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.25)
+        yt-dlp is up to date (2022.03.08)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.12.25**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2022.03.08**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -55,19 +55,19 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output **using one of the example URLs provided above**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.25 (exe)
+        [debug] yt-dlp version 2022.03.08 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.25)
+        yt-dlp is up to date (2022.03.08)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a site feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.12.25**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2022.03.08**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -32,7 +32,7 @@ body:
       label: Example URLs
       description: |
         Example URLs that can be used to demonstrate the requested feature
-      value: |
+      placeholder: |
         https://www.youtube.com/watch?v=BaW_jenozKc
     validations:
       required: true
@@ -53,19 +53,19 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.25 (exe)
+        [debug] yt-dlp version 2022.03.08 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.25)
+        yt-dlp is up to date (2022.03.08)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.12.25**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2022.03.08**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -38,19 +38,19 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.25 (exe)
+        [debug] yt-dlp version 2022.03.08 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.25)
+        yt-dlp is up to date (2022.03.08)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -11,7 +11,9 @@ body:
       options:
         - label: I'm reporting a feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.12.25**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
+          required: true
+        - label: I've verified that I'm running yt-dlp version **2022.03.08**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE/6_question.yml

@@ -25,7 +25,8 @@ body:
         Ask your question in an arbitrary form.
         Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
         Provide any additional information and as much context and examples as possible.
-        If your question contains "isn't working" or "can you add", this is most likely the wrong template
+        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
+        If you are in doubt if this is the right template, use another template!
       placeholder: WRITE QUESTION HERE
     validations:
       required: true
@@ -35,10 +36,10 @@ body:
       label: Verbose log
       description: |
         If your question involes a yt-dlp command, provide the complete verbose output of that command.
-        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/ISSUE_TEMPLATE/config.yml

@@ -3,3 +3,6 @@ contact_links:
   - name: Get help from the community on Discord
     url: https://discord.gg/H5MNcFW63r
     about: Join the yt-dlp Discord for community-powered support!
+  - name: Matrix Bridge to the Discord server
+    url: https://matrix.to/#/#yt-dlp:matrix.org
+    about: For those who do not want to use Discord

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -1,4 +1,4 @@
-name: Broken site support
+name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
@@ -44,10 +44,10 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -55,10 +55,10 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output **using one of the example URLs provided above**.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -32,7 +32,7 @@ body:
       label: Example URLs
       description: |
         Example URLs that can be used to demonstrate the requested feature
-      value: |
+      placeholder: |
         https://www.youtube.com/watch?v=BaW_jenozKc
     validations:
       required: true
@@ -53,10 +53,10 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -38,10 +38,10 @@ body:
       label: Verbose log
       description: |
         Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -11,6 +11,8 @@ body:
       options:
         - label: I'm reporting a feature request
           required: true
+        - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
+          required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -25,7 +25,8 @@ body:
         Ask your question in an arbitrary form.
         Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
         Provide any additional information and as much context and examples as possible.
-        If your question contains "isn't working" or "can you add", this is most likely the wrong template
+        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
+        If you are in doubt if this is the right template, use another template!
       placeholder: WRITE QUESTION HERE
     validations:
       required: true
@@ -35,10 +36,10 @@ body:
       label: Verbose log
       description: |
         If your question involes a yt-dlp command, provide the complete verbose output of that command.
-        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
-        [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
+        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252

.github/workflows/build.yml

@@ -96,7 +96,7 @@ jobs:
       env:
         BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
       if: "env.BREW_TOKEN != ''"
-      uses: webfactory/ssh-agent@v0.5.3
+      uses: yt-dlp/ssh-agent@v0.5.3
       with:
           ssh-private-key: ${{ env.BREW_TOKEN }}
     - name: Update Homebrew Formulae
@@ -161,11 +161,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     # In order to create a universal2 application, the version of python3 in /usr/bin has to be used
-    # Pyinstaller is pinned to 4.5.1 because the builds are failing in 4.6, 4.7
     - name: Install Requirements
       run: |
           brew install coreutils
-          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.5.1 mutagen pycryptodomex websockets
+          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.10 -r requirements.txt
     - name: Bump version
       id: bump_version
       run: /usr/bin/python3 devscripts/update-version.py
@@ -192,11 +191,9 @@ jobs:
       run: echo "::set-output name=sha512_macos::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')"
 
     - name: Run PyInstaller Script with --onedir
-      run: /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
+      run: |
-    - uses: papeloto/action-zip@v1
+        /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir
-      with:
+        zip ./dist/yt-dlp_macos.zip ./dist/yt-dlp_macos
-        files: ./dist/yt-dlp_macos
-        dest: ./dist/yt-dlp_macos.zip
     - name: Upload yt-dlp MacOS onedir
       id: upload-release-macos-zip
       uses: actions/upload-release-asset@v1
@@ -210,7 +207,7 @@ jobs:
     - name: Get SHA2-256SUMS for yt-dlp_macos.zip
       id: sha256_macos_zip
       run: echo "::set-output name=sha256_macos_zip::$(sha256sum dist/yt-dlp_macos.zip | awk '{print $1}')"
-    - name: Get SHA2-512SUMS for yt-dlp_macos
+    - name: Get SHA2-512SUMS for yt-dlp_macos.zip
       id: sha512_macos_zip
       run: echo "::set-output name=sha512_macos_zip::$(sha512sum dist/yt-dlp_macos.zip | awk '{print $1}')"
 
@@ -236,7 +233,7 @@ jobs:
       # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
       run: |
           python -m pip install --upgrade pip setuptools wheel py2exe
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt
     - name: Bump version
       id: bump_version
       env:
@@ -265,11 +262,9 @@ jobs:
       run: echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
 
     - name: Run PyInstaller Script with --onedir
-      run: python pyinst.py --onedir
+      run: |
-    - uses: papeloto/action-zip@v1
+        python pyinst.py --onedir
-      with:
+        Compress-Archive -LiteralPath ./dist/yt-dlp -DestinationPath ./dist/yt-dlp_win.zip
-        files: ./dist/yt-dlp
-        dest: ./dist/yt-dlp_win.zip
     - name: Upload yt-dlp Windows onedir
       id: upload-release-windows-zip
       uses: actions/upload-release-asset@v1
@@ -325,7 +320,7 @@ jobs:
     - name: Install Requirements
       run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt
     - name: Bump version
       id: bump_version
       env:

.gitignore

@@ -14,13 +14,17 @@ cookies
 *.frag.urls
 *.info.json
 *.live_chat.json
+*.meta
 *.part*
+*.tmp
+*.temp
 *.unknown_video
 *.ytdl
 .cache/
 
 *.3gp
 *.ape
+*.ass
 *.avi
 *.desktop
 *.flac
@@ -89,7 +93,7 @@ README.txt
 *.tar.gz
 *.zsh
 *.spec
-test/testdata/player-*.js
+test/testdata/sigs/player-*.js
 
 # Binary
 /youtube-dl
@@ -103,6 +107,7 @@ yt-dlp.zip
 *.iml
 .vscode
 *.sublime-*
+*.code-workspace
 
 # Lazy extractors
 */extractor/lazy_extractors.py

CONTRIBUTING.md

@@ -11,6 +11,7 @@
     - [Is anyone going to need the feature?](#is-anyone-going-to-need-the-feature)
     - [Is your question about yt-dlp?](#is-your-question-about-yt-dlp)
     - [Are you willing to share account details if needed?](#are-you-willing-to-share-account-details-if-needed)
+    - [Is the website primarily used for piracy](#is-the-website-primarily-used-for-piracy)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
     - [Adding new feature or making overarching changes](#adding-new-feature-or-making-overarching-changes)
     - [Adding support for a new site](#adding-support-for-a-new-site)
@@ -19,10 +20,12 @@
         - [Provide fallbacks](#provide-fallbacks)
         - [Regular expressions](#regular-expressions)
         - [Long lines policy](#long-lines-policy)
+        - [Quotes](#quotes)
         - [Inline values](#inline-values)
         - [Collapse fallbacks](#collapse-fallbacks)
         - [Trailing parentheses](#trailing-parentheses)
         - [Use convenience conversion and parsing functions](#use-convenience-conversion-and-parsing-functions)
+    - [My pull request is labeled pending-fixes](#my-pull-request-is-labeled-pending-fixes)
 - [EMBEDDING YT-DLP](README.md#embedding-yt-dlp)
 
 
@@ -31,9 +34,9 @@
 
 Bugs and suggestions should be reported at: [yt-dlp/yt-dlp/issues](https://github.com/yt-dlp/yt-dlp/issues). Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in our [discord server](https://discord.gg/H5MNcFW63r).
 
-**Please include the full output of yt-dlp when run with `-Uv`**, i.e. **add** `-Uv` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+**Please include the full output of yt-dlp when run with `-vU`**, i.e. **add** `-vU` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
-$ yt-dlp -Uv <your command line>
+$ yt-dlp -vU <your command line>
 [debug] Command-line config: ['-v', 'demo.com']
 [debug] Encodings: locale UTF-8, fs utf-8, out utf-8, pref UTF-8
 [debug] yt-dlp version 2021.09.25 (zip)
@@ -64,7 +67,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. We often get frustrated by these issues, since the only possible way for us to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the **complete** output of yt-dlp when called with the `-Uv` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the **complete** output of yt-dlp when called with the `-vU` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
 If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--write-pages` and upload the `.dump` files you get [somewhere](https://gist.github.com).
 
@@ -112,7 +115,7 @@ If the issue is with `youtube-dl` (the upstream fork of yt-dlp) and not with yt-
 
 ### Are you willing to share account details if needed?
 
-The maintainers and potential contributors of the project often do not have an account for the website you are asking support for. So any developer interested in solving your issue may ask you for account details. It is your personal discression whether you are willing to share the account in order for the developer to try and solve your issue. However, if you are unwilling or unable to provide details, they obviously cannot work on the issue and it cannot be solved unless some developer who both has an account and is willing/able to contribute decides to solve it.
+The maintainers and potential contributors of the project often do not have an account for the website you are asking support for. So any developer interested in solving your issue may ask you for account details. It is your personal discretion whether you are willing to share the account in order for the developer to try and solve your issue. However, if you are unwilling or unable to provide details, they obviously cannot work on the issue and it cannot be solved unless some developer who both has an account and is willing/able to contribute decides to solve it.
 
 By sharing an account with anyone, you agree to bear all risks associated with it. The maintainers and yt-dlp can't be held responsible for any misuse of the credentials.
 
@@ -122,6 +125,10 @@ While these steps won't necessarily ensure that no misuse of the account takes p
 - Change the password before sharing the account to something random (use [this](https://passwordsgenerator.net/) if you don't have a random password generator).
 - Change the password after receiving the account back.
 
+### Is the website primarily used for piracy?
+
+We follow [youtube-dl's policy](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) to not support services that is primarily used for infringing copyright. Additionally, it has been decided to not to support porn sites that specialize in deep fake. We also cannot support any service that serves only [DRM protected content](https://en.wikipedia.org/wiki/Digital_rights_management). 
+
 
 
 
@@ -209,7 +216,7 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 1. Add an import in [`yt_dlp/extractor/extractors.py`](yt_dlp/extractor/extractors.py).
-1. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
+1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
 1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L91-L426). Add tests and code for as many as you want.
 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
@@ -251,7 +258,11 @@ For extraction to work yt-dlp relies on metadata your extractor extracts and pro
  - `title` (media title)
  - `url` (media download URL) or `formats`
 
-The aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken. While, in fact, only `id` is technically mandatory, due to compatibility reasons, yt-dlp also treats `title` as mandatory. The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - Eg: when the video is a live stream that has not started yet.
+The aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken. While all extractors must return a `title`, they must also allow it's extraction to be non-fatal.
+
+For pornographic sites, appropriate `age_limit` must also be returned.
+
+The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - Eg: when the video is a live stream that has not started yet.
 
 [Any field](yt_dlp/extractor/common.py#219-L426) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
@@ -452,10 +463,14 @@ Here the presence or absence of other attributes including `style` is irrelevent
 
 ### Long lines policy
 
-There is a soft limit to keep lines of code under 100 characters long. This means it should be respected if possible and if it does not make readability and code maintenance worse. Sometimes, it may be reasonable to go upto 120 characters and sometimes even 80 can be unreadable. Keep in mind that this is not a hard limit and is just one of many tools to make the code more readable
+There is a soft limit to keep lines of code under 100 characters long. This means it should be respected if possible and if it does not make readability and code maintenance worse. Sometimes, it may be reasonable to go upto 120 characters and sometimes even 80 can be unreadable. Keep in mind that this is not a hard limit and is just one of many tools to make the code more readable.
 
 For example, you should **never** split long string literals like URLs or some other often copied entities over multiple lines to fit this limit:
 
+Conversely, don't unecessarily split small lines further. As a rule of thumb, if removing the line split keeps the code under 80 characters, it should be a single line.
+
+##### Examples
+
 Correct:
 
 ```python
@@ -469,6 +484,47 @@ Incorrect:
 'PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4'
 ```
 
+Correct:
+
+```python
+uploader = traverse_obj(info, ('uploader', 'name'), ('author', 'fullname'))
+```
+
+Incorrect:
+
+```python
+uploader = traverse_obj(
+    info,
+    ('uploader', 'name'),
+    ('author', 'fullname'))
+```
+
+Correct:
+
+```python
+formats = self._extract_m3u8_formats(
+    m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
+    note='Downloading HD m3u8 information', errnote='Unable to download HD m3u8 information')
+```
+
+Incorrect:
+
+```python
+formats = self._extract_m3u8_formats(m3u8_url,
+                                     video_id,
+                                     'mp4',
+                                     'm3u8_native',
+                                     m3u8_id='hls',
+                                     note='Downloading HD m3u8 information',
+                                     errnote='Unable to download HD m3u8 information')
+```
+
+
+### Quotes
+
+Always use single quotes for strings (even if the string has `'`) and double quotes for docstrings. Use `'''` only for multi-line strings. An exception can be made if a string has multiple single quotes in it and escaping makes it significantly harder to read. For f-strings, use you can use double quotes on the inside. But avoid f-strings that have too many quotes inside.
+
+
 ### Inline values
 
 Extracting variables is acceptable for reducing code duplication and improving readability of complex expressions. However, you should avoid extracting variables used only once and moving them to opposite parts of the extractor file, which makes reading the linear flow difficult.
@@ -518,27 +574,68 @@ Methods supporting list of patterns are: `_search_regex`, `_html_search_regex`,
 
 ### Trailing parentheses
 
-Always move trailing parentheses after the last argument.
+Always move trailing parentheses used for grouping/functions after the last argument. On the other hand, literal list/tuple/dict/set should closed be in a new line. Generators and list/dict comprehensions may use either style
 
-Note that this *does not* apply to braces `}` or square brackets `]` both of which should closed be in a new line
+#### Examples
-
-#### Example
 
 Correct:
 
 ```python
+url = try_get(
+    info,
     lambda x: x['ResultSet']['Result'][0]['VideoUrlSet']['VideoUrl'],
     list)
 ```
+Correct:
+
+```python
+url = try_get(info,
+              lambda x: x['ResultSet']['Result'][0]['VideoUrlSet']['VideoUrl'],
+              list)
+```
 
 Incorrect:
 
 ```python
+url = try_get(
+    info,
     lambda x: x['ResultSet']['Result'][0]['VideoUrlSet']['VideoUrl'],
     list,
 )
 ```
 
+Correct:
+
+```python
+f = {
+    'url': url,
+    'format_id': format_id,
+}
+```
+
+Incorrect:
+
+```python
+f = {'url': url,
+     'format_id': format_id}
+```
+
+Correct:
+
+```python
+formats = [process_formats(f) for f in format_data
+           if f.get('type') in ('hls', 'dash', 'direct') and f.get('downloadable')]
+```
+
+Correct:
+
+```python
+formats = [
+    process_formats(f) for f in format_data
+    if f.get('type') in ('hls', 'dash', 'direct') and f.get('downloadable')
+]
+```
+
 
 ### Use convenience conversion and parsing functions
 
@@ -567,6 +664,10 @@ duration = float_or_none(video.get('durationMs'), scale=1000)
 view_count = int_or_none(video.get('views'))
 ```
 
+# My pull request is labeled pending-fixes
+
+The `pending-fixes` label is added when there are changes requested to a PR. When the necessary changes are made, the label should be removed. However, despite our best efforts, it may sometimes happen that the maintainer did not see the changes or forgot to remove the label. If your PR is still marked as `pending-fixes` a few days after all requested changes have been made, feel free to ping the maintainer who labeled your issue and ask them to re-review and remove the label.
+
 
 
 

CONTRIBUTORS

@@ -2,6 +2,7 @@ pukkandan (owner)
 shirt-dev (collaborator)
 coletdjnz/colethedj (collaborator)
 Ashish0804 (collaborator)
+nao20010128nao/Lesmiscore (collaborator)
 h-h-h-h
 pauldubois98
 nixxo
@@ -19,7 +20,6 @@ samiksome
 alxnull
 FelixFrog
 Zocker1999NET
-nao20010128nao
 kurumigi
 bbepis
 animelover1984/horahoradev
@@ -146,7 +146,7 @@ chio0hai
 cntrl-s
 Deer-Spangle
 DEvmIb
-Grabien
+Grabien/MaximVol
 j54vc1bk
 mpeter50
 mrpapersonic
@@ -160,7 +160,7 @@ PilzAdam
 zmousm
 iw0nderhow
 unit193
-TwoThousandHedgehogs
+TwoThousandHedgehogs/KathrynElrod
 Jertzukka
 cypheron
 Hyeeji
@@ -177,3 +177,40 @@ Sematre
 jaller94
 r5d
 julien-hadleyjack
+git-anony-mouse
+mdawar
+trassshhub
+foghawk
+k3ns1n
+teridon
+mozlima
+timendum
+ischmidt20
+CreaValix
+sian1468
+arkamar
+hyano
+KiberInfinity
+tejing1
+Bricio
+lazypete365
+Aniruddh-J
+blackgear
+CplPwnies
+cyberfox1691
+FestplattenSchnitzel
+hatienl0i261299
+iphoting
+jakeogh
+lukasfink1
+lyz-code
+marieell
+mdpauley
+Mipsters
+mxmehl
+ofkz
+P-reducible
+pycabbage
+regarten
+Ronnnny
+schn0sch

Changelog.md

@@ -11,6 +11,339 @@
 -->
 
 
+### 2022.03.08
+
+* Merge youtube-dl: Upto [commit/6508688](https://github.com/ytdl-org/youtube-dl/commit/6508688e88c83bb811653083db9351702cd39a6a) (except NDR)
+* Add regex operator and quoting to format filters by [lukasfink1](https://github.com/lukasfink1)
+* Add brotli content-encoding support by [coletdjnz](https://github.com/coletdjnz)
+* Add pre-processor stage `after_filter`
+* Better error message when no `--live-from-start` format
+* Create necessary directories for `--print-to-file`
+* Fill more fields for playlists by [Lesmiscore](https://github.com/Lesmiscore)
+* Fix `-all` for `--sub-langs`
+* Fix doubling of `video_id` in `ExtractorError`
+* Fix for when stdout/stderr encoding is `None`
+* Handle negative duration from extractor
+* Implement `--add-header` without modifying `std_headers`
+* Obey `--abort-on-error` for "ffmpeg not installed"
+* Set `webpage_url_...` from `webpage_url` and not input URL
+* Tolerate failure to `--write-link` due to unknown URL
+* [aria2c] Add `--http-accept-gzip=true`
+* [build] Update pyinstaller to 4.10 by [shirt-dev](https://github.com/shirt-dev)
+* [cookies] Update MacOS12 `Cookies.binarycookies` location by [mdpauley](https://github.com/mdpauley)
+* [devscripts] Improve `prepare_manpage`
+* [downloader] Do not use aria2c for non-native `m3u8`
+* [downloader] Obey `--file-access-retries` when deleting/renaming by [ehoogeveen-medweb](https://github.com/ehoogeveen-medweb)
+* [extractor] Allow `http_headers` to be specified for `thumbnails`
+* [extractor] Extract subtitles from manifests for vimeo, globo, kaltura, svt by [fstirlitz](https://github.com/fstirlitz)
+* [extractor] Fix for manifests without period duration by [dirkf,](https://github.com/dirkf,) [pukkandan](https://github.com/pukkandan)
+* [extractor] Support `--mark-watched` without `_NETRC_MACHINE` by [coletdjnz](https://github.com/coletdjnz)
+* [FFmpegConcat] Abort on `--simulate`
+* [FormatSort] Consider `acodec`=`ogg` as `vorbis`
+* [fragment] Fix bugs around resuming with Range by [Lesmiscore](https://github.com/Lesmiscore)
+* [fragment] Improve `--live-from-start` for YouTube livestreams by [Lesmiscore](https://github.com/Lesmiscore)
+* [generic] Pass referer to extracted formats
+* [generic] Set rss `guid` as video id by [Bricio](https://github.com/Bricio)
+* [options] Better ambiguous option resolution
+* [options] Rename `--clean-infojson` to `--clean-info-json`
+* [SponsorBlock] Fixes for highlight and "full video labels" by [nihil-admirari](https://github.com/nihil-admirari)
+* [Sponsorblock] minor fixes by [nihil-admirari](https://github.com/nihil-admirari)
+* [utils] Better traceback for `ExtractorError`
+* [utils] Fix file locking for AOSP by [jakeogh](https://github.com/jakeogh)
+* [utils] Improve file locking
+* [utils] OnDemandPagedList: Do not download pages after error
+* [utils] render_table: Fix character calculation for removing extra gap by [Lesmiscore](https://github.com/Lesmiscore)
+* [utils] Use `locked_file` for `sanitize_open` by [jakeogh](https://github.com/jakeogh)
+* [utils] Validate `DateRange` input
+* [utils] WebSockets wrapper for non-async functions by [Lesmiscore](https://github.com/Lesmiscore)
+* [cleanup] Don't pass protocol to `_extract_m3u8_formats` for live videos
+* [cleanup] Remove extractors for some dead websites by [marieell](https://github.com/marieell)
+* [cleanup, docs] Misc cleanup
+* [AbemaTV] Add extractors by [Lesmiscore](https://github.com/Lesmiscore)
+* [adobepass] Add Suddenlink MSO by [CplPwnies](https://github.com/CplPwnies)
+* [ant1newsgr] Add extractor by [zmousm](https://github.com/zmousm)
+* [bigo] Add extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [Caltrans] Add extractor by [Bricio](https://github.com/Bricio)
+* [daystar] Add extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [fc2:live] Add extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [fptplay] Add extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [murrtube] Add extractor by [cyberfox1691](https://github.com/cyberfox1691)
+* [nfb] Add extractor by [ofkz](https://github.com/ofkz)
+* [niconico] Add playlist extractors and refactor by [Lesmiscore](https://github.com/Lesmiscore)
+* [peekvids] Add extractor by [schn0sch](https://github.com/schn0sch)
+* [piapro] Add extractor by [pycabbage,](https://github.com/pycabbage,) [Lesmiscore](https://github.com/Lesmiscore)
+* [rokfin] Add extractor by [P-reducible,](https://github.com/P-reducible,) [pukkandan](https://github.com/pukkandan)
+* [rokfin] Add stack and channel extractors by [P-reducible,](https://github.com/P-reducible,) [pukkandan](https://github.com/pukkandan)
+* [ruv.is] Add extractor by [iw0nderhow](https://github.com/iw0nderhow)
+* [telegram] Add extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [VideocampusSachsen] Add extractors by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
+* [xinpianchang] Add extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [abc] Support 1080p by [Ronnnny](https://github.com/Ronnnny)
+* [afreecatv] Support password-protected livestreams by [wlritchi](https://github.com/wlritchi)
+* [ard] Fix valid URL
+* [ATVAt] Detect geo-restriction by [marieell](https://github.com/marieell)
+* [bandcamp] Detect acodec
+* [bandcamp] Fix user URLs by [lyz-code](https://github.com/lyz-code)
+* [bbc] Fix extraction of news articles by [ajj8](https://github.com/ajj8)
+* [beeg] Fix extractor by [Bricio](https://github.com/Bricio)
+* [bigo] Fix extractor to not to use `form_params`
+* [Bilibili] Pass referer for all formats by [blackgear](https://github.com/blackgear)
+* [Biqle] Fix extractor by [Bricio](https://github.com/Bricio)
+* [ccma] Fix timestamp parsing by [nyuszika7h](https://github.com/nyuszika7h)
+* [crunchyroll] Better error reporting on login failure by [tejing1](https://github.com/tejing1)
+* [cspan] Support of C-Span congress videos by [Grabien](https://github.com/Grabien)
+* [dropbox] fix regex by [zenerdi0de](https://github.com/zenerdi0de)
+* [fc2] Fix extraction by [Lesmiscore](https://github.com/Lesmiscore)
+* [fujitv] Extract resolution for free sources by [YuenSzeHong](https://github.com/YuenSzeHong)
+* [Gettr] Add `GettrStreamingIE` by [i6t](https://github.com/i6t)
+* [Gettr] Fix formats order by [i6t](https://github.com/i6t)
+* [Gettr] Improve extractor by [i6t](https://github.com/i6t)
+* [globo] Expand valid URL by [Bricio](https://github.com/Bricio)
+* [lbry] Fix `--ignore-no-formats-error`
+* [manyvids] Extract `uploader` by [regarten](https://github.com/regarten)
+* [mildom] Fix linter
+* [mildom] Rework extractors by [Lesmiscore](https://github.com/Lesmiscore)
+* [mirrativ] Cleanup extractor code by [Lesmiscore](https://github.com/Lesmiscore)
+* [nhk] Add support for NHK for School by [Lesmiscore](https://github.com/Lesmiscore)
+* [niconico:tag] Add support for searching tags
+* [nrk] Add fallback API
+* [peekvids] Use JSON-LD by [schn0sch](https://github.com/schn0sch)
+* [peertube] Add media.fsfe.org by [mxmehl](https://github.com/mxmehl)
+* [rtvs] Fix extractor by [Bricio](https://github.com/Bricio)
+* [spiegel] Fix `_VALID_URL`
+* [ThumbnailsConvertor] Support `webp`
+* [tiktok] Fix `vm.tiktok`/`vt.tiktok` URLs
+* [tubitv] Fix/improve TV series extraction by [bbepis](https://github.com/bbepis)
+* [tumblr] Fix extractor by [foghawk](https://github.com/foghawk)
+* [twitcasting] Add fallback for finding running live by [Lesmiscore](https://github.com/Lesmiscore)
+* [TwitCasting] Check for password protection by [Lesmiscore](https://github.com/Lesmiscore)
+* [twitcasting] Fix extraction by [Lesmiscore](https://github.com/Lesmiscore)
+* [twitch] Fix field name of `view_count`
+* [twitter] Fix for private videos by [iphoting](https://github.com/iphoting)
+* [washingtonpost] Fix extractor by [Bricio](https://github.com/Bricio)
+* [youtube:tab] Add `approximate_date` extractor-arg
+* [youtube:tab] Follow redirect to regional channel  by [coletdjnz](https://github.com/coletdjnz)
+* [youtube:tab] Reject webpage data if redirected to home page
+* [youtube] De-prioritize potentially damaged formats
+* [youtube] Differentiate descriptive audio by language code
+* [youtube] Ensure subtitle urls are absolute by [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Escape possible `$` in `_extract_n_function_name` regex by [Lesmiscore](https://github.com/Lesmiscore)
+* [youtube] Fix automatic captions
+* [youtube] Fix n-sig extraction for phone player JS by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [youtube] Further de-prioritize 3gp format
+* [youtube] Label original auto-subs
+* [youtube] Prefer UTC upload date for videos by [coletdjnz](https://github.com/coletdjnz)
+* [zaq1] Remove dead extractor by [marieell](https://github.com/marieell)
+* [zee5] Support web-series by [Aniruddh-J](https://github.com/Aniruddh-J)
+* [zingmp3] Fix extractor by [hatienl0i261299](https://github.com/hatienl0i261299)
+* [zoom] Add support for screen cast by [Mipsters](https://github.com/Mipsters)
+
+
+### 2022.02.04
+
+* [youtube:search] Fix extractor by [coletdjnz](https://github.com/coletdjnz)
+* [youtube:search] Add tests
+* [twitcasting] Enforce UTF-8 for POST payload by [Lesmiscore](https://github.com/Lesmiscore)
+* [mediaset] Fix extractor by [nixxo](https://github.com/nixxo)
+* [websocket] Make syntax error in `websockets` module non-fatal
+
+### 2022.02.03
+
+* Merge youtube-dl: Upto [commit/78ce962](https://github.com/ytdl-org/youtube-dl/commit/78ce962f4fe020994c216dd2671546fbe58a5c67)
+* Add option `--print-to-file`
+* Make nested --config-locations relative to parent file
+* Ensure `_type` is present in `info.json`
+* Fix `--compat-options list-formats`
+* Fix/improve `InAdvancePagedList`
+* [downloader/ffmpeg] Handle unknown formats better
+* [outtmpl] Handle `-o ""` better
+* [outtmpl] Handle hard-coded file extension better
+* [extractor] Add convinience function `_yes_playlist`
+* [extractor] Allow non-fatal `title` extraction
+* [extractor] Extract video inside `Article` json_ld
+* [generic] Allow further processing of json_ld URL
+* [cookies] Fix keyring selection for unsupported desktops
+* [utils] Strip double spaces in `clean_html` by [dirkf](https://github.com/dirkf)
+* [aes] Add `unpad_pkcs7`
+* [test] Fix `test_youtube_playlist_noplaylist`
+* [docs,cleanup] Misc cleanup
+* [dplay] Add extractors for site changes by [Sipherdrakon](https://github.com/Sipherdrakon)
+* [ertgr] Add  extractors by [zmousm](https://github.com/zmousm), [dirkf](https://github.com/dirkf)
+* [Musicdex] Add extractors by [Ashish0804](https://github.com/Ashish0804)
+* [YandexVideoPreview] Add extractor by [KiberInfinity](https://github.com/KiberInfinity)
+* [youtube] Add extractor `YoutubeMusicSearchURLIE`
+* [archive.org] Ignore unnecessary files
+* [Bilibili] Add 8k support by [u-spec-png](https://github.com/u-spec-png)
+* [bilibili] Fix extractor, make anthology title non-fatal
+* [CAM4] Add thumbnail extraction by [alerikaisattera](https://github.com/alerikaisattera)
+* [cctv] De-prioritize sample format
+* [crunchyroll:beta] Add cookies support by [tejing1](https://github.com/tejing1)
+* [crunchyroll] Fix login by [tejing1](https://github.com/tejing1)
+* [doodstream] Fix extractor
+* [fc2] Fix extraction by [Lesmiscore](https://github.com/Lesmiscore)
+* [FFmpegConcat] Abort on --skip-download and download errors
+* [Fujitv] Extract metadata and support premium by [YuenSzeHong](https://github.com/YuenSzeHong)
+* [globo] Fix extractor by [Bricio](https://github.com/Bricio)
+* [glomex] Simplify embed detection
+* [GoogleSearch] Fix extractor
+* [Instagram] Fix extraction when logged in by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [iq.com] Add VIP support by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [mildom] Fix extractor by [lazypete365](https://github.com/lazypete365)
+* [MySpass] Fix video url processing by [trassshhub](https://github.com/trassshhub)
+* [Odnoklassniki] Improve embedded players extraction by [KiberInfinity](https://github.com/KiberInfinity)
+* [orf:tvthek] Lazy playlist extraction and obey --no-playlist
+* [Pladform] Fix redirection to external player by [KiberInfinity](https://github.com/KiberInfinity)
+* [ThisOldHouse] Improve Premium URL check by [Ashish0804](https://github.com/Ashish0804)
+* [TikTok] Iterate through app versions by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [tumblr] Fix 403 errors and handle vimeo embeds by [foghawk](https://github.com/foghawk)
+* [viki] Fix "Bad request" for manifest by [nyuszika7h](https://github.com/nyuszika7h)
+* [Vimm] add recording extractor by [alerikaisattera](https://github.com/alerikaisattera)
+* [web.archive:youtube] Add `ytarchive:` prefix and misc cleanup
+* [youtube:api] Do not use seek when reading HTTPError response by [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Fix n-sig for player e06dea74
+* [youtube, cleanup] Misc fixes and cleanup
+
+
+### 2022.01.21
+
+* Add option `--concat-playlist` to **concat videos in a playlist**
+* Allow **multiple and nested configuration files**
+* Add more post-processing stages (`after_video`, `playlist`)
+* Allow `--exec` to be run at any post-processing stage (Deprecates `--exec-before-download`)
+* Allow `--print` to be run at any post-processing stage
+* Allow listing formats, thumbnails, subtitles using `--print` by [pukkandan](https://github.com/pukkandan), [Zirro](https://github.com/Zirro)
+* Add fields `video_autonumber`, `modified_date`, `modified_timestamp`, `playlist_count`, `channel_follower_count`
+* Add key `requested_downloads` in the root `info_dict`
+* Write `download_archive` only after all formats are downloaded
+* [FfmpegMetadata] Allow setting metadata of individual streams using `meta<n>_` prefix
+* Add option `--legacy-server-connect` by [xtkoba](https://github.com/xtkoba)
+* Allow escaped `,` in `--extractor-args`
+* Allow unicode characters in `info.json`
+* Check for existing thumbnail/subtitle in final directory
+* Don't treat empty containers as `None` in `sanitize_info`
+* Fix `-s --ignore-no-formats --force-write-archive`
+* Fix live title for multiple formats
+* List playlist thumbnails in `--list-thumbnails`
+* Raise error if subtitle download fails
+* [cookies] Fix bug when keyring is unspecified
+* [ffmpeg] Ignore unknown streams, standardize use of `-map 0`
+* [outtmpl] Alternate form for `D` and fix suffix's case
+* [utils] Add `Sec-Fetch-Mode` to `std_headers`
+* [utils] Fix `format_bytes` output for Bytes by [pukkandan](https://github.com/pukkandan), [mdawar](https://github.com/mdawar)
+* [utils] Handle `ss:xxx` in `parse_duration`
+* [utils] Improve parsing for nested HTML elements by [zmousm](https://github.com/zmousm), [pukkandan](https://github.com/pukkandan)
+* [utils] Use key `None` in `traverse_obj` to return as-is
+* [extractor] Detect more subtitle codecs in MPD manifests by [fstirlitz](https://github.com/fstirlitz)
+* [extractor] Extract chapters from JSON-LD by [iw0nderhow](https://github.com/iw0nderhow), [pukkandan](https://github.com/pukkandan)
+* [extractor] Extract thumbnails from JSON-LD by [nixxo](https://github.com/nixxo)
+* [extractor] Improve `url_result` and related
+* [generic] Improve KVS player extraction by [trassshhub](https://github.com/trassshhub)
+* [build] Reduce dependency on third party workflows
+* [extractor,cleanup] Use `_search_nextjs_data`, `format_field`
+* [cleanup] Minor fixes and cleanup
+* [docs] Improvements
+* [test] Fix TestVerboseOutput
+* [afreecatv] Add livestreams extractor by [wlritchi](https://github.com/wlritchi)
+* [callin] Add extractor by [foghawk](https://github.com/foghawk)
+* [CrowdBunker] Add extractors by [Ashish0804](https://github.com/Ashish0804)
+* [daftsex] Add extractors by [k3ns1n](https://github.com/k3ns1n)
+* [digitalconcerthall] Add extractor by [teridon](https://github.com/teridon)
+* [Drooble] Add extractor by [u-spec-png](https://github.com/u-spec-png)
+* [EuropeanTour] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [iq.com] Add extractors by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [KelbyOne] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [LnkIE] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [MainStreaming] Add extractor by [coletdjnz](https://github.com/coletdjnz)
+* [megatvcom] Add extractors by [zmousm](https://github.com/zmousm)
+* [Newsy] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [noodlemagazine] Add extractor by [trassshhub](https://github.com/trassshhub)
+* [PokerGo] Add extractors by [Ashish0804](https://github.com/Ashish0804)
+* [Pornez] Add extractor by [mozlima](https://github.com/mozlima)
+* [PRX] Add Extractors by [coletdjnz](https://github.com/coletdjnz)
+* [RTNews] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [Rule34video] Add extractor by [trassshhub](https://github.com/trassshhub)
+* [tvopengr] Add extractors by [zmousm](https://github.com/zmousm)
+* [Vimm] Add extractor by [alerikaisattera](https://github.com/alerikaisattera)
+* [glomex] Add extractors by [zmousm](https://github.com/zmousm)
+* [instagram] Add story/highlight extractor by [u-spec-png](https://github.com/u-spec-png)
+* [openrec] Add movie extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [rai] Add Raiplaysound extractors by [nixxo](https://github.com/nixxo), [pukkandan](https://github.com/pukkandan)
+* [aparat] Fix extractor
+* [ard] Extract subtitles by [fstirlitz](https://github.com/fstirlitz)
+* [BiliIntl] Add login by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [CeskaTelevize] Use `http` for manifests
+* [CTVNewsIE] Add fallback for video search by [Ashish0804](https://github.com/Ashish0804)
+* [dplay] Migrate DiscoveryPlusItaly to DiscoveryPlus by [timendum](https://github.com/timendum)
+* [dplay] Re-structure DiscoveryPlus extractors
+* [Dropbox] Support password protected files and more formats by [zenerdi0de](https://github.com/zenerdi0de)
+* [facebook] Fix extraction from groups
+* [facebook] Improve title and uploader extraction
+* [facebook] Parse dash manifests
+* [fox] Extract m3u8 from preview by [ischmidt20](https://github.com/ischmidt20)
+* [funk] Support origin URLs
+* [gfycat] Fix `uploader`
+* [gfycat] Support embeds by [coletdjnz](https://github.com/coletdjnz)
+* [hotstar] Add extractor args to ignore tags by [Ashish0804](https://github.com/Ashish0804)
+* [hrfernsehen] Fix ardloader extraction by [CreaValix](https://github.com/CreaValix)
+* [instagram] Fix username extraction for stories and highlights by [nyuszika7h](https://github.com/nyuszika7h)
+* [kakao] Detect geo-restriction
+* [line] Remove `tv.line.me` by [sian1468](https://github.com/sian1468)
+* [mixch] Add `MixchArchiveIE` by [Lesmiscore](https://github.com/Lesmiscore)
+* [mixcloud] Detect restrictions by [llacb47](https://github.com/llacb47)
+* [NBCSports] Fix extraction of platform URLs by [ischmidt20](https://github.com/ischmidt20)
+* [Nexx] Extract more metadata by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [Nexx] Support 3q CDN by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [pbs] de-prioritize AD formats
+* [PornHub,YouTube] Refresh onion addresses by [unit193](https://github.com/unit193)
+* [RedBullTV] Parse subtitles from manifest by [Ashish0804](https://github.com/Ashish0804)
+* [streamcz] Fix extractor by [arkamar](https://github.com/arkamar), [pukkandan](https://github.com/pukkandan)
+* [Ted] Rewrite extractor by [pukkandan](https://github.com/pukkandan), [trassshhub](https://github.com/trassshhub)
+* [Theta] Fix valid URL by [alerikaisattera](https://github.com/alerikaisattera)
+* [ThisOldHouseIE] Add support for premium videos by [Ashish0804](https://github.com/Ashish0804)
+* [TikTok] Fix extraction for sigi-based webpages, add API fallback by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [TikTok] Pass cookies to formats, and misc fixes by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [TikTok] Extract captions, user thumbnail by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [TikTok] Change app version by [MinePlayersPE](https://github.com/MinePlayersPE), [llacb47](https://github.com/llacb47)
+* [TVer] Extract message for unaired live by [Lesmiscore](https://github.com/Lesmiscore)
+* [twitcasting] Refactor extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [twitter] Fix video in quoted tweets
+* [veoh] Improve extractor by [foghawk](https://github.com/foghawk)
+* [vk] Capture `clip` URLs
+* [vk] Fix VKUserVideosIE by [Ashish0804](https://github.com/Ashish0804)
+* [vk] Improve `_VALID_URL` by [k3ns1n](https://github.com/k3ns1n)
+* [VrtNU] Handle empty title by [pgaig](https://github.com/pgaig)
+* [XVideos] Check HLS formats by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [yahoo:gyao] Improved playlist handling by [hyano](https://github.com/hyano)
+* [youtube:tab] Extract more playlist metadata by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
+* [youtube:tab] Raise error on tab redirect by [krichbanana](https://github.com/krichbanana), [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Update Innertube clients by [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Detect live-stream embeds
+* [youtube] Do not return `upload_date` for playlists
+* [youtube] Extract channel subscriber count by [coletdjnz](https://github.com/coletdjnz)
+* [youtube] Make invalid storyboard URL non-fatal
+* [youtube] Enforce UTC, update innertube clients and tests by [coletdjnz](https://github.com/coletdjnz)
+* [zdf] Add chapter extraction by [iw0nderhow](https://github.com/iw0nderhow)
+* [zee5] Add geo-bypass
+
+
+### 2021.12.27
+
+* Avoid recursion error when re-extracting info
+* [ffmpeg] Fix position of `--ppa`
+* [aria2c] Don't show progress when `--no-progress`
+* [cookies] Support other keyrings by [mbway](https://github.com/mbway)
+* [EmbedThumbnail] Prefer AtomicParsley over ffmpeg if available
+* [generic] Fix HTTP KVS Player by [git-anony-mouse](https://github.com/git-anony-mouse)
+* [ThumbnailsConvertor] Fix for when there are no thumbnails
+* [docs] Add examples for using `TYPES:` in `-P`/`-o`
+* [PixivSketch] Add extractors by [nao20010128nao](https://github.com/nao20010128nao)
+* [tiktok] Add music, sticker and tag IEs by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [BiliIntl] Fix extractor by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [CBC] Fix URL regex
+* [tiktok] Fix `extractor_key` used in archive
+* [youtube] **End `live-from-start` properly when stream ends with 403**
+* [Zee5] Fix VALID_URL for tv-shows by [Ashish0804](https://github.com/Ashish0804)
+
 ### 2021.12.25
 
 * [dash,youtube] **Download live from start to end** by [nao20010128nao](https://github.com/nao20010128nao), [pukkandan](https://github.com/pukkandan)
@@ -104,7 +437,7 @@
 * [youtube:comments] Add more options for limiting number of comments extracted by [coletdjnz](https://github.com/coletdjnz)
 * [youtube:tab] Extract more metadata from feeds/channels/playlists by [coletdjnz](https://github.com/coletdjnz)
 * [youtube:tab] Extract video thumbnails from playlist by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
-* [youtube:tab] Ignore query when redirecting channel to playlist and cleanup of related code Closes #2046
+* [youtube:tab] Ignore query when redirecting channel to playlist and cleanup of related code
 * [youtube] Fix `ytsearchdate`
 * [zdf] Support videos with different ptmd location by [iw0nderhow](https://github.com/iw0nderhow)
 * [zee5] Support /episodes in URL

Collaborators.md

@@ -36,5 +36,15 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
 
 [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/ashish0804)
 
-* Added support for new websites Zee5, MXPlayer, DiscoveryPlusIndia, ShemarooMe, Utreon etc
+* Added support for new websites BiliIntl, DiscoveryPlusIndia, OlympicsReplay, PlanetMarathi, ShemarooMe, Utreon, Zee5 etc
-* Added playlist/series downloads for TubiTv, SonyLIV, Voot, HotStar etc
+* Added playlist/series downloads for Hotstar, ParamountPlus, Rumble, SonyLIV, Trovo, TubiTv, Voot etc
+* Improved/fixed support for HiDive, HotStar, Hungama, LBRY, LinkedInLearning, Mxplayer, SonyLiv, TV2, Vimeo, VLive etc
+
+
+## [Lesmiscore](https://github.com/Lesmiscore) (nao20010128nao)
+
+**Bitcoin**: bc1qfd02r007cutfdjwjmyy9w23rjvtls6ncve7r3s  
+**Monacoin**: mona1q3tf7dzvshrhfe3md379xtvt2n22duhglv5dskr
+
+* Download live from start to end for YouTube
+* Added support for new websites mildom, PixivSketch, skeb, radiko, voicy, mirrativ, openrec, whowatch, damtomo, 17.live, mixch etc

Makefile

@@ -1,5 +1,6 @@
 all: lazy-extractors yt-dlp doc pypi-files
-clean: clean-test clean-dist clean-cache
+clean: clean-test clean-dist
+clean-all: clean clean-cache
 completions: completion-bash completion-fish completion-zsh
 doc: README.md CONTRIBUTING.md issuetemplates supportedsites
 ot: offlinetest
@@ -13,15 +14,15 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites com
 .PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites
 
 clean-test:
-	rm -rf test/testdata/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
+	rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
-	*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.part* *.unknown_video *.ytdl \
+	*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \
-	*.3gp *.ape *.avi *.desktop *.flac *.flv *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
+	*.3gp *.ape *.ass *.avi *.desktop *.flac *.flv *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
 	*.mp4 *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
 clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
 clean-cache:
-	find . -name "*.pyc" -o -name "*.class" -delete
+	find . \( -name "*.pyc" -o -name "*.class" \) -delete
 
 completion-bash: completions/bash/yt-dlp
 completion-fish: completions/fish/yt-dlp.fish

README.md

@@ -3,17 +3,17 @@
 
 [![YT-DLP](https://raw.githubusercontent.com/yt-dlp/yt-dlp/master/.github/banner.svg)](#readme)
 
-[![Release version](https://img.shields.io/github/v/release/yt-dlp/yt-dlp?color=blue&label=&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases/latest)
+[![Release version](https://img.shields.io/github/v/release/yt-dlp/yt-dlp?color=blue&label=Download&style=for-the-badge)](#release-files "Release")
-[![CI Status](https://img.shields.io/github/workflow/status/yt-dlp/yt-dlp/Core%20Tests/master?label=&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/actions)
+[![License: Unlicense](https://img.shields.io/badge/-Unlicense-brightgreen.svg?style=for-the-badge)](LICENSE "License")
-[![License: Unlicense](https://img.shields.io/badge/-Unlicense-blue.svg?style=for-the-badge)](LICENSE)
+[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](Collaborators.md#collaborators "Donate")
-[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](Collaborators.md#collaborators)
+[![Docs](https://img.shields.io/badge/-Docs-blue.svg?color=blue&style=for-the-badge)](https://readthedocs.org/projects/yt-dlp/ "Docs")
-[![Supported Sites](https://img.shields.io/badge/-Supported_Sites-brightgreen.svg?style=for-the-badge)](supportedsites.md)
+[![Supported Sites](https://img.shields.io/badge/-Supported_Sites-brightgreen.svg?style=for-the-badge)](supportedsites.md "Supported Sites")
-[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&label=​&logo=discord&style=for-the-badge)](https://discord.gg/H5MNcFW63r)
+[![PyPi](https://img.shields.io/badge/-PyPi-blue.svg?logo=pypi&labelColor=555555&style=for-the-badge)](https://pypi.org/project/yt-dlp "PyPi")
-[![Doc Status](https://readthedocs.org/projects/yt-dlp/badge/?version=latest&style=for-the-badge)](https://yt-dlp.readthedocs.io)
+[![CI Status](https://img.shields.io/github/workflow/status/yt-dlp/yt-dlp/Core%20Tests/master?label=Tests&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/actions "CI Status")
-[![Commits](https://img.shields.io/github/commit-activity/m/yt-dlp/yt-dlp?label=commits&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/commits)
+[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)](https://discord.gg/H5MNcFW63r "Discord")
-[![Last Commit](https://img.shields.io/github/last-commit/yt-dlp/yt-dlp/master?label=&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/commits)
+[![Matrix](https://img.shields.io/matrix/yt-dlp:matrix.org?color=brightgreen&labelColor=555555&label=&logo=element&style=for-the-badge)](https://matrix.to/#/#yt-dlp:matrix.org "Matrix")
-[![Downloads](https://img.shields.io/github/downloads/yt-dlp/yt-dlp/total?style=for-the-badge&color=blue)](https://github.com/yt-dlp/yt-dlp/releases/latest)
+[![Commits](https://img.shields.io/github/commit-activity/m/yt-dlp/yt-dlp?label=commits&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/commits "Commit History")
-[![PyPi Downloads](https://img.shields.io/pypi/dm/yt-dlp?label=PyPi&style=for-the-badge)](https://pypi.org/project/yt-dlp)
+[![Last Commit](https://img.shields.io/github/last-commit/yt-dlp/yt-dlp/master?label=&style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/commits "Commit History")
 
 </div>
 <!-- MANPAGE: END EXCLUDED SECTION -->
@@ -71,7 +71,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 # NEW FEATURES
 
-* Based on **youtube-dl 2021.12.17 [commit/5014bd6](https://github.com/ytdl-org/youtube-dl/commit/5014bd67c22b421207b2650d4dc874b95b36dda1)** and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Based on **youtube-dl 2021.12.17 [commit/5add3f4](https://github.com/ytdl-org/youtube-dl/commit/5add3f4373287e6346ca3551239edab549284db3)** and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -88,9 +88,9 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
     * Redirect channel's home URL automatically to `/video` to preserve the old behaviour
     * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
     * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
-    * Download livestreams from the start using `--live-from-start`
+    * Download livestreams from the start using `--live-from-start` (experimental)
 
-* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[:PROFILE]`
+* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE]`
 
 * **Split video by chapters**: Videos can be split into multiple files based on chapters using `--split-chapters`
 
@@ -110,9 +110,9 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 * **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata`
 
-* **Other new options**: Many new options have been added such as `--print`, `--wait-for-video`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
+* **Other new options**: Many new options have been added such as `--concat-playlist`, `--print`, `--wait-for-video`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc
 
-* **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio etc
+* **Improvements**: Regex and other operators in `--format`/`--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, `--exec` at different stages, etc
 
 * **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details
 
@@ -130,15 +130,15 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
 * The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
 * Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both
-* `--ignore-errors` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead
+* `--no-abort-on-error` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead
 * When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files
 * `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-info-json`. Use `--no-embed-info-json` or `--compat-options no-attach-info-json` to revert this
-* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](https://github.com/yt-dlp/yt-dlp#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
+* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
 * `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior
 * The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this
 * All *experiences* of a funimation episode are considered as a single video. This behavior breaks existing archives. Use `--compat-options seperate-video-versions` to extract information from only the default player
 * Youtube live chat (if available) is considered as a subtitle. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent live chat from downloading
-* Youtube channel URLs are automatically redirected to `/video`. Append a `/featured` to the URL to download only the videos in the home page. If the channel does not have a videos tab, we try to download the equivalent `UU` playlist instead. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections
+* Youtube channel URLs are automatically redirected to `/video`. Append a `/featured` to the URL to download only the videos in the home page. If the channel does not have a videos tab, we try to download the equivalent `UU` playlist instead. For all other tabs, if the channel does not show the requested tab, an error will be raised. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections
 * Unavailable videos are also listed for youtube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this
 * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
@@ -157,8 +157,19 @@ You can install yt-dlp using one of the following methods:
 
 ### Using the release binary
 
-You can simply download the [correct binary file](#release-files) for your OS: **[[Windows](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)] [[UNIX-like](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)]**
+You can simply download the [correct binary file](#release-files) for your OS
 
+<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
+[![Windows](https://img.shields.io/badge/-Windows_x64-blue.svg?style=for-the-badge&logo=windows)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)
+[![Linux](https://img.shields.io/badge/-Linux/MacOS/BSD-red.svg?style=for-the-badge&logo=linux)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)
+[![Source Tarball](https://img.shields.io/badge/-Source_tar-green.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+[![Other variants](https://img.shields.io/badge/-Other-grey.svg?style=for-the-badge)](#release-files)
+[![ALl versions](https://img.shields.io/badge/-All_Versions-lightgrey.svg?style=for-the-badge)](https://github.com/yt-dlp/yt-dlp/releases)
+<!-- MANPAGE: END EXCLUDED SECTION -->
+
+Note: The manpages, shell completion files etc. are available in the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+
+<!-- TODO: Move to Wiki -->
 In UNIX-like OSes (MacOS, Linux, BSD), you can also install the same in one of the following ways:
 
 ```
@@ -176,7 +187,6 @@ sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp --d
 sudo chmod a+rx /usr/local/bin/yt-dlp
 ```
 
-PS: The manpages, shell completion files etc. are available in [yt-dlp.tar.gz](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
 
 ### With [PIP](https://pypi.org/project/pip)
 
@@ -197,6 +207,7 @@ python3 -m pip install --force-reinstall https://github.com/yt-dlp/yt-dlp/archiv
 
 Note that on some systems, you may need to use `py` or `python` instead of `python3`
 
+<!-- TODO: Add to Wiki, Remove Taps -->
 ### With [Homebrew](https://brew.sh)
 
 macOS or Linux users that are using Homebrew can also install it by:
@@ -255,8 +266,9 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 * [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
 * [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
 * [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licensed under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
-* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licensed under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
+* [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
-* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
+* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen/ffmpeg cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
+* [**brotli**](https://github.com/google/brotli) or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
 * [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
 * [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
 * [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD3](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
@@ -267,13 +279,14 @@ To use or redistribute the dependencies, you must agree to their respective lice
 
 The Windows and MacOS standalone release binaries are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
 
-**Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds/wiki/Latest#latest-autobuilds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
+<!-- TODO: ffmpeg has merged this patch. Remove this note once there is new release -->
+**Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds#ffmpeg-static-auto-builds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
 
 
 ## COMPILE
 
 **For Windows**:
-To build the Windows executable, you must have pyinstaller (and optionally mutagen, pycryptodomex, websockets). Once you have all the necessary dependencies installed, (optionally) build lazy extractors using `devscripts/make_lazy_extractors.py`, and then just run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the python used to build it.
+To build the Windows executable, you must have pyinstaller (and any of yt-dlp's optional dependencies if needed). Once you have all the necessary dependencies installed, (optionally) build lazy extractors using `devscripts/make_lazy_extractors.py`, and then just run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the python used to build it.
 
     py -m pip install -U pyinstaller -r requirements.txt
     py devscripts/make_lazy_extractors.py
@@ -327,22 +340,27 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      an error. The default value "fixup_error"
                                      repairs broken URLs, but emits an error if
                                      this is not possible instead of searching
-    --ignore-config, --no-config     Disable loading any configuration files
+    --ignore-config                  Don't load any more configuration files
-                                     except the one provided by --config-location.
+                                     except those given by --config-locations.
-                                     When given inside a configuration
+                                     For backward compatibility, if this option
-                                     file, no further configuration files are
+                                     is found inside the system configuration
-                                     loaded. Additionally, (for backward
+                                     file, the user configuration is not loaded.
-                                     compatibility) if this option is found
+                                     (Alias: --no-config)
-                                     inside the system configuration file, the
+    --no-config-locations            Do not load any custom configuration files
-                                     user configuration is not loaded
+                                     (default). When given inside a
-    --config-location PATH           Location of the main configuration file;
+                                     configuration file, ignore all previous
+                                     --config-locations defined in the current
+                                     file
+    --config-locations PATH          Location of the main configuration file;
                                      either the path to the config or its
-                                     containing directory
+                                     containing directory. Can be used multiple
+                                     times and inside other configuration files
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them
     --no-flat-playlist               Extract the videos of a playlist
     --live-from-start                Download livestreams from the start.
                                      Currently only supported for YouTube
+                                     (Experimental)
     --no-live-from-start             Download livestreams from the current time
                                      (default)
     --wait-for-video MIN[-MAX]       Wait for scheduled streams to become
@@ -363,8 +381,9 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --proxy URL                      Use the specified HTTP/HTTPS/SOCKS proxy.
                                      To enable SOCKS proxy, specify a proper
                                      scheme. For example
-                                     socks5://127.0.0.1:1080/. Pass in an empty
+                                     socks5://user:pass@127.0.0.1:1080/. Pass in
-                                     string (--proxy "") for direct connection
+                                     an empty string (--proxy "") for direct
+                                     connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
     --source-address IP              Client-side IP address to bind to
     -4, --force-ipv4                 Make all connections via IPv4
@@ -377,7 +396,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      option is not present) is used for the
                                      actual downloading
     --geo-bypass                     Bypass geographic restriction via faking
-                                     X-Forwarded-For HTTP header
+                                     X-Forwarded-For HTTP header (default)
     --no-geo-bypass                  Do not bypass geographic restriction via
                                      faking X-Forwarded-For HTTP header
     --geo-bypass-country CODE        Force bypass geographic restriction with
@@ -514,8 +533,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      example, --downloader aria2c --downloader
                                      "dash,m3u8:native" will use aria2c for
                                      http/ftp downloads, and the native
-                                     downloader for dash/m3u8 downloads
+                                     downloader for dash/m3u8 downloads (Alias:
-                                     (Alias: --external-downloader)
+                                     --external-downloader)
     --downloader-args NAME:ARGS      Give these arguments to the external
                                      downloader. Specify the downloader name and
                                      the arguments separated by a colon ":". For
@@ -523,8 +542,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      different positions using the same syntax
                                      as --postprocessor-args. You can use this
                                      option multiple times to give different
-                                     arguments to different downloaders
+                                     arguments to different downloaders (Alias:
-                                     (Alias: --external-downloader-args)
+                                     --external-downloader-args)
 
 ## Filesystem Options:
     -a, --batch-file FILE            File containing URLs to download ("-" for
@@ -535,7 +554,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     -P, --paths [TYPES:]PATH         The paths where the files should be
                                      downloaded. Specify the type of file and
                                      the path separated by a colon ":". All the
-                                     same types as --output are supported.
+                                     same TYPES as --output are supported.
                                      Additionally, you can also provide "home"
                                      (default) and "temp" paths. All
                                      intermediary files are first downloaded to
@@ -588,18 +607,18 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      --write-description etc. (default)
     --no-write-playlist-metafiles    Do not write playlist metadata when using
                                      --write-info-json, --write-description etc.
-    --clean-infojson                 Remove some private fields such as
+    --clean-info-json                Remove some private fields such as
                                      filenames from the infojson. Note that it
                                      could still contain some personal
                                      information (default)
-    --no-clean-infojson              Write all fields to the infojson
+    --no-clean-info-json             Write all fields to the infojson
     --write-comments                 Retrieve video comments to be placed in the
                                      infojson. The comments are fetched even
                                      without this option if the extraction is
                                      known to be quick (Alias: --get-comments)
     --no-write-comments              Do not retrieve video comments unless the
-                                     extraction is known to be quick
+                                     extraction is known to be quick (Alias:
-                                     (Alias: --no-get-comments)
+                                     --no-get-comments)
     --load-info-json FILE            JSON file containing the video information
                                      (created with the "--write-info-json"
                                      option)
@@ -607,16 +626,19 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      from and dump cookie jar in
     --no-cookies                     Do not read/dump cookies from/to file
                                      (default)
-    --cookies-from-browser BROWSER[:PROFILE]
+    --cookies-from-browser BROWSER[+KEYRING][:PROFILE]
-                                     Load cookies from a user profile of the
+                                     The name of the browser and (optionally)
-                                     given web browser. Currently supported
+                                     the name/path of the profile to load
-                                     browsers are: brave, chrome, chromium,
+                                     cookies from, separated by a ":". Currently
-                                     edge, firefox, opera, safari, vivaldi. You
+                                     supported browsers are: brave, chrome,
-                                     can specify the user profile name or
+                                     chromium, edge, firefox, opera, safari,
-                                     directory using "BROWSER:PROFILE_NAME" or
+                                     vivaldi. By default, the most recently
-                                     "BROWSER:PROFILE_PATH". If no profile is
+                                     accessed profile is used. The keyring used
-                                     given, the most recently accessed one is
+                                     for decrypting Chromium cookies on Linux
-                                     used
+                                     can be (optionally) specified after the
+                                     browser name separated by a "+". Currently
+                                     supported keyrings are: basictext,
+                                     gnomekeyring, kwallet
     --no-cookies-from-browser        Do not load cookies from browser (default)
     --cache-dir DIR                  Location in the filesystem where youtube-dl
                                      can store some downloaded information (such
@@ -659,10 +681,20 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      formats are found (default)
     --skip-download                  Do not download the video but write all
                                      related files (Alias: --no-download)
-    -O, --print TEMPLATE             Quiet, but print the given fields for each
+    -O, --print [WHEN:]TEMPLATE      Field name or output template to print to
-                                     video. Simulate unless --no-simulate is
+                                     screen, optionally prefixed with when to
-                                     used. Either a field name or same syntax as
+                                     print it, separated by a ":". Supported
-                                     the output template can be used
+                                     values of "WHEN" are the same as that of
+                                     --use-postprocessor, and "video" (default).
+                                     Implies --quiet and --simulate (unless
+                                     --no-simulate is used). This option can be
+                                     used multiple times
+    --print-to-file [WHEN:]TEMPLATE FILE
+                                     Append given template to the file. The
+                                     values of WHEN and TEMPLATE are same as
+                                     that of --print. FILE uses the same syntax
+                                     as the output template. This option can be
+                                     used multiple times
     -j, --dump-json                  Quiet, but print JSON information for each
                                      video. Simulate unless --no-simulate is
                                      used. See "OUTPUT TEMPLATE" for a
@@ -700,13 +732,13 @@ You can also fork the project on github and run your fork's [build workflow](.gi
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
+    --legacy-server-connect          Explicitly allow HTTPS connection to
+                                     servers that do not support RFC 5746 secure
+                                     renegotiation
     --no-check-certificates          Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve
                                      information about the video (Currently
                                      supported only for YouTube)
-    --user-agent UA                  Specify a custom user agent
-    --referer URL                    Specify a custom referer, use if the video
-                                     access is restricted to one domain
     --add-header FIELD:VALUE         Specify a custom HTTP header and its value,
                                      separated by a colon ":". You can use this
                                      option multiple times
@@ -778,9 +810,9 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      be regex) or "all" separated by commas.
                                      (Eg: --sub-langs "en.*,ja") You can prefix
                                      the language code with a "-" to exempt it
-                                     from the requested languages. (Eg: --sub-
+                                     from the requested languages. (Eg:
-                                     langs all,-live_chat) Use --list-subs for a
+                                     --sub-langs all,-live_chat) Use --list-subs
-                                     list of available language tags
+                                     for a list of available language tags
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
@@ -882,6 +914,15 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      multiple times
     --xattrs                         Write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
+    --concat-playlist POLICY         Concatenate videos in a playlist. One of
+                                     "never", "always", or "multi_video"
+                                     (default; only when the videos form a
+                                     single show). All the video files must have
+                                     same codecs and number of streams to be
+                                     concatable. The "pl_video:" prefix can be
+                                     used with "--paths" and "--output" to set
+                                     the output filename for the split files.
+                                     See "OUTPUT TEMPLATE" for details
     --fixup POLICY                   Automatically correct known faults of the
                                      file. One of never (do nothing), warn (only
                                      emit a warning), detect_or_warn (the
@@ -891,28 +932,25 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --ffmpeg-location PATH           Location of the ffmpeg binary; either the
                                      path to the binary or its containing
                                      directory
-    --exec CMD                       Execute a command on the file after
+    --exec [WHEN:]CMD                Execute a command, optionally prefixed with
-                                     downloading and post-processing. Same
+                                     when to execute it (after_move if
-                                     syntax as the output template can be used
+                                     unspecified), separated by a ":". Supported
-                                     to pass any field as arguments to the
+                                     values of "WHEN" are the same as that of
-                                     command. An additional field "filepath"
+                                     --use-postprocessor. Same syntax as the
+                                     output template can be used to pass any
+                                     field as arguments to the command. After
+                                     download, an additional field "filepath"
                                      that contains the final path of the
-                                     downloaded file is also available. If no
+                                     downloaded file is also available, and if
-                                     fields are passed, %(filepath)q is appended
+                                     no fields are passed, %(filepath)q is
-                                     to the end of the command. This option can
+                                     appended to the end of the command. This
-                                     be used multiple times
-    --no-exec                        Remove any previously defined --exec
-    --exec-before-download CMD       Execute a command before the actual
-                                     download. The syntax is the same as --exec
-                                     but "filepath" is not available. This
                                      option can be used multiple times
-    --no-exec-before-download        Remove any previously defined
+    --no-exec                        Remove any previously defined --exec
-                                     --exec-before-download
     --convert-subs FORMAT            Convert the subtitles to another format
                                      (currently supported: srt|vtt|ass|lrc)
                                      (Alias: --convert-subtitles)
     --convert-thumbnails FORMAT      Convert the thumbnails to another format
-                                     (currently supported: jpg|png)
+                                     (currently supported: jpg|png|webp)
     --split-chapters                 Split video into multiple files based on
                                      internal chapters. The "chapter:" prefix
                                      can be used with "--paths" and "--output"
@@ -943,13 +981,17 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                      semicolon ";" delimited list of NAME=VALUE.
                                      The "when" argument determines when the
                                      postprocessor is invoked. It can be one of
-                                     "pre_process" (after extraction),
+                                     "pre_process" (after video extraction),
-                                     "before_dl" (before video download),
+                                     "after_filter" (after video passes filter),
-                                     "post_process" (after video download;
+                                     "before_dl" (before each video download),
-                                     default) or "after_move" (after moving file
+                                     "post_process" (after each video download;
-                                     to their final locations). This option can
+                                     default), "after_move" (after moving video
-                                     be used multiple times to add different
+                                     file to it's final locations),
-                                     postprocessors
+                                     "after_video" (after downloading and
+                                     processing all formats of a video), or
+                                     "playlist" (at end of playlist). This
+                                     option can be used multiple times to add
+                                     different postprocessors
 
 ## SponsorBlock Options:
 Make chapter entries for, or remove various segments (sponsor,
@@ -1009,7 +1051,7 @@ You can configure yt-dlp by placing any supported command line option to a confi
 
 1. **Main Configuration**: The file given by `--config-location`
 1. **Portable Configuration**: `yt-dlp.conf` in the same directory as the bundled binary. If you are running from source-code (`<root dir>/yt_dlp/__main__.py`), the root directory is used instead.
-1. **Home Configuration**: `yt-dlp.conf` in the home path given by `-P "home:<path>"`, or in the current directory if no such path is given
+1. **Home Configuration**: `yt-dlp.conf` in the home path given by `-P`, or in the current directory if no such path is given
 1. **User Configuration**:
     * `%XDG_CONFIG_HOME%/yt-dlp/config` (recommended on Linux/macOS)
     * `%XDG_CONFIG_HOME%/yt-dlp.conf`
@@ -1087,7 +1129,7 @@ The field names themselves (the part inside the parenthesis) can also have some
 
 1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
 
-1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q`, `D`, `S` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma separated **l**ist (flag `#` for `\n` newline-separated), a string **q**uoted for the terminal (flag `#` to split a list into different arguments), to add **D**ecimal suffixes (Eg: 10M), and to **S**anitize as filename (flag `#` for restricted), respectively
+1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q`, `D`, `S` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma separated **l**ist (flag `#` for `\n` newline-separated), a string **q**uoted for the terminal (flag `#` to split a list into different arguments), to add **D**ecimal suffixes (Eg: 10M) (flag `#` to use 1024 as factor), and to **S**anitize as filename (flag `#` for restricted), respectively
 
 1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
 
@@ -1096,12 +1138,13 @@ To summarize, the general syntax for a field is:
 %(name[.keys][addition][>strf][,alternate][&replacement][|default])[flags][width][.precision][length]type
 ```
 
-Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"`  will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
+Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"`  will put the thumbnails in a folder with the same name as the video. If any of the templates is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
 
 The available fields are:
 
  - `id` (string): Video identifier
  - `title` (string): Video title
+ - `fulltitle` (string): Video title ignoring live timestamp and generic title
  - `url` (string): Video URL
  - `ext` (string): Video filename extension
  - `alt_title` (string): A secondary title of the video
@@ -1112,11 +1155,14 @@ The available fields are:
  - `creator` (string): The creator of the video
  - `timestamp` (numeric): UNIX timestamp of the moment the video became available
  - `upload_date` (string): Video upload date (YYYYMMDD)
- - `release_date` (string): The date (YYYYMMDD) when the video was released
  - `release_timestamp` (numeric): UNIX timestamp of the moment the video was released
+ - `release_date` (string): The date (YYYYMMDD) when the video was released
+ - `modified_timestamp` (numeric): UNIX timestamp of the moment the video was last modified
+ - `modified_date` (string): The date (YYYYMMDD) when the video was last modified
  - `uploader_id` (string): Nickname or id of the video uploader
  - `channel` (string): Full name of the channel the video is uploaded on
  - `channel_id` (string): Id of the channel
+ - `channel_follower_count` (numeric): Number of followers of the channel
  - `location` (string): Physical location where the video was filmed
  - `duration` (numeric): Length of the video in seconds
  - `duration_string` (string): Length of the video (HH:mm:ss)
@@ -1154,14 +1200,16 @@ The available fields are:
  - `protocol` (string): The protocol that will be used for the actual download
  - `extractor` (string): Name of the extractor
  - `extractor_key` (string): Key name of the extractor
- - `epoch` (numeric): Unix epoch when creating the file
+ - `epoch` (numeric): Unix epoch of when the information extraction was completed
  - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start`
+ - `video_autonumber` (numeric): Number that will be increased with each video
  - `n_entries` (numeric): Total number of extracted items in the playlist
- - `playlist` (string): Name or id of the playlist that contains the video
+ - `playlist_id` (string): Identifier of the playlist that contains the video
+ - `playlist_title` (string): Name of the playlist that contains the video
+ - `playlist` (string): `playlist_id` or `playlist_title`
+ - `playlist_count` (numeric): Total number of items in the playlist. May not be known if entire playlist is not extracted
  - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according the final index
  - `playlist_autonumber` (numeric): Position of the video in the playlist download queue padded with leading zeros according to the total length of the playlist
- - `playlist_id` (string): Playlist identifier
- - `playlist_title` (string): Playlist title
  - `playlist_uploader` (string): Full name of the playlist uploader
  - `playlist_uploader_id` (string): Nickname or id of the playlist uploader
  - `webpage_url` (string): A URL to the video webpage which if given to yt-dlp should allow to get the same result again
@@ -1209,6 +1257,11 @@ Available only when used in `--print`:
 
  - `urls` (string): The URLs of all requested formats, one in each line
  - `filename` (string): Name of the video file. Note that the actual filename may be different due to post-processing. Use `--exec echo` to get the name after all postprocessing is complete
+ - `formats_table` (table): The video format table as printed by `--list-formats`
+ - `thumbnails_table` (table): The thumbnail format table as printed by `--list-thumbnails`
+ - `subtitles_table` (table): The subtitle format table as printed by `--list-subs`
+ - `automatic_captions_table` (table): The automatic subtitle format table as printed by `--list-subs`
+ 
  
 Available only in `--sponsorblock-chapter-title`:
 
@@ -1260,7 +1313,7 @@ $ yt-dlp -o "%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s" "https://www.y
 $ yt-dlp -o "%(upload_date>%Y)s/%(title)s.%(ext)s" "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
 
 # Prefix playlist index with " - " separator, but only if it is available
-$ yt-dlp -o '%(playlist_index|)s%(playlist_index& - |)s%(title)s.%(ext)s' BaW_jenozKc https://www.youtube.com/user/TheLinuxFoundation/playlists
+$ yt-dlp -o '%(playlist_index|)s%(playlist_index& - |)s%(title)s.%(ext)s' BaW_jenozKc "https://www.youtube.com/user/TheLinuxFoundation/playlists"
 
 # Download all playlists of YouTube channel/user keeping each playlist in separate directory:
 $ yt-dlp -o "%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s" "https://www.youtube.com/user/TheLinuxFoundation/playlists"
@@ -1271,6 +1324,13 @@ $ yt-dlp -u user -p password -P "~/MyVideos" -o "%(playlist)s/%(chapter_number)s
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
 $ yt-dlp -P "C:/MyVideos" -o "%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" "https://videomore.ru/kino_v_detalayah/5_sezon/367617"
 
+# Download video as "C:\MyVideos\uploader\title.ext", subtitles as "C:\MyVideos\subs\uploader\title.ext"
+# and put all temporary files in "C:\MyVideos\tmp"
+$ yt-dlp -P "C:/MyVideos" -P "temp:tmp" -P "subtitle:subs" -o "%(uploader)s/%(title)s.%(ext)s" BaW_jenoz --write-subs
+
+# Download video as "C:\MyVideos\uploader\title.ext" and subtitles as "C:\MyVideos\uploader\subs\title.ext"
+$ yt-dlp -P "C:/MyVideos" -o "%(uploader)s/%(title)s.%(ext)s" -o "subtitle:%(uploader)s/subs/%(title)s.%(ext)s" BaW_jenozKc --write-subs
+
 # Stream the video being downloaded to stdout
 $ yt-dlp -o - BaW_jenozKc
 ```
@@ -1340,7 +1400,7 @@ The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `
  - `asr`: Audio sampling rate in Hertz
  - `fps`: Frame rate
 
-Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends with), `*=` (contains) and following string meta fields:
+Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends with), `*=` (contains), `~=` (matches regex) and following string meta fields:
 
  - `ext`: File extension
  - `acodec`: Name of the audio codec in use
@@ -1350,7 +1410,7 @@ Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends
  - `format_id`: A short description of the format
  - `language`: Language code
 
-Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain).
+Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain). The comparand of a string comparison needs to be quoted with either double or single quotes if it contains spaces or special characters other than `._-`.
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the website. Any other field made available by the extractor can also be used for filtering.
 
@@ -1366,10 +1426,10 @@ The available fields are:
 
  - `hasvid`: Gives priority to formats that has a video stream
  - `hasaud`: Gives priority to formats that has a audio stream
- - `ie_pref`: The format preference as given by the extractor
+ - `ie_pref`: The format preference
- - `lang`: Language preference as given by the extractor
+ - `lang`: The language preference
- - `quality`: The quality of the format as given by the extractor
+ - `quality`: The quality of the format
- - `source`: Preference of the source as given by the extractor
+ - `source`: The preference of the source
  - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > `mms`/`rtsp` > `f4f`/`f4m`)
  - `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other)
  - `acodec`: Audio Codec (`flac`/`alac` > `wav`/`aiff` > `opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other)
@@ -1493,8 +1553,9 @@ $ yt-dlp -S "proto"
 
 
 
-# Download the best video with h264 codec, or the best video if there is no such video
+# Download the best video with either h264 or h265 codec,
-$ yt-dlp -f "(bv*+ba/b)[vcodec^=avc1] / (bv*+ba/b)"
+# or the best video if there is no such video
+$ yt-dlp -f "(bv*[vcodec~='^((he|a)vc|h26[45])']+ba) / (bv*+ba/b)"
 
 # Download the best video with best codec no better than h264,
 # or the best video with worst codec if there is no such video
@@ -1537,27 +1598,30 @@ Note that any field created by this can be used in the [output template](#output
 
 This option also has a few special uses:
 * You can download an additional URL based on the metadata of the currently downloaded video. To do this, set the field `additional_urls` to the URL that you want to download. Eg: `--parse-metadata "description:(?P<additional_urls>https?://www\.vimeo\.com/\d+)` will download the first vimeo video found in the description
-* You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file. For example, you can use this to set a different "description" and "synopsis". Any value set to the `meta_` field will overwrite all default values.
+* You can use this to change the metadata that is embedded in the media file. To do this, set the value of the corresponding field with a `meta_` prefix. For example, any value you set to `meta_description` field will be added to the `description` field in the file. For example, you can use this to set a different "description" and "synopsis". To modify the metadata of individual streams, use the `meta<n>_` prefix (Eg: `meta1_language`). Any value set to the `meta_` field will overwrite all default values.
+
+**Note**: Metadata modification happens before format selection, post-extraction and other post-processing operations. Some fields may be added or changed during these steps, overriding your changes.
 
 For reference, these are the fields yt-dlp adds by default to the file metadata:
 
-Metadata fields|From
+Metadata fields            | From
-:---|:---
+:--------------------------|:------------------------------------------------
-`title`|`track` or `title`
+`title`                    | `track` or `title`
-`date`|`upload_date`
+`date`                     | `upload_date`
-`description`,  `synopsis`|`description`
+`description`,  `synopsis` | `description`
-`purl`, `comment`|`webpage_url`
+`purl`, `comment`          | `webpage_url`
-`track`|`track_number`
+`track`                    | `track_number`
-`artist`|`artist`, `creator`, `uploader` or `uploader_id`
+`artist`                   | `artist`, `creator`, `uploader` or `uploader_id`
-`genre`|`genre`
+`genre`                    | `genre`
-`album`|`album`
+`album`                    | `album`
-`album_artist`|`album_artist`
+`album_artist`             | `album_artist`
-`disc`|`disc_number`
+`disc`                     | `disc_number`
-`show`|`series`
+`show`                     | `series`
-`season_number`|`season_number`
+`season_number`            | `season_number`
-`episode_id`|`episode` or `episode_id`
+`episode_id`               | `episode` or `episode_id`
-`episode_sort`|`episode_number`
+`episode_sort`             | `episode_number`
-`language` of each stream|From the format's `language`
+`language` of each stream  | the format's `language`
+
 **Note**: The file format may not support some of these fields
 
 
@@ -1603,6 +1667,7 @@ The following extractors use this feature:
 
 #### youtubetab (YouTube playlists, channels, feeds, etc.)
 * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
+* `approximate_date`: Extract approximate `upload_date` in flat-playlist. This may cause date-based filters to be slightly off
 
 #### funimation
 * `language`: Languages to extract. Eg: `funimation:language=english,japanese`
@@ -1612,6 +1677,11 @@ The following extractors use this feature:
 * `language`: Languages to extract. Eg: `crunchyroll:language=jaJp`
 * `hardsub`: Which hard-sub versions to extract. Eg: `crunchyroll:hardsub=None,enUS`
 
+#### crunchyroll:beta
+* `format`: Which stream type(s) to extract. Default is `adaptive_hls` Eg: `crunchyrollbeta:format=vo_adaptive_hls`
+    * Potentially useful values include `adaptive_hls`, `adaptive_dash`, `vo_adaptive_hls`, `vo_adaptive_dash`, `download_hls`, `trailer_hls`, `trailer_dash`
+* `hardsub`: Preference order for which hardsub versions to extract. Default is `None` (no hardsubs). Eg: `crunchyrollbeta:hardsub=en-US,None`
+
 #### vikichannel
 * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
 
@@ -1621,6 +1691,19 @@ The following extractors use this feature:
 #### gamejolt
 * `comment_sort`: `hot` (default), `you` (cookies needed), `top`, `new` - choose comment sorting mode (on GameJolt's side)
 
+#### hotstar
+* `res`: resolution to ignore - one or more of `sd`, `hd`, `fhd`
+* `vcodec`: vcodec to ignore - one or more of `h264`, `h265`, `dvh265`
+* `dr`: dynamic range to ignore - one or more of `sdr`, `hdr10`, `dv`
+
+#### tiktok
+* `app_version`: App version to call mobile APIs with - should be set along with `manifest_app_version`. (e.g. `20.2.1`)
+* `manifest_app_version`: Numeric app version to call mobile APIs with. (e.g. `221`)
+
+#### rokfinchannel
+* `tab`: Which tab to download. One of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`. (E.g. `rokfinchannel:tab=streams`)
+
+
 NOTE: These options may be changed/removed in the future without concern for backward compatibility
 
 <!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
@@ -1656,7 +1739,7 @@ with YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L162).
+Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L191).
 
 Here's a more complete example demonstrating various functionality:
 
@@ -1737,12 +1820,11 @@ ydl_opts = {
     }],
     'logger': MyLogger(),
     'progress_hooks': [my_hook],
+    # Add custom headers
+    'http_headers': {'Referer': 'https://www.google.com'}
 }
 
 
-# Add custom headers
-yt_dlp.utils.std_headers.update({'Referer': 'https://www.google.com'})
-
 # ℹ️ See the public functions in yt_dlp.YoutubeDL for for other available functions.
 # Eg: "ydl.download", "ydl.download_with_info_file"
 with yt_dlp.YoutubeDL(ydl_opts) as ydl:
@@ -1762,6 +1844,14 @@ with yt_dlp.YoutubeDL(ydl_opts) as ydl:
 
 These are all the deprecated options and the current alternative to achieve the same effect
 
+#### Almost redundant options
+While these options are almost the same as their new counterparts, there are some differences that prevents them being redundant
+
+    -j, --dump-json                  --print "%()j"
+    -F, --list-formats               --print formats_table
+    --list-thumbnails                --print thumbnails_table --print playlist:thumbnails_table
+    --list-subs                      --print automatic_captions_table --print subtitles_table
+
 #### Redundant options
 While these options are redundant, they are still expected to be used due to their ease of use
 
@@ -1773,16 +1863,19 @@ While these options are redundant, they are still expected to be used due to the
     --get-thumbnail                  --print thumbnail
     -e, --get-title                  --print title
     -g, --get-url                    --print urls
-    -j, --dump-json                  --print "%()j"
     --match-title REGEX              --match-filter "title ~= (?i)REGEX"
     --reject-title REGEX             --match-filter "title !~= (?i)REGEX"
     --min-views COUNT                --match-filter "view_count >=? COUNT"
     --max-views COUNT                --match-filter "view_count <=? COUNT"
+    --user-agent UA                  --add-header "User-Agent:UA"
+    --referer URL                    --add-header "Referer:URL"
 
 
 #### Not recommended
 While these options still work, their use is not recommended since there are other alternatives to achieve the same
 
+    --exec-before-download CMD       --exec "before_dl:CMD"
+    --no-exec-before-download        --no-exec
     --all-formats                    -f all
     --all-subs                       --sub-langs all --write-subs
     --print-json                     -j --no-simulate
@@ -1813,11 +1906,13 @@ These options are not intended to be used by the end-user
 These are aliases that are no longer documented for various reasons
 
     --avconv-location                --ffmpeg-location
+    --clean-infojson                 --clean-info-json
     --cn-verification-proxy URL      --geo-verification-proxy URL
     --dump-headers                   --print-traffic
     --dump-intermediate-pages        --dump-pages
     --force-write-download-archive   --force-write-archive
     --load-info                      --load-info-json
+    --no-clean-infojson              --no-clean-info-json
     --no-split-tracks                --no-split-chapters
     --no-write-srt                   --no-write-subs
     --prefer-unsecure                --prefer-insecure

devscripts/prepare_manpage.py

@@ -75,21 +75,21 @@ def filter_options(readme):
     section = re.search(r'(?sm)^# USAGE AND OPTIONS\n.+?(?=^# )', readme).group(0)
     options = '# OPTIONS\n'
     for line in section.split('\n')[1:]:
-        if line.lstrip().startswith('-'):
+        mobj = re.fullmatch(r'''(?x)
-            split = re.split(r'\s{2,}', line.lstrip())
+                \s{4}(?P<opt>-(?:,\s|[^\s])+)
-            # Description string may start with `-` as well. If there is
+                (?:\s(?P<meta>(?:[^\s]|\s(?!\s))+))?
-            # only one piece then it's a description bit not an option.
+                (\s{2,}(?P<desc>.+))?
-            if len(split) > 1:
+            ''', line)
-                option, description = split
+        if not mobj:
-                split_option = option.split(' ')
+            options += f'{line.lstrip()}\n'
+            continue
+        option, metavar, description = mobj.group('opt', 'meta', 'desc')
 
-                if not split_option[-1].startswith('-'):  # metavar
+        # Pandoc's definition_lists. See http://pandoc.org/README.html
-                    option = ' '.join(split_option[:-1] + [f'*{split_option[-1]}*'])
+        option = f'{option} *{metavar}*' if metavar else option
-
+        description = f'{description}\n' if description else ''
-                # Pandoc's definition_lists. See http://pandoc.org/README.html
+        options += f'\n{option}\n:   {description}'
-                options += f'\n{option}\n:   {description}\n'
+        continue
-                continue
-        options += line.lstrip() + '\n'
 
     return readme.replace(section, options, 1)
 

pyinst.py

@@ -74,7 +74,7 @@ def version_to_list(version):
 
 
 def dependency_options():
-    dependencies = [pycryptodome_module(), 'mutagen'] + collect_submodules('websockets')
+    dependencies = [pycryptodome_module(), 'mutagen', 'brotli'] + collect_submodules('websockets')
     excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc']
 
     yield from (f'--hidden-import={module}' for module in dependencies)

requirements.txt

@@ -1,3 +1,5 @@
 mutagen
 pycryptodomex
 websockets
+brotli; platform_python_implementation=='CPython'
+brotlicffi; platform_python_implementation!='CPython'

setup.py

@@ -21,9 +21,9 @@ DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',
     '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
-    open('README.md', 'r', encoding='utf-8').read()))
+    open('README.md').read()))
 
-REQUIREMENTS = ['mutagen', 'pycryptodomex', 'websockets']
+REQUIREMENTS = open('requirements.txt').read().splitlines()
 
 
 if sys.argv[1:2] == ['py2exe']:

supportedsites.md

@@ -3,7 +3,6 @@
  - **17live:clip**
  - **1tv**: Первый канал
  - **20min**
- - **220.ro**
  - **23video**
  - **247sports**
  - **24video**
@@ -11,7 +10,6 @@
  - **3sat**
  - **4tube**
  - **56.com**
- - **5min**
  - **6play**
  - **7plus**
  - **8tracks**
@@ -26,6 +24,8 @@
  - **abcnews:video**
  - **abcotvs**: ABC Owned Television Stations
  - **abcotvs:clips**
+ - **AbemaTV**
+ - **AbemaTVTitle**
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
@@ -41,11 +41,14 @@
  - **aenetworks:collection**
  - **aenetworks:show**
  - **afreecatv**: afreecatv.com
+ - **afreecatv:live**: afreecatv.com
  - **AirMozilla**
  - **AliExpressLive**
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **Alsace20TV**
+ - **Alsace20TVEmbed**
  - **Alura**
  - **AluraCourse**
  - **Amara**
@@ -53,11 +56,15 @@
  - **AMCNetworks**
  - **AmericasTestKitchen**
  - **AmericasTestKitchenSeason**
+ - **AmHistoryChannel**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimalPlanet**
  - **AnimeLab**
  - **AnimeLabShows**
  - **AnimeOnDemand**
+ - **ant1newsgr:article**: ant1news.gr articles
+ - **ant1newsgr:embed**: ant1news.gr embedded videos
+ - **ant1newsgr:watch**: ant1news.gr videos
  - **Anvato**
  - **aol.com**: Yahoo screen and movies
  - **APA**
@@ -75,6 +82,7 @@
  - **Arkena**
  - **arte.sky.it**
  - **ArteTV**
+ - **ArteTVCategory**
  - **ArteTVEmbed**
  - **ArteTVPlaylist**
  - **AsianCrush**
@@ -99,8 +107,8 @@
  - **bandaichannel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **Bandcamp:user**
  - **Bandcamp:weekly**
- - **BandcampMusic**
  - **bangumi.bilibili.com**: BiliBili番剧
  - **BannedVideo**
  - **bbc**: BBC
@@ -122,6 +130,7 @@
  - **bfmtv:live**
  - **BibelTV**
  - **Bigflix**
+ - **Bigo**
  - **Bild**: Bild.de
  - **BiliBili**
  - **Bilibili category extractor**
@@ -162,6 +171,8 @@
  - **BuzzFeed**
  - **BYUtv**
  - **CableAV**
+ - **Callin**
+ - **Caltrans**
  - **CAM4**
  - **Camdemy**
  - **CamdemyFolder**
@@ -225,18 +236,24 @@
  - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **CONtv**
+ - **CookingChannel**
  - **Corus**
  - **Coub**
  - **CozyTV**
  - **cp24**
+ - **cpac**
+ - **cpac:playlist**
  - **Cracked**
  - **Crackle**
  - **CrooksAndLiars**
+ - **CrowdBunker**
+ - **CrowdBunkerChannel**
  - **crunchyroll**
  - **crunchyroll:beta**
  - **crunchyroll:playlist**
  - **crunchyroll:playlist:beta**
  - **CSpan**: C-SPAN
+ - **CSpanCongress**
  - **CtsNews**: 華視新聞
  - **CTV**
  - **CTVNews**
@@ -246,6 +263,7 @@
  - **curiositystream:collections**
  - **curiositystream:series**
  - **CWTV**
+ - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
  - **dailymotion**
@@ -257,26 +275,27 @@
  - **daum.net:clip**
  - **daum.net:playlist**
  - **daum.net:user**
+ - **daystar:clip**
  - **DBTV**
  - **DctpTv**
  - **DeezerAlbum**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
  - **democracynow**
+ - **DestinationAmerica**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Digg**
+ - **DigitalConcertHall**: DigitalConcertHall extractor
  - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
- - **DiscoveryGo**
+ - **DiscoveryLife**
- - **DiscoveryGoPlaylist**
  - **DiscoveryNetworksDe**
  - **DiscoveryPlus**
  - **DiscoveryPlusIndia**
  - **DiscoveryPlusIndiaShow**
  - **DiscoveryPlusItaly**
  - **DiscoveryPlusItalyShow**
- - **DiscoveryVR**
  - **Disney**
  - **DIYNetwork**
  - **dlive:stream**
@@ -288,6 +307,7 @@
  - **DouyuTV**: 斗鱼
  - **DPlay**
  - **DRBonanza**
+ - **Drooble**
  - **Dropbox**
  - **Dropout**
  - **DropoutSeason**
@@ -324,12 +344,16 @@
  - **Eporner**
  - **EroProfile**
  - **EroProfile:album**
+ - **ertflix**: ERTFLIX videos
+ - **ertflix:codename**: ERTFLIX videos by codename
+ - **ertwebtv:embed**: ert.gr webtv embedded videos
  - **Escapist**
  - **ESPN**
  - **ESPNArticle**
  - **ESPNCricInfo**
  - **EsriVideo**
  - **Europa**
+ - **EuropeanTour**
  - **EUScreen**
  - **EWETV**
  - **ExpoTV**
@@ -343,6 +367,7 @@
  - **faz.net**
  - **fc2**
  - **fc2:embed**
+ - **fc2:live**
  - **Fczenit**
  - **Filmmodu**
  - **filmon**
@@ -352,6 +377,7 @@
  - **FiveTV**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
+ - **FoodNetwork**
  - **FootyRoom**
  - **Formula1**
  - **FOX**
@@ -361,6 +387,7 @@
  - **foxnews**: Fox News and Fox Business Video
  - **foxnews:article**
  - **FoxSports**
+ - **fptplay**: fptplay.vn
  - **FranceCulture**
  - **FranceInter**
  - **FranceTV**
@@ -368,7 +395,6 @@
  - **FranceTVSite**
  - **Freesound**
  - **freespeech.org**
- - **FreshLive**
  - **FrontendMasters**
  - **FrontendMastersCourse**
  - **FrontendMastersLesson**
@@ -400,6 +426,7 @@
  - **gem.cbc.ca:playlist**
  - **generic**: Generic downloader that works on some sites
  - **Gettr**
+ - **GettrStreaming**
  - **Gfycat**
  - **GiantBomb**
  - **Giga**
@@ -407,7 +434,10 @@
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GloboArticle**
+ - **glomex**: Glomex videos
+ - **glomex:embed**: Glomex embedded videos
  - **Go**
+ - **GoDiscovery**
  - **GodTube**
  - **Gofile**
  - **Golem**
@@ -429,6 +459,7 @@
  - **hetklokhuis**
  - **hgtv.com:show**
  - **HGTVDe**
+ - **HGTVUsa**
  - **HiDive**
  - **HistoricFilms**
  - **history:player**
@@ -437,7 +468,6 @@
  - **hitbox:live**
  - **HitRecord**
  - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
- - **HornBunny**
  - **HotNewHipHop**
  - **hotstar**
  - **hotstar:playlist**
@@ -470,15 +500,18 @@
  - **IndavideoEmbed**
  - **InfoQ**
  - **Instagram**
+ - **instagram:story**
  - **instagram:tag**: Instagram hashtag search URLs
  - **instagram:user**: Instagram user profile
  - **InstagramIOS**: IOS instagram:// URL
  - **Internazionale**
  - **InternetVideoArchive**
+ - **InvestigationDiscovery**
  - **IPrima**
  - **IPrimaCNN**
+ - **iq.com**: International version of iQiyi
+ - **iq.com:album**
  - **iqiyi**: 爱奇艺
- - **Ir90Tv**
  - **ITTF**
  - **ITV**
  - **ITVBTCC**
@@ -495,11 +528,11 @@
  - **JWPlatform**
  - **Kakao**
  - **Kaltura**
- - **Kankan**
  - **Karaoketv**
  - **KarriereVideos**
  - **Katsomo**
  - **KeezMovies**
+ - **KelbyOne**
  - **Ketnet**
  - **khanacademy**
  - **khanacademy:unit**
@@ -545,7 +578,6 @@
  - **limelight:channel_list**
  - **LineLive**
  - **LineLiveChannel**
- - **LineTV**
  - **LinkedIn**
  - **linkedin:learning**
  - **linkedin:learning:course**
@@ -554,6 +586,7 @@
  - **LiveJournal**
  - **livestream**
  - **livestream:original**
+ - **Lnk**
  - **LnkGo**
  - **loc**: Library of Congress
  - **LocalNews8**
@@ -566,6 +599,7 @@
  - **mailru**: Видео@Mail.Ru
  - **mailru:music**: Музыка@Mail.Ru
  - **mailru:music:search**: Музыка@Mail.Ru
+ - **MainStreaming**: MainStreaming Player
  - **MallTV**
  - **mangomolo:live**
  - **mangomolo:video**
@@ -592,6 +626,8 @@
  - **MediasiteNamedCatalog**
  - **Medici**
  - **megaphone.fm**: megaphone.fm embedded players
+ - **megatvcom**: megatv.com videos
+ - **megatvcom:embed**: megatv.com embedded videos
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -603,8 +639,9 @@
  - **MiaoPai**
  - **microsoftstream**: Microsoft Stream
  - **mildom**: Record ongoing live by specific user in Mildom
+ - **mildom:clip**: Clip in Mildom
  - **mildom:user:vod**: Download all VODs from specific user in Mildom
- - **mildom:vod**: Download a VOD in Mildom
+ - **mildom:vod**: VOD in Mildom
  - **minds**
  - **minds:channel**
  - **minds:group**
@@ -615,6 +652,7 @@
  - **mirrativ:user**
  - **MiTele**: mitele.es
  - **mixch**
+ - **mixch:archive**
  - **mixcloud**
  - **mixcloud:playlist**
  - **mixcloud:user**
@@ -646,7 +684,13 @@
  - **mtvservices:embedded**
  - **MTVUutisetArticle**
  - **MuenchenTV**: münchen.tv
+ - **Murrtube**
+ - **MurrtubeUser**: Murrtube user profile
  - **MuseScore**
+ - **MusicdexAlbum**
+ - **MusicdexArtist**
+ - **MusicdexPlaylist**
+ - **MusicdexSong**
  - **mva**: Microsoft Virtual Academy videos
  - **mva:course**: Microsoft Virtual Academy courses
  - **Mwave**
@@ -704,14 +748,19 @@
  - **Newgrounds:playlist**
  - **Newgrounds:user**
  - **Newstube**
+ - **Newsy**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **NextTV**: 壹電視
  - **Nexx**
  - **NexxEmbed**
+ - **NFB**
  - **NFHSNetwork**
  - **nfl.com** (Currently broken)
  - **nfl.com:article** (Currently broken)
+ - **NhkForSchoolBangumi**
+ - **NhkForSchoolProgramList**
+ - **NhkForSchoolSubject**: Portal page for each school subjects, like Japanese (kokugo, 国語) or math (sansuu/suugaku or 算数・数学)
  - **NhkVod**
  - **NhkVodProgram**
  - **nhl.com**
@@ -721,7 +770,10 @@
  - **nickelodeonru**
  - **nicknight**
  - **niconico**: ニコニコ動画
- - **NiconicoPlaylist**
+ - **niconico:history**: NicoNico user history. Requires cookies.
+ - **niconico:playlist**
+ - **niconico:series**
+ - **niconico:tag**: NicoNico video tag URLs
  - **NiconicoUser**
  - **nicovideo:search**: Nico video search; "nicosearch:" prefix
  - **nicovideo:search:date**: Nico video search, newest first; "nicosearchdate:" prefix
@@ -733,6 +785,7 @@
  - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **NonkTube**
+ - **NoodleMagazine**
  - **Noovo**
  - **Normalboots**
  - **NosVideo**
@@ -785,6 +838,7 @@
  - **OpencastPlaylist**
  - **openrec**
  - **openrec:capture**
+ - **openrec:movie**
  - **OraTV**
  - **orf:burgenland**: Radio Burgenland
  - **orf:fm4**: radio FM4
@@ -818,6 +872,7 @@
  - **PatreonUser**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **PearVideo**
+ - **PeekVids**
  - **peer.tv**
  - **PeerTube**
  - **PeerTube:Playlist**
@@ -830,12 +885,15 @@
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
+ - **Piapro**
  - **Picarto**
  - **PicartoVod**
  - **Piksel**
  - **Pinkbike**
  - **Pinterest**
  - **PinterestCollection**
+ - **pixiv:sketch**
+ - **pixiv:sketch:user**
  - **Pladform**
  - **PlanetMarathi**
  - **Platzi**
@@ -847,6 +905,7 @@
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
+ - **PlayVids**
  - **Playwire**
  - **pluralsight**
  - **pluralsight:course**
@@ -854,6 +913,8 @@
  - **podomatic**
  - **Pokemon**
  - **PokemonWatch**
+ - **PokerGo**
+ - **PokerGoCollection**
  - **PolsatGo**
  - **PolskieRadio**
  - **polskieradio:kierowcow**
@@ -865,6 +926,7 @@
  - **PopcornTV**
  - **PornCom**
  - **PornerBros**
+ - **Pornez**
  - **PornFlip**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
@@ -879,6 +941,11 @@
  - **PressTV**
  - **ProjectVeritas**
  - **prosiebensat1**: ProSiebenSat.1 Digital
+ - **PRXAccount**
+ - **PRXSeries**
+ - **prxseries:search**: PRX Series Search; "prxseries:" prefix
+ - **prxstories:search**: PRX Stories Search; "prxstories:" prefix
+ - **PRXStory**
  - **puhutv**
  - **puhutv:serie**
  - **Puls4**
@@ -912,8 +979,9 @@
  - **RaiPlay**
  - **RaiPlayLive**
  - **RaiPlayPlaylist**
- - **RaiPlayRadio**
+ - **RaiPlaySound**
- - **RaiPlayRadioPlaylist**
+ - **RaiPlaySoundLive**
+ - **RaiPlaySoundPlaylist**
  - **RayWenderlich**
  - **RayWenderlichCourse**
  - **RBMARadio**
@@ -942,18 +1010,23 @@
  - **RICE**
  - **RMCDecouverte**
  - **RockstarGames**
+ - **Rokfin**
+ - **rokfin:channel**
+ - **rokfin:stack**
  - **RoosterTeeth**
  - **RoosterTeethSeries**
  - **RottenTomatoes**
- - **Roxwel**
  - **Rozhlas**
  - **RTBF**
+ - **RTDocumentry**
+ - **RTDocumentryPlaylist**
  - **rte**: Raidió Teilifís Éireann TV
  - **rte:radio**: Raidió Teilifís Éireann radio
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **rtl2**
  - **rtl2:you**
  - **rtl2:you:series**
+ - **RTNews**
  - **RTP**
  - **RTRFM**
  - **RTS**: RTS.ch
@@ -965,8 +1038,10 @@
  - **RTVNH**
  - **RTVS**
  - **RUHD**
+ - **Rule34Video**
  - **RumbleChannel**
  - **RumbleEmbed**
+ - **Ruptly**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channel
  - **rutube:embed**: Rutube embedded videos
@@ -977,6 +1052,7 @@
  - **RUTV**: RUTV.RU
  - **Ruutu**
  - **Ruv**
+ - **ruv.is:spila**
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
@@ -1107,12 +1183,16 @@
  - **TeamTreeHouse**
  - **TechTalks**
  - **techtv.mit.edu**
- - **ted**
+ - **TedEmbed**
+ - **TedPlaylist**
+ - **TedSeries**
+ - **TedTalk**
  - **Tele13**
  - **Tele5**
  - **TeleBruxelles**
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
+ - **telegram:embed**
  - **TeleMB**
  - **Telemundo**
  - **TeleQuebec**
@@ -1129,7 +1209,6 @@
  - **TheIntercept**
  - **ThePlatform**
  - **ThePlatformFeed**
- - **TheScene**
  - **TheStar**
  - **TheSun**
  - **ThetaStream**
@@ -1141,8 +1220,12 @@
  - **ThreeSpeak**
  - **ThreeSpeakUser**
  - **TikTok**
+ - **tiktok:effect**
+ - **tiktok:sound**
+ - **tiktok:tag**
  - **tiktok:user**
  - **tinypic**: tinypic.com videos
+ - **TLC**
  - **TMZ**
  - **TNAFlix**
  - **TNAFlixNetworkEmbed**
@@ -1155,6 +1238,7 @@
  - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
+ - **TravelChannel**
  - **Trilulilu**
  - **Trovo**
  - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
@@ -1202,6 +1286,8 @@
  - **TVNowNew**
  - **TVNowSeason**
  - **TVNowShow**
+ - **tvopengr:embed**: tvopen.gr embedded videos
+ - **tvopengr:watch**: tvopen.gr (and ethnos.gr) videos
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
@@ -1265,9 +1351,11 @@
  - **Viddler**
  - **Videa**
  - **video.arnes.si**: Arnes Video
- - **video.google:search**: Google Video search; "gvsearch:" prefix (Currently broken)
+ - **video.google:search**: Google Video search; "gvsearch:" prefix
  - **video.sky.it**
  - **video.sky.it:live**
+ - **VideocampusSachsen**
+ - **VideocampusSachsenEmbed**
  - **VideoDetective**
  - **videofy.me**
  - **videomore**
@@ -1294,6 +1382,8 @@
  - **vimeo:review**: Review pages on vimeo
  - **vimeo:user**
  - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
+ - **Vimm:recording**
+ - **Vimm:stream**
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
@@ -1308,6 +1398,7 @@
  - **vlive**
  - **vlive:channel**
  - **vlive:post**
+ - **vm.tiktok**
  - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
@@ -1327,7 +1418,6 @@
  - **VShare**
  - **VTM**
  - **VTXTV**
- - **vube**: Vube.com
  - **VuClip**
  - **Vupload**
  - **VVVVID**
@@ -1343,10 +1433,10 @@
  - **WatchBox**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
- - **wdr:mobile**
+ - **wdr:mobile** (Currently broken)
  - **WDRElefant**
  - **WDRPage**
- - **web.archive:youtube**: web.archive.org saved youtube videos
+ - **web.archive:youtube**: web.archive.org saved youtube videos, "ytarchive:" prefix
  - **Webcaster**
  - **WebcasterFeed**
  - **WebOfStories**
@@ -1378,6 +1468,7 @@
  - **xiami:song**: 虾米音乐
  - **ximalaya**: 喜马拉雅FM
  - **ximalaya:album**: 喜马拉雅FM 专辑
+ - **xinpianchang**: xinpianchang.com
  - **XMinus**
  - **XNXX**
  - **Xstream**
@@ -1397,6 +1488,7 @@
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YandexVideo**
+ - **YandexVideoPreview**
  - **YapFiles**
  - **YesJapan**
  - **yinyuetai:video**: 音悦Tai
@@ -1413,6 +1505,7 @@
  - **youtube**: YouTube
  - **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)
  - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies)
+ - **youtube:music:search_url**: YouTube music search URLs with selectable sections (Eg: #songs)
  - **youtube:playlist**: YouTube playlists
  - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword
  - **youtube:search**: YouTube search; "ytsearch:" prefix
@@ -1420,9 +1513,10 @@
  - **youtube:search_url**: YouTube search URLs with sorting and filter support
  - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)
  - **youtube:tab**: YouTube Tabs
+ - **youtube:user**: YouTube user videos; "ytuser:" prefix
  - **youtube:watchlater**: Youtube watch later list; ":ytwatchlater" keyword (requires cookies)
+ - **YoutubeLivestreamEmbed**: YouTube livestream embeds
  - **YoutubeYtBe**: youtu.be
- - **YoutubeYtUser**: YouTube user videos; "ytuser:" prefix
  - **Zapiks**
  - **Zattoo**
  - **ZattooLive**
@@ -1433,7 +1527,7 @@
  - **ZenYandex**
  - **ZenYandexChannel**
  - **Zhihu**
- - **zingmp3**: mp3.zing.vn
+ - **zingmp3**: zingmp3.vn
  - **zingmp3:album**
  - **zoom**
  - **Zype**

test/helper.py

@@ -211,7 +211,7 @@ def sanitize_got_info_dict(got_dict):
 
         # Auto-generated
         'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch',
-        'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url',
+        'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url', 'n_entries',
 
         # Only live_status needs to be checked
         'is_live', 'was_live',
@@ -220,10 +220,12 @@ def sanitize_got_info_dict(got_dict):
     IGNORED_PREFIXES = ('', 'playlist', 'requested', 'webpage')
 
     def sanitize(key, value):
-        if isinstance(value, str) and len(value) > 100:
+        if isinstance(value, str) and len(value) > 100 and key != 'thumbnail':
             return f'md5:{md5(value)}'
         elif isinstance(value, list) and len(value) > 10:
             return f'count:{len(value)}'
+        elif key.endswith('_count') and isinstance(value, int):
+            return int
         return value
 
     test_info_dict = {
@@ -233,7 +235,7 @@ def sanitize_got_info_dict(got_dict):
     }
 
     # display_id may be generated from id
-    if test_info_dict.get('display_id') == test_info_dict['id']:
+    if test_info_dict.get('display_id') == test_info_dict.get('id'):
         test_info_dict.pop('display_id')
 
     return test_info_dict
@@ -259,6 +261,8 @@ def expect_info_dict(self, got_dict, expected_dict):
         def _repr(v):
             if isinstance(v, compat_str):
                 return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
+            elif isinstance(v, type):
+                return v.__name__
             else:
                 return repr(v)
         info_dict_str = ''

test/test_InfoExtractor.py

@@ -208,6 +208,91 @@ class TestInfoExtractor(unittest.TestCase):
                 },
                 {'expected_type': 'NewsArticle'},
             ),
+            (
+                r'''<script type="application/ld+json">
+                {"url":"/vrtnu/a-z/het-journaal/2021/het-journaal-het-journaal-19u-20211231/",
+                "name":"Het journaal 19u",
+                "description":"Het journaal 19u van vrijdag 31 december 2021.",
+                "potentialAction":{"url":"https://vrtnu.page.link/pfVy6ihgCAJKgHqe8","@type":"ShareAction"},
+                "mainEntityOfPage":{"@id":"1640092242445","@type":"WebPage"},
+                "publication":[{
+                    "startDate":"2021-12-31T19:00:00.000+01:00",
+                    "endDate":"2022-01-30T23:55:00.000+01:00",
+                    "publishedBy":{"name":"een","@type":"Organization"},
+                    "publishedOn":{"url":"https://www.vrt.be/vrtnu/","name":"VRT NU","@type":"BroadcastService"},
+                    "@id":"pbs-pub-3a7ec233-da95-4c1e-9b2b-cf5fdfebcbe8",
+                    "@type":"BroadcastEvent"
+                    }],
+                "video":{
+                    "name":"Het journaal - Aflevering 365 (Seizoen 2021)",
+                    "description":"Het journaal 19u van vrijdag 31 december 2021. Bekijk aflevering 365 van seizoen 2021 met VRT NU via de site of app.",
+                    "thumbnailUrl":"//images.vrt.be/width1280/2021/12/31/80d5ed00-6a64-11ec-b07d-02b7b76bf47f.jpg",
+                    "expires":"2022-01-30T23:55:00.000+01:00",
+                    "hasPart":[
+                        {"name":"Explosie Turnhout","startOffset":70,"@type":"Clip"},
+                        {"name":"Jaarwisseling","startOffset":440,"@type":"Clip"},
+                        {"name":"Natuurbranden Colorado","startOffset":1179,"@type":"Clip"},
+                        {"name":"Klimaatverandering","startOffset":1263,"@type":"Clip"},
+                        {"name":"Zacht weer","startOffset":1367,"@type":"Clip"},
+                        {"name":"Financiële balans","startOffset":1383,"@type":"Clip"},
+                        {"name":"Club Brugge","startOffset":1484,"@type":"Clip"},
+                        {"name":"Mentale gezondheid bij topsporters","startOffset":1575,"@type":"Clip"},
+                        {"name":"Olympische Winterspelen","startOffset":1728,"@type":"Clip"},
+                        {"name":"Sober oudjaar in Nederland","startOffset":1873,"@type":"Clip"}
+                        ],
+                    "duration":"PT34M39.23S",
+                    "uploadDate":"2021-12-31T19:00:00.000+01:00",
+                    "@id":"vid-9457d0c6-b8ac-4aba-b5e1-15aa3a3295b5",
+                    "@type":"VideoObject"
+                },
+                "genre":["Nieuws en actua"],
+                "episodeNumber":365,
+                "partOfSeries":{"name":"Het journaal","@id":"222831405527","@type":"TVSeries"},
+                "partOfSeason":{"name":"Seizoen 2021","@id":"961809365527","@type":"TVSeason"},
+                "@context":"https://schema.org","@id":"961685295527","@type":"TVEpisode"}</script>
+                ''',
+                {
+                    'chapters': [
+                        {"title": "Explosie Turnhout", "start_time": 70, "end_time": 440},
+                        {"title": "Jaarwisseling", "start_time": 440, "end_time": 1179},
+                        {"title": "Natuurbranden Colorado", "start_time": 1179, "end_time": 1263},
+                        {"title": "Klimaatverandering", "start_time": 1263, "end_time": 1367},
+                        {"title": "Zacht weer", "start_time": 1367, "end_time": 1383},
+                        {"title": "Financiële balans", "start_time": 1383, "end_time": 1484},
+                        {"title": "Club Brugge", "start_time": 1484, "end_time": 1575},
+                        {"title": "Mentale gezondheid bij topsporters", "start_time": 1575, "end_time": 1728},
+                        {"title": "Olympische Winterspelen", "start_time": 1728, "end_time": 1873},
+                        {"title": "Sober oudjaar in Nederland", "start_time": 1873, "end_time": 2079.23}
+                    ],
+                    'title': 'Het journaal - Aflevering 365 (Seizoen 2021)'
+                }, {}
+            ),
+            (
+                # test multiple thumbnails in a list
+                r'''
+<script type="application/ld+json">
+{"@context":"https://schema.org",
+"@type":"VideoObject",
+"thumbnailUrl":["https://www.rainews.it/cropgd/640x360/dl/img/2021/12/30/1640886376927_GettyImages.jpg"]}
+</script>''',
+                {
+                    'thumbnails': [{'url': 'https://www.rainews.it/cropgd/640x360/dl/img/2021/12/30/1640886376927_GettyImages.jpg'}],
+                },
+                {},
+            ),
+            (
+                # test single thumbnail
+                r'''
+<script type="application/ld+json">
+{"@context":"https://schema.org",
+"@type":"VideoObject",
+"thumbnailUrl":"https://www.rainews.it/cropgd/640x360/dl/img/2021/12/30/1640886376927_GettyImages.jpg"}
+</script>''',
+                {
+                    'thumbnails': [{'url': 'https://www.rainews.it/cropgd/640x360/dl/img/2021/12/30/1640886376927_GettyImages.jpg'}],
+                },
+                {},
+            )
         ]
         for html, expected_dict, search_json_ld_kwargs in _TESTS:
             expect_dict(

test/test_YoutubeDL.py

@@ -30,8 +30,7 @@ class YDL(FakeYDL):
         self.msgs = []
 
     def process_info(self, info_dict):
-        info_dict.pop('__original_infodict', None)
+        self.downloaded_info_dicts.append(info_dict.copy())
-        self.downloaded_info_dicts.append(info_dict)
 
     def to_screen(self, msg):
         self.msgs.append(msg)
@@ -645,6 +644,7 @@ class TestYoutubeDL(unittest.TestCase):
         'ext': 'mp4',
         'width': None,
         'height': 1080,
+        'filesize': 1024,
         'title1': '$PATH',
         'title2': '%PATH%',
         'title3': 'foo/bar\\test',
@@ -778,8 +778,9 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
         test('%(title5)+U', 'áéí A')
         test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
-        test('%(height)D', '1K')
+        test('%(height)D', '1k')
-        test('%(height)5.2D', ' 1.08K')
+        test('%(filesize)#D', '1Ki')
+        test('%(height)5.2D', ' 1.08k')
         test('%(title4)#S', 'foo_bar_test')
         test('%(title4).10S', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' ')))
         if compat_os_name == 'nt':
@@ -895,20 +896,6 @@ class TestYoutubeDL(unittest.TestCase):
         os.unlink(filename)
 
     def test_match_filter(self):
-        class FilterYDL(YDL):
-            def __init__(self, *args, **kwargs):
-                super(FilterYDL, self).__init__(*args, **kwargs)
-                self.params['simulate'] = True
-
-            def process_info(self, info_dict):
-                super(YDL, self).process_info(info_dict)
-
-            def _match_entry(self, info_dict, incomplete=False):
-                res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
-                if res is None:
-                    self.downloaded_info_dicts.append(info_dict)
-                return res
-
         first = {
             'id': '1',
             'url': TEST_URL,
@@ -936,7 +923,7 @@ class TestYoutubeDL(unittest.TestCase):
         videos = [first, second]
 
         def get_videos(filter_=None):
-            ydl = FilterYDL({'match_filter': filter_})
+            ydl = YDL({'match_filter': filter_, 'simulate': True})
             for v in videos:
                 ydl.process_ie_result(v, download=True)
             return [v['id'] for v in ydl.downloaded_info_dicts]
@@ -1151,6 +1138,7 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertTrue(entries[1] is None)
         self.assertEqual(len(ydl.downloaded_info_dicts), 1)
         downloaded = ydl.downloaded_info_dicts[0]
+        entries[2].pop('requested_downloads', None)
         self.assertEqual(entries[2], downloaded)
         self.assertEqual(downloaded['url'], TEST_URL)
         self.assertEqual(downloaded['title'], 'Video Transparent 2')

test/test_cookies.py

@@ -8,6 +8,8 @@ from yt_dlp.cookies import (
     WindowsChromeCookieDecryptor,
     parse_safari_cookies,
     pbkdf2_sha1,
+    _get_linux_desktop_environment,
+    _LinuxDesktopEnvironment,
 )
 
 
@@ -42,6 +44,37 @@ class MonkeyPatch:
 
 
 class TestCookies(unittest.TestCase):
+    def test_get_desktop_environment(self):
+        """ based on https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util_unittest.cc """
+        test_cases = [
+            ({}, _LinuxDesktopEnvironment.OTHER),
+
+            ({'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME),
+            ({'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME),
+            ({'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE),
+            ({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE),
+            ({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),
+
+            ({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
+            ({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE),
+
+            ({'XDG_CURRENT_DESKTOP': 'X-Cinnamon'}, _LinuxDesktopEnvironment.CINNAMON),
+            ({'XDG_CURRENT_DESKTOP': 'GNOME'}, _LinuxDesktopEnvironment.GNOME),
+            ({'XDG_CURRENT_DESKTOP': 'GNOME:GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
+            ({'XDG_CURRENT_DESKTOP': 'GNOME : GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
+
+            ({'XDG_CURRENT_DESKTOP': 'Unity', 'DESKTOP_SESSION': 'gnome-fallback'}, _LinuxDesktopEnvironment.GNOME),
+            ({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '5'}, _LinuxDesktopEnvironment.KDE),
+            ({'XDG_CURRENT_DESKTOP': 'KDE'}, _LinuxDesktopEnvironment.KDE),
+            ({'XDG_CURRENT_DESKTOP': 'Pantheon'}, _LinuxDesktopEnvironment.PANTHEON),
+            ({'XDG_CURRENT_DESKTOP': 'Unity'}, _LinuxDesktopEnvironment.UNITY),
+            ({'XDG_CURRENT_DESKTOP': 'Unity:Unity7'}, _LinuxDesktopEnvironment.UNITY),
+            ({'XDG_CURRENT_DESKTOP': 'Unity:Unity8'}, _LinuxDesktopEnvironment.UNITY),
+        ]
+
+        for env, expected_desktop_environment in test_cases:
+            self.assertEqual(_get_linux_desktop_environment(env), expected_desktop_environment)
+
     def test_chrome_cookie_decryptor_linux_derive_key(self):
         key = LinuxChromeCookieDecryptor.derive_key(b'abc')
         self.assertEqual(key, b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17')
@@ -58,8 +91,7 @@ class TestCookies(unittest.TestCase):
             self.assertEqual(decryptor.decrypt(encrypted_value), value)
 
     def test_chrome_cookie_decryptor_linux_v11(self):
-        with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b'',
+        with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}):
-                                   'KEYRING_AVAILABLE': True}):
             encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd'
             value = 'tz=Europe.London'
             decryptor = LinuxChromeCookieDecryptor('Chrome', Logger())

test/test_download.py

@@ -53,7 +53,7 @@ class YoutubeDL(yt_dlp.YoutubeDL):
         raise ExtractorError(message)
 
     def process_info(self, info_dict):
-        self.processed_info_dicts.append(info_dict)
+        self.processed_info_dicts.append(info_dict.copy())
         return super(YoutubeDL, self).process_info(info_dict)
 
 

test/test_options.py

@@ -1,26 +0,0 @@
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from yt_dlp.options import _hide_login_info
-
-
-class TestOptions(unittest.TestCase):
-    def test_hide_login_info(self):
-        self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
-                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
-        self.assertEqual(_hide_login_info(['-u']), ['-u'])
-        self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
-                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
-        self.assertEqual(_hide_login_info(['--username=foo']),
-                         ['--username=PRIVATE'])
-
-
-if __name__ == '__main__':
-    unittest.main()

test/test_subtitles.py

@@ -13,7 +13,7 @@ from test.helper import FakeYDL, md5, is_download_test
 from yt_dlp.extractor import (
     YoutubeIE,
     DailymotionIE,
-    TEDIE,
+    TedTalkIE,
     VimeoIE,
     WallaIE,
     CeskaTelevizeIE,
@@ -141,7 +141,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
 @is_download_test
 class TestTedSubtitles(BaseTestSubtitles):
     url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
-    IE = TEDIE
+    IE = TedTalkIE
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True

test/test_utils.py

@@ -23,6 +23,7 @@ from yt_dlp.utils import (
     caesar,
     clean_html,
     clean_podcast_url,
+    Config,
     date_from_str,
     datetime_from_str,
     DateRange,
@@ -37,11 +38,18 @@ from yt_dlp.utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
+    format_bytes,
     float_or_none,
     get_element_by_class,
     get_element_by_attribute,
     get_elements_by_class,
     get_elements_by_attribute,
+    get_element_html_by_class,
+    get_element_html_by_attribute,
+    get_elements_html_by_class,
+    get_elements_html_by_attribute,
+    get_elements_text_and_html_by_attribute,
+    get_element_text_and_html_by_tag,
     InAdvancePagedList,
     int_or_none,
     intlist_to_bytes,
@@ -116,6 +124,7 @@ from yt_dlp.compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_getenv,
+    compat_HTMLParseError,
     compat_os_name,
     compat_setenv,
 )
@@ -634,6 +643,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
         self.assertEqual(parse_duration('PT00H03M30SZ'), 210)
         self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88)
+        self.assertEqual(parse_duration('01:02:03:050'), 3723.05)
+        self.assertEqual(parse_duration('103:050'), 103.05)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
@@ -1122,7 +1133,7 @@ class TestUtil(unittest.TestCase):
 
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
-        self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+        self.assertEqual(clean_html('a:\n   "b"'), 'a: "b"')
         self.assertEqual(clean_html('a<br>\xa0b'), 'a\nb')
 
     def test_intlist_to_bytes(self):
@@ -1573,46 +1584,116 @@ Line 1
         self.assertEqual(urshift(3, 1), 1)
         self.assertEqual(urshift(-3, 1), 2147483646)
 
+    GET_ELEMENT_BY_CLASS_TEST_STRING = '''
+        <span class="foo bar">nice</span>
+    '''
+
     def test_get_element_by_class(self):
-        html = '''
+        html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
-            <span class="foo bar">nice</span>
-        '''
 
         self.assertEqual(get_element_by_class('foo', html), 'nice')
         self.assertEqual(get_element_by_class('no-such-class', html), None)
 
+    def test_get_element_html_by_class(self):
+        html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
+
+        self.assertEqual(get_element_html_by_class('foo', html), html.strip())
+        self.assertEqual(get_element_by_class('no-such-class', html), None)
+
+    GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING = '''
+        <div itemprop="author" itemscope>foo</div>
+    '''
+
     def test_get_element_by_attribute(self):
-        html = '''
+        html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
-            <span class="foo bar">nice</span>
-        '''
 
         self.assertEqual(get_element_by_attribute('class', 'foo bar', html), 'nice')
         self.assertEqual(get_element_by_attribute('class', 'foo', html), None)
         self.assertEqual(get_element_by_attribute('class', 'no-such-foo', html), None)
 
-        html = '''
+        html = self.GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING
-            <div itemprop="author" itemscope>foo</div>
-        '''
 
         self.assertEqual(get_element_by_attribute('itemprop', 'author', html), 'foo')
 
+    def test_get_element_html_by_attribute(self):
+        html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
+
+        self.assertEqual(get_element_html_by_attribute('class', 'foo bar', html), html.strip())
+        self.assertEqual(get_element_html_by_attribute('class', 'foo', html), None)
+        self.assertEqual(get_element_html_by_attribute('class', 'no-such-foo', html), None)
+
+        html = self.GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING
+
+        self.assertEqual(get_element_html_by_attribute('itemprop', 'author', html), html.strip())
+
+    GET_ELEMENTS_BY_CLASS_TEST_STRING = '''
+        <span class="foo bar">nice</span><span class="foo bar">also nice</span>
+    '''
+    GET_ELEMENTS_BY_CLASS_RES = ['<span class="foo bar">nice</span>', '<span class="foo bar">also nice</span>']
+
     def test_get_elements_by_class(self):
-        html = '''
+        html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
-            <span class="foo bar">nice</span><span class="foo bar">also nice</span>
-        '''
 
         self.assertEqual(get_elements_by_class('foo', html), ['nice', 'also nice'])
         self.assertEqual(get_elements_by_class('no-such-class', html), [])
 
+    def test_get_elements_html_by_class(self):
+        html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
+
+        self.assertEqual(get_elements_html_by_class('foo', html), self.GET_ELEMENTS_BY_CLASS_RES)
+        self.assertEqual(get_elements_html_by_class('no-such-class', html), [])
+
     def test_get_elements_by_attribute(self):
-        html = '''
+        html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
-            <span class="foo bar">nice</span><span class="foo bar">also nice</span>
-        '''
 
         self.assertEqual(get_elements_by_attribute('class', 'foo bar', html), ['nice', 'also nice'])
         self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
         self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
 
+    def test_get_elements_html_by_attribute(self):
+        html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
+
+        self.assertEqual(get_elements_html_by_attribute('class', 'foo bar', html), self.GET_ELEMENTS_BY_CLASS_RES)
+        self.assertEqual(get_elements_html_by_attribute('class', 'foo', html), [])
+        self.assertEqual(get_elements_html_by_attribute('class', 'no-such-foo', html), [])
+
+    def test_get_elements_text_and_html_by_attribute(self):
+        html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
+
+        self.assertEqual(
+            list(get_elements_text_and_html_by_attribute('class', 'foo bar', html)),
+            list(zip(['nice', 'also nice'], self.GET_ELEMENTS_BY_CLASS_RES)))
+        self.assertEqual(list(get_elements_text_and_html_by_attribute('class', 'foo', html)), [])
+        self.assertEqual(list(get_elements_text_and_html_by_attribute('class', 'no-such-foo', html)), [])
+
+    GET_ELEMENT_BY_TAG_TEST_STRING = '''
+    random text lorem ipsum</p>
+    <div>
+        this should be returned
+        <span>this should also be returned</span>
+        <div>
+            this should also be returned
+        </div>
+        closing tag above should not trick, so this should also be returned
+    </div>
+    but this text should not be returned
+    '''
+    GET_ELEMENT_BY_TAG_RES_OUTERDIV_HTML = GET_ELEMENT_BY_TAG_TEST_STRING.strip()[32:276]
+    GET_ELEMENT_BY_TAG_RES_OUTERDIV_TEXT = GET_ELEMENT_BY_TAG_RES_OUTERDIV_HTML[5:-6]
+    GET_ELEMENT_BY_TAG_RES_INNERSPAN_HTML = GET_ELEMENT_BY_TAG_TEST_STRING.strip()[78:119]
+    GET_ELEMENT_BY_TAG_RES_INNERSPAN_TEXT = GET_ELEMENT_BY_TAG_RES_INNERSPAN_HTML[6:-7]
+
+    def test_get_element_text_and_html_by_tag(self):
+        html = self.GET_ELEMENT_BY_TAG_TEST_STRING
+
+        self.assertEqual(
+            get_element_text_and_html_by_tag('div', html),
+            (self.GET_ELEMENT_BY_TAG_RES_OUTERDIV_TEXT, self.GET_ELEMENT_BY_TAG_RES_OUTERDIV_HTML))
+        self.assertEqual(
+            get_element_text_and_html_by_tag('span', html),
+            (self.GET_ELEMENT_BY_TAG_RES_INNERSPAN_TEXT, self.GET_ELEMENT_BY_TAG_RES_INNERSPAN_HTML))
+        self.assertRaises(compat_HTMLParseError, get_element_text_and_html_by_tag, 'article', html)
+
     def test_iri_to_uri(self):
         self.assertEqual(
             iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'),
@@ -1688,6 +1769,27 @@ Line 1
         ll = reversed(ll)
         test(ll, -15, 14, range(15))
 
+    def test_format_bytes(self):
+        self.assertEqual(format_bytes(0), '0.00B')
+        self.assertEqual(format_bytes(1000), '1000.00B')
+        self.assertEqual(format_bytes(1024), '1.00KiB')
+        self.assertEqual(format_bytes(1024**2), '1.00MiB')
+        self.assertEqual(format_bytes(1024**3), '1.00GiB')
+        self.assertEqual(format_bytes(1024**4), '1.00TiB')
+        self.assertEqual(format_bytes(1024**5), '1.00PiB')
+        self.assertEqual(format_bytes(1024**6), '1.00EiB')
+        self.assertEqual(format_bytes(1024**7), '1.00ZiB')
+        self.assertEqual(format_bytes(1024**8), '1.00YiB')
+
+    def test_hide_login_info(self):
+        self.assertEqual(Config.hide_login_info(['-u', 'foo', '-p', 'bar']),
+                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
+        self.assertEqual(Config.hide_login_info(['-u']), ['-u'])
+        self.assertEqual(Config.hide_login_info(['-u', 'foo', '-u', 'bar']),
+                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
+        self.assertEqual(Config.hide_login_info(['--username=foo']),
+                         ['--username=PRIVATE'])
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_verbose_output.py

@@ -19,52 +19,52 @@ class TestVerboseOutput(unittest.TestCase):
             [
                 sys.executable, 'yt_dlp/__main__.py', '-v',
                 '--username', 'johnsmith@gmail.com',
-                '--password', 'secret',
+                '--password', 'my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
         self.assertTrue(b'--username' in serr)
         self.assertTrue(b'johnsmith' not in serr)
         self.assertTrue(b'--password' in serr)
-        self.assertTrue(b'secret' not in serr)
+        self.assertTrue(b'my_secret_password' not in serr)
 
     def test_private_info_shortarg(self):
         outp = subprocess.Popen(
             [
                 sys.executable, 'yt_dlp/__main__.py', '-v',
                 '-u', 'johnsmith@gmail.com',
-                '-p', 'secret',
+                '-p', 'my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
         self.assertTrue(b'-u' in serr)
         self.assertTrue(b'johnsmith' not in serr)
         self.assertTrue(b'-p' in serr)
-        self.assertTrue(b'secret' not in serr)
+        self.assertTrue(b'my_secret_password' not in serr)
 
     def test_private_info_eq(self):
         outp = subprocess.Popen(
             [
                 sys.executable, 'yt_dlp/__main__.py', '-v',
                 '--username=johnsmith@gmail.com',
-                '--password=secret',
+                '--password=my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
         self.assertTrue(b'--username' in serr)
         self.assertTrue(b'johnsmith' not in serr)
         self.assertTrue(b'--password' in serr)
-        self.assertTrue(b'secret' not in serr)
+        self.assertTrue(b'my_secret_password' not in serr)
 
     def test_private_info_shortarg_eq(self):
         outp = subprocess.Popen(
             [
                 sys.executable, 'yt_dlp/__main__.py', '-v',
                 '-u=johnsmith@gmail.com',
-                '-p=secret',
+                '-p=my_secret_password',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
         self.assertTrue(b'-u' in serr)
         self.assertTrue(b'johnsmith' not in serr)
         self.assertTrue(b'-p' in serr)
-        self.assertTrue(b'secret' not in serr)
+        self.assertTrue(b'my_secret_password' not in serr)
 
 
 if __name__ == '__main__':

test/test_youtube_lists.py

@@ -9,11 +9,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import FakeYDL, is_download_test
 
-
 from yt_dlp.extractor import (
-    YoutubePlaylistIE,
-    YoutubeTabIE,
     YoutubeIE,
+    YoutubeTabIE,
 )
 
 
@@ -27,21 +25,10 @@ class TestYoutubeLists(unittest.TestCase):
         dl = FakeYDL()
         dl.params['noplaylist'] = True
         ie = YoutubeTabIE(dl)
-        result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        result = ie.extract('https://www.youtube.com/watch?v=OmJ-4B-mS-Y&list=PLydZ2Hrp_gPRJViZjLFKaBMgCQOYEEkyp&index=2')
         self.assertEqual(result['_type'], 'url')
-        self.assertEqual(YoutubeIE.extract_id(result['url']), 'FXxLjLQi3Fg')
+        self.assertEqual(result['ie_key'], YoutubeIE.ie_key())
-
+        self.assertEqual(YoutubeIE.extract_id(result['url']), 'OmJ-4B-mS-Y')
-    def test_youtube_course(self):
-        print('Skipping: Course URLs no longer exists')
-        return
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        # TODO find a > 100 (paginating?) videos course
-        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = list(result['entries'])
-        self.assertEqual(YoutubeIE.extract_id(entries[0]['url']), 'j9WZyLZCBzs')
-        self.assertEqual(len(entries), 25)
-        self.assertEqual(YoutubeIE.extract_id(entries[-1]['url']), 'rYefUsYuEp0')
 
     def test_youtube_mix(self):
         dl = FakeYDL()
@@ -52,15 +39,6 @@ class TestYoutubeLists(unittest.TestCase):
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'tyITL_exICo')
 
-    def test_youtube_toptracks(self):
-        print('Skipping: The playlist page gives error 500')
-        return
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
-        entries = result['entries']
-        self.assertEqual(len(entries), 100)
-
     def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()
         dl.params['extract_flat'] = True

test/test_youtube_signature.py

@@ -86,6 +86,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
         'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
     ),
+    (
+        'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
+        'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
+    ),
+    (
+        'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
+        'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
+    ),
 ]
 
 
@@ -116,10 +124,17 @@ class TestPlayerInfo(unittest.TestCase):
 class TestSignature(unittest.TestCase):
     def setUp(self):
         TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-        self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
+        self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata/sigs')
         if not os.path.exists(self.TESTDATA_DIR):
             os.mkdir(self.TESTDATA_DIR)
 
+    def tearDown(self):
+        try:
+            for f in os.listdir(self.TESTDATA_DIR):
+                os.remove(f)
+        except OSError:
+            pass
+
 
 def t_factory(name, sig_func, url_pattern):
     def make_tfunc(url, sig_input, expected_sig):

yt_dlp/__init__.py

@@ -22,7 +22,7 @@ from .compat import (
     compat_shlex_quote,
     workaround_optparse_bug9161,
 )
-from .cookies import SUPPORTED_BROWSERS
+from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
 from .utils import (
     DateRange,
     decodeOption,
@@ -41,6 +41,7 @@ from .utils import (
     SameFileError,
     setproctitle,
     std_headers,
+    traverse_obj,
     write_string,
 )
 from .update import run_update
@@ -75,20 +76,15 @@ def _real_main(argv=None):
     parser, opts, args = parseOpts(argv)
     warnings, deprecation_warnings = [], []
 
-    # Set user agent
     if opts.user_agent is not None:
-        std_headers['User-Agent'] = opts.user_agent
+        opts.headers.setdefault('User-Agent', opts.user_agent)
-
-    # Set referer
     if opts.referer is not None:
-        std_headers['Referer'] = opts.referer
+        opts.headers.setdefault('Referer', opts.referer)
-
-    # Custom HTTP headers
-    std_headers.update(opts.headers)
 
     # Dump user agent
     if opts.dump_user_agent:
-        write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
+        ua = traverse_obj(opts.headers, 'User-Agent', casesense=False, default=std_headers['User-Agent'])
+        write_string(f'{ua}\n', out=sys.stdout)
         sys.exit(0)
 
     # Batch file verification
@@ -143,6 +139,8 @@ def _real_main(argv=None):
             '"-f best" selects the best pre-merged format which is often not the best option',
             'To let yt-dlp download and merge the best available formats, simply do not pass any format selection',
             'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning')))
+    if opts.exec_cmd.get('before_dl') and opts.exec_before_dl_cmd:
+        parser.error('using "--exec-before-download" conflicts with "--exec before_dl:"')
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error('using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
@@ -266,10 +264,20 @@ def _real_main(argv=None):
         if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
             parser.error('invalid thumbnail format specified')
     if opts.cookiesfrombrowser is not None:
-        opts.cookiesfrombrowser = [
+        mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
-            part.strip() or None for part in opts.cookiesfrombrowser.split(':', 1)]
+        if mobj is None:
-        if opts.cookiesfrombrowser[0].lower() not in SUPPORTED_BROWSERS:
+            parser.error(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
-            parser.error('unsupported browser specified for cookies')
+        browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
+        browser_name = browser_name.lower()
+        if browser_name not in SUPPORTED_BROWSERS:
+            parser.error(f'unsupported browser specified for cookies: "{browser_name}". '
+                         f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
+        if keyring is not None:
+            keyring = keyring.upper()
+            if keyring not in SUPPORTED_KEYRINGS:
+                parser.error(f'unsupported keyring specified for cookies: "{keyring}". '
+                             f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
+        opts.cookiesfrombrowser = (browser_name, profile, keyring)
     geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
     if geo_bypass_code is not None:
         try:
@@ -323,6 +331,9 @@ def _real_main(argv=None):
     if _video_multistreams_set is False and _audio_multistreams_set is False:
         _unused_compat_opt('multistreams')
     outtmpl_default = opts.outtmpl.get('default')
+    if outtmpl_default == '':
+        outtmpl_default, opts.skip_download = None, True
+        del opts.outtmpl['default']
     if opts.useid:
         if outtmpl_default is None:
             outtmpl_default = opts.outtmpl['default'] = '%(id)s.%(ext)s'
@@ -341,9 +352,13 @@ def _real_main(argv=None):
 
     for k, tmpl in opts.outtmpl.items():
         validate_outtmpl(tmpl, f'{k} output template')
-    opts.forceprint = opts.forceprint or []
+    for type_, tmpl_list in opts.forceprint.items():
-    for tmpl in opts.forceprint or []:
+        for tmpl in tmpl_list:
-        validate_outtmpl(tmpl, 'print template')
+            validate_outtmpl(tmpl, f'{type_} print template')
+    for type_, tmpl_list in opts.print_to_file.items():
+        for tmpl, file in tmpl_list:
+            validate_outtmpl(tmpl, f'{type_} print-to-file template')
+            validate_outtmpl(file, f'{type_} print-to-file filename')
     validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
     for k, tmpl in opts.progress_template.items():
         k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
@@ -385,7 +400,10 @@ def _real_main(argv=None):
         opts.parse_metadata.append('title:%s' % opts.metafromtitle)
     opts.parse_metadata = list(itertools.chain(*map(metadataparser_actions, opts.parse_metadata)))
 
-    any_getting = opts.forceprint or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_getting = (any(opts.forceprint.values()) or opts.dumpjson or opts.dump_single_json
+                   or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail
+                   or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration)
+
     any_printing = opts.print_json
     download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
@@ -452,8 +470,8 @@ def _real_main(argv=None):
             'key': 'SponsorBlock',
             'categories': sponsorblock_query,
             'api': opts.sponsorblock_api,
-            # Run this immediately after extraction is complete
+            # Run this after filtering videos
-            'when': 'pre_process'
+            'when': 'after_filter'
         })
     if opts.parse_metadata:
         postprocessors.append({
@@ -476,13 +494,6 @@ def _real_main(argv=None):
             # Run this before the actual video download
             'when': 'before_dl'
         })
-    # Must be after all other before_dl
-    if opts.exec_before_dl_cmd:
-        postprocessors.append({
-            'key': 'Exec',
-            'exec_cmd': opts.exec_before_dl_cmd,
-            'when': 'before_dl'
-        })
     if opts.extractaudio:
         postprocessors.append({
             'key': 'FFmpegExtractAudio',
@@ -583,13 +594,21 @@ def _real_main(argv=None):
     # XAttrMetadataPP should be run after post-processors that may change file contents
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
-    # Exec must be the last PP
+    if opts.concat_playlist != 'never':
-    if opts.exec_cmd:
+        postprocessors.append({
+            'key': 'FFmpegConcat',
+            'only_multi_video': opts.concat_playlist != 'always',
+            'when': 'playlist',
+        })
+    # Exec must be the last PP of each category
+    if opts.exec_before_dl_cmd:
+        opts.exec_cmd.setdefault('before_dl', opts.exec_before_dl_cmd)
+    for when, exec_cmd in opts.exec_cmd.items():
         postprocessors.append({
             'key': 'Exec',
-            'exec_cmd': opts.exec_cmd,
+            'exec_cmd': exec_cmd,
             # Run this only after the files have been moved to their final locations
-            'when': 'after_move'
+            'when': when,
         })
 
     def report_args_compat(arg, name):
@@ -647,6 +666,7 @@ def _real_main(argv=None):
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
         'forceprint': opts.forceprint,
+        'print_to_file': opts.print_to_file,
         'forcejson': opts.dumpjson or opts.print_json,
         'dump_single_json': opts.dump_single_json,
         'force_write_download_archive': opts.force_write_download_archive,
@@ -740,8 +760,10 @@ def _real_main(argv=None):
         'skip_playlist_after_errors': opts.skip_playlist_after_errors,
         'cookiefile': opts.cookiefile,
         'cookiesfrombrowser': opts.cookiesfrombrowser,
+        'legacyserverconnect': opts.legacy_server_connect,
         'nocheckcertificate': opts.no_check_certificate,
         'prefer_insecure': opts.prefer_insecure,
+        'http_headers': opts.headers,
         'proxy': opts.proxy,
         'socket_timeout': opts.socket_timeout,
         'bidi_workaround': opts.bidi_workaround,

yt_dlp/aes.py

@@ -2,8 +2,15 @@ from __future__ import unicode_literals
 
 from math import ceil
 
-from .compat import compat_b64decode, compat_pycrypto_AES
+from .compat import (
-from .utils import bytes_to_intlist, intlist_to_bytes
+    compat_b64decode,
+    compat_ord,
+    compat_pycrypto_AES,
+)
+from .utils import (
+    bytes_to_intlist,
+    intlist_to_bytes,
+)
 
 
 if compat_pycrypto_AES:
@@ -25,6 +32,10 @@ else:
         return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
 
 
+def unpad_pkcs7(data):
+    return data[:-compat_ord(data[-1])]
+
+
 BLOCK_SIZE_BYTES = 16
 
 
@@ -506,5 +517,6 @@ __all__ = [
     'aes_encrypt',
     'aes_gcm_decrypt_and_verify',
     'aes_gcm_decrypt_and_verify_bytes',
-    'key_expansion'
+    'key_expansion',
+    'unpad_pkcs7',
 ]

yt_dlp/compat.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import base64
+import collections
 import ctypes
 import getpass
 import html
@@ -133,6 +134,16 @@ except AttributeError:
     asyncio.run = compat_asyncio_run
 
 
+try:  # >= 3.7
+    asyncio.tasks.all_tasks
+except AttributeError:
+    asyncio.tasks.all_tasks = asyncio.tasks.Task.all_tasks
+
+try:
+    import websockets as compat_websockets
+except ImportError:
+    compat_websockets = None
+
 # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
 # See https://github.com/yt-dlp/yt-dlp/issues/792
 # https://docs.python.org/3/library/os.path.html#os.path.expanduser
@@ -159,6 +170,13 @@ except ImportError:
     except ImportError:
         compat_pycrypto_AES = None
 
+try:
+    import brotlicffi as compat_brotli
+except ImportError:
+    try:
+        import brotli as compat_brotli
+    except ImportError:
+        compat_brotli = None
 
 WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
 
@@ -180,14 +198,17 @@ def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.pytho
 
 compat_basestring = str
 compat_chr = chr
+compat_filter = filter
 compat_input = input
 compat_integer_types = (int, )
 compat_kwargs = lambda kwargs: kwargs
+compat_map = map
 compat_numeric_types = (int, float, complex)
 compat_str = str
 compat_xpath = lambda xpath: xpath
 compat_zip = zip
 
+compat_collections_abc = collections.abc
 compat_HTMLParser = html.parser.HTMLParser
 compat_HTTPError = urllib.error.HTTPError
 compat_Struct = struct.Struct
@@ -244,7 +265,9 @@ __all__ = [
     'compat_asyncio_run',
     'compat_b64decode',
     'compat_basestring',
+    'compat_brotli',
     'compat_chr',
+    'compat_collections_abc',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
@@ -254,6 +277,7 @@ __all__ = [
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -265,6 +289,7 @@ __all__ = [
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
@@ -296,6 +321,7 @@ __all__ = [
     'compat_urllib_response',
     'compat_urlparse',
     'compat_urlretrieve',
+    'compat_websockets',
     'compat_xml_parse_error',
     'compat_xpath',
     'compat_zip',

yt_dlp/cookies.py

@@ -1,3 +1,4 @@
+import contextlib
 import ctypes
 import json
 import os
@@ -7,15 +8,19 @@ import subprocess
 import sys
 import tempfile
 from datetime import datetime, timedelta, timezone
+from enum import Enum, auto
 from hashlib import pbkdf2_hmac
 
-from .aes import aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes
+from .aes import (
+    aes_cbc_decrypt_bytes,
+    aes_gcm_decrypt_and_verify_bytes,
+    unpad_pkcs7,
+)
 from .compat import (
     compat_b64decode,
     compat_cookiejar_Cookie,
 )
 from .utils import (
-    bug_reports_message,
     expand_path,
     Popen,
     YoutubeDLCookieJar,
@@ -31,19 +36,16 @@ except ImportError:
 
 
 try:
-    import keyring
+    import secretstorage
-    KEYRING_AVAILABLE = True
+    SECRETSTORAGE_AVAILABLE = True
-    KEYRING_UNAVAILABLE_REASON = f'due to unknown reasons{bug_reports_message()}'
 except ImportError:
-    KEYRING_AVAILABLE = False
+    SECRETSTORAGE_AVAILABLE = False
-    KEYRING_UNAVAILABLE_REASON = (
+    SECRETSTORAGE_UNAVAILABLE_REASON = (
-        'as the `keyring` module is not installed. '
+        'as the `secretstorage` module is not installed. '
-        'Please install by running `python3 -m pip install keyring`. '
+        'Please install by running `python3 -m pip install secretstorage`.')
-        'Depending on your platform, additional packages may be required '
-        'to access the keyring; see  https://pypi.org/project/keyring')
 except Exception as _err:
-    KEYRING_AVAILABLE = False
+    SECRETSTORAGE_AVAILABLE = False
-    KEYRING_UNAVAILABLE_REASON = 'as the `keyring` module could not be initialized: %s' % _err
+    SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}'
 
 
 CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
@@ -74,8 +76,8 @@ class YDLLogger:
 def load_cookies(cookie_file, browser_specification, ydl):
     cookie_jars = []
     if browser_specification is not None:
-        browser_name, profile = _parse_browser_specification(*browser_specification)
+        browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
-        cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl)))
+        cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
 
     if cookie_file is not None:
         cookie_file = expand_path(cookie_file)
@@ -87,13 +89,13 @@ def load_cookies(cookie_file, browser_specification, ydl):
     return _merge_cookie_jars(cookie_jars)
 
 
-def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()):
+def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
     if browser_name == 'firefox':
         return _extract_firefox_cookies(profile, logger)
     elif browser_name == 'safari':
         return _extract_safari_cookies(profile, logger)
     elif browser_name in CHROMIUM_BASED_BROWSERS:
-        return _extract_chrome_cookies(browser_name, profile, logger)
+        return _extract_chrome_cookies(browser_name, profile, keyring, logger)
     else:
         raise ValueError('unknown browser: {}'.format(browser_name))
 
@@ -207,7 +209,7 @@ def _get_chromium_based_browser_settings(browser_name):
     }
 
 
-def _extract_chrome_cookies(browser_name, profile, logger):
+def _extract_chrome_cookies(browser_name, profile, keyring, logger):
     logger.info('Extracting cookies from {}'.format(browser_name))
 
     if not SQLITE_AVAILABLE:
@@ -234,7 +236,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 
-    decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
+    decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
 
     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
@@ -247,6 +249,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
                            'expires_utc, {} FROM cookies'.format(secure_column))
             jar = YoutubeDLCookieJar()
             failed_cookies = 0
+            unencrypted_cookies = 0
             for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
                 host_key = host_key.decode('utf-8')
                 name = name.decode('utf-8')
@@ -258,6 +261,8 @@ def _extract_chrome_cookies(browser_name, profile, logger):
                     if value is None:
                         failed_cookies += 1
                         continue
+                else:
+                    unencrypted_cookies += 1
 
                 cookie = compat_cookiejar_Cookie(
                     version=0, name=name, value=value, port=None, port_specified=False,
@@ -270,6 +275,9 @@ def _extract_chrome_cookies(browser_name, profile, logger):
             else:
                 failed_message = ''
             logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
+            counts = decryptor.cookie_counts.copy()
+            counts['unencrypted'] = unencrypted_cookies
+            logger.debug('cookie version breakdown: {}'.format(counts))
             return jar
         finally:
             if cursor is not None:
@@ -305,10 +313,14 @@ class ChromeCookieDecryptor:
     def decrypt(self, encrypted_value):
         raise NotImplementedError
 
+    @property
+    def cookie_counts(self):
+        raise NotImplementedError
 
-def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
+
+def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
     if sys.platform in ('linux', 'linux2'):
-        return LinuxChromeCookieDecryptor(browser_keyring_name, logger)
+        return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
     elif sys.platform == 'darwin':
         return MacChromeCookieDecryptor(browser_keyring_name, logger)
     elif sys.platform == 'win32':
@@ -319,13 +331,12 @@ def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
 
 
 class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
-    def __init__(self, browser_keyring_name, logger):
+    def __init__(self, browser_keyring_name, logger, *, keyring=None):
         self._logger = logger
         self._v10_key = self.derive_key(b'peanuts')
-        if KEYRING_AVAILABLE:
+        password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
-            self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name))
+        self._v11_key = None if password is None else self.derive_key(password)
-        else:
+        self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
-            self._v11_key = None
 
     @staticmethod
     def derive_key(password):
@@ -333,20 +344,27 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
 
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
+
     def decrypt(self, encrypted_value):
         version = encrypted_value[:3]
         ciphertext = encrypted_value[3:]
 
         if version == b'v10':
+            self._cookie_counts['v10'] += 1
             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 
         elif version == b'v11':
+            self._cookie_counts['v11'] += 1
             if self._v11_key is None:
-                self._logger.warning(f'cannot decrypt cookie {KEYRING_UNAVAILABLE_REASON}', only_once=True)
+                self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
                 return None
             return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
 
         else:
+            self._cookie_counts['other'] += 1
             return None
 
 
@@ -355,6 +373,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
         self._logger = logger
         password = _get_mac_keyring_password(browser_keyring_name, logger)
         self._v10_key = None if password is None else self.derive_key(password)
+        self._cookie_counts = {'v10': 0, 'other': 0}
 
     @staticmethod
     def derive_key(password):
@@ -362,11 +381,16 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
         # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
         return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
 
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
+
     def decrypt(self, encrypted_value):
         version = encrypted_value[:3]
         ciphertext = encrypted_value[3:]
 
         if version == b'v10':
+            self._cookie_counts['v10'] += 1
             if self._v10_key is None:
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
@@ -374,6 +398,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
             return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
 
         else:
+            self._cookie_counts['other'] += 1
             # other prefixes are considered 'old data' which were stored as plaintext
             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
             return encrypted_value
@@ -383,12 +408,18 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
     def __init__(self, browser_root, logger):
         self._logger = logger
         self._v10_key = _get_windows_v10_key(browser_root, logger)
+        self._cookie_counts = {'v10': 0, 'other': 0}
+
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
 
     def decrypt(self, encrypted_value):
         version = encrypted_value[:3]
         ciphertext = encrypted_value[3:]
 
         if version == b'v10':
+            self._cookie_counts['v10'] += 1
             if self._v10_key is None:
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
@@ -408,6 +439,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
             return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
 
         else:
+            self._cookie_counts['other'] += 1
             # any other prefix means the data is DPAPI encrypted
             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
             return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
@@ -422,7 +454,10 @@ def _extract_safari_cookies(profile, logger):
     cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
 
     if not os.path.isfile(cookies_path):
-        raise FileNotFoundError('could not find safari cookies database')
+        logger.debug('Trying secondary cookie location')
+        cookies_path = os.path.expanduser('~/Library/Containers/com.apple.Safari/Data/Library/Cookies/Cookies.binarycookies')
+        if not os.path.isfile(cookies_path):
+            raise FileNotFoundError('could not find safari cookies database')
 
     with open(cookies_path, 'rb') as f:
         cookies_data = f.read()
@@ -577,42 +612,220 @@ def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
     return jar
 
 
-def _get_linux_keyring_password(browser_keyring_name):
+class _LinuxDesktopEnvironment(Enum):
-    password = keyring.get_password('{} Keys'.format(browser_keyring_name),
+    """
-                                    '{} Safe Storage'.format(browser_keyring_name))
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
-    if password is None:
+    DesktopEnvironment
-        # this sometimes occurs in KDE because chrome does not check hasEntry and instead
+    """
-        # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry
+    OTHER = auto()
-        # to verify this:
+    CINNAMON = auto()
-        # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
+    GNOME = auto()
-        # while starting chrome.
+    KDE = auto()
-        # this may be a bug as the intended behaviour is to generate a random password and store
+    PANTHEON = auto()
-        # it, but that doesn't matter here.
+    UNITY = auto()
-        password = ''
+    XFCE = auto()
-    return password.encode('utf-8')
+
+
+class _LinuxKeyring(Enum):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
+    SelectedLinuxBackend
+    """
+    KWALLET = auto()
+    GNOMEKEYRING = auto()
+    BASICTEXT = auto()
+
+
+SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
+
+
+def _get_linux_desktop_environment(env):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
+    GetDesktopEnvironment
+    """
+    xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
+    desktop_session = env.get('DESKTOP_SESSION', None)
+    if xdg_current_desktop is not None:
+        xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
+
+        if xdg_current_desktop == 'Unity':
+            if desktop_session is not None and 'gnome-fallback' in desktop_session:
+                return _LinuxDesktopEnvironment.GNOME
+            else:
+                return _LinuxDesktopEnvironment.UNITY
+        elif xdg_current_desktop == 'GNOME':
+            return _LinuxDesktopEnvironment.GNOME
+        elif xdg_current_desktop == 'X-Cinnamon':
+            return _LinuxDesktopEnvironment.CINNAMON
+        elif xdg_current_desktop == 'KDE':
+            return _LinuxDesktopEnvironment.KDE
+        elif xdg_current_desktop == 'Pantheon':
+            return _LinuxDesktopEnvironment.PANTHEON
+        elif xdg_current_desktop == 'XFCE':
+            return _LinuxDesktopEnvironment.XFCE
+    elif desktop_session is not None:
+        if desktop_session in ('mate', 'gnome'):
+            return _LinuxDesktopEnvironment.GNOME
+        elif 'kde' in desktop_session:
+            return _LinuxDesktopEnvironment.KDE
+        elif 'xfce' in desktop_session:
+            return _LinuxDesktopEnvironment.XFCE
+    else:
+        if 'GNOME_DESKTOP_SESSION_ID' in env:
+            return _LinuxDesktopEnvironment.GNOME
+        elif 'KDE_FULL_SESSION' in env:
+            return _LinuxDesktopEnvironment.KDE
+    return _LinuxDesktopEnvironment.OTHER
+
+
+def _choose_linux_keyring(logger):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
+    SelectBackend
+    """
+    desktop_environment = _get_linux_desktop_environment(os.environ)
+    logger.debug('detected desktop environment: {}'.format(desktop_environment.name))
+    if desktop_environment == _LinuxDesktopEnvironment.KDE:
+        linux_keyring = _LinuxKeyring.KWALLET
+    elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
+        linux_keyring = _LinuxKeyring.BASICTEXT
+    else:
+        linux_keyring = _LinuxKeyring.GNOMEKEYRING
+    return linux_keyring
+
+
+def _get_kwallet_network_wallet(logger):
+    """ The name of the wallet used to store network passwords.
+
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
+    KWalletDBus::NetworkWallet
+    which does a dbus call to the following function:
+    https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
+    Wallet::NetworkWallet
+    """
+    default_wallet = 'kdewallet'
+    try:
+        proc = Popen([
+            'dbus-send', '--session', '--print-reply=literal',
+            '--dest=org.kde.kwalletd5',
+            '/modules/kwalletd5',
+            'org.kde.KWallet.networkWallet'
+        ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+        stdout, stderr = proc.communicate_or_kill()
+        if proc.returncode != 0:
+            logger.warning('failed to read NetworkWallet')
+            return default_wallet
+        else:
+            network_wallet = stdout.decode('utf-8').strip()
+            logger.debug('NetworkWallet = "{}"'.format(network_wallet))
+            return network_wallet
+    except BaseException as e:
+        logger.warning('exception while obtaining NetworkWallet: {}'.format(e))
+        return default_wallet
+
+
+def _get_kwallet_password(browser_keyring_name, logger):
+    logger.debug('using kwallet-query to obtain password from kwallet')
+
+    if shutil.which('kwallet-query') is None:
+        logger.error('kwallet-query command not found. KWallet and kwallet-query '
+                     'must be installed to read from KWallet. kwallet-query should be'
+                     'included in the kwallet package for your distribution')
+        return b''
+
+    network_wallet = _get_kwallet_network_wallet(logger)
+
+    try:
+        proc = Popen([
+            'kwallet-query',
+            '--read-password', '{} Safe Storage'.format(browser_keyring_name),
+            '--folder', '{} Keys'.format(browser_keyring_name),
+            network_wallet
+        ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+        stdout, stderr = proc.communicate_or_kill()
+        if proc.returncode != 0:
+            logger.error('kwallet-query failed with return code {}. Please consult '
+                         'the kwallet-query man page for details'.format(proc.returncode))
+            return b''
+        else:
+            if stdout.lower().startswith(b'failed to read'):
+                logger.debug('failed to read password from kwallet. Using empty string instead')
+                # this sometimes occurs in KDE because chrome does not check hasEntry and instead
+                # just tries to read the value (which kwallet returns "") whereas kwallet-query
+                # checks hasEntry. To verify this:
+                # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
+                # while starting chrome.
+                # this may be a bug as the intended behaviour is to generate a random password and store
+                # it, but that doesn't matter here.
+                return b''
+            else:
+                logger.debug('password found')
+                if stdout[-1:] == b'\n':
+                    stdout = stdout[:-1]
+                return stdout
+    except BaseException as e:
+        logger.warning(f'exception running kwallet-query: {type(e).__name__}({e})')
+        return b''
+
+
+def _get_gnome_keyring_password(browser_keyring_name, logger):
+    if not SECRETSTORAGE_AVAILABLE:
+        logger.error('secretstorage not available {}'.format(SECRETSTORAGE_UNAVAILABLE_REASON))
+        return b''
+    # the Gnome keyring does not seem to organise keys in the same way as KWallet,
+    # using `dbus-monitor` during startup, it can be observed that chromium lists all keys
+    # and presumably searches for its key in the list. It appears that we must do the same.
+    # https://github.com/jaraco/keyring/issues/556
+    with contextlib.closing(secretstorage.dbus_init()) as con:
+        col = secretstorage.get_default_collection(con)
+        for item in col.get_all_items():
+            if item.get_label() == '{} Safe Storage'.format(browser_keyring_name):
+                return item.get_secret()
+        else:
+            logger.error('failed to read from keyring')
+            return b''
+
+
+def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
+    # note: chrome/chromium can be run with the following flags to determine which keyring backend
+    # it has chosen to use
+    # chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
+    # Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
+    # will not be sufficient in all cases.
+
+    keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
+    logger.debug(f'Chosen keyring: {keyring.name}')
+
+    if keyring == _LinuxKeyring.KWALLET:
+        return _get_kwallet_password(browser_keyring_name, logger)
+    elif keyring == _LinuxKeyring.GNOMEKEYRING:
+        return _get_gnome_keyring_password(browser_keyring_name, logger)
+    elif keyring == _LinuxKeyring.BASICTEXT:
+        # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
+        return None
+    assert False, f'Unknown keyring {keyring}'
 
 
 def _get_mac_keyring_password(browser_keyring_name, logger):
-    if KEYRING_AVAILABLE:
+    logger.debug('using find-generic-password to obtain password from OSX keychain')
-        logger.debug('using keyring to obtain password')
+    try:
-        password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name)
-        return password.encode('utf-8')
-    else:
-        logger.debug('using find-generic-password to obtain password')
         proc = Popen(
             ['security', 'find-generic-password',
              '-w',  # write password to stdout
              '-a', browser_keyring_name,  # match 'account'
              '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
             stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
-        try:
+
-            stdout, stderr = proc.communicate_or_kill()
+        stdout, stderr = proc.communicate_or_kill()
-            if stdout[-1:] == b'\n':
+        if stdout[-1:] == b'\n':
-                stdout = stdout[:-1]
+            stdout = stdout[:-1]
-            return stdout
+        return stdout
-        except BaseException as e:
+    except BaseException as e:
-            logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
+        logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
-            return None
+        return None
 
 
 def _get_windows_v10_key(browser_root, logger):
@@ -640,10 +853,9 @@ def pbkdf2_sha1(password, salt, iterations, key_length):
 
 
 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
-    plaintext = aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)
+    plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
-    padding_length = plaintext[-1]
     try:
-        return plaintext[:-padding_length].decode('utf-8')
+        return plaintext.decode('utf-8')
     except UnicodeDecodeError:
         logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
         return None
@@ -736,10 +948,11 @@ def _is_path(value):
     return os.path.sep in value
 
 
-def _parse_browser_specification(browser_name, profile=None):
+def _parse_browser_specification(browser_name, profile=None, keyring=None):
-    browser_name = browser_name.lower()
     if browser_name not in SUPPORTED_BROWSERS:
         raise ValueError(f'unsupported browser: "{browser_name}"')
+    if keyring not in (None, *SUPPORTED_KEYRINGS):
+        raise ValueError(f'unsupported keyring: "{keyring}"')
     if profile is not None and _is_path(profile):
         profile = os.path.expanduser(profile)
-    return browser_name, profile
+    return browser_name, profile, keyring

yt_dlp/downloader/__init__.py

@@ -30,6 +30,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
 from .common import FileDownloader
 from .dash import DashSegmentsFD
 from .f4m import F4mFD
+from .fc2 import FC2LiveFD
 from .hls import HlsFD
 from .http import HttpFD
 from .rtmp import RtmpFD
@@ -58,6 +59,7 @@ PROTOCOL_MAP = {
     'ism': IsmFD,
     'mhtml': MhtmlFD,
     'niconico_dmc': NiconicoDmcFD,
+    'fc2_live': FC2LiveFD,
     'websocket_frag': WebSocketFragmentFD,
     'youtube_live_chat': YoutubeLiveChatFD,
     'youtube_live_chat_replay': YoutubeLiveChatFD,
@@ -117,7 +119,7 @@ def _get_suitable_downloader(info_dict, protocol, params, default):
             return FFmpegFD
         elif (external_downloader or '').lower() == 'native':
             return HlsFD
-        elif get_suitable_downloader(
+        elif protocol == 'm3u8_native' and get_suitable_downloader(
                 info_dict, params, None, protocol='m3u8_frag_urls', to_stdout=info_dict['to_stdout']):
             return HlsFD
         elif params.get('hls_prefer_native') is True:

yt_dlp/downloader/common.py

@@ -210,28 +210,41 @@ class FileDownloader(object):
     def ytdl_filename(self, filename):
         return filename + '.ytdl'
 
-    def sanitize_open(self, filename, open_mode):
+    def wrap_file_access(action, *, fatal=False):
-        file_access_retries = self.params.get('file_access_retries', 10)
+        def outer(func):
-        retry = 0
+            def inner(self, *args, **kwargs):
-        while True:
+                file_access_retries = self.params.get('file_access_retries', 0)
-            try:
+                retry = 0
-                return sanitize_open(filename, open_mode)
+                while True:
-            except (IOError, OSError) as err:
+                    try:
-                retry = retry + 1
+                        return func(self, *args, **kwargs)
-                if retry > file_access_retries or err.errno not in (errno.EACCES,):
+                    except (IOError, OSError) as err:
-                    raise
+                        retry = retry + 1
-                self.to_screen(
+                        if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL):
-                    '[download] Got file access error. Retrying (attempt %d of %s) ...'
+                            if not fatal:
-                    % (retry, self.format_retries(file_access_retries)))
+                                self.report_error(f'unable to {action} file: {err}')
-                time.sleep(0.01)
+                                return
+                            raise
+                        self.to_screen(
+                            f'[download] Unable to {action} file due to file access error. '
+                            f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
+                        time.sleep(0.01)
+            return inner
+        return outer
 
+    @wrap_file_access('open', fatal=True)
+    def sanitize_open(self, filename, open_mode):
+        return sanitize_open(filename, open_mode)
+
+    @wrap_file_access('remove')
+    def try_remove(self, filename):
+        os.remove(filename)
+
+    @wrap_file_access('rename')
     def try_rename(self, old_filename, new_filename):
         if old_filename == new_filename:
             return
-        try:
+        os.replace(old_filename, new_filename)
-            os.replace(old_filename, new_filename)
-        except (IOError, OSError) as err:
-            self.report_error(f'unable to rename file: {err}')
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""

yt_dlp/downloader/external.py

@@ -17,11 +17,13 @@ from ..utils import (
     cli_valueless_option,
     cli_bool_option,
     _configuration_args,
+    determine_ext,
     encodeFilename,
     encodeArgument,
     handle_youtubedl_headers,
     check_executable,
     Popen,
+    remove_end,
 )
 
 
@@ -157,9 +159,9 @@ class ExternalFD(FragmentFD):
             dest.write(decrypt_fragment(fragment, src.read()))
             src.close()
             if not self.params.get('keep_fragments', False):
-                os.remove(encodeFilename(fragment_filename))
+                self.try_remove(encodeFilename(fragment_filename))
         dest.close()
-        os.remove(encodeFilename('%s.frag.urls' % tmpfilename))
+        self.try_remove(encodeFilename('%s.frag.urls' % tmpfilename))
         return 0
 
 
@@ -251,7 +253,7 @@ class Aria2cFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c',
                '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
-               '--file-allocation=none', '-x16', '-j16', '-s16']
+               '--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
         if 'fragments' in info_dict:
             cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
         else:
@@ -265,6 +267,7 @@ class Aria2cFD(ExternalFD):
         cmd += self._option('--all-proxy', 'proxy')
         cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
         cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
+        cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
         cmd += self._configuration_args()
 
         # aria2c strips out spaces from the beginning/end of filenames and paths.
@@ -303,7 +306,7 @@ class HttpieFD(ExternalFD):
 
     @classmethod
     def available(cls, path=None):
-        return ExternalFD.available(cls, path or 'http')
+        return super().available(path or 'http')
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
@@ -462,6 +465,15 @@ class FFmpegFD(ExternalFD):
             args += ['-f', 'flv']
         elif ext == 'mp4' and tmpfilename == '-':
             args += ['-f', 'mpegts']
+        elif ext == 'unknown_video':
+            ext = determine_ext(remove_end(tmpfilename, '.part'))
+            if ext == 'unknown_video':
+                self.report_warning(
+                    'The video format is unknown and cannot be downloaded by ffmpeg. '
+                    'Explicitly set the extension in the filename to attempt download in that format')
+            else:
+                self.report_warning(f'The video format is unknown. Trying to download as {ext} according to the filename')
+                args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
         else:
             args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
 

yt_dlp/downloader/fc2.py

@@ -0,0 +1,41 @@
+from __future__ import division, unicode_literals
+
+import threading
+
+from .common import FileDownloader
+from .external import FFmpegFD
+
+
+class FC2LiveFD(FileDownloader):
+    """
+    Downloads FC2 live without being stopped. <br>
+    Note, this is not a part of public API, and will be removed without notice.
+    DO NOT USE
+    """
+
+    def real_download(self, filename, info_dict):
+        ws = info_dict['ws']
+
+        heartbeat_lock = threading.Lock()
+        heartbeat_state = [None, 1]
+
+        def heartbeat():
+            try:
+                heartbeat_state[1] += 1
+                ws.send('{"name":"heartbeat","arguments":{},"id":%d}' % heartbeat_state[1])
+            except Exception:
+                self.to_screen('[fc2:live] Heartbeat failed')
+
+            with heartbeat_lock:
+                heartbeat_state[0] = threading.Timer(30, heartbeat)
+                heartbeat_state[0]._daemonic = True
+                heartbeat_state[0].start()
+
+        heartbeat()
+
+        new_info_dict = info_dict.copy()
+        new_info_dict.update({
+            'ws': None,
+            'protocol': 'live_ffmpeg',
+        })
+        return FFmpegFD(self.ydl, self.params or {}).download(filename, new_info_dict)

yt_dlp/downloader/fragment.py

@@ -14,7 +14,7 @@ except ImportError:
 
 from .common import FileDownloader
 from .http import HttpFD
-from ..aes import aes_cbc_decrypt_bytes
+from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
 from ..compat import (
     compat_os_name,
     compat_urllib_error,
@@ -25,6 +25,7 @@ from ..utils import (
     error_to_compat_str,
     encodeFilename,
     sanitized_Request,
+    traverse_obj,
 )
 
 
@@ -136,7 +137,12 @@ class FragmentFD(FileDownloader):
         if fragment_info_dict.get('filetime'):
             ctx['fragment_filetime'] = fragment_info_dict.get('filetime')
         ctx['fragment_filename_sanitized'] = fragment_filename
-        return True, self._read_fragment(ctx)
+        try:
+            return True, self._read_fragment(ctx)
+        except FileNotFoundError:
+            if not info_dict.get('is_live'):
+                raise
+            return False, None
 
     def _read_fragment(self, ctx):
         down, frag_sanitized = self.sanitize_open(ctx['fragment_filename_sanitized'], 'rb')
@@ -153,7 +159,7 @@ class FragmentFD(FileDownloader):
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
-                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
+                self.try_remove(encodeFilename(ctx['fragment_filename_sanitized']))
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
@@ -172,7 +178,7 @@ class FragmentFD(FileDownloader):
         dl = HttpQuietDownloader(
             self.ydl,
             {
-                'continuedl': True,
+                'continuedl': self.params.get('continuedl', True),
                 'quiet': self.params.get('quiet'),
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
@@ -299,7 +305,7 @@ class FragmentFD(FileDownloader):
         if self.__do_ytdl_file(ctx):
             ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
             if os.path.isfile(ytdl_filename):
-                os.remove(ytdl_filename)
+                self.try_remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
 
         if ctx['tmpfilename'] == '-':
@@ -366,8 +372,7 @@ class FragmentFD(FileDownloader):
             # not what it decrypts to.
             if self.params.get('test', False):
                 return frag_content
-            decrypted_data = aes_cbc_decrypt_bytes(frag_content, decrypt_info['KEY'], iv)
+            return unpad_pkcs7(aes_cbc_decrypt_bytes(frag_content, decrypt_info['KEY'], iv))
-            return decrypted_data[:-decrypted_data[-1]]
 
         return decrypt_fragment
 
@@ -383,6 +388,7 @@ class FragmentFD(FileDownloader):
         max_workers = self.params.get('concurrent_fragment_downloads', 1)
         if max_progress > 1:
             self._prepare_multiline_status(max_progress)
+        is_live = any(traverse_obj(args, (..., 2, 'is_live'), default=[]))
 
         def thread_func(idx, ctx, fragments, info_dict, tpe):
             ctx['max_progress'] = max_progress
@@ -396,25 +402,43 @@ class FragmentFD(FileDownloader):
             def __exit__(self, exc_type, exc_val, exc_tb):
                 pass
 
-        spins = []
         if compat_os_name == 'nt':
-            self.report_warning('Ctrl+C does not work on Windows when used with parallel threads. '
+            def bindoj_result(future):
-                                'This is a known issue and patches are welcome')
+                while True:
+                    try:
+                        return future.result(0.1)
+                    except KeyboardInterrupt:
+                        raise
+                    except concurrent.futures.TimeoutError:
+                        continue
+        else:
+            def bindoj_result(future):
+                return future.result()
+
+        def interrupt_trigger_iter(fg):
+            for f in fg:
+                if not interrupt_trigger[0]:
+                    break
+                yield f
+
+        spins = []
         for idx, (ctx, fragments, info_dict) in enumerate(args):
             tpe = FTPE(math.ceil(max_workers / max_progress))
-            job = tpe.submit(thread_func, idx, ctx, fragments, info_dict, tpe)
+            job = tpe.submit(thread_func, idx, ctx, interrupt_trigger_iter(fragments), info_dict, tpe)
             spins.append((tpe, job))
 
         result = True
         for tpe, job in spins:
             try:
-                result = result and job.result()
+                result = result and bindoj_result(job)
             except KeyboardInterrupt:
                 interrupt_trigger[0] = False
             finally:
                 tpe.shutdown(wait=True)
-        if not interrupt_trigger[0]:
+        if not interrupt_trigger[0] and not is_live:
             raise KeyboardInterrupt()
+        # we expect the user wants to stop and DO WANT the preceding postprocessors to run;
+        # so returning a intermediate result here instead of KeyboardInterrupt on live
         return result
 
     def download_and_append_fragments(
@@ -432,9 +456,11 @@ class FragmentFD(FileDownloader):
             pack_func = lambda frag_content, _: frag_content
 
         def download_fragment(fragment, ctx):
-            frag_index = ctx['fragment_index'] = fragment['frag_index']
             if not interrupt_trigger[0]:
-                return False, frag_index
+                return False, fragment['frag_index']
+
+            frag_index = ctx['fragment_index'] = fragment['frag_index']
+            ctx['last_error'] = None
             headers = info_dict.get('http_headers', {}).copy()
             byte_range = fragment.get('byte_range')
             if byte_range:
@@ -455,6 +481,7 @@ class FragmentFD(FileDownloader):
                     # See https://github.com/ytdl-org/youtube-dl/issues/10165,
                     # https://github.com/ytdl-org/youtube-dl/issues/10448).
                     count += 1
+                    ctx['last_error'] = err
                     if count <= fragment_retries:
                         self.report_retry_fragment(err, frag_index, count, fragment_retries)
                 except DownloadError:
@@ -499,8 +526,6 @@ class FragmentFD(FileDownloader):
             self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome')
             with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
                 for fragment, frag_content, frag_index, frag_filename in pool.map(_download_fragment, fragments):
-                    if not interrupt_trigger[0]:
-                        break
                     ctx['fragment_filename_sanitized'] = frag_filename
                     ctx['fragment_index'] = frag_index
                     result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)

yt_dlp/downloader/http.py

@@ -5,7 +5,6 @@ import os
 import socket
 import time
 import random
-import re
 
 from .common import FileDownloader
 from ..compat import (
@@ -16,6 +15,7 @@ from ..utils import (
     ContentTooShortError,
     encodeFilename,
     int_or_none,
+    parse_http_range,
     sanitized_Request,
     ThrottledDownload,
     write_xattr,
@@ -59,6 +59,9 @@ class HttpFD(FileDownloader):
         ctx.chunk_size = None
         throttle_start = None
 
+        # parse given Range
+        req_start, req_end, _ = parse_http_range(headers.get('Range'))
+
         if self.params.get('continuedl', True):
             # Establish possible resume length
             if os.path.isfile(encodeFilename(ctx.tmpfilename)):
@@ -91,6 +94,9 @@ class HttpFD(FileDownloader):
                               if not is_test and chunk_size else chunk_size)
             if ctx.resume_len > 0:
                 range_start = ctx.resume_len
+                if req_start is not None:
+                    # offset the beginning of Range to be within request
+                    range_start += req_start
                 if ctx.is_resume:
                     self.report_resuming_byte(ctx.resume_len)
                 ctx.open_mode = 'ab'
@@ -99,7 +105,17 @@ class HttpFD(FileDownloader):
             else:
                 range_start = None
             ctx.is_resume = False
-            range_end = range_start + ctx.chunk_size - 1 if ctx.chunk_size else None
+
+            if ctx.chunk_size:
+                chunk_aware_end = range_start + ctx.chunk_size - 1
+                # we're not allowed to download outside Range
+                range_end = chunk_aware_end if req_end is None else min(chunk_aware_end, req_end)
+            elif req_end is not None:
+                # there's no need for chunked downloads, so download until the end of Range
+                range_end = req_end
+            else:
+                range_end = None
+
             if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
                 range_end = ctx.data_len - 1
             has_range = range_start is not None
@@ -124,23 +140,19 @@ class HttpFD(FileDownloader):
                 # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
                 if has_range:
                     content_range = ctx.data.headers.get('Content-Range')
-                    if content_range:
+                    content_range_start, content_range_end, content_len = parse_http_range(content_range)
-                        content_range_m = re.search(r'bytes (\d+)-(\d+)?(?:/(\d+))?', content_range)
+                    if content_range_start is not None and range_start == content_range_start:
                         # Content-Range is present and matches requested Range, resume is possible
-                        if content_range_m:
+                        accept_content_len = (
-                            if range_start == int(content_range_m.group(1)):
+                            # Non-chunked download
-                                content_range_end = int_or_none(content_range_m.group(2))
+                            not ctx.chunk_size
-                                content_len = int_or_none(content_range_m.group(3))
+                            # Chunked download and requested piece or
-                                accept_content_len = (
+                            # its part is promised to be served
-                                    # Non-chunked download
+                            or content_range_end == range_end
-                                    not ctx.chunk_size
+                            or content_len < range_end)
-                                    # Chunked download and requested piece or
+                        if accept_content_len:
-                                    # its part is promised to be served
+                            ctx.data_len = content_len
-                                    or content_range_end == range_end
+                            return
-                                    or content_len < range_end)
-                                if accept_content_len:
-                                    ctx.data_len = content_len
-                                    return
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload

yt_dlp/downloader/websocket.py

@@ -5,9 +5,12 @@ import threading
 
 try:
     import websockets
-    has_websockets = True
+except (ImportError, SyntaxError):
-except ImportError:
+    # websockets 3.10 on python 3.6 causes SyntaxError
+    # See https://github.com/yt-dlp/yt-dlp/issues/2633
     has_websockets = False
+else:
+    has_websockets = True
 
 from .common import FileDownloader
 from .external import FFmpegFD

yt_dlp/downloader/youtube_live_chat.py

@@ -22,6 +22,9 @@ class YoutubeLiveChatFD(FragmentFD):
     def real_download(self, filename, info_dict):
         video_id = info_dict['video_id']
         self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
+        if not self.params.get('skip_download'):
+            self.report_warning('Live chat download runs until the livestream ends. '
+                                'If you wish to download the video simultaneously, run a separate yt-dlp instance')
 
         fragment_retries = self.params.get('fragment_retries', 0)
         test = self.params.get('test', False)

yt_dlp/extractor/abc.py

@@ -213,7 +213,7 @@ class ABCIViewIE(InfoExtractor):
                 'hdnea': token,
             })
 
-        for sd in ('720', 'sd', 'sd-low'):
+        for sd in ('1080', '720', 'sd', 'sd-low'):
             sd_url = try_get(
                 stream, lambda x: x['streams']['hls'][sd], compat_str)
             if not sd_url:
@@ -300,11 +300,10 @@ class ABCIViewShowSeriesIE(InfoExtractor):
             unescapeHTML(webpage_data).encode('utf-8').decode('unicode_escape'), show_id)
         video_data = video_data['route']['pageData']['_embedded']
 
-        if self.get_param('noplaylist') and 'highlightVideo' in video_data:
+        highlight = try_get(video_data, lambda x: x['highlightVideo']['shareUrl'])
-            self.to_screen('Downloading just the highlight video because of --no-playlist')
+        if not self._yes_playlist(show_id, bool(highlight), video_label='highlight video'):
-            return self.url_result(video_data['highlightVideo']['shareUrl'], ie=ABCIViewIE.ie_key())
+            return self.url_result(highlight, ie=ABCIViewIE.ie_key())
 
-        self.to_screen(f'Downloading playlist {show_id} - add --no-playlist to just download the highlight video')
         series = video_data['selectedSeries']
         return {
             '_type': 'playlist',

yt_dlp/extractor/abematv.py

@@ -0,0 +1,484 @@
+import io
+import json
+import time
+import hashlib
+import hmac
+import re
+import struct
+from base64 import urlsafe_b64encode
+from binascii import unhexlify
+
+from .common import InfoExtractor
+from ..aes import aes_ecb_decrypt
+from ..compat import (
+    compat_urllib_response,
+    compat_urllib_parse_urlparse,
+    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    decode_base,
+    int_or_none,
+    random_uuidv4,
+    request_to_url,
+    time_seconds,
+    update_url_query,
+    traverse_obj,
+    intlist_to_bytes,
+    bytes_to_intlist,
+    urljoin,
+)
+
+
+# NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862)
+
+def add_opener(ydl, handler):
+    ''' Add a handler for opening URLs, like _download_webpage '''
+    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
+    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
+    assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector)
+    ydl._opener.add_handler(handler)
+
+
+def remove_opener(ydl, handler):
+    '''
+    Remove handler(s) for opening URLs
+    @param handler Either handler object itself or handler type.
+    Specifying handler type will remove all handler which isinstance returns True.
+    '''
+    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
+    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
+    opener = ydl._opener
+    assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector)
+    if isinstance(handler, (type, tuple)):
+        find_cp = lambda x: isinstance(x, handler)
+    else:
+        find_cp = lambda x: x is handler
+
+    removed = []
+    for meth in dir(handler):
+        if meth in ["redirect_request", "do_open", "proxy_open"]:
+            # oops, coincidental match
+            continue
+
+        i = meth.find("_")
+        protocol = meth[:i]
+        condition = meth[i + 1:]
+
+        if condition.startswith("error"):
+            j = condition.find("_") + i + 1
+            kind = meth[j + 1:]
+            try:
+                kind = int(kind)
+            except ValueError:
+                pass
+            lookup = opener.handle_error.get(protocol, {})
+            opener.handle_error[protocol] = lookup
+        elif condition == "open":
+            kind = protocol
+            lookup = opener.handle_open
+        elif condition == "response":
+            kind = protocol
+            lookup = opener.process_response
+        elif condition == "request":
+            kind = protocol
+            lookup = opener.process_request
+        else:
+            continue
+
+        handlers = lookup.setdefault(kind, [])
+        if handlers:
+            handlers[:] = [x for x in handlers if not find_cp(x)]
+
+        removed.append(x for x in handlers if find_cp(x))
+
+    if removed:
+        for x in opener.handlers:
+            if find_cp(x):
+                x.add_parent(None)
+        opener.handlers[:] = [x for x in opener.handlers if not find_cp(x)]
+
+
+class AbemaLicenseHandler(compat_urllib_request.BaseHandler):
+    handler_order = 499
+    STRTABLE = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+    HKEY = b'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E'
+
+    def __init__(self, ie: 'AbemaTVIE'):
+        # the protcol that this should really handle is 'abematv-license://'
+        # abematv_license_open is just a placeholder for development purposes
+        # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510
+        setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open'))
+        self.ie = ie
+
+    def _get_videokey_from_ticket(self, ticket):
+        to_show = self.ie._downloader.params.get('verbose', False)
+        media_token = self.ie._get_media_token(to_show=to_show)
+
+        license_response = self.ie._download_json(
+            'https://license.abema.io/abematv-hls', None, note='Requesting playback license' if to_show else False,
+            query={'t': media_token},
+            data=json.dumps({
+                'kv': 'a',
+                'lt': ticket
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
+
+        res = decode_base(license_response['k'], self.STRTABLE)
+        encvideokey = bytes_to_intlist(struct.pack('>QQ', res >> 64, res & 0xffffffffffffffff))
+
+        h = hmac.new(
+            unhexlify(self.HKEY),
+            (license_response['cid'] + self.ie._DEVICE_ID).encode('utf-8'),
+            digestmod=hashlib.sha256)
+        enckey = bytes_to_intlist(h.digest())
+
+        return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey))
+
+    def abematv_license_open(self, url):
+        url = request_to_url(url)
+        ticket = compat_urllib_parse_urlparse(url).netloc
+        response_data = self._get_videokey_from_ticket(ticket)
+        return compat_urllib_response.addinfourl(io.BytesIO(response_data), headers={
+            'Content-Length': len(response_data),
+        }, url=url, code=200)
+
+
+class AbemaTVBaseIE(InfoExtractor):
+    def _extract_breadcrumb_list(self, webpage, video_id):
+        for jld in re.finditer(
+                r'(?is)</span></li></ul><script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
+                webpage):
+            jsonld = self._parse_json(jld.group('json_ld'), video_id, fatal=False)
+            if jsonld:
+                if jsonld.get('@type') != 'BreadcrumbList':
+                    continue
+                trav = traverse_obj(jsonld, ('itemListElement', ..., 'name'))
+                if trav:
+                    return trav
+        return []
+
+
+class AbemaTVIE(AbemaTVBaseIE):
+    _VALID_URL = r'https?://abema\.tv/(?P<type>now-on-air|video/episode|channels/.+?/slots)/(?P<id>[^?/]+)'
+    _NETRC_MACHINE = 'abematv'
+    _TESTS = [{
+        'url': 'https://abema.tv/video/episode/194-25_s2_p1',
+        'info_dict': {
+            'id': '194-25_s2_p1',
+            'title': '第1話 「チーズケーキ」　「モーニング再び」',
+            'series': '異世界食堂２',
+            'series_number': 2,
+            'episode': '第1話 「チーズケーキ」　「モーニング再び」',
+            'episode_number': 1,
+        },
+        'skip': 'expired',
+    }, {
+        'url': 'https://abema.tv/channels/anime-live2/slots/E8tvAnMJ7a9a5d',
+        'info_dict': {
+            'id': 'E8tvAnMJ7a9a5d',
+            'title': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】',
+            'series': 'ゆるキャン△ SEASON２',
+            'episode': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】',
+            'series_number': 2,
+            'episode_number': 1,
+            'description': 'md5:9c5a3172ae763278f9303922f0ea5b17',
+        },
+        'skip': 'expired',
+    }, {
+        'url': 'https://abema.tv/video/episode/87-877_s1282_p31047',
+        'info_dict': {
+            'id': 'E8tvAnMJ7a9a5d',
+            'title': '第5話『光射す』',
+            'description': 'md5:56d4fc1b4f7769ded5f923c55bb4695d',
+            'thumbnail': r're:https://hayabusa\.io/.+',
+            'series': '相棒',
+            'episode': '第5話『光射す』',
+        },
+        'skip': 'expired',
+    }, {
+        'url': 'https://abema.tv/now-on-air/abema-anime',
+        'info_dict': {
+            'id': 'abema-anime',
+            # this varies
+            # 'title': '女子高生の無駄づかい 全話一挙【無料ビデオ72時間】',
+            'description': 'md5:55f2e61f46a17e9230802d7bcc913d5f',
+            'is_live': True,
+        },
+        'skip': 'Not supported until yt-dlp implements native live downloader OR AbemaTV can start a local HTTP server',
+    }]
+    _USERTOKEN = None
+    _DEVICE_ID = None
+    _TIMETABLE = None
+    _MEDIATOKEN = None
+
+    _SECRETKEY = b'v+Gjs=25Aw5erR!J8ZuvRrCx*rGswhB&qdHd_SYerEWdU&a?3DzN9BRbp5KwY4hEmcj5#fykMjJ=AuWz5GSMY-d@H7DMEh3M@9n2G552Us$$k9cD=3TxwWe86!x#Zyhe'
+
+    def _generate_aks(self, deviceid):
+        deviceid = deviceid.encode('utf-8')
+        # add 1 hour and then drop minute and secs
+        ts_1hour = int((time_seconds(hours=9) // 3600 + 1) * 3600)
+        time_struct = time.gmtime(ts_1hour)
+        ts_1hour_str = str(ts_1hour).encode('utf-8')
+
+        tmp = None
+
+        def mix_once(nonce):
+            nonlocal tmp
+            h = hmac.new(self._SECRETKEY, digestmod=hashlib.sha256)
+            h.update(nonce)
+            tmp = h.digest()
+
+        def mix_tmp(count):
+            nonlocal tmp
+            for i in range(count):
+                mix_once(tmp)
+
+        def mix_twist(nonce):
+            nonlocal tmp
+            mix_once(urlsafe_b64encode(tmp).rstrip(b'=') + nonce)
+
+        mix_once(self._SECRETKEY)
+        mix_tmp(time_struct.tm_mon)
+        mix_twist(deviceid)
+        mix_tmp(time_struct.tm_mday % 5)
+        mix_twist(ts_1hour_str)
+        mix_tmp(time_struct.tm_hour % 5)
+
+        return urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8')
+
+    def _get_device_token(self):
+        if self._USERTOKEN:
+            return self._USERTOKEN
+
+        self._DEVICE_ID = random_uuidv4()
+        aks = self._generate_aks(self._DEVICE_ID)
+        user_data = self._download_json(
+            'https://api.abema.io/v1/users', None, note='Authorizing',
+            data=json.dumps({
+                'deviceId': self._DEVICE_ID,
+                'applicationKeySecret': aks,
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
+        self._USERTOKEN = user_data['token']
+
+        # don't allow adding it 2 times or more, though it's guarded
+        remove_opener(self._downloader, AbemaLicenseHandler)
+        add_opener(self._downloader, AbemaLicenseHandler(self))
+
+        return self._USERTOKEN
+
+    def _get_media_token(self, invalidate=False, to_show=True):
+        if not invalidate and self._MEDIATOKEN:
+            return self._MEDIATOKEN
+
+        self._MEDIATOKEN = self._download_json(
+            'https://api.abema.io/v1/media/token', None, note='Fetching media token' if to_show else False,
+            query={
+                'osName': 'android',
+                'osVersion': '6.0.1',
+                'osLang': 'ja_JP',
+                'osTimezone': 'Asia/Tokyo',
+                'appId': 'tv.abema',
+                'appVersion': '3.27.1'
+            }, headers={
+                'Authorization': 'bearer ' + self._get_device_token()
+            })['token']
+
+        return self._MEDIATOKEN
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        # No authentication to be performed
+        if not username:
+            return True
+
+        if '@' in username:  # don't strictly check if it's email address or not
+            ep, method = 'user/email', 'email'
+        else:
+            ep, method = 'oneTimePassword', 'userId'
+
+        login_response = self._download_json(
+            f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in',
+            data=json.dumps({
+                method: username,
+                'password': password
+            }).encode('utf-8'), headers={
+                'Authorization': 'bearer ' + self._get_device_token(),
+                'Origin': 'https://abema.tv',
+                'Referer': 'https://abema.tv/',
+                'Content-Type': 'application/json',
+            })
+
+        self._USERTOKEN = login_response['token']
+        self._get_media_token(True)
+
+    def _real_extract(self, url):
+        # starting download using infojson from this extractor is undefined behavior,
+        # and never be fixed in the future; you must trigger downloads by directly specifing URL.
+        # (unless there's a way to hook before downloading by extractor)
+        video_id, video_type = self._match_valid_url(url).group('id', 'type')
+        headers = {
+            'Authorization': 'Bearer ' + self._get_device_token(),
+        }
+        video_type = video_type.split('/')[-1]
+
+        webpage = self._download_webpage(url, video_id)
+        canonical_url = self._search_regex(
+            r'<link\s+rel="canonical"\s*href="(.+?)"', webpage, 'canonical URL',
+            default=url)
+        info = self._search_json_ld(webpage, video_id, default={})
+
+        title = self._search_regex(
+            r'<span\s*class=".+?EpisodeTitleBlock__title">(.+?)</span>', webpage, 'title', default=None)
+        if not title:
+            jsonld = None
+            for jld in re.finditer(
+                    r'(?is)<span\s*class="com-m-Thumbnail__image">(?:</span>)?<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
+                    webpage):
+                jsonld = self._parse_json(jld.group('json_ld'), video_id, fatal=False)
+                if jsonld:
+                    break
+            if jsonld:
+                title = jsonld.get('caption')
+        if not title and video_type == 'now-on-air':
+            if not self._TIMETABLE:
+                # cache the timetable because it goes to 5MiB in size (!!)
+                self._TIMETABLE = self._download_json(
+                    'https://api.abema.io/v1/timetable/dataSet?debug=false', video_id,
+                    headers=headers)
+            now = time_seconds(hours=9)
+            for slot in self._TIMETABLE.get('slots', []):
+                if slot.get('channelId') != video_id:
+                    continue
+                if slot['startAt'] <= now and now < slot['endAt']:
+                    title = slot['title']
+                    break
+
+        # read breadcrumb on top of page
+        breadcrumb = self._extract_breadcrumb_list(webpage, video_id)
+        if breadcrumb:
+            # breadcrumb list translates to: (example is 1st test for this IE)
+            # Home > Anime (genre) > Isekai Shokudo 2 (series name) > Episode 1 "Cheese cakes" "Morning again" (episode title)
+            # hence this works
+            info['series'] = breadcrumb[-2]
+            info['episode'] = breadcrumb[-1]
+            if not title:
+                title = info['episode']
+
+        description = self._html_search_regex(
+            (r'<p\s+class="com-video-EpisodeDetailsBlock__content"><span\s+class=".+?">(.+?)</span></p><div',
+             r'<span\s+class=".+?SlotSummary.+?">(.+?)</span></div><div',),
+            webpage, 'description', default=None, group=1)
+        if not description:
+            og_desc = self._html_search_meta(
+                ('description', 'og:description', 'twitter:description'), webpage)
+            if og_desc:
+                description = re.sub(r'''(?sx)
+                    ^(.+?)(?:
+                        アニメの動画を無料で見るならABEMA！| # anime
+                        等、.+ # applies for most of categories
+                    )?
+                ''', r'\1', og_desc)
+
+        # canonical URL may contain series and episode number
+        mobj = re.search(r's(\d+)_p(\d+)$', canonical_url)
+        if mobj:
+            seri = int_or_none(mobj.group(1), default=float('inf'))
+            epis = int_or_none(mobj.group(2), default=float('inf'))
+            info['series_number'] = seri if seri < 100 else None
+            # some anime like Detective Conan (though not available in AbemaTV)
+            # has more than 1000 episodes (1026 as of 2021/11/15)
+            info['episode_number'] = epis if epis < 2000 else None
+
+        is_live, m3u8_url = False, None
+        if video_type == 'now-on-air':
+            is_live = True
+            channel_url = 'https://api.abema.io/v1/channels'
+            if video_id == 'news-global':
+                channel_url = update_url_query(channel_url, {'division': '1'})
+            onair_channels = self._download_json(channel_url, video_id)
+            for ch in onair_channels['channels']:
+                if video_id == ch['id']:
+                    m3u8_url = ch['playback']['hls']
+                    break
+            else:
+                raise ExtractorError(f'Cannot find on-air {video_id} channel.', expected=True)
+        elif video_type == 'episode':
+            api_response = self._download_json(
+                f'https://api.abema.io/v1/video/programs/{video_id}', video_id,
+                note='Checking playability',
+                headers=headers)
+            ondemand_types = traverse_obj(api_response, ('terms', ..., 'onDemandType'), default=[])
+            if 3 not in ondemand_types:
+                # cannot acquire decryption key for these streams
+                self.report_warning('This is a premium-only stream')
+
+            m3u8_url = f'https://vod-abematv.akamaized.net/program/{video_id}/playlist.m3u8'
+        elif video_type == 'slots':
+            api_response = self._download_json(
+                f'https://api.abema.io/v1/media/slots/{video_id}', video_id,
+                note='Checking playability',
+                headers=headers)
+            if not traverse_obj(api_response, ('slot', 'flags', 'timeshiftFree'), default=False):
+                self.report_warning('This is a premium-only stream')
+
+            m3u8_url = f'https://vod-abematv.akamaized.net/slot/{video_id}/playlist.m3u8'
+        else:
+            raise ExtractorError('Unreachable')
+
+        if is_live:
+            self.report_warning("This is a livestream; yt-dlp doesn't support downloading natively, but FFmpeg cannot handle m3u8 manifests from AbemaTV")
+            self.report_warning('Please consider using Streamlink to download these streams (https://github.com/streamlink/streamlink)')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', live=is_live)
+
+        info.update({
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'is_live': is_live,
+        })
+        return info
+
+
+class AbemaTVTitleIE(AbemaTVBaseIE):
+    _VALID_URL = r'https?://abema\.tv/video/title/(?P<id>[^?/]+)'
+
+    _TESTS = [{
+        'url': 'https://abema.tv/video/title/90-1597',
+        'info_dict': {
+            'id': '90-1597',
+            'title': 'シャッフルアイランド',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'https://abema.tv/video/title/193-132',
+        'info_dict': {
+            'id': '193-132',
+            'title': '真心が届く~僕とスターのオフィス・ラブ!?~',
+        },
+        'playlist_mincount': 16,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        playlist_title, breadcrumb = None, self._extract_breadcrumb_list(webpage, video_id)
+        if breadcrumb:
+            playlist_title = breadcrumb[-1]
+
+        playlist = [
+            self.url_result(urljoin('https://abema.tv/', mobj.group(1)))
+            for mobj in re.finditer(r'<li\s*class=".+?EpisodeList.+?"><a\s*href="(/[^"]+?)"', webpage)]
+
+        return self.playlist_result(playlist, playlist_title=playlist_title, playlist_id=video_id)

yt_dlp/extractor/adn.py

@@ -8,11 +8,10 @@ import os
 import random
 
 from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt
+from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
 from ..compat import (
     compat_HTTPError,
     compat_b64decode,
-    compat_ord,
 )
 from ..utils import (
     ass_subtitles_timecode,
@@ -84,14 +83,11 @@ class ADNIE(InfoExtractor):
             return None
 
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
-        dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
+        dec_subtitles = unpad_pkcs7(aes_cbc_decrypt_bytes(
-            bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
+            compat_b64decode(enc_subtitles[24:]),
-            bytes_to_intlist(binascii.unhexlify(self._K + 'ab9f52f5baae7c72')),
+            binascii.unhexlify(self._K + 'ab9f52f5baae7c72'),
-            bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
+            compat_b64decode(enc_subtitles[:24])))
-        ))
+        subtitles_json = self._parse_json(dec_subtitles.decode(), None, fatal=False)
-        subtitles_json = self._parse_json(
-            dec_subtitles[:-compat_ord(dec_subtitles[-1])].decode(),
-            None, fatal=False)
         if not subtitles_json:
             return None
 

yt_dlp/extractor/adobepass.py

@@ -1345,6 +1345,11 @@ MSO_INFO = {
         'username_field': 'username',
         'password_field': 'password',
     },
+    'Suddenlink': {
+        'name': 'Suddenlink',
+        'username_field': 'username',
+        'password_field': 'password',
+    },
 }
 
 
@@ -1635,6 +1640,52 @@ class AdobePassIE(InfoExtractor):
                         urlh.geturl(), video_id, 'Sending final bookend',
                         query=hidden_data)
 
+                    post_form(mvpd_confirm_page_res, 'Confirming Login')
+                elif mso_id == 'Suddenlink':
+                    # Suddenlink is similar to SlingTV in using a tab history count and a meta refresh,
+                    # but they also do a dynmaic redirect using javascript that has to be followed as well
+                    first_bookend_page, urlh = post_form(
+                        provider_redirect_page_res, 'Pressing Continue...')
+
+                    hidden_data = self._hidden_inputs(first_bookend_page)
+                    hidden_data['history_val'] = 1
+
+                    provider_login_redirect_page = self._download_webpage(
+                        urlh.geturl(), video_id, 'Sending First Bookend',
+                        query=hidden_data)
+
+                    provider_tryauth_url = self._html_search_regex(
+                        r'url:\s*[\'"]([^\'"]+)', provider_login_redirect_page, 'ajaxurl')
+
+                    provider_tryauth_page = self._download_webpage(
+                        provider_tryauth_url, video_id, 'Submitting TryAuth',
+                        query=hidden_data)
+
+                    provider_login_page_res = self._download_webpage_handle(
+                        f'https://authorize.suddenlink.net/saml/module.php/authSynacor/login.php?AuthState={provider_tryauth_page}',
+                        video_id, 'Getting Login Page',
+                        query=hidden_data)
+
+                    provider_association_redirect, urlh = post_form(
+                        provider_login_page_res, 'Logging in', {
+                            mso_info['username_field']: username,
+                            mso_info['password_field']: password
+                        })
+
+                    provider_refresh_redirect_url = extract_redirect_url(
+                        provider_association_redirect, url=urlh.geturl())
+
+                    last_bookend_page, urlh = self._download_webpage_handle(
+                        provider_refresh_redirect_url, video_id,
+                        'Downloading Auth Association Redirect Page')
+
+                    hidden_data = self._hidden_inputs(last_bookend_page)
+                    hidden_data['history_val'] = 3
+
+                    mvpd_confirm_page_res = self._download_webpage_handle(
+                        urlh.geturl(), video_id, 'Sending Final Bookend',
+                        query=hidden_data)
+
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
                 else:
                     # Some providers (e.g. DIRECTV NOW) have another meta refresh

yt_dlp/extractor/afreecatv.py

@@ -10,7 +10,11 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    qualities,
+    traverse_obj,
     unified_strdate,
+    unified_timestamp,
+    update_url_query,
     url_or_none,
     urlencode_postdata,
     xpath_text,
@@ -380,3 +384,105 @@ class AfreecaTVIE(InfoExtractor):
             })
 
         return info
+
+
+class AfreecaTVLiveIE(AfreecaTVIE):
+
+    IE_NAME = 'afreecatv:live'
+    _VALID_URL = r'https?://play\.afreeca(?:tv)?\.com/(?P<id>[^/]+)(?:/(?P<bno>\d+))?'
+    _TESTS = [{
+        'url': 'https://play.afreecatv.com/pyh3646/237852185',
+        'info_dict': {
+            'id': '237852185',
+            'ext': 'mp4',
+            'title': '【 우루과이 오늘은 무슨일이? 】',
+            'uploader': '박진우[JINU]',
+            'uploader_id': 'pyh3646',
+            'timestamp': 1640661495,
+            'is_live': True,
+        },
+        'skip': 'Livestream has ended',
+    }, {
+        'url': 'http://play.afreeca.com/pyh3646/237852185',
+        'only_matching': True,
+    }, {
+        'url': 'http://play.afreeca.com/pyh3646',
+        'only_matching': True,
+    }]
+
+    _LIVE_API_URL = 'https://live.afreecatv.com/afreeca/player_live_api.php'
+
+    _QUALITIES = ('sd', 'hd', 'hd2k', 'original')
+
+    def _real_extract(self, url):
+        broadcaster_id, broadcast_no = self._match_valid_url(url).group('id', 'bno')
+        password = self.get_param('videopassword')
+
+        info = self._download_json(self._LIVE_API_URL, broadcaster_id, fatal=False,
+                                   data=urlencode_postdata({'bid': broadcaster_id})) or {}
+        channel_info = info.get('CHANNEL') or {}
+        broadcaster_id = channel_info.get('BJID') or broadcaster_id
+        broadcast_no = channel_info.get('BNO') or broadcast_no
+        password_protected = channel_info.get('BPWD')
+        if not broadcast_no:
+            raise ExtractorError(f'Unable to extract broadcast number ({broadcaster_id} may not be live)', expected=True)
+        if password_protected == 'Y' and password is None:
+            raise ExtractorError(
+                'This livestream is protected by a password, use the --video-password option',
+                expected=True)
+
+        formats = []
+        quality_key = qualities(self._QUALITIES)
+        for quality_str in self._QUALITIES:
+            params = {
+                'bno': broadcast_no,
+                'stream_type': 'common',
+                'type': 'aid',
+                'quality': quality_str,
+            }
+            if password is not None:
+                params['pwd'] = password
+            aid_response = self._download_json(
+                self._LIVE_API_URL, broadcast_no, fatal=False,
+                data=urlencode_postdata(params),
+                note=f'Downloading access token for {quality_str} stream',
+                errnote=f'Unable to download access token for {quality_str} stream')
+            aid = traverse_obj(aid_response, ('CHANNEL', 'AID'))
+            if not aid:
+                continue
+
+            stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.afreecatv.com'
+            stream_info = self._download_json(
+                f'{stream_base_url}/broad_stream_assign.html', broadcast_no, fatal=False,
+                query={
+                    'return_type': channel_info.get('CDN', 'gcp_cdn'),
+                    'broad_key': f'{broadcast_no}-common-{quality_str}-hls',
+                },
+                note=f'Downloading metadata for {quality_str} stream',
+                errnote=f'Unable to download metadata for {quality_str} stream') or {}
+
+            if stream_info.get('view_url'):
+                formats.append({
+                    'format_id': quality_str,
+                    'url': update_url_query(stream_info['view_url'], {'aid': aid}),
+                    'ext': 'mp4',
+                    'protocol': 'm3u8',
+                    'quality': quality_key(quality_str),
+                })
+
+        self._sort_formats(formats)
+
+        station_info = self._download_json(
+            'https://st.afreecatv.com/api/get_station_status.php', broadcast_no,
+            query={'szBjId': broadcaster_id}, fatal=False,
+            note='Downloading channel metadata', errnote='Unable to download channel metadata') or {}
+
+        return {
+            'id': broadcast_no,
+            'title': channel_info.get('TITLE') or station_info.get('station_title'),
+            'uploader': channel_info.get('BJNICK') or station_info.get('station_name'),
+            'uploader_id': broadcaster_id,
+            'timestamp': unified_timestamp(station_info.get('broad_start')),
+            'formats': formats,
+            'is_live': True,
+        }

yt_dlp/extractor/aliexpress.py

@@ -18,7 +18,7 @@ class AliExpressLiveIE(InfoExtractor):
             'id': '2800002704436634',
             'ext': 'mp4',
             'title': 'CASIMA7.22',
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'uploader': 'CASIMA Official Store',
             'timestamp': 1500717600,
             'upload_date': '20170722',

yt_dlp/extractor/alsace20tv.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    dict_get,
+    get_element_by_class,
+    int_or_none,
+    unified_strdate,
+    url_or_none,
+)
+
+
+class Alsace20TVBaseIE(InfoExtractor):
+    def _extract_video(self, video_id, url=None):
+        info = self._download_json(
+            'https://www.alsace20.tv/visionneuse/visio_v9_js.php?key=%s&habillage=0&mode=html' % (video_id, ),
+            video_id) or {}
+        title = info.get('titre')
+
+        formats = []
+        for res, fmt_url in (info.get('files') or {}).items():
+            formats.extend(
+                self._extract_smil_formats(fmt_url, video_id, fatal=False)
+                if '/smil:_' in fmt_url
+                else self._extract_mpd_formats(fmt_url, video_id, mpd_id=res, fatal=False))
+        self._sort_formats(formats)
+
+        webpage = (url and self._download_webpage(url, video_id, fatal=False)) or ''
+        thumbnail = url_or_none(dict_get(info, ('image', 'preview', )) or self._og_search_thumbnail(webpage))
+        upload_date = self._search_regex(r'/(\d{6})_', thumbnail, 'upload_date', default=None)
+        upload_date = unified_strdate('20%s-%s-%s' % (upload_date[:2], upload_date[2:4], upload_date[4:])) if upload_date else None
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': clean_html(get_element_by_class('wysiwyg', webpage)),
+            'upload_date': upload_date,
+            'thumbnail': thumbnail,
+            'duration': int_or_none(self._og_search_property('video:duration', webpage) if webpage else None),
+            'view_count': int_or_none(info.get('nb_vues')),
+        }
+
+
+class Alsace20TVIE(Alsace20TVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?alsace20\.tv/(?:[\w-]+/)+[\w-]+-(?P<id>[\w]+)'
+    _TESTS = [{
+        'url': 'https://www.alsace20.tv/VOD/Actu/JT/Votre-JT-jeudi-3-fevrier-lyNHCXpYJh.html',
+        'info_dict': {
+            'id': 'lyNHCXpYJh',
+            'ext': 'mp4',
+            'description': 'md5:fc0bc4a0692d3d2dba4524053de4c7b7',
+            'title': 'Votre JT du jeudi 3 février',
+            'upload_date': '20220203',
+            'thumbnail': r're:https?://.+\.jpg',
+            'duration': 1073,
+            'view_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_video(video_id, url)
+
+
+class Alsace20TVEmbedIE(Alsace20TVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?alsace20\.tv/emb/(?P<id>[\w]+)'
+    _TESTS = [{
+        'url': 'https://www.alsace20.tv/emb/lyNHCXpYJh',
+        # 'md5': 'd91851bf9af73c0ad9b2cdf76c127fbb',
+        'info_dict': {
+            'id': 'lyNHCXpYJh',
+            'ext': 'mp4',
+            'title': 'Votre JT du jeudi 3 février',
+            'upload_date': '20220203',
+            'thumbnail': r're:https?://.+\.jpg',
+            'view_count': int,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_video(video_id)

yt_dlp/extractor/ant1newsgr.py

@@ -0,0 +1,143 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    ExtractorError,
+    determine_ext,
+    scale_thumbnails_to_max_format_width,
+    unescapeHTML,
+)
+
+
+class Ant1NewsGrBaseIE(InfoExtractor):
+    def _download_and_extract_api_data(self, video_id, netloc, cid=None):
+        url = f'{self.http_scheme()}//{netloc}{self._API_PATH}'
+        info = self._download_json(url, video_id, query={'cid': cid or video_id})
+        try:
+            source = info['url']
+        except KeyError:
+            raise ExtractorError('no source found for %s' % video_id)
+        formats, subs = (self._extract_m3u8_formats_and_subtitles(source, video_id, 'mp4')
+                         if determine_ext(source) == 'm3u8' else ([{'url': source}], {}))
+        self._sort_formats(formats)
+        thumbnails = scale_thumbnails_to_max_format_width(
+            formats, [{'url': info['thumb']}], r'(?<=/imgHandler/)\d+')
+        return {
+            'id': video_id,
+            'title': info.get('title'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subs,
+        }
+
+
+class Ant1NewsGrWatchIE(Ant1NewsGrBaseIE):
+    IE_NAME = 'ant1newsgr:watch'
+    IE_DESC = 'ant1news.gr videos'
+    _VALID_URL = r'https?://(?P<netloc>(?:www\.)?ant1news\.gr)/watch/(?P<id>\d+)/'
+    _API_PATH = '/templates/data/player'
+
+    _TESTS = [{
+        'url': 'https://www.ant1news.gr/watch/1506168/ant1-news-09112021-stis-18-45',
+        'md5': '95925e6b32106754235f2417e0d2dfab',
+        'info_dict': {
+            'id': '1506168',
+            'ext': 'mp4',
+            'title': 'md5:0ad00fa66ecf8aa233d26ab0dba7514a',
+            'description': 'md5:18665af715a6dcfeac1d6153a44f16b0',
+            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/26d46bf6-8158-4f02-b197-7096c714b2de.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id, netloc = self._match_valid_url(url).group('id', 'netloc')
+        webpage = self._download_webpage(url, video_id)
+        info = self._download_and_extract_api_data(video_id, netloc)
+        info['description'] = self._og_search_description(webpage)
+        return info
+
+
+class Ant1NewsGrArticleIE(Ant1NewsGrBaseIE):
+    IE_NAME = 'ant1newsgr:article'
+    IE_DESC = 'ant1news.gr articles'
+    _VALID_URL = r'https?://(?:www\.)?ant1news\.gr/[^/]+/article/(?P<id>\d+)/'
+
+    _TESTS = [{
+        'url': 'https://www.ant1news.gr/afieromata/article/549468/o-tzeims-mpont-sta-meteora-oi-apeiles-kai-o-xesikomos-ton-kalogeron',
+        'md5': '294f18331bb516539d72d85a82887dcc',
+        'info_dict': {
+            'id': '_xvg/m_cmbatw=',
+            'ext': 'mp4',
+            'title': 'md5:a93e8ecf2e4073bfdffcb38f59945411',
+            'timestamp': 1603092840,
+            'upload_date': '20201019',
+            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
+        },
+    }, {
+        'url': 'https://ant1news.gr/Society/article/620286/symmoria-anilikon-dikigoros-thymaton-ithelan-na-toys-apoteleiosoyn',
+        'info_dict': {
+            'id': '620286',
+            'title': 'md5:91fe569e952e4d146485740ae927662b',
+        },
+        'playlist_mincount': 2,
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        info = self._search_json_ld(webpage, video_id, expected_type='NewsArticle')
+        embed_urls = list(Ant1NewsGrEmbedIE._extract_urls(webpage))
+        if not embed_urls:
+            raise ExtractorError('no videos found for %s' % video_id, expected=True)
+        return self.playlist_from_matches(
+            embed_urls, video_id, info.get('title'), ie=Ant1NewsGrEmbedIE.ie_key(),
+            video_kwargs={'url_transparent': True, 'timestamp': info.get('timestamp')})
+
+
+class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
+    IE_NAME = 'ant1newsgr:embed'
+    IE_DESC = 'ant1news.gr embedded videos'
+    _BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'
+    _VALID_URL = rf'{_BASE_PLAYER_URL_RE}\?([^#]+&)?cid=(?P<id>[^#&]+)'
+    _API_PATH = '/news/templates/data/jsonPlayer'
+
+    _TESTS = [{
+        'url': 'https://www.antenna.gr/templates/pages/player?cid=3f_li_c_az_jw_y_u=&w=670&h=377',
+        'md5': 'dfc58c3a11a5a9aad2ba316ed447def3',
+        'info_dict': {
+            'id': '3f_li_c_az_jw_y_u=',
+            'ext': 'mp4',
+            'title': 'md5:a30c93332455f53e1e84ae0724f0adf7',
+            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/bbe31201-3f09-4a4e-87f5-8ad2159fffe2.jpg',
+        },
+    }]
+
+    @classmethod
+    def _extract_urls(cls, webpage):
+        _EMBED_URL_RE = rf'{cls._BASE_PLAYER_URL_RE}\?(?:(?!(?P=_q1)).)+'
+        _EMBED_RE = rf'<iframe[^>]+?src=(?P<_q1>["\'])(?P<url>{_EMBED_URL_RE})(?P=_q1)'
+        for mobj in re.finditer(_EMBED_RE, webpage):
+            url = unescapeHTML(mobj.group('url'))
+            if not cls.suitable(url):
+                continue
+            yield url
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        canonical_url = self._request_webpage(
+            HEADRequest(url), video_id,
+            note='Resolve canonical player URL',
+            errnote='Could not resolve canonical player URL').geturl()
+        _, netloc, _, _, query, _ = urllib.parse.urlparse(canonical_url)
+        cid = urllib.parse.parse_qs(query)['cid'][0]
+
+        return self._download_and_extract_api_data(video_id, netloc, cid=cid)

yt_dlp/extractor/aparat.py

@@ -33,19 +33,22 @@ class AparatIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _parse_options(self, webpage, video_id, fatal=True):
+        return self._parse_json(self._search_regex(
+            r'options\s*=\s*({.+?})\s*;', webpage, 'options', default='{}'), video_id)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Provides more metadata
+        # If available, provides more metadata
         webpage = self._download_webpage(url, video_id, fatal=False)
+        options = self._parse_options(webpage, video_id, fatal=False)
 
-        if not webpage:
+        if not options:
             webpage = self._download_webpage(
                 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
-                video_id)
+                video_id, 'Downloading embed webpage')
-
+            options = self._parse_options(webpage, video_id)
-        options = self._parse_json(self._search_regex(
-            r'options\s*=\s*({.+?})\s*;', webpage, 'options'), video_id)
 
         formats = []
         for sources in (options.get('multiSRC') or []):

yt_dlp/extractor/applepodcasts.py

@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     clean_podcast_url,
+    get_element_by_class,
     int_or_none,
     parse_iso8601,
     try_get,
@@ -14,16 +16,17 @@ class ApplePodcastsIE(InfoExtractor):
     _VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
-        'md5': 'df02e6acb11c10e844946a39e7222b08',
+        'md5': '41dc31cd650143e530d9423b6b5a344f',
         'info_dict': {
             'id': '1000482637777',
             'ext': 'mp3',
             'title': '207 - Whitney Webb Returns',
-            'description': 'md5:13a73bade02d2e43737751e3987e1399',
+            'description': 'md5:75ef4316031df7b41ced4e7b987f79c6',
             'upload_date': '20200705',
-            'timestamp': 1593921600,
+            'timestamp': 1593932400,
-            'duration': 6425,
+            'duration': 6454,
             'series': 'The Tim Dillon Show',
+            'thumbnail': 're:.+[.](png|jpe?g|webp)',
         }
     }, {
         'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -39,24 +42,47 @@ class ApplePodcastsIE(InfoExtractor):
     def _real_extract(self, url):
         episode_id = self._match_id(url)
         webpage = self._download_webpage(url, episode_id)
-        ember_data = self._parse_json(self._search_regex(
+        episode_data = {}
-            r'id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
+        ember_data = {}
-            webpage, 'ember data'), episode_id)
+        # new page type 2021-11
-        ember_data = ember_data.get(episode_id) or ember_data
+        amp_data = self._parse_json(self._search_regex(
-        episode = ember_data['data']['attributes']
+            r'(?s)id="shoebox-media-api-cache-amp-podcasts"[^>]*>\s*({.+?})\s*<',
+            webpage, 'AMP data', default='{}'), episode_id, fatal=False) or {}
+        amp_data = try_get(amp_data,
+                           lambda a: self._parse_json(
+                               next(a[x] for x in iter(a) if episode_id in x),
+                               episode_id),
+                           dict) or {}
+        amp_data = amp_data.get('d') or []
+        episode_data = try_get(
+            amp_data,
+            lambda a: next(x for x in a
+                           if x['type'] == 'podcast-episodes' and x['id'] == episode_id),
+            dict)
+        if not episode_data:
+            # try pre 2021-11 page type: TODO: consider deleting if no longer used
+            ember_data = self._parse_json(self._search_regex(
+                r'(?s)id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
+                webpage, 'ember data'), episode_id) or {}
+            ember_data = ember_data.get(episode_id) or ember_data
+            episode_data = try_get(ember_data, lambda x: x['data'], dict)
+        episode = episode_data['attributes']
         description = episode.get('description') or {}
 
         series = None
-        for inc in (ember_data.get('included') or []):
+        for inc in (amp_data or ember_data.get('included') or []):
             if inc.get('type') == 'media/podcast':
                 series = try_get(inc, lambda x: x['attributes']['name'])
+        series = series or clean_html(get_element_by_class('podcast-header__identity', webpage))
 
         return {
             'id': episode_id,
-            'title': episode['name'],
+            'title': episode.get('name'),
             'url': clean_podcast_url(episode['assetUrl']),
             'description': description.get('standard') or description.get('short'),
             'timestamp': parse_iso8601(episode.get('releaseDateTime')),
             'duration': int_or_none(episode.get('durationInMilliseconds'), 1000),
             'series': series,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'vcodec': 'none',
         }

yt_dlp/extractor/archiveorg.py

@@ -19,6 +19,7 @@ from ..utils import (
     get_element_by_id,
     HEADRequest,
     int_or_none,
+    join_nonempty,
     KNOWN_EXTENSIONS,
     merge_dicts,
     mimetype2ext,
@@ -64,7 +65,7 @@ class ArchiveOrgIE(InfoExtractor):
             'description': 'md5:43a603fd6c5b4b90d12a96b921212b9c',
             'uploader': 'yorkmba99@hotmail.com',
             'timestamp': 1387699629,
-            'upload_date': "20131222",
+            'upload_date': '20131222',
         },
     }, {
         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
@@ -150,8 +151,7 @@ class ArchiveOrgIE(InfoExtractor):
 
         # Archive.org metadata API doesn't clearly demarcate playlist entries
         # or subtitle tracks, so we get them from the embeddable player.
-        embed_page = self._download_webpage(
+        embed_page = self._download_webpage(f'https://archive.org/embed/{identifier}', identifier)
-            'https://archive.org/embed/' + identifier, identifier)
         playlist = self._playlist_data(embed_page)
 
         entries = {}
@@ -166,17 +166,17 @@ class ArchiveOrgIE(InfoExtractor):
                 'thumbnails': [],
                 'artist': p.get('artist'),
                 'track': p.get('title'),
-                'subtitles': {}}
+                'subtitles': {},
+            }
 
             for track in p.get('tracks', []):
                 if track['kind'] != 'subtitles':
                     continue
-
                 entries[p['orig']][track['label']] = {
-                    'url': 'https://archive.org/' + track['file'].lstrip('/')}
+                    'url': 'https://archive.org/' + track['file'].lstrip('/')
+                }
 
-        metadata = self._download_json(
+        metadata = self._download_json('http://archive.org/metadata/' + identifier, identifier)
-            'http://archive.org/metadata/' + identifier, identifier)
         m = metadata['metadata']
         identifier = m['identifier']
 
@@ -189,7 +189,7 @@ class ArchiveOrgIE(InfoExtractor):
             'license': m.get('licenseurl'),
             'release_date': unified_strdate(m.get('date')),
             'timestamp': unified_timestamp(dict_get(m, ['publicdate', 'addeddate'])),
-            'webpage_url': 'https://archive.org/details/' + identifier,
+            'webpage_url': f'https://archive.org/details/{identifier}',
             'location': m.get('venue'),
             'release_year': int_or_none(m.get('year'))}
 
@@ -207,7 +207,7 @@ class ArchiveOrgIE(InfoExtractor):
                     'discnumber': int_or_none(f.get('disc')),
                     'release_year': int_or_none(f.get('year'))})
                 entry = entries[f['name']]
-            elif f.get('original') in entries:
+            elif traverse_obj(f, 'original', expected_type=str) in entries:
                 entry = entries[f['original']]
             else:
                 continue
@@ -230,13 +230,12 @@ class ArchiveOrgIE(InfoExtractor):
                     'filesize': int_or_none(f.get('size')),
                     'protocol': 'https'})
 
-        # Sort available formats by filesize
         for entry in entries.values():
-            entry['formats'] = list(sorted(entry['formats'], key=lambda x: x.get('filesize', -1)))
+            self._sort_formats(entry['formats'])
 
         if len(entries) == 1:
             # If there's only one item, use it as the main info dict
-            only_video = entries[list(entries.keys())[0]]
+            only_video = next(iter(entries.values()))
             if entry_id:
                 info = merge_dicts(only_video, info)
             else:
@@ -261,19 +260,19 @@ class ArchiveOrgIE(InfoExtractor):
 
 class YoutubeWebArchiveIE(InfoExtractor):
     IE_NAME = 'web.archive:youtube'
-    IE_DESC = 'web.archive.org saved youtube videos'
+    IE_DESC = 'web.archive.org saved youtube videos, "ytarchive:" prefix'
-    _VALID_URL = r"""(?x)^
+    _VALID_URL = r'''(?x)(?:(?P<prefix>ytarchive:)|
-                (?:https?://)?web\.archive\.org/
+            (?:https?://)?web\.archive\.org/
-                    (?:web/)?
+            (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
-                    (?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
+            (?:https?(?::|%3[Aa])//)?(?:
-
+                (?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD])  # Youtube URL
-                (?:https?(?::|%3[Aa])//)?
+                |(?:wayback-fakeurl\.archive\.org/yt/)  # Or the internal fake url
-                (?:
+            )
-                    (?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD])  # Youtube URL
+        )(?P<id>[0-9A-Za-z_-]{11})
-                    |(?:wayback-fakeurl\.archive\.org/yt/)  # Or the internal fake url
+        (?(prefix)
-                )
+            (?::(?P<date2>[0-9]{14}))?$|
-                (?P<id>[0-9A-Za-z_-]{11})(?:%26|\#|&|$)
+            (?:%26|[#&]|$)
-                """
+        )'''
 
     _TESTS = [
         {
@@ -438,7 +437,13 @@ class YoutubeWebArchiveIE(InfoExtractor):
         }, {
             'url': 'https://web.archive.org/http://www.youtube.com:80/watch?v=-05VVye-ffg',
             'only_matching': True
-        }
+        }, {
+            'url': 'ytarchive:BaW_jenozKc:20050214000000',
+            'only_matching': True
+        }, {
+            'url': 'ytarchive:BaW_jenozKc',
+            'only_matching': True
+        },
     ]
     _YT_INITIAL_DATA_RE = r'(?:(?:(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;)|%s)' % YoutubeBaseInfoExtractor._YT_INITIAL_DATA_RE
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'(?:(?:(?:window\s*\[\s*["\']ytInitialPlayerResponse["\']\s*\]|ytInitialPlayerResponse)\s*=[(\s]*({.+?})[)\s]*;)|%s)' % YoutubeBaseInfoExtractor._YT_INITIAL_PLAYER_RESPONSE_RE
@@ -484,7 +489,6 @@ class YoutubeWebArchiveIE(InfoExtractor):
             page_title, 'title', default='')
 
     def _extract_metadata(self, video_id, webpage):
-
         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None)) if webpage else (lambda x: None))
         player_response = self._extract_yt_initial_variable(
             webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE, video_id, 'initial player response') or {}
@@ -596,7 +600,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
 
         # Prefer the new polymer UI captures as we support extracting more metadata from them
         # WBM captures seem to all switch to this layout ~July 2020
-        modern_captures = list(filter(lambda x: x >= 20200701000000, all_captures))
+        modern_captures = [x for x in all_captures if x >= 20200701000000]
         if modern_captures:
             capture_dates.append(modern_captures[0])
         capture_dates.append(url_date)
@@ -608,11 +612,11 @@ class YoutubeWebArchiveIE(InfoExtractor):
 
         # Fallbacks if any of the above fail
         capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE])
-        return orderedSet(capture_dates)
+        return orderedSet(filter(None, capture_dates))
 
     def _real_extract(self, url):
-
+        video_id, url_date, url_date_2 = self._match_valid_url(url).group('id', 'date', 'date2')
-        url_date, video_id = self._match_valid_url(url).groups()
+        url_date = url_date or url_date_2
 
         urlh = None
         try:
@@ -629,11 +633,9 @@ class YoutubeWebArchiveIE(InfoExtractor):
                 raise
 
         capture_dates = self._get_capture_dates(video_id, int_or_none(url_date))
-        self.write_debug('Captures to try: ' + ', '.join(str(i) for i in capture_dates if i is not None))
+        self.write_debug('Captures to try: ' + join_nonempty(*capture_dates, delim=', '))
         info = {'id': video_id}
         for capture in capture_dates:
-            if not capture:
-                continue
             webpage = self._download_webpage(
                 (self._WAYBACK_BASE_URL + 'http://www.youtube.com/watch?v=%s') % (capture, video_id),
                 video_id=video_id, fatal=False, errnote='unable to download capture webpage (it may not be archived)',
@@ -648,7 +650,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
         info['thumbnails'] = self._extract_thumbnails(video_id)
 
         if urlh:
-            url = compat_urllib_parse_unquote(urlh.url)
+            url = compat_urllib_parse_unquote(urlh.geturl())
             video_file_url_qs = parse_qs(url)
             # Attempt to recover any ext & format info from playback url & response headers
             format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}

yt_dlp/extractor/arcpublishing.py

@@ -124,8 +124,7 @@ class ArcPublishingIE(InfoExtractor):
                 formats.extend(smil_formats)
             elif stream_type in ('ts', 'hls'):
                 m3u8_formats = self._extract_m3u8_formats(
-                    s_url, uuid, 'mp4', 'm3u8' if is_live else 'm3u8_native',
+                    s_url, uuid, 'mp4', live=is_live, m3u8_id='hls', fatal=False)
-                    m3u8_id='hls', fatal=False)
                 if all([f.get('acodec') == 'none' for f in m3u8_formats]):
                     continue
                 for f in m3u8_formats:

yt_dlp/extractor/ard.py

@@ -376,9 +376,24 @@ class ARDIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
+        _SUB_FORMATS = (
+            ('./dataTimedText', 'ttml'),
+            ('./dataTimedTextNoOffset', 'ttml'),
+            ('./dataTimedTextVtt', 'vtt'),
+        )
+
+        subtitles = {}
+        for subsel, subext in _SUB_FORMATS:
+            for node in video_node.findall(subsel):
+                subtitles.setdefault('de', []).append({
+                    'url': node.attrib['url'],
+                    'ext': subext,
+                })
+
         return {
             'id': xpath_text(video_node, './videoId', default=display_id),
             'formats': formats,
+            'subtitles': subtitles,
             'display_id': display_id,
             'title': video_node.find('./title').text,
             'duration': parse_duration(video_node.find('./duration').text),
@@ -392,8 +407,9 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         (?:(?:beta|www)\.)?ardmediathek\.de/
         (?:(?P<client>[^/]+)/)?
         (?:player|live|video|(?P<playlist>sendung|sammlung))/
-        (?:(?P<display_id>[^?#]+)/)?
+        (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
-        (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)'''
+        (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
+        (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
 
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
@@ -421,6 +437,13 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
             'upload_date': '20211108',
         },
+    }, {
+        'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
+        'playlist_count': 6,
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
+            'title': 'beforeigners/beforeigners/staffel-1',
+        },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
         'only_matching': True,
@@ -546,14 +569,15 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
                 break
             pageNumber = pageNumber + 1
 
-        return self.playlist_result(entries, playlist_title=display_id)
+        return self.playlist_result(entries, playlist_id, playlist_title=display_id)
 
     def _real_extract(self, url):
-        video_id, display_id, playlist_type, client = self._match_valid_url(url).group(
+        video_id, display_id, playlist_type, client, season_number = self._match_valid_url(url).group(
-            'id', 'display_id', 'playlist', 'client')
+            'id', 'display_id', 'playlist', 'client', 'season')
         display_id, client = display_id or video_id, client or 'ard'
 
         if playlist_type:
+            # TODO: Extract only specified season
             return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
 
         player_page = self._download_json(

yt_dlp/extractor/arnes.py

@@ -7,6 +7,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    format_field,
     float_or_none,
     int_or_none,
     parse_iso8601,
@@ -92,7 +93,7 @@ class ArnesIE(InfoExtractor):
             'timestamp': parse_iso8601(video.get('creationTime')),
             'channel': channel.get('name'),
             'channel_id': channel_id,
-            'channel_url': self._BASE_URL + '/?channel=' + channel_id if channel_id else None,
+            'channel_url': format_field(channel_id, template=f'{self._BASE_URL}/?channel=%s'),
             'duration': float_or_none(video.get('duration'), 1000),
             'view_count': int_or_none(video.get('views')),
             'tags': video.get('hashtags'),

yt_dlp/extractor/arte.py

@@ -12,6 +12,7 @@ from ..utils import (
     int_or_none,
     parse_qs,
     qualities,
+    strip_or_none,
     try_get,
     unified_strdate,
     url_or_none,
@@ -253,3 +254,44 @@ class ArteTVPlaylistIE(ArteTVBaseIE):
         title = collection.get('title')
         description = collection.get('shortDescription') or collection.get('teaserText')
         return self.playlist_result(entries, playlist_id, title, description)
+
+
+class ArteTVCategoryIE(ArteTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>[\w-]+(?:/[\w-]+)*)/?\s*$' % ArteTVBaseIE._ARTE_LANGUAGES
+    _TESTS = [{
+        'url': 'https://www.arte.tv/en/videos/politics-and-society/',
+        'info_dict': {
+            'id': 'politics-and-society',
+            'title': 'Politics and society',
+            'description': 'Investigative documentary series, geopolitical analysis, and international commentary',
+        },
+        'playlist_mincount': 13,
+    },
+    ]
+
+    @classmethod
+    def suitable(cls, url):
+        return (
+            not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE, ))
+            and super(ArteTVCategoryIE, cls).suitable(url))
+
+    def _real_extract(self, url):
+        lang, playlist_id = self._match_valid_url(url).groups()
+        webpage = self._download_webpage(url, playlist_id)
+
+        items = []
+        for video in re.finditer(
+                r'<a\b[^>]*?href\s*=\s*(?P<q>"|\'|\b)(?P<url>https?://www\.arte\.tv/%s/videos/[\w/-]+)(?P=q)' % lang,
+                webpage):
+            video = video.group('url')
+            if video == url:
+                continue
+            if any(ie.suitable(video) for ie in (ArteTVIE, ArteTVPlaylistIE, )):
+                items.append(video)
+
+        title = (self._og_search_title(webpage, default=None)
+                 or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title>', default=None))
+        title = strip_or_none(title.rsplit('|', 1)[0]) or self._generic_title(url)
+
+        return self.playlist_from_matches(items, playlist_id=playlist_id, playlist_title=title,
+                                          description=self._og_search_description(webpage, default=None))

yt_dlp/extractor/atvat.py

@@ -8,6 +8,7 @@ from ..utils import (
     float_or_none,
     jwt_encode_hs256,
     try_get,
+    ExtractorError,
 )
 
 
@@ -94,6 +95,11 @@ class ATVAtIE(InfoExtractor):
             })
 
         video_id, videos_data = list(videos['data'].items())[0]
+        error_msg = try_get(videos_data, lambda x: x['error']['title'])
+        if error_msg == 'Geo check failed':
+            self.raise_geo_restricted(error_msg)
+        elif error_msg:
+            raise ExtractorError(error_msg)
         entries = [
             self._extract_video_info(url, contentResource[video['id']], video)
             for video in videos_data]

yt_dlp/extractor/audiomack.py

@@ -29,6 +29,7 @@ class AudiomackIE(InfoExtractor):
             }
         },
         # audiomack wrapper around soundcloud song
+        # Needs new test URL.
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/hip-hop-daily/black-mamba-freestyle',

yt_dlp/extractor/awaan.py

@@ -9,6 +9,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    format_field,
     int_or_none,
     parse_iso8601,
     smuggle_url,
@@ -43,7 +44,7 @@ class AWAANBaseIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'description': video_data.get('description_en') or video_data.get('description_ar'),
-            'thumbnail': 'http://admin.mangomolo.com/analytics/%s' % img if img else None,
+            'thumbnail': format_field(img, template='http://admin.mangomolo.com/analytics/%s'),
             'duration': int_or_none(video_data.get('duration')),
             'timestamp': parse_iso8601(video_data.get('create_time'), ' '),
             'is_live': is_live,

yt_dlp/extractor/bandcamp.py

@@ -183,6 +183,7 @@ class BandcampIE(InfoExtractor):
                             'format_note': f.get('description'),
                             'filesize': parse_filesize(f.get('size_mb')),
                             'vcodec': 'none',
+                            'acodec': format_id.split('-')[0],
                         })
 
         self._sort_formats(formats)
@@ -212,7 +213,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(BandcampIE):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?!/music)(?:/album/(?P<id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com/album/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -257,14 +258,6 @@ class BandcampAlbumIE(BandcampIE):
             'id': 'hierophany-of-the-open-grave',
         },
         'playlist_mincount': 9,
-    }, {
-        'url': 'http://dotscale.bandcamp.com',
-        'info_dict': {
-            'title': 'Loom',
-            'id': 'dotscale',
-            'uploader_id': 'dotscale',
-        },
-        'playlist_mincount': 7,
     }, {
         # with escaped quote in title
         'url': 'https://jstrecords.bandcamp.com/album/entropy-ep',
@@ -391,41 +384,63 @@ class BandcampWeeklyIE(BandcampIE):
         }
 
 
-class BandcampMusicIE(InfoExtractor):
+class BandcampUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<id>[^/]+)\.bandcamp\.com/music'
+    IE_NAME = 'Bandcamp:user'
+    _VALID_URL = r'https?://(?!www\.)(?P<id>[^.]+)\.bandcamp\.com(?:/music)?/?(?:[#?]|$)'
+
     _TESTS = [{
+        # Type 1 Bandcamp user page.
+        'url': 'https://adrianvonziegler.bandcamp.com',
+        'info_dict': {
+            'id': 'adrianvonziegler',
+            'title': 'Discography of adrianvonziegler',
+        },
+        'playlist_mincount': 23,
+    }, {
+        # Bandcamp user page with only one album
+        'url': 'http://dotscale.bandcamp.com',
+        'info_dict': {
+            'id': 'dotscale',
+            'title': 'Discography of dotscale'
+        },
+        'playlist_count': 1,
+    }, {
+        # Type 2 Bandcamp user page.
+        'url': 'https://nightcallofficial.bandcamp.com',
+        'info_dict': {
+            'id': 'nightcallofficial',
+            'title': 'Discography of nightcallofficial',
+        },
+        'playlist_count': 4,
+    }, {
         'url': 'https://steviasphere.bandcamp.com/music',
         'playlist_mincount': 47,
         'info_dict': {
             'id': 'steviasphere',
+            'title': 'Discography of steviasphere',
         },
     }, {
         'url': 'https://coldworldofficial.bandcamp.com/music',
         'playlist_mincount': 10,
         'info_dict': {
             'id': 'coldworldofficial',
+            'title': 'Discography of coldworldofficial',
         },
     }, {
         'url': 'https://nuclearwarnowproductions.bandcamp.com/music',
         'playlist_mincount': 399,
         'info_dict': {
             'id': 'nuclearwarnowproductions',
+            'title': 'Discography of nuclearwarnowproductions',
         },
-    }
+    }]
-    ]
-
-    _TYPE_IE_DICT = {
-        'album': BandcampAlbumIE.ie_key(),
-        'track': BandcampIE.ie_key()
-    }
 
     def _real_extract(self, url):
-        id = self._match_id(url)
+        uploader = self._match_id(url)
-        webpage = self._download_webpage(url, id)
+        webpage = self._download_webpage(url, uploader)
-        items = re.findall(r'href\=\"\/(?P<path>(?P<type>album|track)+/[^\"]+)', webpage)
+
-        entries = [
+        discography_data = (re.findall(r'<li data-item-id=["\'][^>]+>\s*<a href=["\']([^"\']+)', webpage)
-            self.url_result(
+                            or re.findall(r'<div[^>]+trackTitle["\'][^"\']+["\']([^"\']+)', webpage))
-                f'https://{id}.bandcamp.com/{item[0]}',
+
-                ie=self._TYPE_IE_DICT[item[1]])
+        return self.playlist_from_matches(
-            for item in items]
+            discography_data, uploader, f'Discography of {uploader}', getter=lambda x: urljoin(url, x))
-        return self.playlist_result(entries, id)

yt_dlp/extractor/bbc.py

@@ -11,6 +11,7 @@ from ..compat import (
     compat_etree_Element,
     compat_HTTPError,
     compat_str,
+    compat_urllib_error,
     compat_urlparse,
 )
 from ..utils import (
@@ -38,7 +39,7 @@ from ..utils import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _ID_REGEX = r'(?:[pbm][\da-z]{7}|w[\da-z]{7,14})'
+    _ID_REGEX = r'(?:[pbml][\da-z]{7}|w[\da-z]{7,14})'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?bbc\.co\.uk/
@@ -394,9 +395,17 @@ class BBCCoUkIE(InfoExtractor):
                         formats.extend(self._extract_mpd_formats(
                             href, programme_id, mpd_id=format_id, fatal=False))
                     elif transfer_format == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
+                        # TODO: let expected_status be passed into _extract_xxx_formats() instead
-                            href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                        try:
-                            m3u8_id=format_id, fatal=False))
+                            fmts = self._extract_m3u8_formats(
+                                href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                                m3u8_id=format_id, fatal=False)
+                        except ExtractorError as e:
+                            if not (isinstance(e.exc_info[1], compat_urllib_error.HTTPError)
+                                    and e.exc_info[1].code in (403, 404)):
+                                raise
+                            fmts = []
+                        formats.extend(fmts)
                     elif transfer_format == 'hds':
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))
@@ -784,21 +793,33 @@ class BBCIE(BBCCoUkIE):
             'timestamp': 1437785037,
             'upload_date': '20150725',
         },
+    }, {
+        # video with window.__INITIAL_DATA__ and value as JSON string
+        'url': 'https://www.bbc.com/news/av/world-europe-59468682',
+        'info_dict': {
+            'id': 'p0b71qth',
+            'ext': 'mp4',
+            'title': 'Why France is making this woman a national hero',
+            'description': 'md5:7affdfab80e9c3a1f976230a1ff4d5e4',
+            'thumbnail': r're:https?://.+/.+\.jpg',
+            'timestamp': 1638230731,
+            'upload_date': '20211130',
+        },
     }, {
         # single video article embedded with data-media-vpid
         'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
         'only_matching': True,
     }, {
+        # bbcthreeConfig
         'url': 'https://www.bbc.co.uk/bbcthree/clip/73d0bbd0-abc3-4cea-b3c0-cdae21905eb1',
         'info_dict': {
             'id': 'p06556y7',
             'ext': 'mp4',
-            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'title': 'Things Not To Say to people that live on council estates',
-            'description': 'md5:4b7dfd063d5a789a1512e99662be3ddd',
+            'description': "From being labelled a 'chav', to the presumption that they're 'scroungers', people who live on council estates encounter all kinds of prejudices and false assumptions about themselves, their families, and their lifestyles. Here, eight people discuss the common statements, misconceptions, and clichés that they're tired of hearing.",
+            'duration': 360,
+            'thumbnail': r're:https?://.+/.+\.jpg',
         },
-        'params': {
-            'skip_download': True,
-        }
     }, {
         # window.__PRELOADED_STATE__
         'url': 'https://www.bbc.co.uk/radio/play/b0b9z4yl',
@@ -1171,9 +1192,16 @@ class BBCIE(BBCCoUkIE):
                 return self.playlist_result(
                     entries, playlist_id, playlist_title, playlist_description)
 
-        initial_data = self._parse_json(self._search_regex(
+        initial_data = self._search_regex(
-            r'window\.__INITIAL_DATA__\s*=\s*({.+?});', webpage,
+            r'window\.__INITIAL_DATA__\s*=\s*("{.+?}")\s*;', webpage,
-            'preload state', default='{}'), playlist_id, fatal=False)
+            'quoted preload state', default=None)
+        if initial_data is None:
+            initial_data = self._search_regex(
+                r'window\.__INITIAL_DATA__\s*=\s*({.+?})\s*;', webpage,
+                'preload state', default={})
+        else:
+            initial_data = self._parse_json(initial_data or '"{}"', playlist_id, fatal=False)
+        initial_data = self._parse_json(initial_data, playlist_id, fatal=False)
         if initial_data:
             def parse_media(media):
                 if not media:
@@ -1214,7 +1242,10 @@ class BBCIE(BBCCoUkIE):
                 if name == 'media-experience':
                     parse_media(try_get(resp, lambda x: x['data']['initialItem']['mediaItem'], dict))
                 elif name == 'article':
-                    for block in (try_get(resp, lambda x: x['data']['blocks'], list) or []):
+                    for block in (try_get(resp,
+                                          (lambda x: x['data']['blocks'],
+                                           lambda x: x['data']['content']['model']['blocks'],),
+                                          list) or []):
                         if block.get('type') != 'media':
                             continue
                         parse_media(block.get('model'))

yt_dlp/extractor/beeg.py

@@ -1,32 +1,45 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
+
-    compat_str,
-)
 from ..utils import (
     int_or_none,
-    parse_qs,
+    traverse_obj,
+    try_get,
     unified_timestamp,
 )
 
 
 class BeegIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?beeg\.(?:com|porn(?:/video)?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?beeg\.(?:com(?:/video)?)/-?(?P<id>\d+)'
     _TESTS = [{
-        # api/v6 v1
+        'url': 'https://beeg.com/-0983946056129650',
-        'url': 'http://beeg.com/5416503',
+        'md5': '51d235147c4627cfce884f844293ff88',
-        'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
         'info_dict': {
-            'id': '5416503',
+            'id': '0983946056129650',
             'ext': 'mp4',
-            'title': 'Sultry Striptease',
+            'title': 'sucked cock and fucked in a private plane',
-            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
+            'duration': 927,
-            'timestamp': 1391813355,
-            'upload_date': '20140207',
-            'duration': 383,
             'tags': list,
             'age_limit': 18,
+            'upload_date': '20220131',
+            'timestamp': 1643656455,
+            'display_id': 2540839,
+        }
+    }, {
+        'url': 'https://beeg.com/-0599050563103750?t=4-861',
+        'md5': 'bd8b5ea75134f7f07fad63008db2060e',
+        'info_dict': {
+            'id': '0599050563103750',
+            'ext': 'mp4',
+            'title': 'Bad Relatives',
+            'duration': 2060,
+            'tags': list,
+            'age_limit': 18,
+            'description': 'md5:b4fc879a58ae6c604f8f259155b7e3b9',
+            'timestamp': 1643623200,
+            'display_id': 2569965,
+            'upload_date': '20220131',
         }
     }, {
         # api/v6 v2
@@ -36,12 +49,6 @@ class BeegIE(InfoExtractor):
         # api/v6 v2 w/o t
         'url': 'https://beeg.com/1277207756',
         'only_matching': True,
-    }, {
-        'url': 'https://beeg.porn/video/5416503',
-        'only_matching': True,
-    }, {
-        'url': 'https://beeg.porn/5416503',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -49,68 +56,38 @@ class BeegIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        beeg_version = self._search_regex(
+        video = self._download_json(
-            r'beeg_version\s*=\s*([\da-zA-Z_-]+)', webpage, 'beeg version',
+            'https://store.externulls.com/facts/file/%s' % video_id,
-            default='1546225636701')
+            video_id, 'Downloading JSON for %s' % video_id)
 
-        if len(video_id) >= 10:
+        fc_facts = video.get('fc_facts')
-            query = {
+        first_fact = {}
-                'v': 2,
+        for fact in fc_facts:
-            }
+            if not first_fact or try_get(fact, lambda x: x['id'] < first_fact['id']):
-            qs = parse_qs(url)
+                first_fact = fact
-            t = qs.get('t', [''])[0].split('-')
-            if len(t) > 1:
-                query.update({
-                    's': t[0],
-                    'e': t[1],
-                })
-        else:
-            query = {'v': 1}
 
-        for api_path in ('', 'api.'):
+        resources = traverse_obj(video, ('file', 'hls_resources')) or first_fact.get('hls_resources')
-            video = self._download_json(
-                'https://%sbeeg.com/api/v6/%s/video/%s'
-                % (api_path, beeg_version, video_id), video_id,
-                fatal=api_path == 'api.', query=query)
-            if video:
-                break
 
         formats = []
-        for format_id, video_url in video.items():
+        for format_id, video_uri in resources.items():
-            if not video_url:
+            if not video_uri:
                 continue
-            height = self._search_regex(
+            height = int_or_none(self._search_regex(r'fl_cdn_(\d+)', format_id, 'height', default=None))
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            current_formats = self._extract_m3u8_formats(f'https://video.beeg.com/{video_uri}', video_id, ext='mp4', m3u8_id=str(height))
-            if not height:
+            for f in current_formats:
-                continue
+                f['height'] = height
-            formats.append({
+            formats.extend(current_formats)
-                'url': self._proto_relative_url(
+
-                    video_url.replace('{DATA_MARKERS}', 'data=pc_XX__%s_0' % beeg_version), 'https:'),
-                'format_id': format_id,
-                'height': int(height),
-            })
         self._sort_formats(formats)
 
-        title = video['title']
-        video_id = compat_str(video.get('id') or video_id)
-        display_id = video.get('code')
-        description = video.get('desc')
-        series = video.get('ps_name')
-
-        timestamp = unified_timestamp(video.get('date'))
-        duration = int_or_none(video.get('duration'))
-
-        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None
-
         return {
             'id': video_id,
-            'display_id': display_id,
+            'display_id': first_fact.get('id'),
-            'title': title,
+            'title': traverse_obj(video, ('file', 'stuff', 'sf_name')),
-            'description': description,
+            'description': traverse_obj(video, ('file', 'stuff', 'sf_story')),
-            'series': series,
+            'timestamp': unified_timestamp(first_fact.get('fc_created')),
-            'timestamp': timestamp,
+            'duration': int_or_none(traverse_obj(video, ('file', 'fl_duration'))),
-            'duration': duration,
+            'tags': traverse_obj(video, ('tags', ..., 'tg_name')),
-            'tags': tags,
             'formats': formats,
             'age_limit': self._rta_search(webpage),
         }

yt_dlp/extractor/bigo.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, urlencode_postdata
+
+
+class BigoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bigo\.tv/(?:[a-z]{2,}/)?(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'https://www.bigo.tv/ja/221338632',
+        'info_dict': {
+            'id': '6576287577575737440',
+            'title': '土よ〜💁‍♂️ 休憩室/REST room',
+            'thumbnail': r're:https?://.+',
+            'uploader': '✨Shin💫',
+            'uploader_id': '221338632',
+            'is_live': True,
+        },
+        'skip': 'livestream',
+    }, {
+        'url': 'https://www.bigo.tv/th/Tarlerm1304',
+        'only_matching': True,
+    }, {
+        'url': 'https://bigo.tv/115976881',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+
+        info_raw = self._download_json(
+            'https://bigo.tv/studio/getInternalStudioInfo',
+            user_id, data=urlencode_postdata({'siteId': user_id}))
+
+        if not isinstance(info_raw, dict):
+            raise ExtractorError('Received invalid JSON data')
+        if info_raw.get('code'):
+            raise ExtractorError(
+                'Bigo says: %s (code %s)' % (info_raw.get('msg'), info_raw.get('code')), expected=True)
+        info = info_raw.get('data') or {}
+
+        if not info.get('alive'):
+            raise ExtractorError('This user is offline.', expected=True)
+
+        return {
+            'id': info.get('roomId') or user_id,
+            'title': info.get('roomTopic') or info.get('nick_name') or user_id,
+            'formats': [{
+                'url': info.get('hls_src'),
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+            'thumbnail': info.get('snapshot'),
+            'uploader': info.get('nick_name'),
+            'uploader_id': user_id,
+            'is_live': True,
+        }

yt_dlp/extractor/bilibili.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+import base64
 import hashlib
 import itertools
 import functools
@@ -16,17 +17,18 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
+    mimetype2ext,
     parse_iso8601,
     traverse_obj,
-    try_get,
+    parse_count,
     smuggle_url,
     srt_subtitles_timecode,
     str_or_none,
-    str_to_int,
     strip_jsonp,
     unified_timestamp,
     unsmuggle_url,
     urlencode_postdata,
+    url_or_none,
     OnDemandPagedList
 )
 
@@ -50,16 +52,14 @@ class BiliBiliIE(InfoExtractor):
         'url': 'http://www.bilibili.com/video/av1074402/',
         'md5': '5f7d29e1a2872f3df0cf76b1f87d3788',
         'info_dict': {
-            'id': '1074402',
+            'id': '1074402_part1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '【金坷垃】金泡沫',
-            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
-            'duration': 308.067,
-            'timestamp': 1398012678,
-            'upload_date': '20140420',
-            'thumbnail': r're:^https?://.+\.jpg',
-            'uploader': '菊子桑',
             'uploader_id': '156160',
+            'uploader': '菊子桑',
+            'upload_date': '20140420',
+            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
+            'timestamp': 1398012678,
         },
     }, {
         # Tested in BiliBiliBangumiIE
@@ -73,49 +73,27 @@ class BiliBiliIE(InfoExtractor):
         'url': 'http://bangumi.bilibili.com/anime/5802/play#100643',
         'md5': '3f721ad1e75030cc06faf73587cfec57',
         'info_dict': {
-            'id': '100643',
+            'id': '100643_part1',
             'ext': 'mp4',
             'title': 'CHAOS;CHILD',
             'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
         },
         'skip': 'Geo-restricted to China',
     }, {
-        # Title with double quotes
         'url': 'http://www.bilibili.com/video/av8903802/',
         'info_dict': {
-            'id': '8903802',
+            'id': '8903802_part1',
+            'ext': 'mp4',
             'title': '阿滴英文｜英文歌分享#6 "Closer',
+            'upload_date': '20170301',
             'description': '滴妹今天唱Closer給你聽! 有史以来，被推最多次也是最久的歌曲，其实歌词跟我原本想像差蛮多的，不过还是好听！ 微博@阿滴英文',
+            'timestamp': 1488382634,
+            'uploader_id': '65880958',
+            'uploader': '阿滴英文',
+        },
+        'params': {
+            'skip_download': True,
         },
-        'playlist': [{
-            'info_dict': {
-                'id': '8903802_part1',
-                'ext': 'flv',
-                'title': '阿滴英文｜英文歌分享#6 "Closer',
-                'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a',
-                'uploader': '阿滴英文',
-                'uploader_id': '65880958',
-                'timestamp': 1488382634,
-                'upload_date': '20170301',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        }, {
-            'info_dict': {
-                'id': '8903802_part2',
-                'ext': 'flv',
-                'title': '阿滴英文｜英文歌分享#6 "Closer',
-                'description': 'md5:3b1b9e25b78da4ef87e9b548b88ee76a',
-                'uploader': '阿滴英文',
-                'uploader_id': '65880958',
-                'timestamp': 1488382634,
-                'upload_date': '20170301',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        }]
     }, {
         # new BV video id format
         'url': 'https://www.bilibili.com/video/BV1JE411F741',
@@ -150,6 +128,7 @@ class BiliBiliIE(InfoExtractor):
         av_id, bv_id = self._get_video_id_set(video_id, mobj.group('id_bv') is not None)
         video_id = av_id
 
+        info = {}
         anime_id = mobj.group('anime_id')
         page_id = mobj.group('page')
         webpage = self._download_webpage(url, video_id)
@@ -201,66 +180,95 @@ class BiliBiliIE(InfoExtractor):
         }
         headers.update(self.geo_verification_headers())
 
+        video_info = self._parse_json(
+            self._search_regex(r'window.__playinfo__\s*=\s*({.+?})</script>', webpage, 'video info', default=None) or '{}',
+            video_id, fatal=False)
+        video_info = video_info.get('data') or {}
+
+        durl = traverse_obj(video_info, ('dash', 'video'))
+        audios = traverse_obj(video_info, ('dash', 'audio')) or []
         entries = []
 
         RENDITIONS = ('qn=80&quality=80&type=', 'quality=2&type=mp4')
         for num, rendition in enumerate(RENDITIONS, start=1):
             payload = 'appkey=%s&cid=%s&otype=json&%s' % (self._APP_KEY, cid, rendition)
             sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
-
-            video_info = self._download_json(
-                'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (payload, sign),
-                video_id, note='Downloading video info page',
-                headers=headers, fatal=num == len(RENDITIONS))
-
             if not video_info:
-                continue
+                video_info = self._download_json(
+                    'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (payload, sign),
+                    video_id, note='Downloading video info page',
+                    headers=headers, fatal=num == len(RENDITIONS))
+                if not video_info:
+                    continue
 
-            if 'durl' not in video_info:
+            if not durl and 'durl' not in video_info:
                 if num < len(RENDITIONS):
                     continue
                 self._report_error(video_info)
 
-            for idx, durl in enumerate(video_info['durl']):
+            formats = []
-                formats = [{
+            for idx, durl in enumerate(durl or video_info['durl']):
-                    'url': durl['url'],
+                formats.append({
-                    'filesize': int_or_none(durl['size']),
+                    'url': durl.get('baseUrl') or durl.get('base_url') or durl.get('url'),
-                }]
+                    'ext': mimetype2ext(durl.get('mimeType') or durl.get('mime_type')),
-                for backup_url in durl.get('backup_url', []):
+                    'fps': int_or_none(durl.get('frameRate') or durl.get('frame_rate')),
+                    'width': int_or_none(durl.get('width')),
+                    'height': int_or_none(durl.get('height')),
+                    'vcodec': durl.get('codecs'),
+                    'acodec': 'none' if audios else None,
+                    'tbr': float_or_none(durl.get('bandwidth'), scale=1000),
+                    'filesize': int_or_none(durl.get('size')),
+                })
+                for backup_url in traverse_obj(durl, 'backup_url', expected_type=list) or []:
                     formats.append({
                         'url': backup_url,
-                        # backup URLs have lower priorities
                         'quality': -2 if 'hd.mp4' in backup_url else -3,
                     })
 
-                for a_format in formats:
+            for audio in audios:
-                    a_format.setdefault('http_headers', {}).update({
+                formats.append({
-                        'Referer': url,
+                    'url': audio.get('baseUrl') or audio.get('base_url') or audio.get('url'),
+                    'ext': mimetype2ext(audio.get('mimeType') or audio.get('mime_type')),
+                    'fps': int_or_none(audio.get('frameRate') or audio.get('frame_rate')),
+                    'width': int_or_none(audio.get('width')),
+                    'height': int_or_none(audio.get('height')),
+                    'acodec': audio.get('codecs'),
+                    'vcodec': 'none',
+                    'tbr': float_or_none(audio.get('bandwidth'), scale=1000),
+                    'filesize': int_or_none(audio.get('size'))
+                })
+                for backup_url in traverse_obj(audio, 'backup_url', expected_type=list) or []:
+                    formats.append({
+                        'url': backup_url,
+                        # backup URLs have lower priorities
+                        'quality': -3,
                     })
 
-                self._sort_formats(formats)
+            info.update({
-
+                'id': video_id,
-                entries.append({
+                'duration': float_or_none(durl.get('length'), 1000),
-                    'id': '%s_part%s' % (video_id, idx),
+                'formats': formats,
-                    'duration': float_or_none(durl.get('length'), 1000),
+                'http_headers': {
-                    'formats': formats,
+                    'Referer': url,
-                })
+                },
+            })
             break
 
-        title = self._html_search_regex(
+        self._sort_formats(formats)
-            (r'<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
+
-             r'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
+        title = self._html_search_regex((
-            group='title')
+            r'<h1[^>]+title=(["\'])(?P<content>[^"\']+)',
+            r'(?s)<h1[^>]*>(?P<content>.+?)</h1>',
+            self._meta_regex('title')
+        ), webpage, 'title', group='content', fatal=False)
 
         # Get part title for anthologies
         if page_id is not None:
-            # TODO: The json is already downloaded by _extract_anthology_entries. Don't redownload for each video
+            # TODO: The json is already downloaded by _extract_anthology_entries. Don't redownload for each video.
-            part_title = try_get(
+            part_info = traverse_obj(self._download_json(
-                self._download_json(
+                f'https://api.bilibili.com/x/player/pagelist?bvid={bv_id}&jsonp=jsonp',
-                    f'https://api.bilibili.com/x/player/pagelist?bvid={bv_id}&jsonp=jsonp',
+                video_id, note='Extracting videos in anthology'), 'data', expected_type=list)
-                    video_id, note='Extracting videos in anthology'),
+            title = title if len(part_info) == 1 else traverse_obj(part_info, (int(page_id) - 1, 'part')) or title
-                lambda x: x['data'][int(page_id) - 1]['part'])
-            title = part_title or title
 
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
@@ -270,15 +278,15 @@ class BiliBiliIE(InfoExtractor):
         thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
 
         # TODO 'view_count' requires deobfuscating Javascript
-        info = {
+        info.update({
-            'id': str(video_id) if page_id is None else '%s_part%s' % (video_id, page_id),
+            'id': f'{video_id}_part{page_id or 1}',
             'cid': cid,
             'title': title,
             'description': description,
             'timestamp': timestamp,
             'thumbnail': thumbnail,
             'duration': float_or_none(video_info.get('timelength'), scale=1000),
-        }
+        })
 
         uploader_mobj = re.search(
             r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]*>\s*(?P<name>[^<]+?)\s*<',
@@ -299,7 +307,7 @@ class BiliBiliIE(InfoExtractor):
                 video_id, fatal=False, note='Downloading tags'), ('data', ..., 'tag_name')),
         }
 
-        entries[0]['subtitles'] = {
+        info['subtitles'] = {
             'danmaku': [{
                 'ext': 'xml',
                 'url': f'https://comment.bilibili.com/{cid}.xml',
@@ -334,12 +342,10 @@ class BiliBiliIE(InfoExtractor):
             entry['id'] = '%s_part%d' % (video_id, (idx + 1))
 
         return {
-            '_type': 'multi_video',
             'id': str(video_id),
             'bv_id': bv_id,
             'title': title,
             'description': description,
-            'entries': entries,
             **info, **top_level_info
         }
 
@@ -480,9 +486,9 @@ class BilibiliChannelIE(InfoExtractor):
             data = self._download_json(
                 self._API_URL % (list_id, page_num), list_id, note=f'Downloading page {page_num}')['data']
 
-            max_count = max_count or try_get(data, lambda x: x['page']['count'])
+            max_count = max_count or traverse_obj(data, ('page', 'count'))
 
-            entries = try_get(data, lambda x: x['list']['vlist'])
+            entries = traverse_obj(data, ('list', 'vlist'))
             if not entries:
                 return
             for entry in entries:
@@ -520,7 +526,7 @@ class BilibiliCategoryIE(InfoExtractor):
             api_url, query, query={'Search_key': query, 'pn': page_num},
             note='Extracting results from page %s of %s' % (page_num, num_pages))
 
-        video_list = try_get(parsed_json, lambda x: x['data']['archives'], list)
+        video_list = traverse_obj(parsed_json, ('data', 'archives'), expected_type=list)
         if not video_list:
             raise ExtractorError('Failed to retrieve video list for page %d' % page_num)
 
@@ -550,7 +556,7 @@ class BilibiliCategoryIE(InfoExtractor):
 
         api_url = 'https://api.bilibili.com/x/web-interface/newlist?rid=%d&type=1&ps=20&jsonp=jsonp' % rid_value
         page_json = self._download_json(api_url, query, query={'Search_key': query, 'pn': '1'})
-        page_data = try_get(page_json, lambda x: x['data']['page'], dict)
+        page_data = traverse_obj(page_json, ('data', 'page'), expected_type=dict)
         count, size = int_or_none(page_data.get('count')), int_or_none(page_data.get('size'))
         if count is None or not size:
             raise ExtractorError('Failed to calculate either page count or size')
@@ -722,40 +728,57 @@ class BiliBiliPlayerIE(InfoExtractor):
 
 
 class BiliIntlBaseIE(InfoExtractor):
-    _API_URL = 'https://api.bili{}/intl/gateway{}'
+    _API_URL = 'https://api.bilibili.tv/intl/gateway'
+    _NETRC_MACHINE = 'biliintl'
 
-    def _call_api(self, type, endpoint, id):
+    def _call_api(self, endpoint, *args, **kwargs):
-        return self._download_json(self._API_URL.format(type, endpoint), id)['data']
+        json = self._download_json(self._API_URL + endpoint, *args, **kwargs)
+        if json.get('code'):
+            if json['code'] in (10004004, 10004005, 10023006):
+                self.raise_login_required()
+            elif json['code'] == 10004001:
+                self.raise_geo_restricted()
+            else:
+                if json.get('message') and str(json['code']) != json['message']:
+                    errmsg = f'{kwargs.get("errnote", "Unable to download JSON metadata")}: {self.IE_NAME} said: {json["message"]}'
+                else:
+                    errmsg = kwargs.get('errnote', 'Unable to download JSON metadata')
+                if kwargs.get('fatal'):
+                    raise ExtractorError(errmsg)
+                else:
+                    self.report_warning(errmsg)
+        return json.get('data')
 
     def json2srt(self, json):
         data = '\n\n'.join(
             f'{i + 1}\n{srt_subtitles_timecode(line["from"])} --> {srt_subtitles_timecode(line["to"])}\n{line["content"]}'
-            for i, line in enumerate(json['body']))
+            for i, line in enumerate(json['body']) if line.get('content'))
         return data
 
-    def _get_subtitles(self, type, ep_id):
+    def _get_subtitles(self, ep_id):
-        sub_json = self._call_api(type, f'/m/subtitle?ep_id={ep_id}&platform=web', ep_id)
+        sub_json = self._call_api(f'/web/v2/subtitle?episode_id={ep_id}&platform=web', ep_id)
         subtitles = {}
-        for sub in sub_json.get('subtitles', []):
+        for sub in sub_json.get('subtitles') or []:
             sub_url = sub.get('url')
             if not sub_url:
                 continue
-            sub_data = self._download_json(sub_url, ep_id, fatal=False)
+            sub_data = self._download_json(
+                sub_url, ep_id, errnote='Unable to download subtitles', fatal=False,
+                note='Downloading subtitles%s' % f' for {sub["lang"]}' if sub.get('lang') else '')
             if not sub_data:
                 continue
-            subtitles.setdefault(sub.get('key', 'en'), []).append({
+            subtitles.setdefault(sub.get('lang_key', 'en'), []).append({
                 'ext': 'srt',
                 'data': self.json2srt(sub_data)
             })
         return subtitles
 
-    def _get_formats(self, type, ep_id):
+    def _get_formats(self, ep_id):
-        video_json = self._call_api(type, f'/web/playurl?ep_id={ep_id}&platform=web', ep_id)
+        video_json = self._call_api(f'/web/playurl?ep_id={ep_id}&platform=web', ep_id,
-        if not video_json:
+                                    note='Downloading video formats', errnote='Unable to download video formats')
-            self.raise_login_required(method='cookies')
         video_json = video_json['playurl']
         formats = []
-        for vid in video_json.get('video', []):
+        for vid in video_json.get('video') or []:
             video_res = vid.get('video_resource') or {}
             video_info = vid.get('stream_info') or {}
             if not video_res.get('url'):
@@ -771,7 +794,7 @@ class BiliIntlBaseIE(InfoExtractor):
                 'vcodec': video_res.get('codecs'),
                 'filesize': video_res.get('size'),
             })
-        for aud in video_json.get('audio_resource', []):
+        for aud in video_json.get('audio_resource') or []:
             if not aud.get('url'):
                 continue
             formats.append({
@@ -786,85 +809,144 @@ class BiliIntlBaseIE(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
-    def _extract_ep_info(self, type, episode_data, ep_id):
+    def _extract_ep_info(self, episode_data, ep_id):
         return {
             'id': ep_id,
-            'title': episode_data.get('long_title') or episode_data['title'],
+            'title': episode_data.get('title_display') or episode_data['title'],
             'thumbnail': episode_data.get('cover'),
-            'episode_number': str_to_int(episode_data.get('title')),
+            'episode_number': int_or_none(self._search_regex(
-            'formats': self._get_formats(type, ep_id),
+                r'^E(\d+)(?:$| - )', episode_data.get('title_display'), 'episode number', default=None)),
-            'subtitles': self._get_subtitles(type, ep_id),
+            'formats': self._get_formats(ep_id),
+            'subtitles': self._get_subtitles(ep_id),
             'extractor_key': BiliIntlIE.ie_key(),
         }
 
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        try:
+            from Cryptodome.PublicKey import RSA
+            from Cryptodome.Cipher import PKCS1_v1_5
+        except ImportError:
+            try:
+                from Crypto.PublicKey import RSA
+                from Crypto.Cipher import PKCS1_v1_5
+            except ImportError:
+                raise ExtractorError('pycryptodomex not found. Please install', expected=True)
+
+        key_data = self._download_json(
+            'https://passport.bilibili.tv/x/intl/passport-login/web/key?lang=en-US', None,
+            note='Downloading login key', errnote='Unable to download login key')['data']
+
+        public_key = RSA.importKey(key_data['key'])
+        password_hash = PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8'))
+        login_post = self._download_json(
+            'https://passport.bilibili.tv/x/intl/passport-login/web/login/password?lang=en-US', None, data=urlencode_postdata({
+                'username': username,
+                'password': base64.b64encode(password_hash).decode('ascii'),
+                'keep_me': 'true',
+                's_locale': 'en_US',
+                'isTrusted': 'true'
+            }), note='Logging in', errnote='Unable to log in')
+        if login_post.get('code'):
+            if login_post.get('message'):
+                raise ExtractorError(f'Unable to log in: {self.IE_NAME} said: {login_post["message"]}', expected=True)
+            else:
+                raise ExtractorError('Unable to log in')
+
+    def _real_initialize(self):
+        self._login()
+
 
 class BiliIntlIE(BiliIntlBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?bili(?P<type>bili\.tv|intl.com)/(?:[a-z]{2}/)?play/(?P<season_id>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-z]{2}/)?play/(?P<season_id>\d+)/(?P<id>\d+)'
     _TESTS = [{
+        # Bstation page
         'url': 'https://www.bilibili.tv/en/play/34613/341736',
         'info_dict': {
             'id': '341736',
             'ext': 'mp4',
-            'title': 'The First Night',
+            'title': 'E2 - The First Night',
-            'thumbnail': 'https://i0.hdslb.com/bfs/intl/management/91e30e5521235d9b163339a26a0b030ebda54310.png',
+            'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
             'episode_number': 2,
+        }
+    }, {
+        # Non-Bstation page
+        'url': 'https://www.bilibili.tv/en/play/1033760/11005006',
+        'info_dict': {
+            'id': '11005006',
+            'ext': 'mp4',
+            'title': 'E3 - Who?',
+            'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
+            'episode_number': 3,
+        }
+    }, {
+        # Subtitle with empty content
+        'url': 'https://www.bilibili.tv/en/play/1005144/10131790',
+        'info_dict': {
+            'id': '10131790',
+            'ext': 'mp4',
+            'title': 'E140 - Two Heartbeats: Kabuto\'s Trap',
+            'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
+            'episode_number': 140,
         },
-        'params': {
+        'skip': 'According to the copyright owner\'s request, you may only watch the video after you log in.'
-            'format': 'bv',
-        },
     }, {
         'url': 'https://www.biliintl.com/en/play/34613/341736',
-        'info_dict': {
+        'only_matching': True,
-            'id': '341736',
-            'ext': 'mp4',
-            'title': 'The First Night',
-            'thumbnail': 'https://i0.hdslb.com/bfs/intl/management/91e30e5521235d9b163339a26a0b030ebda54310.png',
-            'episode_number': 2,
-        },
-        'params': {
-            'format': 'bv',
-        },
     }]
 
     def _real_extract(self, url):
-        type, season_id, id = self._match_valid_url(url).groups()
+        season_id, video_id = self._match_valid_url(url).groups()
-        data_json = self._call_api(type, f'/web/view/ogv_collection?season_id={season_id}', id)
+        webpage = self._download_webpage(url, video_id)
-        episode_data = next(
+        # Bstation layout
-            episode for episode in data_json.get('episodes', [])
+        initial_data = self._parse_json(self._search_regex(
-            if str(episode.get('ep_id')) == id)
+            r'window\.__INITIAL_DATA__\s*=\s*({.+?});', webpage,
-        return self._extract_ep_info(type, episode_data, id)
+            'preload state', default='{}'), video_id, fatal=False) or {}
+        episode_data = traverse_obj(initial_data, ('OgvVideo', 'epDetail'), expected_type=dict)
+
+        if not episode_data:
+            # Non-Bstation layout, read through episode list
+            season_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={season_id}&platform=web', video_id)
+            episode_data = next(
+                episode for episode in traverse_obj(season_json, ('sections', ..., 'episodes', ...), expected_type=dict)
+                if str(episode.get('episode_id')) == video_id)
+        return self._extract_ep_info(episode_data, video_id)
 
 
 class BiliIntlSeriesIE(BiliIntlBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?bili(?P<type>bili\.tv|intl.com)/(?:[a-z]{2}/)?play/(?P<id>\d+)$'
+    _VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-z]{2}/)?play/(?P<id>\d+)$'
     _TESTS = [{
         'url': 'https://www.bilibili.tv/en/play/34613',
         'playlist_mincount': 15,
         'info_dict': {
             'id': '34613',
+            'title': 'Fly Me to the Moon',
+            'description': 'md5:a861ee1c4dc0acfad85f557cc42ac627',
+            'categories': ['Romance', 'Comedy', 'Slice of life'],
+            'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
+            'view_count': int,
         },
         'params': {
             'skip_download': True,
-            'format': 'bv',
         },
     }, {
         'url': 'https://www.biliintl.com/en/play/34613',
-        'playlist_mincount': 15,
+        'only_matching': True,
-        'info_dict': {
-            'id': '34613',
-        },
-        'params': {
-            'skip_download': True,
-            'format': 'bv',
-        },
     }]
 
-    def _entries(self, id, type):
+    def _entries(self, series_id):
-        data_json = self._call_api(type, f'/web/view/ogv_collection?season_id={id}', id)
+        series_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={series_id}&platform=web', series_id)
-        for episode in data_json.get('episodes', []):
+        for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict, default=[]):
-            episode_id = str(episode.get('ep_id'))
+            episode_id = str(episode.get('episode_id'))
-            yield self._extract_ep_info(type, episode, episode_id)
+            yield self._extract_ep_info(episode, episode_id)
 
     def _real_extract(self, url):
-        type, id = self._match_valid_url(url).groups()
+        series_id = self._match_id(url)
-        return self.playlist_result(self._entries(id, type), playlist_id=id)
+        series_info = self._call_api(f'/web/v2/ogv/play/season_info?season_id={series_id}&platform=web', series_id).get('season') or {}
+        return self.playlist_result(
+            self._entries(series_id), series_id, series_info.get('title'), series_info.get('description'),
+            categories=traverse_obj(series_info, ('styles', ..., 'title'), expected_type=str_or_none),
+            thumbnail=url_or_none(series_info.get('horizontal_cover')), view_count=parse_count(series_info.get('view')))

yt_dlp/extractor/biqle.py

@@ -3,27 +3,28 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from .vk import VKIE
-from ..compat import (
+from ..compat import compat_b64decode
-    compat_b64decode,
+from ..utils import (
-    compat_urllib_parse_unquote,
+    int_or_none,
+    js_to_json,
+    traverse_obj,
+    unified_timestamp,
 )
-from ..utils import int_or_none
 
 
 class BIQLEIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
     _TESTS = [{
-        # Youtube embed
+        'url': 'https://biqle.ru/watch/-2000421746_85421746',
-        'url': 'https://biqle.ru/watch/-115995369_456239081',
+        'md5': 'ae6ef4f04d19ac84e4658046d02c151c',
-        'md5': '97af5a06ee4c29bbf9c001bdb1cf5c06',
         'info_dict': {
-            'id': '8v4f-avW-VI',
+            'id': '-2000421746_85421746',
             'ext': 'mp4',
-            'title': "PASSE-PARTOUT - L'ete c'est fait pour jouer",
+            'title': 'Forsaken By Hope Studio Clip',
-            'description': 'Passe-Partout',
+            'description': 'Forsaken By Hope Studio Clip — Смотреть онлайн',
-            'uploader_id': 'mrsimpsonstef3',
+            'upload_date': '19700101',
-            'uploader': 'Phanolito',
+            'thumbnail': r're:https://[^/]+/impf/7vN3ACwSTgChP96OdOfzFjUCzFR6ZglDQgWsIw/KPaACiVJJxM\.jpg\?size=800x450&quality=96&keep_aspect_ratio=1&background=000000&sign=b48ea459c4d33dbcba5e26d63574b1cb&type=video_thumb',
-            'upload_date': '20120822',
+            'timestamp': 0,
         },
     }, {
         'url': 'http://biqle.org/watch/-44781847_168547604',
@@ -32,53 +33,62 @@ class BIQLEIE(InfoExtractor):
             'id': '-44781847_168547604',
             'ext': 'mp4',
             'title': 'Ребенок в шоке от автоматической мойки',
+            'description': 'Ребенок в шоке от автоматической мойки — Смотреть онлайн',
             'timestamp': 1396633454,
-            'uploader': 'Dmitry Kotov',
             'upload_date': '20140404',
-            'uploader_id': '47850140',
+            'thumbnail': r're:https://[^/]+/c535507/u190034692/video/l_b84df002\.jpg',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        embed_url = self._proto_relative_url(self._search_regex(
+
-            r'<iframe.+?src="((?:https?:)?//(?:daxab\.com|dxb\.to|[^/]+/player)/[^"]+)".*?></iframe>',
+        title = self._html_search_meta('name', webpage, 'Title', fatal=False)
-            webpage, 'embed url'))
+        timestamp = unified_timestamp(self._html_search_meta('uploadDate', webpage, 'Upload Date', default=None))
+        description = self._html_search_meta('description', webpage, 'Description', default=None)
+
+        global_embed_url = self._search_regex(
+            r'<script[^<]+?window.globEmbedUrl\s*=\s*\'((?:https?:)?//(?:daxab\.com|dxb\.to|[^/]+/player)/[^\']+)\'',
+            webpage, 'global Embed url')
+        hash = self._search_regex(
+            r'<script id="data-embed-video[^<]+?hash: "([^"]+)"[^<]*</script>', webpage, 'Hash')
+
+        embed_url = global_embed_url + hash
+
         if VKIE.suitable(embed_url):
             return self.url_result(embed_url, VKIE.ie_key(), video_id)
 
         embed_page = self._download_webpage(
-            embed_url, video_id, headers={'Referer': url})
+            embed_url, video_id, 'Downloading embed webpage', headers={'Referer': url})
-        video_ext = self._get_cookies(embed_url).get('video_ext')
+
-        if video_ext:
+        glob_params = self._parse_json(self._search_regex(
-            video_ext = compat_urllib_parse_unquote(video_ext.value)
+            r'<script id="globParams">[^<]*window.globParams = ([^;]+);[^<]+</script>',
-        if not video_ext:
+            embed_page, 'Global Parameters'), video_id, transform_source=js_to_json)
-            video_ext = compat_b64decode(self._search_regex(
+        host_name = compat_b64decode(glob_params['server'][::-1]).decode()
-                r'video_ext\s*:\s*[\'"]([A-Za-z0-9+/=]+)',
+
-                embed_page, 'video_ext')).decode()
-        video_id, sig, _, access_token = video_ext.split(':')
         item = self._download_json(
-            'https://api.vk.com/method/video.get', video_id,
+            f'https://{host_name}/method/video.get/{video_id}', video_id,
-            headers={'User-Agent': 'okhttp/3.4.1'}, query={
+            headers={'Referer': url}, query={
-                'access_token': access_token,
+                'token': glob_params['video']['access_token'],
-                'sig': sig,
-                'v': 5.44,
                 'videos': video_id,
+                'ckey': glob_params['c_key'],
+                'credentials': glob_params['video']['credentials'],
             })['response']['items'][0]
-        title = item['title']
 
         formats = []
         for f_id, f_url in item.get('files', {}).items():
             if f_id == 'external':
                 return self.url_result(f_url)
             ext, height = f_id.split('_')
-            formats.append({
+            height_extra_key = traverse_obj(glob_params, ('video', 'partial', 'quality', height))
-                'format_id': height + 'p',
+            if height_extra_key:
-                'url': f_url,
+                formats.append({
-                'height': int_or_none(height),
+                    'format_id': f'{height}p',
-                'ext': ext,
+                    'url': f'https://{host_name}/{f_url[8:]}&videos={video_id}&extra_key={height_extra_key}',
-            })
+                    'height': int_or_none(height),
+                    'ext': ext,
+                })
         self._sort_formats(formats)
 
         thumbnails = []
@@ -96,10 +106,9 @@ class BIQLEIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'comment_count': int_or_none(item.get('comments')),
-            'description': item.get('description'),
+            'description': description,
             'duration': int_or_none(item.get('duration')),
             'thumbnails': thumbnails,
-            'timestamp': int_or_none(item.get('date')),
+            'timestamp': timestamp,
-            'uploader': item.get('owner_id'),
             'view_count': int_or_none(item.get('views')),
         }

yt_dlp/extractor/callin.py

@@ -0,0 +1,114 @@
+# coding: utf-8
+from .common import InfoExtractor
+from ..utils import (
+    traverse_obj,
+    float_or_none,
+    int_or_none
+)
+
+
+class CallinIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?callin\.com/(episode)/(?P<id>[-a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://www.callin.com/episode/the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
+        'info_dict': {
+            'id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
+            'title': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
+            'ext': 'ts',
+            'display_id': 'the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
+            'thumbnail': 're:https://.+\\.png',
+            'description': 'First episode',
+            'uploader': 'Wesley Yang',
+            'timestamp': 1639404128.65,
+            'upload_date': '20211213',
+            'uploader_id': 'wesyang',
+            'uploader_url': 'http://wesleyyang.substack.com',
+            'channel': 'Conversations in Year Zero',
+            'channel_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
+            'channel_url': 'https://callin.com/show/conversations-in-year-zero-oJNllRFSfx',
+            'duration': 9951.936,
+            'view_count': int,
+            'categories': ['News & Politics', 'History', 'Technology'],
+            'cast': ['Wesley Yang', 'KC Johnson', 'Gabi Abramovich'],
+            'series': 'Conversations in Year Zero',
+            'series_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
+            'episode': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
+            'episode_number': 1,
+            'episode_id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd'
+        }
+    }]
+
+    def try_get_user_name(self, d):
+        names = [d.get(n) for n in ('first', 'last')]
+        if None in names:
+            return next((n for n in names if n), default=None)
+        return ' '.join(names)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        next_data = self._search_nextjs_data(webpage, display_id)
+        episode = next_data['props']['pageProps']['episode']
+
+        id = episode['id']
+        title = (episode.get('title')
+                 or self._og_search_title(webpage, fatal=False)
+                 or self._html_search_regex('<title>(.*?)</title>', webpage, 'title'))
+        url = episode['m3u8']
+        formats = self._extract_m3u8_formats(url, display_id, ext='ts')
+        self._sort_formats(formats)
+
+        show = traverse_obj(episode, ('show', 'title'))
+        show_id = traverse_obj(episode, ('show', 'id'))
+
+        show_json = None
+        app_slug = (self._html_search_regex(
+            '<script\\s+src=["\']/_next/static/([-_a-zA-Z0-9]+)/_',
+            webpage, 'app slug', fatal=False) or next_data.get('buildId'))
+        show_slug = traverse_obj(episode, ('show', 'linkObj', 'resourceUrl'))
+        if app_slug and show_slug and '/' in show_slug:
+            show_slug = show_slug.rsplit('/', 1)[1]
+            show_json_url = f'https://www.callin.com/_next/data/{app_slug}/show/{show_slug}.json'
+            show_json = self._download_json(show_json_url, display_id, fatal=False)
+
+        host = (traverse_obj(show_json, ('pageProps', 'show', 'hosts', 0))
+                or traverse_obj(episode, ('speakers', 0)))
+
+        host_nick = traverse_obj(host, ('linkObj', 'resourceUrl'))
+        host_nick = host_nick.rsplit('/', 1)[1] if (host_nick and '/' in host_nick) else None
+
+        cast = list(filter(None, [
+            self.try_get_user_name(u) for u in
+            traverse_obj(episode, (('speakers', 'callerTags'), ...)) or []
+        ]))
+
+        episode_list = traverse_obj(show_json, ('pageProps', 'show', 'episodes')) or []
+        episode_number = next(
+            (len(episode_list) - i for (i, e) in enumerate(episode_list) if e.get('id') == id),
+            None)
+
+        return {
+            'id': id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': traverse_obj(episode, ('show', 'photo')),
+            'description': episode.get('description'),
+            'uploader': self.try_get_user_name(host) if host else None,
+            'timestamp': episode.get('publishedAt'),
+            'uploader_id': host_nick,
+            'uploader_url': traverse_obj(show_json, ('pageProps', 'show', 'url')),
+            'channel': show,
+            'channel_id': show_id,
+            'channel_url': traverse_obj(episode, ('show', 'linkObj', 'resourceUrl')),
+            'duration': float_or_none(episode.get('runtime')),
+            'view_count': int_or_none(episode.get('plays')),
+            'categories': traverse_obj(episode, ('show', 'categorizations', ..., 'name')),
+            'cast': cast if cast else None,
+            'series': show,
+            'series_id': show_id,
+            'episode': title,
+            'episode_number': episode_number,
+            'episode_id': id
+        }

yt_dlp/extractor/caltrans.py

@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CaltransIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?ca\.gov/vm/loc/[^/]+/(?P<id>[a-z0-9_]+)\.htm'
+    _TEST = {
+        'url': 'https://cwwp2.dot.ca.gov/vm/loc/d3/hwy50at24th.htm',
+        'info_dict': {
+            'id': 'hwy50at24th',
+            'ext': 'ts',
+            'title': 'US-50 : Sacramento : Hwy 50 at 24th',
+            'live_status': 'is_live',
+            'thumbnail': 'https://cwwp2.dot.ca.gov/data/d3/cctv/image/hwy50at24th/hwy50at24th.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        global_vars = self._search_regex(
+            r'<script[^<]+?([^<]+\.m3u8[^<]+)</script>',
+            webpage, 'Global Vars')
+        route_place = self._search_regex(r'routePlace\s*=\s*"([^"]+)"', global_vars, 'Route Place', fatal=False)
+        location_name = self._search_regex(r'locationName\s*=\s*"([^"]+)"', global_vars, 'Location Name', fatal=False)
+        poster_url = self._search_regex(r'posterURL\s*=\s*"([^"]+)"', global_vars, 'Poster Url', fatal=False)
+        video_stream = self._search_regex(r'videoStreamURL\s*=\s*"([^"]+)"', global_vars, 'Video Stream URL', fatal=False)
+
+        formats = self._extract_m3u8_formats(video_stream, video_id, 'ts', live=True)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': f'{route_place} : {location_name}',
+            'is_live': True,
+            'formats': formats,
+            'thumbnail': poster_url,
+        }

yt_dlp/extractor/cam4.py

@@ -13,6 +13,8 @@ class CAM4IE(InfoExtractor):
             'ext': 'mp4',
             'title': 're:^foxynesss [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'age_limit': 18,
+            'live_status': 'is_live',
+            'thumbnail': 'https://snapshots.xcdnpro.com/thumbnails/foxynesss',
         }
     }
 
@@ -29,4 +31,5 @@ class CAM4IE(InfoExtractor):
             'is_live': True,
             'age_limit': 18,
             'formats': formats,
+            'thumbnail': f'https://snapshots.xcdnpro.com/thumbnails/{channel_id}',
         }

yt_dlp/extractor/canalalpha.py

@@ -78,11 +78,11 @@ class CanalAlphaIE(InfoExtractor):
             'height': try_get(video, lambda x: x['res']['height'], expected_type=int),
         } for video in try_get(data_json, lambda x: x['video']['mp4'], expected_type=list) or [] if video.get('$url')]
         if manifests.get('hls'):
-            m3u8_frmts, m3u8_subs = self._parse_m3u8_formats_and_subtitles(manifests['hls'], id)
+            m3u8_frmts, m3u8_subs = self._parse_m3u8_formats_and_subtitles(manifests['hls'], video_id=id)
             formats.extend(m3u8_frmts)
             subtitles = self._merge_subtitles(subtitles, m3u8_subs)
         if manifests.get('dash'):
-            dash_frmts, dash_subs = self._parse_mpd_formats_and_subtitles(manifests['dash'], id)
+            dash_frmts, dash_subs = self._parse_mpd_formats_and_subtitles(manifests['dash'])
             formats.extend(dash_frmts)
             subtitles = self._merge_subtitles(subtitles, dash_subs)
         self._sort_formats(formats)

yt_dlp/extractor/canvas.py

@@ -76,7 +76,7 @@ class CanvasIE(InfoExtractor):
                     'vrtPlayerToken': vrtPlayerToken,
                     'client': 'null',
                 }, expected_status=400)
-            if not data.get('title'):
+            if 'title' not in data:
                 code = data.get('code')
                 if code == 'AUTHENTICATION_REQUIRED':
                     self.raise_login_required()
@@ -84,7 +84,8 @@ class CanvasIE(InfoExtractor):
                     self.raise_geo_restricted(countries=['BE'])
                 raise ExtractorError(data.get('message') or code, expected=True)
 
-        title = data['title']
+        # Note: The title may be an empty string
+        title = data['title'] or f'{site_id} {video_id}'
         description = data.get('description')
 
         formats = []

yt_dlp/extractor/carambatv.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    format_field,
     float_or_none,
     int_or_none,
     try_get,
@@ -43,7 +44,7 @@ class CarambaTVIE(InfoExtractor):
         formats = [{
             'url': base_url + f['fn'],
             'height': int_or_none(f.get('height')),
-            'format_id': '%sp' % f['height'] if f.get('height') else None,
+            'format_id': format_field(f, 'height', '%sp'),
         } for f in video['qualities'] if f.get('fn')]
         self._sort_formats(formats)
 

yt_dlp/extractor/cbc.py

@@ -340,7 +340,8 @@ class CBCGemIE(InfoExtractor):
                 yield {
                     **base_format,
                     'format_id': join_nonempty('sec', height),
-                    'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\1{bitrate}\2', base_url),
+                    # Note: \g<1> is necessary instead of \1 since bitrate is a number
+                    'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\g<1>{bitrate}\2', base_url),
                     'width': int_or_none(video_quality.attrib.get('MaxWidth')),
                     'tbr': bitrate / 1000.0,
                     'height': height,

yt_dlp/extractor/ccma.py

@@ -1,17 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import calendar
-import datetime
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
-    extract_timezone,
     int_or_none,
     parse_duration,
     parse_resolution,
     try_get,
+    unified_timestamp,
     url_or_none,
 )
 
@@ -95,14 +92,8 @@ class CCMAIE(InfoExtractor):
         duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text'))
         tematica = try_get(informacio, lambda x: x['tematica']['text'])
 
-        timestamp = None
         data_utc = try_get(informacio, lambda x: x['data_emissio']['utc'])
-        try:
+        timestamp = unified_timestamp(data_utc)
-            timezone, data_utc = extract_timezone(data_utc)
-            timestamp = calendar.timegm((datetime.datetime.strptime(
-                data_utc, '%Y-%d-%mT%H:%M:%S') - timezone).timetuple())
-        except TypeError:
-            pass
 
         subtitles = {}
         subtitols = media.get('subtitols') or []

yt_dlp/extractor/cctv.py

@@ -162,7 +162,8 @@ class CCTVIE(InfoExtractor):
                         'url': video_url,
                         'format_id': 'http',
                         'quality': quality,
-                        'source_preference': -10
+                        # Sample clip
+                        'preference': -10
                     })
 
         hls_url = try_get(data, lambda x: x['hls_url'], compat_str)

yt_dlp/extractor/ceskatelevize.py

@@ -177,6 +177,7 @@ class CeskaTelevizeIE(InfoExtractor):
                 is_live = item.get('type') == 'LIVE'
                 formats = []
                 for format_id, stream_url in item.get('streamUrls', {}).items():
+                    stream_url = stream_url.replace('https://', 'http://')
                     if 'playerType=flash' in stream_url:
                         stream_formats = self._extract_m3u8_formats(
                             stream_url, playlist_id, 'mp4', 'm3u8_native',

yt_dlp/extractor/common.py

@@ -45,6 +45,7 @@ from ..utils import (
     determine_ext,
     determine_protocol,
     dict_get,
+    encode_data_uri,
     error_to_compat_str,
     extract_attributes,
     ExtractorError,
@@ -74,6 +75,7 @@ from ..utils import (
     str_to_int,
     strip_or_none,
     traverse_obj,
+    try_get,
     unescapeHTML,
     UnsupportedError,
     unified_strdate,
@@ -224,6 +226,7 @@ class InfoExtractor(object):
 
     The following fields are optional:
 
+    direct:         True if a direct video file was given (must only be set by GenericIE)
     alt_title:      A secondary title of the video.
     display_id      An alternative identifier for the video, not necessarily
                     unique, but available before title. Typically, id is
@@ -238,16 +241,22 @@ class InfoExtractor(object):
                         * "resolution" (optional, string "{width}x{height}",
                                         deprecated)
                         * "filesize" (optional, int)
+                        * "http_headers" (dict) - HTTP headers for the request
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
     license:        License name the video is licensed under.
     creator:        The creator of the video.
-    release_timestamp: UNIX timestamp of the moment the video was released.
-    release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video was uploaded
     upload_date:    Video upload date (YYYYMMDD).
-                    If not explicitly set, calculated from timestamp.
+                    If not explicitly set, calculated from timestamp
+    release_timestamp: UNIX timestamp of the moment the video was released.
+                    If it is not clear whether to use timestamp or this, use the former
+    release_date:   The date (YYYYMMDD) when the video was released.
+                    If not explicitly set, calculated from release_timestamp
+    modified_timestamp: UNIX timestamp of the moment the video was last modified.
+    modified_date:   The date (YYYYMMDD) when the video was last modified.
+                    If not explicitly set, calculated from modified_timestamp
     uploader_id:    Nickname or id of the video uploader.
     uploader_url:   Full URL to a personal webpage of the video uploader.
     channel:        Full name of the channel the video is uploaded on.
@@ -255,6 +264,7 @@ class InfoExtractor(object):
                     fields. This depends on a particular extractor.
     channel_id:     Id of the channel.
     channel_url:    Full URL to a channel webpage.
+    channel_follower_count: Number of followers of the channel.
     location:       Physical location where the video was filmed.
     subtitles:      The available subtitles as a dictionary in the format
                     {tag: subformats}. "tag" is usually a language code, and
@@ -265,6 +275,8 @@ class InfoExtractor(object):
                         * "url": A URL pointing to the subtitles file
                     It can optionally also have:
                         * "name": Name or description of the subtitles
+                        * "http_headers": A dictionary of additional HTTP headers
+                                  to add to the request.
                     "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles'; contains automatically generated
                     captions instead of normal subtitles
@@ -370,6 +382,7 @@ class InfoExtractor(object):
     disc_number:    Number of the disc or other physical medium the track belongs to,
                     as an integer.
     release_year:   Year (YYYY) when the album was released.
+    composer:       Composer of the piece
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -383,6 +396,11 @@ class InfoExtractor(object):
     Additionally, playlists can have "id", "title", and any other relevent
     attributes with the same semantics as videos (see above).
 
+    It can also have the following optional fields:
+
+    playlist_count: The total number of videos in a playlist. If not given,
+                    YoutubeDL tries to calculate it from "entries"
+
 
     _type "multi_video" indicates that there are multiple videos that
     form a single show, for examples multiple acts of an opera or TV episode.
@@ -408,8 +426,8 @@ class InfoExtractor(object):
     title, description etc.
 
 
-    Subclasses of this one should re-define the _real_initialize() and
+    Subclasses of this should define a _VALID_URL regexp and, re-define the
-    _real_extract() methods and define a _VALID_URL regexp.
+    _real_extract() and (optionally) _real_initialize() methods.
     Probably, they should also be added to the list of extractors.
 
     Subclasses may also override suitable() if necessary, but ensure the function
@@ -622,7 +640,7 @@ class InfoExtractor(object):
             }
             if hasattr(e, 'countries'):
                 kwargs['countries'] = e.countries
-            raise type(e)(e.msg, **kwargs)
+            raise type(e)(e.orig_msg, **kwargs)
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
@@ -644,7 +662,7 @@ class InfoExtractor(object):
         return False
 
     def set_downloader(self, downloader):
-        """Sets the downloader for this IE."""
+        """Sets a YoutubeDL instance as the downloader for this IE."""
         self._downloader = downloader
 
     def _real_initialize(self):
@@ -653,7 +671,7 @@ class InfoExtractor(object):
 
     def _real_extract(self, url):
         """Real extraction process. Redefine in subclasses."""
-        pass
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     @classmethod
     def ie_key(cls):
@@ -732,7 +750,7 @@ class InfoExtractor(object):
 
             errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
             if fatal:
-                raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
+                raise ExtractorError(errmsg, cause=err)
             else:
                 self.report_warning(errmsg)
                 return False
@@ -1084,6 +1102,7 @@ class InfoExtractor(object):
         if metadata_available and (
                 self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')):
             self.report_warning(msg)
+            return
         if method is not None:
             msg = '%s. %s' % (msg, self._LOGIN_HINTS[method])
         raise ExtractorError(msg, expected=True)
@@ -1108,39 +1127,39 @@ class InfoExtractor(object):
 
     # Methods for following #608
     @staticmethod
-    def url_result(url, ie=None, video_id=None, video_title=None, **kwargs):
+    def url_result(url, ie=None, video_id=None, video_title=None, *, url_transparent=False, **kwargs):
         """Returns a URL that points to a page that should be processed"""
-        # TODO: ie should be the class used for getting the info
+        if ie is not None:
-        video_info = {'_type': 'url',
+            kwargs['ie_key'] = ie if isinstance(ie, str) else ie.ie_key()
-                      'url': url,
-                      'ie_key': ie}
-        video_info.update(kwargs)
         if video_id is not None:
-            video_info['id'] = video_id
+            kwargs['id'] = video_id
         if video_title is not None:
-            video_info['title'] = video_title
+            kwargs['title'] = video_title
-        return video_info
+        return {
+            **kwargs,
+            '_type': 'url_transparent' if url_transparent else 'url',
+            'url': url,
+        }
 
-    def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
+    def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None, video_kwargs=None, **kwargs):
-        urls = orderedSet(
+        urls = (self.url_result(self._proto_relative_url(m), ie, **(video_kwargs or {}))
-            self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
+                for m in orderedSet(map(getter, matches) if getter else matches))
-            for m in matches)
+        return self.playlist_result(urls, playlist_id, playlist_title, **kwargs)
-        return self.playlist_result(
-            urls, playlist_id=playlist_id, playlist_title=playlist_title)
 
     @staticmethod
-    def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, **kwargs):
+    def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, *, multi_video=False, **kwargs):
         """Returns a playlist"""
-        video_info = {'_type': 'playlist',
-                      'entries': entries}
-        video_info.update(kwargs)
         if playlist_id:
-            video_info['id'] = playlist_id
+            kwargs['id'] = playlist_id
         if playlist_title:
-            video_info['title'] = playlist_title
+            kwargs['title'] = playlist_title
         if playlist_description is not None:
-            video_info['description'] = playlist_description
+            kwargs['description'] = playlist_description
-        return video_info
+        return {
+            **kwargs,
+            '_type': 'multi_video' if multi_video else 'playlist',
+            'entries': entries,
+        }
 
     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
@@ -1278,6 +1297,7 @@ class InfoExtractor(object):
         return self._og_search_property('description', html, fatal=False, **kargs)
 
     def _og_search_title(self, html, **kargs):
+        kargs.setdefault('fatal', False)
         return self._og_search_property('title', html, **kargs)
 
     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
@@ -1289,6 +1309,10 @@ class InfoExtractor(object):
     def _og_search_url(self, html, **kargs):
         return self._og_search_property('url', html, **kargs)
 
+    def _html_extract_title(self, html, name, **kwargs):
+        return self._html_search_regex(
+            r'(?s)<title>(.*?)</title>', html, name, **kwargs)
+
     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
         name = variadic(name)
         if display_name is None:
@@ -1429,6 +1453,23 @@ class InfoExtractor(object):
                     continue
                 info[count_key] = interaction_count
 
+        def extract_chapter_information(e):
+            chapters = [{
+                'title': part.get('name'),
+                'start_time': part.get('startOffset'),
+                'end_time': part.get('endOffset'),
+            } for part in variadic(e.get('hasPart') or []) if part.get('@type') == 'Clip']
+            for idx, (last_c, current_c, next_c) in enumerate(zip(
+                    [{'end_time': 0}] + chapters, chapters, chapters[1:])):
+                current_c['end_time'] = current_c['end_time'] or next_c['start_time']
+                current_c['start_time'] = current_c['start_time'] or last_c['end_time']
+                if None in current_c.values():
+                    self.report_warning(f'Chapter {idx} contains broken data. Not extracting chapters')
+                    return
+            if chapters:
+                chapters[-1]['end_time'] = chapters[-1]['end_time'] or info['duration']
+                info['chapters'] = chapters
+
         def extract_video_object(e):
             assert e['@type'] == 'VideoObject'
             author = e.get('author')
@@ -1436,7 +1477,8 @@ class InfoExtractor(object):
                 'url': url_or_none(e.get('contentUrl')),
                 'title': unescapeHTML(e.get('name')),
                 'description': unescapeHTML(e.get('description')),
-                'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
+                'thumbnails': [{'url': url_or_none(url)}
+                               for url in variadic(traverse_obj(e, 'thumbnailUrl', 'thumbnailURL'))],
                 'duration': parse_duration(e.get('duration')),
                 'timestamp': unified_timestamp(e.get('uploadDate')),
                 # author can be an instance of 'Organization' or 'Person' types.
@@ -1451,6 +1493,7 @@ class InfoExtractor(object):
                 'view_count': int_or_none(e.get('interactionCount')),
             })
             extract_interaction_statistic(e)
+            extract_chapter_information(e)
 
         def traverse_json_ld(json_ld, at_top_level=True):
             for e in json_ld:
@@ -1496,6 +1539,8 @@ class InfoExtractor(object):
                         'title': unescapeHTML(e.get('headline')),
                         'description': unescapeHTML(e.get('articleBody') or e.get('description')),
                     })
+                    if traverse_obj(e, ('video', 0, '@type')) == 'VideoObject':
+                        extract_video_object(e['video'][0])
                 elif item_type == 'VideoObject':
                     extract_video_object(e)
                     if expected_type is None:
@@ -1513,12 +1558,12 @@ class InfoExtractor(object):
 
         return dict((k, v) for k, v in info.items() if v is not None)
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
+    def _search_nextjs_data(self, webpage, video_id, *, transform_source=None, fatal=True, **kw):
         return self._parse_json(
             self._search_regex(
                 r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
-                webpage, 'next.js data', **kw),
+                webpage, 'next.js data', fatal=fatal, **kw),
-            video_id, **kw)
+            video_id, transform_source=transform_source, fatal=fatal)
 
     def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__'):
         ''' Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function. '''
@@ -1574,7 +1619,7 @@ class InfoExtractor(object):
             'vcodec': {'type': 'ordered', 'regex': True,
                        'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
             'acodec': {'type': 'ordered', 'regex': True,
-                       'order': ['[af]lac', 'wav|aiff', 'opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
+                       'order': ['[af]lac', 'wav|aiff', 'opus', 'vorbis|ogg', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
             'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
                     'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
             'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
@@ -1617,31 +1662,31 @@ class InfoExtractor(object):
             'format_id': {'type': 'alias', 'field': 'id'},
             'preference': {'type': 'alias', 'field': 'ie_pref'},
             'language_preference': {'type': 'alias', 'field': 'lang'},
+            'source_preference': {'type': 'alias', 'field': 'source'},
+            'protocol': {'type': 'alias', 'field': 'proto'},
+            'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
 
             # Deprecated
-            'dimension': {'type': 'alias', 'field': 'res'},
+            'dimension': {'type': 'alias', 'field': 'res', 'deprecated': True},
-            'resolution': {'type': 'alias', 'field': 'res'},
+            'resolution': {'type': 'alias', 'field': 'res', 'deprecated': True},
-            'extension': {'type': 'alias', 'field': 'ext'},
+            'extension': {'type': 'alias', 'field': 'ext', 'deprecated': True},
-            'bitrate': {'type': 'alias', 'field': 'br'},
+            'bitrate': {'type': 'alias', 'field': 'br', 'deprecated': True},
-            'total_bitrate': {'type': 'alias', 'field': 'tbr'},
+            'total_bitrate': {'type': 'alias', 'field': 'tbr', 'deprecated': True},
-            'video_bitrate': {'type': 'alias', 'field': 'vbr'},
+            'video_bitrate': {'type': 'alias', 'field': 'vbr', 'deprecated': True},
-            'audio_bitrate': {'type': 'alias', 'field': 'abr'},
+            'audio_bitrate': {'type': 'alias', 'field': 'abr', 'deprecated': True},
-            'framerate': {'type': 'alias', 'field': 'fps'},
+            'framerate': {'type': 'alias', 'field': 'fps', 'deprecated': True},
-            'protocol': {'type': 'alias', 'field': 'proto'},
+            'filesize_estimate': {'type': 'alias', 'field': 'size', 'deprecated': True},
-            'source_preference': {'type': 'alias', 'field': 'source'},
+            'samplerate': {'type': 'alias', 'field': 'asr', 'deprecated': True},
-            'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
+            'video_ext': {'type': 'alias', 'field': 'vext', 'deprecated': True},
-            'filesize_estimate': {'type': 'alias', 'field': 'size'},
+            'audio_ext': {'type': 'alias', 'field': 'aext', 'deprecated': True},
-            'samplerate': {'type': 'alias', 'field': 'asr'},
+            'video_codec': {'type': 'alias', 'field': 'vcodec', 'deprecated': True},
-            'video_ext': {'type': 'alias', 'field': 'vext'},
+            'audio_codec': {'type': 'alias', 'field': 'acodec', 'deprecated': True},
-            'audio_ext': {'type': 'alias', 'field': 'aext'},
+            'video': {'type': 'alias', 'field': 'hasvid', 'deprecated': True},
-            'video_codec': {'type': 'alias', 'field': 'vcodec'},
+            'has_video': {'type': 'alias', 'field': 'hasvid', 'deprecated': True},
-            'audio_codec': {'type': 'alias', 'field': 'acodec'},
+            'audio': {'type': 'alias', 'field': 'hasaud', 'deprecated': True},
-            'video': {'type': 'alias', 'field': 'hasvid'},
+            'has_audio': {'type': 'alias', 'field': 'hasaud', 'deprecated': True},
-            'has_video': {'type': 'alias', 'field': 'hasvid'},
+            'extractor': {'type': 'alias', 'field': 'ie_pref', 'deprecated': True},
-            'audio': {'type': 'alias', 'field': 'hasaud'},
+            'extractor_preference': {'type': 'alias', 'field': 'ie_pref', 'deprecated': True},
-            'has_audio': {'type': 'alias', 'field': 'hasaud'},
-            'extractor': {'type': 'alias', 'field': 'ie_pref'},
-            'extractor_preference': {'type': 'alias', 'field': 'ie_pref'},
         }
 
         def __init__(self, ie, field_preference):
@@ -1741,7 +1786,7 @@ class InfoExtractor(object):
                     continue
                 if self._get_field_setting(field, 'type') == 'alias':
                     alias, field = field, self._get_field_setting(field, 'field')
-                    if alias not in ('format_id', 'preference', 'language_preference'):
+                    if self._get_field_setting(alias, 'deprecated'):
                         self.ydl.deprecation_warning(
                             f'Format sorting alias {alias} is deprecated '
                             f'and may be removed in a future version. Please use {field} instead')
@@ -2076,7 +2121,7 @@ class InfoExtractor(object):
             headers=headers, query=query, video_id=video_id)
 
     def _parse_m3u8_formats_and_subtitles(
-            self, m3u8_doc, m3u8_url, ext=None, entry_protocol='m3u8_native',
+            self, m3u8_doc, m3u8_url=None, ext=None, entry_protocol='m3u8_native',
             preference=None, quality=None, m3u8_id=None, live=False, note=None,
             errnote=None, fatal=True, data=None, headers={}, query={},
             video_id=None):
@@ -2126,7 +2171,7 @@ class InfoExtractor(object):
             formats = [{
                 'format_id': join_nonempty(m3u8_id, idx),
                 'format_index': idx,
-                'url': m3u8_url,
+                'url': m3u8_url or encode_data_uri(m3u8_doc.encode('utf-8'), 'application/x-mpegurl'),
                 'ext': ext,
                 'protocol': entry_protocol,
                 'preference': preference,
@@ -2712,11 +2757,15 @@ class InfoExtractor(object):
                     mime_type = representation_attrib['mimeType']
                     content_type = representation_attrib.get('contentType', mime_type.split('/')[0])
 
-                    codecs = representation_attrib.get('codecs', '')
+                    codecs = parse_codecs(representation_attrib.get('codecs', ''))
                     if content_type not in ('video', 'audio', 'text'):
                         if mime_type == 'image/jpeg':
                             content_type = mime_type
-                        elif codecs.split('.')[0] == 'stpp':
+                        elif codecs['vcodec'] != 'none':
+                            content_type = 'video'
+                        elif codecs['acodec'] != 'none':
+                            content_type = 'audio'
+                        elif codecs.get('tcodec', 'none') != 'none':
                             content_type = 'text'
                         elif mimetype2ext(mime_type) in ('tt', 'dfxp', 'ttml', 'xml', 'json'):
                             content_type = 'text'
@@ -2762,8 +2811,8 @@ class InfoExtractor(object):
                             'format_note': 'DASH %s' % content_type,
                             'filesize': filesize,
                             'container': mimetype2ext(mime_type) + '_dash',
+                            **codecs
                         }
-                        f.update(parse_codecs(codecs))
                     elif content_type == 'text':
                         f = {
                             'ext': mimetype2ext(mime_type),
@@ -2836,7 +2885,8 @@ class InfoExtractor(object):
                             segment_duration = None
                             if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
                                 segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
-                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
+                                representation_ms_info['total_number'] = int(math.ceil(
+                                    float_or_none(period_duration, segment_duration, default=0)))
                             representation_ms_info['fragments'] = [{
                                 media_location_key: media_template % {
                                     'Number': segment_number,
@@ -2927,6 +2977,10 @@ class InfoExtractor(object):
                                 f['url'] = initialization_url
                             f['fragments'].append({location_key(initialization_url): initialization_url})
                         f['fragments'].extend(representation_ms_info['fragments'])
+                        if not period_duration:
+                            period_duration = try_get(
+                                representation_ms_info,
+                                lambda r: sum(frag['duration'] for frag in r['fragments']), float)
                     else:
                         # Assuming direct URL to unfragmented media.
                         f['url'] = base_url
@@ -3069,7 +3123,7 @@ class InfoExtractor(object):
                     })
         return formats, subtitles
 
-    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None, quality=None):
+    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8_native', mpd_id=None, preference=None, quality=None):
         def absolute_url(item_url):
             return urljoin(base_url, item_url)
 
@@ -3468,8 +3522,6 @@ class InfoExtractor(object):
 
     def _int(self, v, name, fatal=False, **kwargs):
         res = int_or_none(v, **kwargs)
-        if 'get_attr' in kwargs:
-            print(getattr(v, kwargs['get_attr']))
         if res is None:
             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
             if fatal:
@@ -3628,7 +3680,7 @@ class InfoExtractor(object):
     def mark_watched(self, *args, **kwargs):
         if not self.get_param('mark_watched', False):
             return
-        if (self._get_login_info()[0] is not None
+        if (hasattr(self, '_NETRC_MACHINE') and self._get_login_info()[0] is not None
                 or self.get_param('cookiefile')
                 or self.get_param('cookiesfrombrowser')):
             self._mark_watched(*args, **kwargs)
@@ -3676,6 +3728,22 @@ class InfoExtractor(object):
             return [] if default is NO_DEFAULT else default
         return list(val) if casesense else [x.lower() for x in val]
 
+    def _yes_playlist(self, playlist_id, video_id, smuggled_data=None, *, playlist_label='playlist', video_label='video'):
+        if not playlist_id or not video_id:
+            return not video_id
+
+        no_playlist = (smuggled_data or {}).get('force_noplaylist')
+        if no_playlist is not None:
+            return not no_playlist
+
+        video_id = '' if video_id is True else f' {video_id}'
+        playlist_id = '' if playlist_id is True else f' {playlist_id}'
+        if self.get_param('noplaylist'):
+            self.to_screen(f'Downloading just the {video_label}{video_id} because of --no-playlist')
+            return False
+        self.to_screen(f'Downloading {playlist_label}{playlist_id} - add --no-playlist to download just the {video_label}{video_id}')
+        return True
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

yt_dlp/extractor/cpac.py

@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    str_or_none,
+    try_get,
+    unified_timestamp,
+    update_url_query,
+    urljoin,
+)
+
+# compat_range
+try:
+    if callable(xrange):
+        range = xrange
+except (NameError, TypeError):
+    pass
+
+
+class CPACIE(InfoExtractor):
+    IE_NAME = 'cpac'
+    _VALID_URL = r'https?://(?:www\.)?cpac\.ca/(?P<fr>l-)?episode\?id=(?P<id>[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12})'
+    _TEST = {
+        # 'url': 'http://www.cpac.ca/en/programs/primetime-politics/episodes/65490909',
+        'url': 'https://www.cpac.ca/episode?id=fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+        'md5': 'e46ad699caafd7aa6024279f2614e8fa',
+        'info_dict': {
+            'id': 'fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+            'ext': 'mp4',
+            'upload_date': '20220215',
+            'title': 'News Conference to Celebrate National Kindness Week – February 15, 2022',
+            'description': 'md5:466a206abd21f3a6f776cdef290c23fb',
+            'timestamp': 1644901200,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'hls_prefer_native': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if '/l-episode?' in url else 'en'
+
+        content = self._download_json(
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/episode/index.xml&crafterSite=cpacca&id=' + video_id,
+            video_id)
+        video_url = try_get(content, lambda x: x['page']['details']['videoUrl'], compat_str)
+        formats = []
+        if video_url:
+            content = content['page']
+            title = str_or_none(content['details']['title_%s_t' % (url_lang, )])
+            formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', ext='mp4')
+            for fmt in formats:
+                # prefer language to match URL
+                fmt_lang = fmt.get('language')
+                if fmt_lang == url_lang:
+                    fmt['language_preference'] = 10
+                elif not fmt_lang:
+                    fmt['language_preference'] = -1
+                else:
+                    fmt['language_preference'] = -10
+
+        self._sort_formats(formats)
+
+        category = str_or_none(content['details']['category_%s_t' % (url_lang, )])
+
+        def is_live(v_type):
+            return (v_type == 'live') if v_type is not None else None
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'description': str_or_none(content['details'].get('description_%s_t' % (url_lang, ))),
+            'timestamp': unified_timestamp(content['details'].get('liveDateTime')),
+            'category': [category] if category else None,
+            'thumbnail': urljoin(url, str_or_none(content['details'].get('image_%s_s' % (url_lang, )))),
+            'is_live': is_live(content['details'].get('type')),
+        }
+
+
+class CPACPlaylistIE(InfoExtractor):
+    IE_NAME = 'cpac:playlist'
+    _VALID_URL = r'(?i)https?://(?:www\.)?cpac\.ca/(?:program|search|(?P<fr>emission|rechercher))\?(?:[^&]+&)*?(?P<id>(?:id=\d+|programId=\d+|key=[^&]+))'
+
+    _TESTS = [{
+        'url': 'https://www.cpac.ca/program?id=6',
+        'info_dict': {
+            'id': 'id=6',
+            'title': 'Headline Politics',
+            'description': 'Watch CPAC’s signature long-form coverage of the day’s pressing political events as they unfold.',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.cpac.ca/search?key=hudson&type=all&order=desc',
+        'info_dict': {
+            'id': 'key=hudson',
+            'title': 'hudson',
+        },
+        'playlist_count': 22,
+    }, {
+        'url': 'https://www.cpac.ca/search?programId=50',
+        'info_dict': {
+            'id': 'programId=50',
+            'title': '50',
+        },
+        'playlist_count': 9,
+    }, {
+        'url': 'https://www.cpac.ca/emission?id=6',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cpac.ca/rechercher?key=hudson&type=all&order=desc',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if any(x in url for x in ('/emission?', '/rechercher?')) else 'en'
+        pl_type, list_type = ('program', 'itemList') if any(x in url for x in ('/program?', '/emission?')) else ('search', 'searchResult')
+        api_url = (
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/%s/index.xml&crafterSite=cpacca&%s'
+            % (pl_type, video_id, ))
+        content = self._download_json(api_url, video_id)
+        entries = []
+        total_pages = int_or_none(try_get(content, lambda x: x['page'][list_type]['totalPages']), default=1)
+        for page in range(1, total_pages + 1):
+            if page > 1:
+                api_url = update_url_query(api_url, {'page': '%d' % (page, ), })
+                content = self._download_json(
+                    api_url, video_id,
+                    note='Downloading continuation - %d' % (page, ),
+                    fatal=False)
+
+            for item in try_get(content, lambda x: x['page'][list_type]['item'], list) or []:
+                episode_url = urljoin(url, try_get(item, lambda x: x['url_%s_s' % (url_lang, )]))
+                if episode_url:
+                    entries.append(episode_url)
+
+        return self.playlist_result(
+            (self.url_result(entry) for entry in entries),
+            playlist_id=video_id,
+            playlist_title=try_get(content, lambda x: x['page']['program']['title_%s_t' % (url_lang, )]) or video_id.split('=')[-1],
+            playlist_description=try_get(content, lambda x: x['page']['program']['description_%s_t' % (url_lang, )]),
+        )

yt_dlp/extractor/crowdbunker.py

@@ -0,0 +1,113 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    try_get,
+    unified_strdate,
+)
+
+
+class CrowdBunkerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?crowdbunker\.com/v/(?P<id>[^/?#$&]+)'
+
+    _TESTS = [{
+        'url': 'https://crowdbunker.com/v/0z4Kms8pi8I',
+        'info_dict': {
+            'id': '0z4Kms8pi8I',
+            'ext': 'mp4',
+            'title': '117) Pass vax et solutions',
+            'description': 'md5:86bcb422c29475dbd2b5dcfa6ec3749c',
+            'view_count': int,
+            'duration': 5386,
+            'uploader': 'Jérémie Mercier',
+            'uploader_id': 'UCeN_qQV829NYf0pvPJhW5dQ',
+            'like_count': int,
+            'upload_date': '20211218',
+            'thumbnail': 'https://scw.divulg.org/cb-medias4/images/0z4Kms8pi8I/maxres.jpg'
+        },
+        'params': {'skip_download': True}
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        data_json = self._download_json(f'https://api.divulg.org/post/{id}/details',
+                                        id, headers={'accept': 'application/json, text/plain, */*'})
+        video_json = data_json['video']
+        formats, subtitles = [], {}
+        for sub in video_json.get('captions') or []:
+            sub_url = try_get(sub, lambda x: x['file']['url'])
+            if not sub_url:
+                continue
+            subtitles.setdefault(sub.get('languageCode', 'fr'), []).append({
+                'url': sub_url,
+            })
+
+        mpd_url = try_get(video_json, lambda x: x['dashManifest']['url'])
+        if mpd_url:
+            fmts, subs = self._extract_mpd_formats_and_subtitles(mpd_url, id)
+            formats.extend(fmts)
+            subtitles = self._merge_subtitles(subtitles, subs)
+        m3u8_url = try_get(video_json, lambda x: x['hlsManifest']['url'])
+        if m3u8_url:
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(mpd_url, id)
+            formats.extend(fmts)
+            subtitles = self._merge_subtitles(subtitles, subs)
+
+        thumbnails = [{
+            'url': image['url'],
+            'height': int_or_none(image.get('height')),
+            'width': int_or_none(image.get('width')),
+        } for image in video_json.get('thumbnails') or [] if image.get('url')]
+
+        self._sort_formats(formats)
+        return {
+            'id': id,
+            'title': video_json.get('title'),
+            'description': video_json.get('description'),
+            'view_count': video_json.get('viewCount'),
+            'duration': video_json.get('duration'),
+            'uploader': try_get(data_json, lambda x: x['channel']['name']),
+            'uploader_id': try_get(data_json, lambda x: x['channel']['id']),
+            'like_count': data_json.get('likesCount'),
+            'upload_date': unified_strdate(video_json.get('publishedAt') or video_json.get('createdAt')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+
+class CrowdBunkerChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?crowdbunker\.com/@(?P<id>[^/?#$&]+)'
+
+    _TESTS = [{
+        'url': 'https://crowdbunker.com/@Milan_UHRIN',
+        'playlist_mincount': 14,
+        'info_dict': {
+            'id': 'Milan_UHRIN',
+        },
+    }]
+
+    def _entries(self, id):
+        last = None
+
+        for page in itertools.count():
+            channel_json = self._download_json(
+                f'https://api.divulg.org/organization/{id}/posts', id, headers={'accept': 'application/json, text/plain, */*'},
+                query={'after': last} if last else {}, note=f'Downloading Page {page}')
+            for item in channel_json.get('items') or []:
+                v_id = item.get('uid')
+                if not v_id:
+                    continue
+                yield self.url_result(
+                    'https://crowdbunker.com/v/%s' % v_id, ie=CrowdBunkerIE.ie_key(), video_id=v_id)
+            last = channel_json.get('last')
+            if not last:
+                break
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        return self.playlist_result(self._entries(id), playlist_id=id)

yt_dlp/extractor/crunchyroll.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import re
 import json
 import zlib
@@ -23,15 +24,17 @@ from ..utils import (
     bytes_to_intlist,
     extract_attributes,
     float_or_none,
+    format_field,
     intlist_to_bytes,
     int_or_none,
+    join_nonempty,
     lowercase_escape,
     merge_dicts,
     qualities,
     remove_end,
     sanitized_Request,
+    traverse_obj,
     try_get,
-    urlencode_postdata,
     xpath_text,
 )
 from ..aes import (
@@ -40,8 +43,8 @@ from ..aes import (
 
 
 class CrunchyrollBaseIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.crunchyroll.com/login'
+    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
-    _LOGIN_FORM = 'login_form'
+    _API_BASE = 'https://api.crunchyroll.com'
     _NETRC_MACHINE = 'crunchyroll'
 
     def _call_rpc_api(self, method, video_id, note=None, data=None):
@@ -58,50 +61,33 @@ class CrunchyrollBaseIE(InfoExtractor):
         username, password = self._get_login_info()
         if username is None:
             return
-
+        if self._get_cookies(self._LOGIN_URL).get('etp_rt'):
-        login_page = self._download_webpage(
-            self._LOGIN_URL, None, 'Downloading login page')
-
-        def is_logged(webpage):
-            return 'href="/logout"' in webpage
-
-        # Already logged in
-        if is_logged(login_page):
             return
 
-        login_form_str = self._search_regex(
+        upsell_response = self._download_json(
-            r'(?P<form><form[^>]+?id=(["\'])%s\2[^>]*>)' % self._LOGIN_FORM,
+            f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
-            login_page, 'login form', group='form')
+            query={
+                'sess_id': 1,
+                'device_id': 'whatvalueshouldbeforweb',
+                'device_type': 'com.crunchyroll.static',
+                'access_token': 'giKq5eY27ny3cqz',
+                'referer': self._LOGIN_URL
+            })
+        if upsell_response['code'] != 'ok':
+            raise ExtractorError('Could not get session id')
+        session_id = upsell_response['data']['session_id']
 
-        post_url = extract_attributes(login_form_str).get('action')
+        login_response = self._download_json(
-        if not post_url:
+            f'{self._API_BASE}/login.1.json', None, 'Logging in',
-            post_url = self._LOGIN_URL
+            data=compat_urllib_parse_urlencode({
-        elif not post_url.startswith('http'):
+                'account': username,
-            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+                'password': password,
-
+                'session_id': session_id
-        login_form = self._form_hidden_inputs(self._LOGIN_FORM, login_page)
+            }).encode('ascii'))
-
+        if login_response['code'] != 'ok':
-        login_form.update({
+            raise ExtractorError('Login failed. Server message: %s' % login_response['message'], expected=True)
-            'login_form[name]': username,
+        if not self._get_cookies(self._LOGIN_URL).get('etp_rt'):
-            'login_form[password]': password,
+            raise ExtractorError('Login succeeded but did not set etp_rt cookie')
-        })
-
-        response = self._download_webpage(
-            post_url, None, 'Logging in', 'Wrong login info',
-            data=urlencode_postdata(login_form),
-            headers={'Content-Type': 'application/x-www-form-urlencoded'})
-
-        # Successful login
-        if is_logged(response):
-            return
-
-        error = self._html_search_regex(
-            '(?s)<ul[^>]+class=["\']messages["\'][^>]*>(.+?)</ul>',
-            response, 'error message', default=None)
-        if error:
-            raise ExtractorError('Unable to login: %s' % error, expected=True)
-
-        raise ExtractorError('Unable to log in')
 
     def _real_initialize(self):
         self._login()
@@ -733,13 +719,118 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
     def _real_extract(self, url):
         lang, internal_id, display_id = self._match_valid_url(url).group('lang', 'internal_id', 'id')
         webpage = self._download_webpage(url, display_id)
-        episode_data = self._parse_json(
+        initial_state = self._parse_json(
-            self._search_regex(r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'episode data'),
+            self._search_regex(r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'initial state'),
-            display_id)['content']['byId'][internal_id]
+            display_id)
-        video_id = episode_data['external_id'].split('.')[1]
+        episode_data = initial_state['content']['byId'][internal_id]
-        series_id = episode_data['episode_metadata']['series_slug_title']
+        if not self._get_cookies(url).get('etp_rt'):
-        return self.url_result(f'https://www.crunchyroll.com/{lang}{series_id}/{display_id}-{video_id}',
+            video_id = episode_data['external_id'].split('.')[1]
-                               CrunchyrollIE.ie_key(), video_id)
+            series_id = episode_data['episode_metadata']['series_slug_title']
+            return self.url_result(f'https://www.crunchyroll.com/{lang}{series_id}/{display_id}-{video_id}',
+                                   CrunchyrollIE.ie_key(), video_id)
+
+        app_config = self._parse_json(
+            self._search_regex(r'__APP_CONFIG__\s*=\s*({.+?})\s*;', webpage, 'app config'),
+            display_id)
+        client_id = app_config['cxApiParams']['accountAuthClientId']
+        api_domain = app_config['cxApiParams']['apiDomain']
+        basic_token = str(base64.b64encode(('%s:' % client_id).encode('ascii')), 'ascii')
+        auth_response = self._download_json(
+            f'{api_domain}/auth/v1/token', display_id,
+            note='Authenticating with cookie',
+            headers={
+                'Authorization': 'Basic ' + basic_token
+            }, data='grant_type=etp_rt_cookie'.encode('ascii'))
+        policy_response = self._download_json(
+            f'{api_domain}/index/v2', display_id,
+            note='Retrieving signed policy',
+            headers={
+                'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']
+            })
+        bucket = policy_response['cms']['bucket']
+        params = {
+            'Policy': policy_response['cms']['policy'],
+            'Signature': policy_response['cms']['signature'],
+            'Key-Pair-Id': policy_response['cms']['key_pair_id']
+        }
+        locale = traverse_obj(initial_state, ('localization', 'locale'))
+        if locale:
+            params['locale'] = locale
+        episode_response = self._download_json(
+            f'{api_domain}/cms/v2{bucket}/episodes/{internal_id}', display_id,
+            note='Retrieving episode metadata',
+            query=params)
+        if episode_response.get('is_premium_only') and not episode_response.get('playback'):
+            raise ExtractorError('This video is for premium members only.', expected=True)
+        stream_response = self._download_json(
+            episode_response['playback'], display_id,
+            note='Retrieving stream info')
+
+        thumbnails = []
+        for thumbnails_data in traverse_obj(episode_response, ('images', 'thumbnail')):
+            for thumbnail_data in thumbnails_data:
+                thumbnails.append({
+                    'url': thumbnail_data.get('source'),
+                    'width': thumbnail_data.get('width'),
+                    'height': thumbnail_data.get('height'),
+                })
+        subtitles = {}
+        for lang, subtitle_data in stream_response.get('subtitles').items():
+            subtitles[lang] = [{
+                'url': subtitle_data.get('url'),
+                'ext': subtitle_data.get('format')
+            }]
+
+        requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])]
+        hardsub_preference = qualities(requested_hardsubs[::-1])
+        requested_formats = self._configuration_arg('format') or ['adaptive_hls']
+
+        formats = []
+        for stream_type, streams in stream_response.get('streams', {}).items():
+            if stream_type not in requested_formats:
+                continue
+            for stream in streams.values():
+                hardsub_lang = stream.get('hardsub_locale') or ''
+                if hardsub_lang.lower() not in requested_hardsubs:
+                    continue
+                format_id = join_nonempty(
+                    stream_type,
+                    format_field(stream, 'hardsub_locale', 'hardsub-%s'))
+                if not stream.get('url'):
+                    continue
+                if stream_type.split('_')[-1] == 'hls':
+                    adaptive_formats = self._extract_m3u8_formats(
+                        stream['url'], display_id, 'mp4', m3u8_id=format_id,
+                        note='Downloading %s information' % format_id,
+                        fatal=False)
+                elif stream_type.split('_')[-1] == 'dash':
+                    adaptive_formats = self._extract_mpd_formats(
+                        stream['url'], display_id, mpd_id=format_id,
+                        note='Downloading %s information' % format_id,
+                        fatal=False)
+                for f in adaptive_formats:
+                    if f.get('acodec') != 'none':
+                        f['language'] = stream_response.get('audio_locale')
+                    f['quality'] = hardsub_preference(hardsub_lang.lower())
+                formats.extend(adaptive_formats)
+        self._sort_formats(formats)
+
+        return {
+            'id': internal_id,
+            'title': '%s Episode %s – %s' % (episode_response.get('season_title'), episode_response.get('episode'), episode_response.get('title')),
+            'description': episode_response.get('description').replace(r'\r\n', '\n'),
+            'duration': float_or_none(episode_response.get('duration_ms'), 1000),
+            'thumbnails': thumbnails,
+            'series': episode_response.get('series_title'),
+            'series_id': episode_response.get('series_id'),
+            'season': episode_response.get('season_title'),
+            'season_id': episode_response.get('season_id'),
+            'season_number': episode_response.get('season_number'),
+            'episode': episode_response.get('title'),
+            'episode_number': episode_response.get('sequence_number'),
+            'subtitles': subtitles,
+            'formats': formats
+        }
 
 
 class CrunchyrollBetaShowIE(CrunchyrollBaseIE):

yt_dlp/extractor/cspan.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTMLParseError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -11,9 +12,11 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_class,
     int_or_none,
+    join_nonempty,
     js_to_json,
     merge_dicts,
     parse_iso8601,
+    parse_qs,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -126,8 +129,12 @@ class CSpanIE(InfoExtractor):
                         ext = 'vtt'
                     subtitle['ext'] = ext
             ld_info = self._search_json_ld(webpage, video_id, default={})
-            title = get_element_by_class('video-page-title', webpage) or \
+            try:
-                self._og_search_title(webpage)
+                title = get_element_by_class('video-page-title', webpage)
+            except compat_HTMLParseError:
+                title = None
+            if title is None:
+                title = self._og_search_title(webpage)
             description = get_element_by_attribute('itemprop', 'description', webpage) or \
                 self._html_search_meta(['og:description', 'description'], webpage)
             return merge_dicts(info, ld_info, {
@@ -242,3 +249,42 @@ class CSpanIE(InfoExtractor):
                 'title': title,
                 'id': 'c' + video_id if video_type == 'clip' else video_id,
             }
+
+
+class CSpanCongressIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/congress/'
+    _TESTS = [{
+        'url': 'https://www.c-span.org/congress/?chamber=house&date=2017-12-13&t=1513208380',
+        'info_dict': {
+            'id': 'house_2017-12-13',
+            'title': 'Congressional Chronicle - Members of Congress, Hearings and More',
+            'description': 'md5:54c264b7a8f219937987610243305a84',
+            'thumbnail': r're:https://ximage.c-spanvideo.org/.+',
+            'ext': 'mp4'
+        }
+    }]
+
+    def _real_extract(self, url):
+        query = parse_qs(url)
+        video_date = query.get('date', [None])[0]
+        video_id = join_nonempty(query.get('chamber', ['senate'])[0], video_date, delim='_')
+        webpage = self._download_webpage(url, video_id)
+        if not video_date:
+            jwp_date = re.search(r'jwsetup.clipprogdate = \'(?P<date>\d{4}-\d{2}-\d{2})\';', webpage)
+            if jwp_date:
+                video_id = f'{video_id}_{jwp_date.group("date")}'
+        jwplayer_data = self._parse_json(
+            self._search_regex(r'jwsetup\s*=\s*({(?:.|\n)[^;]+});', webpage, 'player config'),
+            video_id, transform_source=js_to_json)
+
+        title = (self._og_search_title(webpage, default=None)
+                 or self._html_search_regex(r'(?s)<title>(.*?)</title>', webpage, 'video title'))
+        description = (self._og_search_description(webpage, default=None)
+                       or self._html_search_meta('description', webpage, 'description', default=None))
+
+        return {
+            **self._parse_jwplayer_data(jwplayer_data, video_id, False),
+            'title': re.sub(r'\s+', ' ', title.split('|')[0]).strip(),
+            'description': description,
+            'http_headers': {'Referer': 'https://www.c-span.org/'},
+        }

yt_dlp/extractor/ctvnews.py

@@ -65,4 +65,9 @@ class CTVNewsIE(InfoExtractor):
             })
             entries = [ninecninemedia_url_result(clip_id) for clip_id in orderedSet(
                 re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
+            if not entries:
+                webpage = self._download_webpage(url, page_id)
+                if 'getAuthStates("' in webpage:
+                    entries = [ninecninemedia_url_result(clip_id) for clip_id in
+                               self._search_regex(r'getAuthStates\("([\d+,]+)"', webpage, 'clip ids').split(',')]
             return self.playlist_result(entries, page_id)

yt_dlp/extractor/daftsex.py

@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_b64decode
+from ..utils import (
+    get_elements_by_class,
+    int_or_none,
+    js_to_json,
+    parse_count,
+    parse_duration,
+    try_get,
+)
+
+
+class DaftsexIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?daftsex\.com/watch/(?P<id>-?\d+_\d+)'
+    _TESTS = [{
+        'url': 'https://daftsex.com/watch/-156601359_456242791',
+        'info_dict': {
+            'id': '-156601359_456242791',
+            'ext': 'mp4',
+            'title': 'Skye Blue - Dinner And A Show',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = get_elements_by_class('heading', webpage)[-1]
+        duration = parse_duration(self._search_regex(
+            r'Duration: ((?:[0-9]{2}:){0,2}[0-9]{2})',
+            webpage, 'duration', fatal=False))
+        views = parse_count(self._search_regex(
+            r'Views: ([0-9 ]+)',
+            webpage, 'views', fatal=False))
+
+        player_hash = self._search_regex(
+            r'DaxabPlayer\.Init\({[\s\S]*hash:\s*"([0-9a-zA-Z_\-]+)"[\s\S]*}',
+            webpage, 'player hash')
+        player_color = self._search_regex(
+            r'DaxabPlayer\.Init\({[\s\S]*color:\s*"([0-9a-z]+)"[\s\S]*}',
+            webpage, 'player color', fatal=False) or ''
+
+        embed_page = self._download_webpage(
+            'https://daxab.com/player/%s?color=%s' % (player_hash, player_color),
+            video_id, headers={'Referer': url})
+        video_params = self._parse_json(
+            self._search_regex(
+                r'window\.globParams\s*=\s*({[\S\s]+})\s*;\s*<\/script>',
+                embed_page, 'video parameters'),
+            video_id, transform_source=js_to_json)
+
+        server_domain = 'https://%s' % compat_b64decode(video_params['server'][::-1]).decode('utf-8')
+        formats = []
+        for format_id, format_data in video_params['video']['cdn_files'].items():
+            ext, height = format_id.split('_')
+            extra_quality_data = format_data.split('.')[-1]
+            url = f'{server_domain}/videos/{video_id.replace("_", "/")}/{height}.mp4?extra={extra_quality_data}'
+            formats.append({
+                'format_id': format_id,
+                'url': url,
+                'height': int_or_none(height),
+                'ext': ext,
+            })
+        self._sort_formats(formats)
+
+        thumbnail = try_get(video_params,
+                            lambda vi: 'https:' + compat_b64decode(vi['video']['thumb']).decode('utf-8'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'view_count': views,
+            'age_limit': 18,
+        }

yt_dlp/extractor/dailymotion.py

@@ -207,12 +207,10 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         video_id, playlist_id = self._match_valid_url(url).groups()
 
         if playlist_id:
-            if not self.get_param('noplaylist'):
+            if self._yes_playlist(playlist_id, video_id):
-                self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % playlist_id)
                 return self.url_result(
                     'http://www.dailymotion.com/playlist/' + playlist_id,
                     'DailymotionPlaylist', playlist_id)
-            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         password = self.get_param('videopassword')
         media = self._call_api(
@@ -261,9 +259,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     continue
                 if media_type == 'application/x-mpegURL':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, video_id, 'mp4',
+                        media_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False))
-                        'm3u8' if is_live else 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
                 else:
                     f = {
                         'url': media_url,

yt_dlp/extractor/daum.py

@@ -157,11 +157,8 @@ class DaumListIE(InfoExtractor):
         query_dict = parse_qs(url)
         if 'clipid' in query_dict:
             clip_id = query_dict['clipid'][0]
-            if self.get_param('noplaylist'):
+            if not self._yes_playlist(list_id, clip_id):
-                self.to_screen('Downloading just video %s because of --no-playlist' % clip_id)
                 return self.url_result(DaumClipIE._URL_TEMPLATE % clip_id, 'DaumClip')
-            else:
-                self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
 
 
 class DaumPlaylistIE(DaumListIE):

yt_dlp/extractor/daystar.py

@@ -0,0 +1,48 @@
+from .common import InfoExtractor
+from ..utils import js_to_json, urljoin
+
+
+class DaystarClipIE(InfoExtractor):
+    IE_NAME = 'daystar:clip'
+    _VALID_URL = r'https?://player\.daystar\.tv/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://player.daystar.tv/0MTO2ITM',
+        'info_dict': {
+            'id': '0MTO2ITM',
+            'ext': 'mp4',
+            'title': 'The Dark World of COVID Pt. 1 | Aaron Siri',
+            'description': 'a420d320dda734e5f29458df3606c5f4',
+            'thumbnail': r're:^https?://.+\.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        src_iframe = self._search_regex(r'\<iframe[^>]+src="([^"]+)"', webpage, 'src iframe')
+        webpage_iframe = self._download_webpage(
+            src_iframe.replace('player.php', 'config2.php'), video_id, headers={'Referer': src_iframe})
+
+        sources = self._parse_json(self._search_regex(
+            r'sources\:\s*(\[.*?\])', webpage_iframe, 'm3u8 source'), video_id, transform_source=js_to_json)
+
+        formats, subtitles = [], {}
+        for source in sources:
+            file = source.get('file')
+            if file and source.get('type') == 'm3u8':
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    urljoin('https://www.lightcast.com/embed/', file),
+                    video_id, 'mp4', fatal=False, headers={'Referer': src_iframe})
+                formats.extend(fmts)
+                subtitles = self._merge_subtitles(subtitles, subs)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
+            'description': self._html_search_meta(['og:description', 'twitter:description'], webpage),
+            'thumbnail': self._search_regex(r'image:\s*"([^"]+)', webpage_iframe, 'thumbnail'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/digitalconcerthall.py

@@ -0,0 +1,143 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+    parse_resolution,
+    traverse_obj,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class DigitalConcertHallIE(InfoExtractor):
+    IE_DESC = 'DigitalConcertHall extractor'
+    _VALID_URL = r'https?://(?:www\.)?digitalconcerthall\.com/(?P<language>[a-z]+)/concert/(?P<id>[0-9]+)'
+    _OAUTH_URL = 'https://api.digitalconcerthall.com/v2/oauth2/token'
+    _ACCESS_TOKEN = None
+    _NETRC_MACHINE = 'digitalconcerthall'
+    _TESTS = [{
+        'note': 'Playlist with only one video',
+        'url': 'https://www.digitalconcerthall.com/en/concert/53201',
+        'info_dict': {
+            'id': '53201-1',
+            'ext': 'mp4',
+            'composer': 'Kurt Weill',
+            'title': '[Magic Night]',
+            'thumbnail': r're:^https?://images.digitalconcerthall.com/cms/thumbnails.*\.jpg$',
+            'upload_date': '20210624',
+            'timestamp': 1624548600,
+            'duration': 2798,
+            'album_artist': 'Members of the Berliner Philharmoniker / Simon Rössler',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'Concert with several works and an interview',
+        'url': 'https://www.digitalconcerthall.com/en/concert/53785',
+        'info_dict': {
+            'id': '53785',
+            'album_artist': 'Berliner Philharmoniker / Kirill Petrenko',
+            'title': 'Kirill Petrenko conducts Mendelssohn and Shostakovich',
+        },
+        'params': {'skip_download': 'm3u8'},
+        'playlist_count': 3,
+    }]
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if not username:
+            self.raise_login_required()
+        token_response = self._download_json(
+            self._OAUTH_URL,
+            None, 'Obtaining token', errnote='Unable to obtain token', data=urlencode_postdata({
+                'affiliate': 'none',
+                'grant_type': 'device',
+                'device_vendor': 'unknown',
+                'app_id': 'dch.webapp',
+                'app_version': '1.0.0',
+                'client_secret': '2ySLN+2Fwb',
+            }), headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+        self._ACCESS_TOKEN = token_response['access_token']
+        try:
+            self._download_json(
+                self._OAUTH_URL,
+                None, note='Logging in', errnote='Unable to login', data=urlencode_postdata({
+                    'grant_type': 'password',
+                    'username': username,
+                    'password': password,
+                }), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                    'Referer': 'https://www.digitalconcerthall.com',
+                    'Authorization': f'Bearer {self._ACCESS_TOKEN}'
+                })
+        except ExtractorError:
+            self.raise_login_required(msg='Login info incorrect')
+
+    def _real_initialize(self):
+        self._login()
+
+    def _entries(self, items, language, **kwargs):
+        for item in items:
+            video_id = item['id']
+            stream_info = self._download_json(
+                self._proto_relative_url(item['_links']['streams']['href']), video_id, headers={
+                    'Accept': 'application/json',
+                    'Authorization': f'Bearer {self._ACCESS_TOKEN}',
+                    'Accept-Language': language
+                })
+
+            m3u8_url = traverse_obj(
+                stream_info, ('channel', lambda x: x.startswith('vod_mixed'), 'stream', 0, 'url'), get_all=False)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', fatal=False)
+            self._sort_formats(formats)
+
+            yield {
+                'id': video_id,
+                'title': item.get('title'),
+                'composer': item.get('name_composer'),
+                'url': m3u8_url,
+                'formats': formats,
+                'duration': item.get('duration_total'),
+                'timestamp': traverse_obj(item, ('date', 'published')),
+                'description': item.get('short_description') or stream_info.get('short_description'),
+                **kwargs,
+                'chapters': [{
+                    'start_time': chapter.get('time'),
+                    'end_time': try_get(chapter, lambda x: x['time'] + x['duration']),
+                    'title': chapter.get('text'),
+                } for chapter in item['cuepoints']] if item.get('cuepoints') else None,
+            }
+
+    def _real_extract(self, url):
+        language, video_id = self._match_valid_url(url).group('language', 'id')
+        if not language:
+            language = 'en'
+
+        thumbnail_url = self._html_search_regex(
+            r'(https?://images\.digitalconcerthall\.com/cms/thumbnails/.*\.jpg)',
+            self._download_webpage(url, video_id), 'thumbnail')
+        thumbnails = [{
+            'url': thumbnail_url,
+            **parse_resolution(thumbnail_url)
+        }]
+
+        vid_info = self._download_json(
+            f'https://api.digitalconcerthall.com/v2/concert/{video_id}', video_id, headers={
+                'Accept': 'application/json',
+                'Accept-Language': language
+            })
+        album_artist = ' / '.join(traverse_obj(vid_info, ('_links', 'artist', ..., 'name')) or '')
+
+        return {
+            '_type': 'playlist',
+            'id': video_id,
+            'title': vid_info.get('title'),
+            'entries': self._entries(traverse_obj(vid_info, ('_embedded', ..., ...)), language,
+                                     thumbnails=thumbnails, album_artist=album_artist),
+            'thumbnails': thumbnails,
+            'album_artist': album_artist,
+        }

yt_dlp/extractor/dispeak.py

@@ -74,13 +74,11 @@ class DigitallySpeakingIE(InfoExtractor):
             tbr = int_or_none(bitrate)
             vbr = int_or_none(self._search_regex(
                 r'-(\d+)\.mp4', video_path, 'vbr', default=None))
-            abr = tbr - vbr if tbr and vbr else None
             video_formats.append({
                 'format_id': bitrate,
                 'url': url,
                 'tbr': tbr,
                 'vbr': vbr,
-                'abr': abr,
             })
         return video_formats
 
@@ -121,6 +119,7 @@ class DigitallySpeakingIE(InfoExtractor):
         video_formats = self._parse_mp4(metadata)
         if video_formats is None:
             video_formats = self._parse_flv(metadata)
+        self._sort_formats(video_formats)
 
         return {
             'id': video_id,

yt_dlp/extractor/doodstream.py

@@ -20,6 +20,16 @@ class DoodStreamIE(InfoExtractor):
             'description': 'Kat Wonders - Monthly May 2020 | DoodStream.com',
             'thumbnail': 'https://img.doodcdn.com/snaps/flyus84qgl2fsk4g.jpg',
         }
+    }, {
+        'url': 'http://dood.watch/d/5s1wmbdacezb',
+        'md5': '4568b83b31e13242b3f1ff96c55f0595',
+        'info_dict': {
+            'id': '5s1wmbdacezb',
+            'ext': 'mp4',
+            'title': 'Kat Wonders - Monthly May 2020',
+            'description': 'Kat Wonders - Monthly May 2020 | DoodStream.com',
+            'thumbnail': 'https://img.doodcdn.com/snaps/flyus84qgl2fsk4g.jpg',
+        }
     }, {
         'url': 'https://dood.to/d/jzrxn12t2s7n',
         'md5': '3207e199426eca7c2aa23c2872e6728a',
@@ -34,31 +44,26 @@ class DoodStreamIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        url = f'https://dood.to/e/{video_id}'
         webpage = self._download_webpage(url, video_id)
 
-        if '/d/' in url:
+        title = self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None)
-            url = "https://dood.to" + self._html_search_regex(
+        thumb = self._html_search_meta(['og:image', 'twitter:image'], webpage, default=None)
-                r'<iframe src="(/e/[a-z0-9]+)"', webpage, 'embed')
-            video_id = self._match_id(url)
-            webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta(['og:title', 'twitter:title'],
-                                       webpage, default=None)
-        thumb = self._html_search_meta(['og:image', 'twitter:image'],
-                                       webpage, default=None)
         token = self._html_search_regex(r'[?&]token=([a-z0-9]+)[&\']', webpage, 'token')
         description = self._html_search_meta(
-            ['og:description', 'description', 'twitter:description'],
+            ['og:description', 'description', 'twitter:description'], webpage, default=None)
-            webpage, default=None)
+
-        auth_url = 'https://dood.to' + self._html_search_regex(
-            r'(/pass_md5.*?)\'', webpage, 'pass_md5')
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/66.0',
             'referer': url
         }
 
-        webpage = self._download_webpage(auth_url, video_id, headers=headers)
+        pass_md5 = self._html_search_regex(r'(/pass_md5.*?)\'', webpage, 'pass_md5')
-        final_url = webpage + ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(10)]) + "?token=" + token + "&expiry=" + str(int(time.time() * 1000))
+        final_url = ''.join((
+            self._download_webpage(f'https://dood.to{pass_md5}', video_id, headers=headers),
+            *(random.choice(string.ascii_letters + string.digits) for _ in range(10)),
+            f'?token={token}&expiry={int(time.time() * 1000)}',
+        ))
 
         return {
             'id': video_id,

yt_dlp/extractor/dplay.py

@@ -347,7 +347,380 @@ class HGTVDeIE(DPlayBaseIE):
             url, display_id, 'eu1-prod.disco-api.com', 'hgtv', 'de')
 
 
-class DiscoveryPlusIE(DPlayBaseIE):
+class DiscoveryPlusBaseIE(DPlayBaseIE):
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['x-disco-client'] = f'WEB:UNKNOWN:{self._PRODUCT}:25.2.6'
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        return self._download_json(
+            disco_base + 'playback/v3/videoPlaybackInfo',
+            video_id, headers=headers, data=json.dumps({
+                'deviceInfo': {
+                    'adBlocker': False,
+                },
+                'videoId': video_id,
+                'wisteriaProperties': {
+                    'platform': 'desktop',
+                    'product': self._PRODUCT,
+                },
+            }).encode('utf-8'))['data']['attributes']['streaming']
+
+    def _real_extract(self, url):
+        return self._get_disco_api_info(url, self._match_id(url), **self._DISCO_API_PARAMS)
+
+
+class GoDiscoveryIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:go\.)?discovery\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://go.discovery.com/video/dirty-jobs-discovery-atve-us/rodbuster-galvanizer',
+        'info_dict': {
+            'id': '4164906',
+            'display_id': 'dirty-jobs-discovery-atve-us/rodbuster-galvanizer',
+            'ext': 'mp4',
+            'title': 'Rodbuster / Galvanizer',
+            'description': 'Mike installs rebar with a team of rodbusters, then he galvanizes steel.',
+            'season_number': 9,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://discovery.com/video/dirty-jobs-discovery-atve-us/rodbuster-galvanizer',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'dsc'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.go.discovery.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class TravelChannelIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:watch\.)?travelchannel\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://watch.travelchannel.com/video/ghost-adventures-travel-channel/ghost-train-of-ely',
+        'info_dict': {
+            'id': '2220256',
+            'display_id': 'ghost-adventures-travel-channel/ghost-train-of-ely',
+            'ext': 'mp4',
+            'title': 'Ghost Train of Ely',
+            'description': 'The crew investigates the dark history of the Nevada Northern Railway.',
+            'season_number': 24,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://watch.travelchannel.com/video/ghost-adventures-travel-channel/ghost-train-of-ely',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'trav'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.watch.travelchannel.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class CookingChannelIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:watch\.)?cookingchanneltv\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://watch.cookingchanneltv.com/video/carnival-eats-cooking-channel/the-postman-always-brings-rice-2348634',
+        'info_dict': {
+            'id': '2348634',
+            'display_id': 'carnival-eats-cooking-channel/the-postman-always-brings-rice-2348634',
+            'ext': 'mp4',
+            'title': 'The Postman Always Brings Rice',
+            'description': 'Noah visits the Maui Fair and the Aurora Winter Festival in Vancouver.',
+            'season_number': 9,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://watch.cookingchanneltv.com/video/carnival-eats-cooking-channel/the-postman-always-brings-rice-2348634',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'cook'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.watch.cookingchanneltv.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class HGTVUsaIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:watch\.)?hgtv\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://watch.hgtv.com/video/home-inspector-joe-hgtv-atve-us/this-mold-house',
+        'info_dict': {
+            'id': '4289736',
+            'display_id': 'home-inspector-joe-hgtv-atve-us/this-mold-house',
+            'ext': 'mp4',
+            'title': 'This Mold House',
+            'description': 'Joe and Noel help take a familys dream home from hazardous to fabulous.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://watch.hgtv.com/video/home-inspector-joe-hgtv-atve-us/this-mold-house',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'hgtv'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.watch.hgtv.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class FoodNetworkIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:watch\.)?foodnetwork\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://watch.foodnetwork.com/video/kids-baking-championship-food-network/float-like-a-butterfly',
+        'info_dict': {
+            'id': '4116449',
+            'display_id': 'kids-baking-championship-food-network/float-like-a-butterfly',
+            'ext': 'mp4',
+            'title': 'Float Like a Butterfly',
+            'description': 'The 12 kid bakers create colorful carved butterfly cakes.',
+            'season_number': 10,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://watch.foodnetwork.com/video/kids-baking-championship-food-network/float-like-a-butterfly',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'food'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.watch.foodnetwork.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class DestinationAmericaIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?destinationamerica\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.destinationamerica.com/video/alaska-monsters-destination-america-atve-us/central-alaskas-bigfoot',
+        'info_dict': {
+            'id': '4210904',
+            'display_id': 'alaska-monsters-destination-america-atve-us/central-alaskas-bigfoot',
+            'ext': 'mp4',
+            'title': 'Central Alaskas Bigfoot',
+            'description': 'A team heads to central Alaska to investigate an aggressive Bigfoot.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.destinationamerica.com/video/alaska-monsters-destination-america-atve-us/central-alaskas-bigfoot',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'dam'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.destinationamerica.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class InvestigationDiscoveryIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?investigationdiscovery\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.investigationdiscovery.com/video/unmasked-investigation-discovery/the-killer-clown',
+        'info_dict': {
+            'id': '2139409',
+            'display_id': 'unmasked-investigation-discovery/the-killer-clown',
+            'ext': 'mp4',
+            'title': 'The Killer Clown',
+            'description': 'A wealthy Florida woman is fatally shot in the face by a clown at her door.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.investigationdiscovery.com/video/unmasked-investigation-discovery/the-killer-clown',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'ids'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.investigationdiscovery.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class AmHistoryChannelIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ahctv\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.ahctv.com/video/modern-sniper-ahc/army',
+        'info_dict': {
+            'id': '2309730',
+            'display_id': 'modern-sniper-ahc/army',
+            'ext': 'mp4',
+            'title': 'Army',
+            'description': 'Snipers today face challenges their predecessors couldve only dreamed of.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.ahctv.com/video/modern-sniper-ahc/army',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'ahc'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.ahctv.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class ScienceChannelIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?sciencechannel\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.sciencechannel.com/video/strangest-things-science-atve-us/nazi-mystery-machine',
+        'info_dict': {
+            'id': '2842849',
+            'display_id': 'strangest-things-science-atve-us/nazi-mystery-machine',
+            'ext': 'mp4',
+            'title': 'Nazi Mystery Machine',
+            'description': 'Experts investigate the secrets of a revolutionary encryption machine.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.sciencechannel.com/video/strangest-things-science-atve-us/nazi-mystery-machine',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'sci'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.sciencechannel.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class DIYNetworkIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:watch\.)?diynetwork\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://watch.diynetwork.com/video/pool-kings-diy-network/bringing-beach-life-to-texas',
+        'info_dict': {
+            'id': '2309730',
+            'display_id': 'pool-kings-diy-network/bringing-beach-life-to-texas',
+            'ext': 'mp4',
+            'title': 'Bringing Beach Life to Texas',
+            'description': 'The Pool Kings give a family a day at the beach in their own backyard.',
+            'season_number': 10,
+            'episode_number': 2,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://watch.diynetwork.com/video/pool-kings-diy-network/bringing-beach-life-to-texas',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'diy'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.watch.diynetwork.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class DiscoveryLifeIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?discoverylife\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.discoverylife.com/video/surviving-death-discovery-life-atve-us/bodily-trauma',
+        'info_dict': {
+            'id': '2218238',
+            'display_id': 'surviving-death-discovery-life-atve-us/bodily-trauma',
+            'ext': 'mp4',
+            'title': 'Bodily Trauma',
+            'description': 'Meet three people who tested the limits of the human body.',
+            'season_number': 1,
+            'episode_number': 2,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.discoverylife.com/video/surviving-death-discovery-life-atve-us/bodily-trauma',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'dlf'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.discoverylife.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class AnimalPlanetIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://www.animalplanet.com/video/north-woods-law-animal-planet/squirrel-showdown',
+        'info_dict': {
+            'id': '3338923',
+            'display_id': 'north-woods-law-animal-planet/squirrel-showdown',
+            'ext': 'mp4',
+            'title': 'Squirrel Showdown',
+            'description': 'A woman is suspected of being in possession of flying squirrel kits.',
+            'season_number': 16,
+            'episode_number': 11,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://www.animalplanet.com/video/north-woods-law-animal-planet/squirrel-showdown',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'apl'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.animalplanet.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class TLCIE(DiscoveryPlusBaseIE):
+    _VALID_URL = r'https?://(?:go\.)?tlc\.com/video' + DPlayBaseIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://go.tlc.com/video/my-600-lb-life-tlc/melissas-story-part-1',
+        'info_dict': {
+            'id': '2206540',
+            'display_id': 'my-600-lb-life-tlc/melissas-story-part-1',
+            'ext': 'mp4',
+            'title': 'Melissas Story (Part 1)',
+            'description': 'At 650 lbs, Melissa is ready to begin her seven-year weight loss journey.',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://go.tlc.com/video/my-600-lb-life-tlc/melissas-story-part-1',
+        'only_matching': True,
+    }]
+
+    _PRODUCT = 'tlc'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'us1-prod-direct.tlc.com',
+        'realm': 'go',
+        'country': 'us',
+    }
+
+
+class DiscoveryPlusIE(DiscoveryPlusBaseIE):
     _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/(?!it/)(?:\w{2}/)?video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
@@ -372,92 +745,14 @@ class DiscoveryPlusIE(DPlayBaseIE):
     }]
 
     _PRODUCT = 'dplus_us'
-    _API_URL = 'us1-prod-direct.discoveryplus.com'
+    _DISCO_API_PARAMS = {
-
+        'disco_host': 'us1-prod-direct.discoveryplus.com',
-    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        'realm': 'go',
-        headers['x-disco-client'] = f'WEB:UNKNOWN:{self._PRODUCT}:25.2.6'
+        'country': 'us',
-
+    }
-    def _download_video_playback_info(self, disco_base, video_id, headers):
-        return self._download_json(
-            disco_base + 'playback/v3/videoPlaybackInfo',
-            video_id, headers=headers, data=json.dumps({
-                'deviceInfo': {
-                    'adBlocker': False,
-                },
-                'videoId': video_id,
-                'wisteriaProperties': {
-                    'platform': 'desktop',
-                    'product': self._PRODUCT,
-                },
-            }).encode('utf-8'))['data']['attributes']['streaming']
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        return self._get_disco_api_info(
-            url, display_id, self._API_URL, 'go', 'us')
 
 
-class ScienceChannelIE(DiscoveryPlusIE):
+class DiscoveryPlusIndiaIE(DiscoveryPlusBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?sciencechannel\.com/video' + DPlayBaseIE._PATH_REGEX
-    _TESTS = [{
-        'url': 'https://www.sciencechannel.com/video/strangest-things-science-atve-us/nazi-mystery-machine',
-        'info_dict': {
-            'id': '2842849',
-            'display_id': 'strangest-things-science-atve-us/nazi-mystery-machine',
-            'ext': 'mp4',
-            'title': 'Nazi Mystery Machine',
-            'description': 'Experts investigate the secrets of a revolutionary encryption machine.',
-            'season_number': 1,
-            'episode_number': 1,
-        },
-        'skip': 'Available for Premium users',
-    }]
-
-    _PRODUCT = 'sci'
-    _API_URL = 'us1-prod-direct.sciencechannel.com'
-
-
-class DIYNetworkIE(DiscoveryPlusIE):
-    _VALID_URL = r'https?://(?:watch\.)?diynetwork\.com/video' + DPlayBaseIE._PATH_REGEX
-    _TESTS = [{
-        'url': 'https://watch.diynetwork.com/video/pool-kings-diy-network/bringing-beach-life-to-texas',
-        'info_dict': {
-            'id': '2309730',
-            'display_id': 'pool-kings-diy-network/bringing-beach-life-to-texas',
-            'ext': 'mp4',
-            'title': 'Bringing Beach Life to Texas',
-            'description': 'The Pool Kings give a family a day at the beach in their own backyard.',
-            'season_number': 10,
-            'episode_number': 2,
-        },
-        'skip': 'Available for Premium users',
-    }]
-
-    _PRODUCT = 'diy'
-    _API_URL = 'us1-prod-direct.watch.diynetwork.com'
-
-
-class AnimalPlanetIE(DiscoveryPlusIE):
-    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/video' + DPlayBaseIE._PATH_REGEX
-    _TESTS = [{
-        'url': 'https://www.animalplanet.com/video/north-woods-law-animal-planet/squirrel-showdown',
-        'info_dict': {
-            'id': '3338923',
-            'display_id': 'north-woods-law-animal-planet/squirrel-showdown',
-            'ext': 'mp4',
-            'title': 'Squirrel Showdown',
-            'description': 'A woman is suspected of being in possession of flying squirrel kits.',
-            'season_number': 16,
-            'episode_number': 11,
-        },
-        'skip': 'Available for Premium users',
-    }]
-
-    _PRODUCT = 'apl'
-    _API_URL = 'us1-prod-direct.animalplanet.com'
-
-
-class DiscoveryPlusIndiaIE(DPlayBaseIE):
     _VALID_URL = r'https?://(?:www\.)?discoveryplus\.in/videos?' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.in/videos/how-do-they-do-it/fugu-and-more?seasonId=8&type=EPISODE',
@@ -467,41 +762,38 @@ class DiscoveryPlusIndiaIE(DPlayBaseIE):
             'display_id': 'how-do-they-do-it/fugu-and-more',
             'title': 'Fugu and More',
             'description': 'The Japanese catch, prepare and eat the deadliest fish on the planet.',
-            'duration': 1319,
+            'duration': 1319.32,
             'timestamp': 1582309800,
             'upload_date': '20200221',
             'series': 'How Do They Do It?',
             'season_number': 8,
             'episode_number': 2,
             'creator': 'Discovery Channel',
+            'thumbnail': r're:https://.+\.jpeg',
+            'episode': 'Episode 2',
+            'season': 'Season 8',
+            'tags': [],
         },
         'params': {
             'skip_download': True,
         }
     }]
 
+    _PRODUCT = 'dplus-india'
+    _DISCO_API_PARAMS = {
+        'disco_host': 'ap2-prod-direct.discoveryplus.in',
+        'realm': 'dplusindia',
+        'country': 'in',
+        'domain': 'https://www.discoveryplus.in/',
+    }
+
     def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
         headers.update({
             'x-disco-params': 'realm=%s' % realm,
-            'x-disco-client': 'WEB:UNKNOWN:dplus-india:17.0.0',
+            'x-disco-client': f'WEB:UNKNOWN:{self._PRODUCT}:17.0.0',
             'Authorization': self._get_auth(disco_base, display_id, realm),
         })
 
-    def _download_video_playback_info(self, disco_base, video_id, headers):
-        return self._download_json(
-            disco_base + 'playback/v3/videoPlaybackInfo',
-            video_id, headers=headers, data=json.dumps({
-                'deviceInfo': {
-                    'adBlocker': False,
-                },
-                'videoId': video_id,
-            }).encode('utf-8'))['data']['attributes']['streaming']
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        return self._get_disco_api_info(
-            url, display_id, 'ap2-prod-direct.discoveryplus.in', 'dplusindia', 'in', 'https://www.discoveryplus.in/')
-
 
 class DiscoveryNetworksDeIE(DPlayBaseIE):
     _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de|dplay\.co\.uk)/(?:programme|show|sendungen)/(?P<programme>[^/]+)/(?:video/)?(?P<alternate_id>[^/]+)'
@@ -515,6 +807,16 @@ class DiscoveryNetworksDeIE(DPlayBaseIE):
             'description': 'md5:61033c12b73286e409d99a41742ef608',
             'timestamp': 1554069600,
             'upload_date': '20190331',
+            'creator': 'TLC',
+            'season': 'Season 1',
+            'series': 'Breaking Amish',
+            'episode_number': 1,
+            'tags': ['new york', 'großstadt', 'amische', 'landleben', 'modern', 'infos', 'tradition', 'herausforderung'],
+            'display_id': 'breaking-amish/die-welt-da-drauen',
+            'episode': 'Episode 1',
+            'duration': 2625.024,
+            'season_number': 1,
+            'thumbnail': r're:https://.+\.jpg',
         },
         'params': {
             'skip_download': True,
@@ -575,16 +877,19 @@ class DiscoveryPlusShowBaseIE(DPlayBaseIE):
         return self.playlist_result(self._entries(show_name), playlist_id=show_name)
 
 
-class DiscoveryPlusItalyIE(InfoExtractor):
+class DiscoveryPlusItalyIE(DiscoveryPlusBaseIE):
     _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/it/video' + DPlayBaseIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.com/it/video/i-signori-della-neve/stagione-2-episodio-1-i-preparativi',
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
+    _PRODUCT = 'dplus_us'
-        video_id = self._match_id(url)
+    _DISCO_API_PARAMS = {
-        return self.url_result(f'https://discoveryplus.it/video/{video_id}', DPlayIE.ie_key(), video_id)
+        'disco_host': 'eu1-prod-direct.discoveryplus.com',
+        'realm': 'dplay',
+        'country': 'it',
+    }
 
 
 class DiscoveryPlusItalyShowIE(DiscoveryPlusShowBaseIE):

yt_dlp/extractor/drooble.py

@@ -0,0 +1,116 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+)
+
+
+class DroobleIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://drooble\.com/(?:
+        (?:(?P<user>[^/]+)/)?(?P<kind>song|videos|music/albums)/(?P<id>\d+)|
+        (?P<user_2>[^/]+)/(?P<kind_2>videos|music))
+    '''
+    _TESTS = [{
+        'url': 'https://drooble.com/song/2858030',
+        'md5': '5ffda90f61c7c318dc0c3df4179eb064',
+        'info_dict': {
+            'id': '2858030',
+            'ext': 'mp3',
+            'title': 'Skankocillin',
+            'upload_date': '20200801',
+            'timestamp': 1596241390,
+            'uploader_id': '95894',
+            'uploader': 'Bluebeat Shelter',
+        }
+    }, {
+        'url': 'https://drooble.com/karl340758/videos/2859183',
+        'info_dict': {
+            'id': 'J6QCQY_I5Tk',
+            'ext': 'mp4',
+            'title': 'Skankocillin',
+            'uploader_id': 'UCrSRoI5vVyeYihtWEYua7rg',
+            'description': 'md5:ffc0bd8ba383db5341a86a6cd7d9bcca',
+            'upload_date': '20200731',
+            'uploader': 'Bluebeat Shelter',
+        }
+    }, {
+        'url': 'https://drooble.com/karl340758/music/albums/2858031',
+        'info_dict': {
+            'id': '2858031',
+        },
+        'playlist_mincount': 8,
+    }, {
+        'url': 'https://drooble.com/karl340758/music',
+        'info_dict': {
+            'id': 'karl340758',
+        },
+        'playlist_mincount': 8,
+    }, {
+        'url': 'https://drooble.com/karl340758/videos',
+        'info_dict': {
+            'id': 'karl340758',
+        },
+        'playlist_mincount': 8,
+    }]
+
+    def _call_api(self, method, video_id, data=None):
+        response = self._download_json(
+            f'https://drooble.com/api/dt/{method}', video_id, data=json.dumps(data).encode())
+        if not response[0]:
+            raise ExtractorError('Unable to download JSON metadata')
+        return response[1]
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        user = mobj.group('user') or mobj.group('user_2')
+        kind = mobj.group('kind') or mobj.group('kind_2')
+        display_id = mobj.group('id') or user
+
+        if mobj.group('kind_2') == 'videos':
+            data = {'from_user': display_id, 'album': -1, 'limit': 18, 'offset': 0, 'order': 'new2old', 'type': 'video'}
+        elif kind in ('music/albums', 'music'):
+            data = {'user': user, 'public_only': True, 'individual_limit': {'singles': 1, 'albums': 1, 'playlists': 1}}
+        else:
+            data = {'url_slug': display_id, 'children': 10, 'order': 'old2new'}
+
+        method = 'getMusicOverview' if kind in ('music/albums', 'music') else 'getElements'
+        json_data = self._call_api(method, display_id, data=data)
+        if kind in ('music/albums', 'music'):
+            json_data = json_data['singles']['list']
+
+        entites = []
+        for media in json_data:
+            url = media.get('external_media_url') or media.get('link')
+            if url.startswith('https://www.youtube.com'):
+                entites.append({
+                    '_type': 'url',
+                    'url': url,
+                    'ie_key': 'Youtube'
+                })
+                continue
+            is_audio = (media.get('type') or '').lower() == 'audio'
+            entites.append({
+                'url': url,
+                'id': media['id'],
+                'title': media['title'],
+                'duration': int_or_none(media.get('duration')),
+                'timestamp': int_or_none(media.get('timestamp')),
+                'album': try_get(media, lambda x: x['album']['title']),
+                'uploader': try_get(media, lambda x: x['creator']['display_name']),
+                'uploader_id': try_get(media, lambda x: x['creator']['id']),
+                'thumbnail': media.get('image_comment'),
+                'like_count': int_or_none(media.get('likes')),
+                'vcodec': 'none' if is_audio else None,
+                'ext': 'mp3' if is_audio else None,
+            })
+
+        if len(entites) > 1:
+            return self.playlist_result(entites, display_id)
+
+        return entites[0]

yt_dlp/extractor/dropbox.py

@@ -6,7 +6,12 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
-from ..utils import url_basename
+from ..utils import (
+    ExtractorError,
+    traverse_obj,
+    try_get,
+    url_basename,
+)
 
 
 class DropboxIE(InfoExtractor):
@@ -28,13 +33,44 @@ class DropboxIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
         fn = compat_urllib_parse_unquote(url_basename(url))
         title = os.path.splitext(fn)[0]
-        video_url = re.sub(r'[?&]dl=0', '', url)
+
-        video_url += ('?' if '?' not in video_url else '&') + 'dl=1'
+        password = self.get_param('videopassword')
+        if (self._og_search_title(webpage) == 'Dropbox - Password Required'
+                or 'Enter the password for this link' in webpage):
+
+            if password:
+                content_id = self._search_regex(r'content_id=(.*?)["\']', webpage, 'content_id')
+                payload = f'is_xhr=true&t={self._get_cookies("https://www.dropbox.com").get("t").value}&content_id={content_id}&password={password}&url={url}'
+                response = self._download_json(
+                    'https://www.dropbox.com/sm/auth', video_id, 'POSTing video password', data=payload.encode('UTF-8'),
+                    headers={'content-type': 'application/x-www-form-urlencoded; charset=UTF-8'})
+
+                if response.get('status') != 'authed':
+                    raise ExtractorError('Authentication failed!', expected=True)
+                webpage = self._download_webpage(url, video_id)
+            elif self._get_cookies('https://dropbox.com').get('sm_auth'):
+                webpage = self._download_webpage(url, video_id)
+            else:
+                raise ExtractorError('Password protected video, use --video-password <password>', expected=True)
+
+        json_string = self._html_search_regex(r'InitReact\.mountComponent\(.*?,\s*(\{.+\})\s*?\)', webpage, 'Info JSON')
+        info_json = self._parse_json(json_string, video_id).get('props')
+        transcode_url = traverse_obj(info_json, ((None, 'preview'), 'file', 'preview', 'content', 'transcode_url'), get_all=False)
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id)
+
+        # downloads enabled we can get the original file
+        if 'anonymous' in (try_get(info_json, lambda x: x['sharePermission']['canDownloadRoles']) or []):
+            video_url = re.sub(r'[?&]dl=0', '', url)
+            video_url += ('?' if '?' not in video_url else '&') + 'dl=1'
+            formats.append({'url': video_url, 'format_id': 'original', 'format_note': 'Original', 'quality': 1})
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
+            'subtitles': subtitles
         }