[version] update

:ci skip all

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -43,7 +43,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
@@ -51,12 +51,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -54,7 +54,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output using one of the example URLs provided above.
+        Provide the complete verbose output **using one of the example URLs provided above**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
@@ -62,12 +62,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a site feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've checked that all provided URLs are alive and playable in a browser
           required: true
@@ -37,20 +37,20 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
+        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
         [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.10.22 (exe)
+        [debug] yt-dlp version 2021.11.10.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.10.22)
+        yt-dlp is up to date (2021.11.10.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a feature request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2021.10.22**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
+        - label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -43,7 +43,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -54,7 +54,7 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output using one of the example URLs provided above.
+        Provide the complete verbose output **using one of the example URLs provided above**.
         Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -37,8 +37,8 @@ body:
     attributes:
       label: Verbose log
       description: |
-        Provide the complete verbose output of yt-dlp that clearly demonstrates the problem.
-        Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
+        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
+        Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
         It should look similar to this:
       placeholder: |
         [debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']

.github/workflows/build.yml

@@ -115,12 +115,12 @@ jobs:
         release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
         commitish: ${{ steps.push_update.outputs.head_sha }}
         body: |
-          ### Changelog:
-          ${{ env.changelog }}
+          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
 
           ---
 
-          ### See [this](https://github.com/yt-dlp/yt-dlp#release-files) for a description of the release files
+          ### Changelog:
+          ${{ env.changelog }}
         draft: false
         prerelease: false
     - name: Upload yt-dlp Unix binary
@@ -146,6 +146,7 @@ jobs:
   build_macos:
     runs-on: macos-11
     needs: build_unix
+    if: False
     outputs:
       sha256_macos: ${{ steps.sha256_macos.outputs.sha256_macos }}
       sha512_macos: ${{ steps.sha512_macos.outputs.sha512_macos }}
@@ -344,7 +345,7 @@ jobs:
 
   finish:
     runs-on: ubuntu-latest
-    needs: [build_unix, build_windows, build_windows32, build_macos]
+    needs: [build_unix, build_windows, build_windows32]
 
     steps:
     - name: Make SHA2-256SUMS file
@@ -364,8 +365,8 @@ jobs:
         echo "${{ env.SHA256_PY2EXE }}  yt-dlp_min.exe" >> SHA2-256SUMS
         echo "${{ env.SHA256_WIN32 }}  yt-dlp_x86.exe" >> SHA2-256SUMS
         echo "${{ env.SHA256_WIN_ZIP }}  yt-dlp_win.zip" >> SHA2-256SUMS
-        echo "${{ env.SHA256_MACOS }}  yt-dlp_macos" >> SHA2-256SUMS
-        echo "${{ env.SHA256_MACOS_ZIP }}  yt-dlp_macos.zip" >> SHA2-256SUMS
+      #  echo "${{ env.SHA256_MACOS }}  yt-dlp_macos" >> SHA2-256SUMS
+      #  echo "${{ env.SHA256_MACOS_ZIP }}  yt-dlp_macos.zip" >> SHA2-256SUMS
     - name: Upload 256SUMS file
       id: upload-sums
       uses: actions/upload-release-asset@v1
@@ -393,8 +394,8 @@ jobs:
         echo "${{ env.SHA512_WIN_ZIP }}  yt-dlp_win.zip" >> SHA2-512SUMS
         echo "${{ env.SHA512_PY2EXE }}  yt-dlp_min.exe" >> SHA2-512SUMS
         echo "${{ env.SHA512_WIN32 }}  yt-dlp_x86.exe" >> SHA2-512SUMS
-        echo "${{ env.SHA512_MACOS }}  yt-dlp_macos" >> SHA2-512SUMS
-        echo "${{ env.SHA512_MACOS_ZIP }}  yt-dlp_macos.zip" >> SHA2-512SUMS
+      #  echo "${{ env.SHA512_MACOS }}  yt-dlp_macos" >> SHA2-512SUMS
+      #  echo "${{ env.SHA512_MACOS_ZIP }}  yt-dlp_macos.zip" >> SHA2-512SUMS
     - name: Upload 512SUMS file
       id: upload-512sums
       uses: actions/upload-release-asset@v1

.gitignore

@@ -41,6 +41,7 @@ cookies
 *.webp
 *.annotations.xml
 *.description
+.cache/
 
 # Allow config/media files in testdata
 !test/**

CONTRIBUTING.md

@@ -105,7 +105,7 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about yt-dlp?
 
-Some bug reports are completely unrelated to yt-dlp and relate to a different, or even the reporter's own, application. Please make sure that you are actually using yt-dlp. If you are using a UI for yt-dlp, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for yt-dlp fails in some way you believe is related to yt-dlp, by all means, go ahead and report the bug.
+Some bug reports are completely unrelated to yt-dlp and relate to a different, or even the reporter's own, application. Please make sure that you are actually using yt-dlp. If you are using a UI for yt-dlp, report the bug to the maintainer of the actual application providing the UI. In general, if you are unable to provide the verbose log, you should not be opening the issue here.
 
 If the issue is with `youtube-dl` (the upstream fork of yt-dlp) and not with yt-dlp, the issue should be raised in the youtube-dl project.
 
@@ -117,7 +117,7 @@ By sharing an account with anyone, you agree to bear all risks associated with i
 
 While these steps won't necessarily ensure that no misuse of the account takes place, these are still some good practices to follow.
 
-- Look for people with `Member` or `Contributor` tag on their messages.
+- Look for people with `Member` (maintainers of the project) or `Contributor` (people who have previously contributed code) tag on their messages.
 - Change the password before sharing the account to something random (use [this](https://passwordsgenerator.net/) if you don't have a random password generator).
 - Change the password after receiving the account back.
 
@@ -148,7 +148,7 @@ If you want to create a build of yt-dlp yourself, you can follow the instruction
 
 Before you start writing code for implementing a new feature, open an issue explaining your feature request and atleast one use case. This allows the maintainers to decide whether such a feature is desired for the project in the first place, and will provide an avenue to discuss some implementation details. If you open a pull request for a new feature without discussing with us first, do not be surprised when we ask for large changes to the code, or even reject it outright.
 
-The same applies for overarching changes to the architecture, documentation or code style
+The same applies for changes to the documentation, code style, or overarching changes to the architecture
 
 
 ## Adding support for a new site

CONTRIBUTORS

@@ -129,3 +129,13 @@ Bojidarist
 nixklai
 smplayer-dev
 Zirro
+CrypticSignal
+flashdagger
+fractalf
+frafra
+kaz-us
+ozburo
+rhendric
+sdomi
+selfisekai
+stanoarn

Changelog.md

@@ -14,6 +14,95 @@
 -->
 
 
+### 2021.11.10.1
+
+* Temporarily disable MacOS Build
+
+### 2021.11.10
+
+* [youtube] **Fix throttling by decrypting n-sig**
+* Merging extractors from [haruhi-dl](https://git.sakamoto.pl/laudom/haruhi-dl) by [selfisekai](https://github.com/selfisekai)
+    * [extractor] Add `_search_nextjs_data`
+    * [tvp] Fix extractors
+    * [tvp] Add TVPStreamIE
+    * [wppilot] Add extractors
+    * [polskieradio] Add extractors
+    * [radiokapital] Add extractors
+    * [polsatgo] Add extractor by [selfisekai](https://github.com/selfisekai), [sdomi](https://github.com/sdomi)
+* Separate `--check-all-formats` from `--check-formats`
+* Approximate filesize from bitrate
+* Don't create console in `windows_enable_vt_mode`
+* Fix bug in `--load-infojson` of playlists
+* [minicurses] Add colors to `-F` and standardize color-printing code
+* [outtmpl] Add type `link` for internet shortcut files
+* [outtmpl] Add alternate forms for `q` and `j`
+* [outtmpl] Do not traverse `None`
+* [fragment] Fix progress display in fragmented downloads
+* [downloader/ffmpeg] Fix vtt download with ffmpeg
+* [ffmpeg] Detect presence of setts and libavformat version
+* [ExtractAudio] Rescale --audio-quality correctly by [CrypticSignal](https://github.com/CrypticSignal), [pukkandan](https://github.com/pukkandan)
+* [ExtractAudio] Use `libfdk_aac` if available by [CrypticSignal](https://github.com/CrypticSignal)
+* [FormatSort] `eac3` is better than `ac3`
+* [FormatSort] Fix some fields' defaults
+* [generic] Detect more json_ld
+* [generic] parse jwplayer with only the json URL
+* [extractor] Add keyword automatically to SearchIE descriptions
+* [extractor] Fix some errors being converted to `ExtractorError`
+* [utils] Add `join_nonempty`
+* [utils] Add `jwt_decode_hs256` by [Ashish0804](https://github.com/Ashish0804)
+* [utils] Create `DownloadCancelled` exception
+* [utils] Parse `vp09` as vp9
+* [utils] Sanitize URL when determining protocol
+* [test/download] Fallback test to `bv`
+* [docs] Minor documentation improvements
+* [cleanup] Improvements to error and debug messages
+* [cleanup] Minor fixes and cleanup
+* [3speak] Add extractors by [Ashish0804](https://github.com/Ashish0804)
+* [AmazonStore] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [Gab] Add extractor by [u-spec-png](https://github.com/u-spec-png)
+* [mediaset] Add playlist support by [nixxo](https://github.com/nixxo)
+* [MLSScoccer] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [N1] Add support for nova.rs by [u-spec-png](https://github.com/u-spec-png)
+* [PlanetMarathi] Add extractor by [Ashish0804](https://github.com/Ashish0804)
+* [RaiplayRadio] Add extractors by [frafra](https://github.com/frafra)
+* [roosterteeth] Add series extractor
+* [sky] Add `SkyNewsStoryIE` by [ajj8](https://github.com/ajj8)
+* [youtube] Fix sorting for some videos
+* [youtube] Populate `thumbnail` with the best "known" thumbnail
+* [youtube] Refactor itag processing
+* [youtube] Remove unnecessary no-playlist warning
+* [youtube:tab] Add Invidious list for playlists/channels by [rhendric](https://github.com/rhendric)
+* [Bilibili:comments] Fix infinite loop by [u-spec-png](https://github.com/u-spec-png)
+* [ceskatelevize] Fix extractor by [flashdagger](https://github.com/flashdagger)
+* [Coub] Fix media format identification by [wlritchi](https://github.com/wlritchi)
+* [crunchyroll] Add extractor-args `language` and `hardsub`
+* [DiscoveryPlus] Allow language codes in URL
+* [imdb] Fix thumbnail by [ozburo](https://github.com/ozburo)
+* [instagram] Add IOS URL support by [u-spec-png](https://github.com/u-spec-png)
+* [instagram] Improve login code by [u-spec-png](https://github.com/u-spec-png)
+* [Instagram] Improve metadata extraction by [u-spec-png](https://github.com/u-spec-png)
+* [iPrima] Fix extractor by [stanoarn](https://github.com/stanoarn)
+* [itv] Add support for ITV News by [ajj8](https://github.com/ajj8)
+* [la7] Fix extractor by [nixxo](https://github.com/nixxo)
+* [linkedin] Don't login multiple times
+* [mtv] Fix some videos by [Sipherdrakon](https://github.com/Sipherdrakon)
+* [Newgrounds] Fix description by [u-spec-png](https://github.com/u-spec-png)
+* [Nrk] Minor fixes by [fractalf](https://github.com/fractalf)
+* [Olympics] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [piksel] Fix sorting
+* [twitter] Do not sort by codec
+* [viewlift] Add cookie-based login and series support by [Ashish0804](https://github.com/Ashish0804), [pukkandan](https://github.com/pukkandan)
+* [vimeo] Detect source extension and misc cleanup by [flashdagger](https://github.com/flashdagger)
+* [vimeo] Fix ondemand videos and direct URLs with hash
+* [vk] Fix login and add subtitles by [kaz-us](https://github.com/kaz-us)
+* [VLive] Add upload_date and thumbnail by [Ashish0804](https://github.com/Ashish0804)
+* [VRT] Fix login by [pgaig](https://github.com/pgaig)
+* [Vupload] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [wakanim] Add support for MPD manifests by [nyuszika7h](https://github.com/nyuszika7h)
+* [wakanim] Detect geo-restriction by [nyuszika7h](https://github.com/nyuszika7h)
+* [ZenYandex] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+
+
 ### 2021.10.22
 
 * [build] Improvements
@@ -61,7 +150,7 @@
 * [AdobePass] Fix RCN MSO by [jfogelman](https://github.com/jfogelman)
 * [CBC] Fix Gem livestream by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
 * [CBC] Support CBC Gem member content by [makeworld-the-better-one](https://github.com/makeworld-the-better-one)
-* [crunchyroll] Add season to flat-playlist Closes #1319
+* [crunchyroll] Add season to flat-playlist
 * [crunchyroll] Add support for `beta.crunchyroll` URLs and fix series URLs with language code
 * [EUScreen] Add Extractor by [Ashish0804](https://github.com/Ashish0804)
 * [Gronkh] Add extractor by [Ashish0804](https://github.com/Ashish0804)

README.md

@@ -61,7 +61,6 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
     * [Opening an Issue](CONTRIBUTING.md#opening-an-issue)
     * [Developer Instructions](CONTRIBUTING.md#developer-instructions)
 * [MORE](#more)
-</div>
 
 
 # NEW FEATURES
@@ -79,8 +78,8 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
     * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`) and private playlists supports downloading multiple pages of content
     * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works
     * Mixes supports downloading multiple pages of content
-    * Most (but not all) age-gated content can be downloaded without cookies
-    * Partial workaround for throttling issue
+    * Some (but not all) age-gated content can be downloaded without cookies
+    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326)
     * Redirect channel's home URL automatically to `/video` to preserve the old behaviour
     * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
     * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
@@ -93,9 +92,13 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats
 
-* **New extractors**: AnimeLab, Philo MSO, Spectrum MSO, SlingTV MSO, Cablevision MSO, RCN MSO, Rcs, Gedi, bitwave.tv, mildom, audius, zee5, mtv.it, wimtv, pluto.tv, niconico users, discoveryplus.in, mediathek, NFHSNetwork, nebula, ukcolumn, whowatch, MxplayerShow, parlview (au), YoutubeWebArchive, fancode, Saitosan, ShemarooMe, telemundo, VootSeries, SonyLIVSeries, HotstarSeries, VidioPremier, VidioLive, RCTIPlus, TBS Live, douyin, pornflip, ParamountPlusSeries, ScienceChannel, Utreon, OpenRec, BandcampMusic, blackboardcollaborate, eroprofile albums, mirrativ, BannedVideo, bilibili categories, Epicon, filmmodu, GabTV, HungamaAlbum, ManotoTV, Niconico search, Patreon User, peloton, ProjectVeritas, radiko, StarTV, tiktok user, Tokentube, voicy, TV2HuSeries, biliintl, 17live, NewgroundsUser, peertube channel/playlist, ZenYandex, CAM4, CGTN, damtomo, gotostage, Koo, Mediaite, Mediaklikk, MuseScore, nzherald, Olympics replay, radlive, SovietsCloset, Streamanity, Theta, Chingari, ciscowebex, Gettr, GoPro, N1, Theta, Veo, Vupload, NovaPlay, SkyNewsAU, EUScreen, Gronkh, microsoftstream, on24, trovo channels
+* **New extractors**: 17live, 3speak, amazonstore, animelab, audius, bandcampmusic, bannedvideo, biliintl, bitwave.tv, blackboardcollaborate, cam4, cgtn, chingari, ciscowebex, damtomo, discoveryplus.in, douyin, epicon, euscreen, fancode, filmmodu, gab, gedi, gettr, gopro, gotostage, gronkh, koo, manototv, mediaite, mediaklikk, mediasetshow, mediathek, microsoftstream, mildom, mirrativ, mlsscoccer, mtv.it, musescore, mxplayershow, n1, nebula, nfhsnetwork, novaplay, nzherald, olympics replay, on24, openrec, parlview-AU, peloton, planetmarathi, pluto.tv, polsatgo, polskieradio, pornflip, projectveritas, radiko, radiokapital, radlive, raiplayradio, rcs, rctiplus, saitosan, sciencechannel, shemaroome, skynews-AU, skynews-story, sovietscloset, startv, streamanity, telemundo, theta, theta, tokentube, tv2huseries, ukcolumn, utreon, veo, vidiolive, vidiopremier, voicy, vupload, whowatch, wim.tv, wppilot, youtube webarchive, zee5, zen.yandex
 
-* **Fixed/improved extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, akamai, ina, rumble, tennistv, amcnetworks, la7 podcasts, linuxacadamy, nitter, twitcasting, viu, crackle, curiositystream, mediasite, rmcdecouverte, sonyliv, tubi, tenplay, patreon, videa, yahoo, BravoTV, crunchyroll, RTP, viki, Hotstar, vidio, vimeo, mediaset, Mxplayer, nbcolympics, ParamountPlus, Newgrounds, SAML Verizon login, Hungama, afreecatv, aljazeera, ATV, bitchute, camtube, CDA, eroprofile, facebook, HearThisAtIE, iwara, kakao, Motherless, Nova, peertube, pornhub, reddit, tiktok, TV2, TV2Hu, tv5mondeplus, VH1, Viafree, XHamster, 9Now, AnimalPlanet, Arte, CBC, Chingari, comedycentral, DIYNetwork, niconico, dw, funimation, globo, HiDive, NDR, Nuvid, Oreilly, pbs, plutotv, reddit, redtube, soundcloud, SpankBang, VrtNU, bbc, Bilibili, LinkedInLearning, parliamentlive, PolskieRadio, Streamable, vidme, francetv, 7plus, tagesschau
+* **New playlist extractors**: bilibili categories, eroprofile albums, hotstar series, hungama albums, newgrounds user, niconico search/users, paramountplus series, patreon user, peertube playlist/channels, roosterteeth series, sonyliv series, tiktok user, trovo channels, voot series
+
+* **Fixed/improved extractors**: 7plus, 9now, afreecatv, akamai, aljazeera, amcnetworks, animalplanet, archive.org, arte, atv, bbc, bilibili, bitchute, bravotv, camtube, cbc, cda, ceskatelevize, chingari, comedycentral, coub, crackle, crunchyroll, curiositystream, diynetwork, dw, eroprofile, facebook, francetv, funimation, globo, hearthisatie, hidive, hotstar, hungama, imdb, ina, instagram, iprima, itv, iwara, kakao, la7, linkedinlearning, linuxacadamy, mediaset, mediasite, motherless, mxplayer, nbcolympics, ndr, newgrounds, niconico, nitter, nova, nrk, nuvid, oreilly, paramountplus, parliamentlive, patreon, pbs, peertube, plutotv, polskieradio, pornhub, reddit, reddit, redtube, rmcdecouverte, roosterteeth, rtp, rumble, saml verizon login, skyit, sonyliv, soundcloud, southparkde, spankbang, spreaker, streamable, tagesschau, tbs, tennistv, tenplay, tiktok, tubi, tv2, tv2hu, tv5mondeplus, tvp, twitcasting, vh1, viafree, videa, vidio, vidme, viewlift, viki, vimeo, viu, vk, vlive, vrt, wakanim, xhamster, yahoo
+
+* **New MSOs**: Philo, Spectrum, SlingTV, Cablevision, RCN
 
 * **Subtitle extraction from manifests**: Subtitles can be extracted from streaming media manifests. See [commit/be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details
 
@@ -109,7 +112,7 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection) etc
 
-* **Plugin extractors**: Extractors can be loaded from an external file. See [plugins](#plugins) for details
+* **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details
 
 * **Self-updater**: The releases can be updated using `yt-dlp -U`
 
@@ -123,11 +126,11 @@ If you are coming from [youtube-dl](https://github.com/ytdl-org/youtube-dl), the
 
 ### Differences in default behavior
 
-Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc.
+Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
 
-* The options `--id`, `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
+* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
 * `avconv` is not supported as as an alternative to `ffmpeg`
-* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s.%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
+* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
 * The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
 * The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be prefered. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
 * Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both
@@ -143,7 +146,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
 * Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
-* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the seperate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this.
+* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the seperate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
 
 For ease of use, a few more compat options are available:
 * `--compat-options all`: Use all compat options
@@ -152,17 +155,14 @@ For ease of use, a few more compat options are available:
 
 
 # INSTALLATION
-yt-dlp is not platform specific. So it should work on your Unix box, on Windows or on macOS
 
 You can install yt-dlp using one of the following methods:
-* Download [the binary](#release-files) from the [latest release](https://github.com/yt-dlp/yt-dlp/releases/latest)
-* With Homebrew, `brew install yt-dlp/taps/yt-dlp`
-* Use [PyPI package](https://pypi.org/project/yt-dlp): `python3 -m pip install --upgrade yt-dlp`
-* Install master branch: `python3 -m pip3 install -U https://github.com/yt-dlp/yt-dlp/archive/master.zip`
 
-Note that on some systems, you may need to use `py` or `python` instead of `python3`
+### Using the release binary
 
-UNIX users (Linux, macOS, BSD) can also install the [latest release](https://github.com/yt-dlp/yt-dlp/releases/latest) one of the following ways:
+You can simply download the [correct binary file](#release-files) for your OS: **[[Windows](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)] [[UNIX-like](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)]**
+
+In UNIX-like OSes (MacOS, Linux, BSD), you can also install the same in one of the following ways:
 
 ```
 sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
@@ -179,35 +179,60 @@ sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o
 sudo chmod a+rx /usr/local/bin/yt-dlp
 ```
 
-macOS or Linux users that are using Homebrew (formerly known as Linuxbrew for Linux users) can also install it by:
+PS: The manpages, shell completion files etc. are available in [yt-dlp.tar.gz](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)
+
+### With [PIP](https://pypi.org/project/pip)
+
+You can install the [PyPI package](https://pypi.org/project/yt-dlp) with:
+```
+python3 -m pip install -U yt-dlp
+```
+
+You can install without any of the optional dependencies using:
+```
+python3 -m pip install --no-deps -U yt-dlp
+```
+
+If you want to be on the cutting edge, you can also install the master branch with:
+```
+python3 -m pip3 install --force-reinstall https://github.com/yt-dlp/yt-dlp/archive/master.zip
+```
+
+Note that on some systems, you may need to use `py` or `python` instead of `python3`
+
+### With [Homebrew](https://brew.sh)
+
+macOS or Linux users that are using Homebrew can also install it by:
 
 ```
 brew install yt-dlp/taps/yt-dlp
 ```
 
-### UPDATE
-You can use `yt-dlp -U` to update if you are using the provided release.
-If you are using `pip`, simply re-run the same command that was used to install the program.
-If you have installed using Homebrew, run `brew upgrade yt-dlp/taps/yt-dlp`
+## UPDATE
+You can use `yt-dlp -U` to update if you are [using the provided release](#using-the-release-binary)
 
-### RELEASE FILES
+If you [installed with pip](#with-pip), simply re-run the same command that was used to install the program
+
+If you [installed using Homebrew](#with-homebrew), run `brew upgrade yt-dlp/taps/yt-dlp`
+
+## RELEASE FILES
 
 #### Recommended
 
 File|Description
 :---|:---
-[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform independant binary. Needs Python (Recommended for **UNIX-like systems**)
-[yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows standalone x64 binary (Recommended for **Windows**)
+[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform-independant binary. Needs Python (recommended for **UNIX-like systems**)
+[yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows (Win7 SP1+) standalone x64 binary (recommended for **Windows**)
 
 #### Alternatives
 
 File|Description
 :---|:---
-[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS standalone executable
-[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows standalone x86 (32bit) binary
-[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`, needs VC++14
-[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged windows executable (No auto-update)
-[yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS executable (No auto-update)
+[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS (10.15+) standalone executable
+[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Vista SP2+) standalone x86 (32-bit) binary
+[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`.<br/> Does not contain `pycryptodomex`, needs VC++14
+[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
+[yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
 
 #### Misc
 
@@ -217,7 +242,7 @@ File|Description
 [SHA2-512SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-512SUMS)|GNU-style SHA512 sums
 [SHA2-256SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS)|GNU-style SHA256 sums
 
-### DEPENDENCIES
+## DEPENDENCIES
 Python versions 3.6+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
 
 <!-- Python 3.5+ uses VC++14 and it is already embedded in the binary created
@@ -227,25 +252,25 @@ On windows, [Microsoft Visual C++ 2010 SP1 Redistributable Package (x86)](https:
 
 While all the other dependancies are optional, `ffmpeg` and `ffprobe` are highly recommended
 * [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging seperate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. Licence [depends on the build](https://www.ffmpeg.org/legal.html)
-* [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licenced under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
-* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licenced under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
-* [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licenced under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
-* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licenced under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
-* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licenced under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
-* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licenced under [GPLv2+](http://rtmpdump.mplayerhq.hu)
-* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licenced under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
-* [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licenced under [BSD3](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
-* [**sponskrub**](https://github.com/faissaloo/SponSkrub) - For using the now **deprecated** [sponskrub options](#sponskrub-options). Licenced under [GPLv3+](https://github.com/faissaloo/SponSkrub/blob/master/LICENCE.md)
+* [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
+* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
+* [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licensed under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
+* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licensed under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
+* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
+* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
+* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
+* [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD3](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
+* [**sponskrub**](https://github.com/faissaloo/SponSkrub) - For using the now **deprecated** [sponskrub options](#sponskrub-options). Licensed under [GPLv3+](https://github.com/faissaloo/SponSkrub/blob/master/LICENCE.md)
 * Any external downloader that you want to use with `--downloader`
 
 To use or redistribute the dependencies, you must agree to their respective licensing terms.
 
-The windows releases are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
+The Windows and MacOS standalone release binaries are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
 
 **Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependancy, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds/wiki/Latest#latest-autobuilds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specifc issues solved by these builds
 
 
-### COMPILE
+## COMPILE
 
 **For Windows**:
 To build the Windows executable, you must have pyinstaller (and optionally mutagen, pycryptodomex, websockets). Once you have all the necessary dependencies installed, (optionally) build lazy extractors using `devscripts/make_lazy_extractors.py`, and then just run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the python used to build it.
@@ -262,6 +287,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
 
 **Note**: In either platform, `devscripts/update-version.py` can be used to automatically update the version number
 
+You can also fork the project on github and push it to a release branch in your fork for the [build workflow](https://github.com/yt-dlp/yt-dlp/blob/master/.github/workflows/build.yml) to automatically make a release for you
+
 # USAGE AND OPTIONS
 
     yt-dlp [OPTIONS] [--] URL [URL...]
@@ -276,7 +303,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      sure that you have sufficient permissions
                                      (run with sudo if needed)
     -i, --ignore-errors              Ignore download and postprocessing errors.
-                                     The download will be considered successfull
+                                     The download will be considered successful
                                      even if the postprocessing fails
     --no-abort-on-error              Continue with next video on download
                                      errors; e.g. to skip unavailable videos in
@@ -366,7 +393,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      SIZE (e.g. 50k or 44.6m)
     --max-filesize SIZE              Do not download any videos larger than SIZE
                                      (e.g. 50k or 44.6m)
-    --date DATE                      Download only videos uploaded in this date.
+    --date DATE                      Download only videos uploaded on this date.
                                      The date can be "YYYYMMDD" or in the format
                                      "(now|today)[+-][0-9](day|week|month|year)(s)?"
     --datebefore DATE                Download only videos uploaded on or before
@@ -510,9 +537,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      filenames
     --no-restrict-filenames          Allow Unicode characters, "&" and spaces in
                                      filenames (default)
-    --windows-filenames              Force filenames to be windows compatible
-    --no-windows-filenames           Make filenames windows compatible only if
-                                     using windows (default)
+    --windows-filenames              Force filenames to be Windows-compatible
+    --no-windows-filenames           Make filenames Windows-compatible only if
+                                     using Windows (default)
     --trim-filenames LENGTH          Limit the filename length (excluding
                                      extension) to the specified number of
                                      characters
@@ -608,9 +635,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      anything to disk
     --no-simulate                    Download the video even if printing/listing
                                      options are used
-    --ignore-no-formats-error        Ignore "No video formats" error. Usefull
-                                     for extracting metadata even if the videos
-                                     are not actually available for download
+    --ignore-no-formats-error        Ignore "No video formats" error. Useful for
+                                     extracting metadata even if the videos are
+                                     not actually available for download
                                      (experimental)
     --no-ignore-no-formats-error     Throw error when no downloadable video
                                      formats are found (default)
@@ -644,7 +671,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      "postprocess:",  or "postprocess-title:".
                                      The video's fields are accessible under the
                                      "info" key and the progress attributes are
-                                     accessible under "progress" key. Eg:
+                                     accessible under "progress" key. E.g.:
                                      --console-title --progress-template
                                      "download-title:%(info.id)s-%(progress.eta)s"
     -v, --verbose                    Print various debugging information
@@ -657,7 +684,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
-    --no-check-certificate           Suppress HTTPS certificate validation
+    --no-check-certificates          Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve
                                      information about the video (Currently
                                      supported only for YouTube)
@@ -706,10 +733,12 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      containers irrespective of quality
     --no-prefer-free-formats         Don't give any special preference to free
                                      containers (default)
-    --check-formats                  Check that the formats selected are
+    --check-formats                  Check that the selected formats are
                                      actually downloadable
-    --no-check-formats               Do not check that the formats selected are
+    --check-all-formats              Check all formats for whether they are
                                      actually downloadable
+    --no-check-formats               Do not check that the formats are actually
+                                     downloadable
     -F, --list-formats               List available formats of each video.
                                      Simulate unless --no-simulate is used
     --merge-output-format FORMAT     If a merge is required (e.g.
@@ -731,7 +760,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      "ass/srt/best"
     --sub-langs LANGS                Languages of the subtitles to download (can
                                      be regex) or "all" separated by commas.
-                                     (Eg: --sub-langs en.*,ja) You can prefix
+                                     (Eg: --sub-langs "en.*,ja") You can prefix
                                      the language code with a "-" to exempt it
                                      from the requested languages. (Eg: --sub-
                                      langs all,-live_chat) Use --list-subs for a
@@ -765,7 +794,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      formats are: best (default) or one of
                                      best|aac|flac|mp3|m4a|opus|vorbis|wav
     --audio-quality QUALITY          Specify ffmpeg audio quality, insert a
-                                     value between 0 (better) and 9 (worse) for
+                                     value between 0 (best) and 10 (worst) for
                                      VBR or a specific bitrate like 128K
                                      (default 5)
     --remux-video FORMAT             Remux the video into another container if
@@ -966,7 +995,7 @@ You can configure yt-dlp by placing any supported command line option to a confi
     * `~/yt-dlp.conf`
     * `~/yt-dlp.conf.txt`
 
-    `%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `~` points to %HOME% if present, `%USERPROFILE%` (generally `C:\Users\<user name>`) or `%HOMEDRIVE%%HOMEPATH%`.
+    `%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `%APPDATA%` generally points to (`C:\Users\<user name>\AppData\Roaming`) and `~` points to `%HOME%` if present, `%USERPROFILE%` (generally `C:\Users\<user name>`), or `%HOMEDRIVE%%HOMEPATH%`
 1. **System Configuration**: `/etc/yt-dlp.conf`
 
 For example, with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
@@ -988,7 +1017,7 @@ For example, with the following configuration file yt-dlp will always extract th
 
 Note that options in configuration file are just the same options aka switches used in regular command line calls; thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
 
-You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of user and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
+You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of home, user, and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
 
 ### Authentication with `.netrc` file
 
@@ -1018,7 +1047,7 @@ The `-o` option is used to indicate a template for the output file names while `
 
 The simplest usage of `-o` is not to set any template arguments when downloading a single file, like in `yt-dlp -o funny_video.flv "https://some/video"` (hard-coding file extension like this is _not_ recommended and could break some post-processing).
 
-It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
+It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
 
 The field names themselves (the part inside the parenthesis) can also have some special formatting:
 1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
@@ -1026,7 +1055,7 @@ The field names themselves (the part inside the parenthesis) can also have some
 1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
 1. **Alternatives**: Alternate fields can be specified seperated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
 1. **Default**: A literal default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
-1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son, a comma seperated **l**ist (alternate form flag `#` makes it new line `\n` seperated) and a string **q**uoted for the terminal, respectively
+1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma seperated **l**ist (flag `#` for `\n` newline-seperated) and a string **q**uoted for the terminal (flag `#` to split a list into different arguments), respectively
 1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
 
 To summarize, the general syntax for a field is:
@@ -1034,7 +1063,7 @@ To summarize, the general syntax for a field is:
 %(name[.keys][addition][>strf][,alternate][|default])[flags][width][.precision][length]type
 ```
 
-Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'`  will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
+Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'`  will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
 
 The available fields are:
 
@@ -1159,7 +1188,7 @@ Each aforementioned sequence when referenced in an output template will be repla
 
 Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default).
 
-**Tip**: Look at the `-j` output to identify which fields are available for the purticular URL
+**Tip**: Look at the `-j` output to identify which fields are available for the particular URL
 
 For numeric sequences you can use [numeric related formatting](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting), for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
 
@@ -1222,19 +1251,19 @@ You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`,
 
 You can also use special names to select particular edge case formats:
 
- - `all`: Select all formats
- - `mergeall`: Select and merge all formats (Must be used with `--audio-multistreams`, `--video-multistreams` or both)
- - `b*`, `best*`: Select the best quality format irrespective of whether it contains video or audio
- - `w*`, `worst*`: Select the worst quality format irrespective of whether it contains video or audio
- - `b`, `best`: Select the best quality format that contains both video and audio. Equivalent to `best*[vcodec!=none][acodec!=none]`
+ - `all`: Select **all formats** separately
+ - `mergeall`: Select and **merge all formats** (Must be used with `--audio-multistreams`, `--video-multistreams` or both)
+ - `b*`, `best*`: Select the best quality format that **contains either** a video or an audio
+ - `b`, `best`: Select the best quality format that **contains both** video and audio. Equivalent to `best*[vcodec!=none][acodec!=none]`
+ - `bv`, `bestvideo`: Select the best quality **video-only** format. Equivalent to `best*[acodec=none]`
+ - `bv*`, `bestvideo*`: Select the best quality format that **contains video**. It may also contain audio. Equivalent to `best*[vcodec!=none]`
+ - `ba`, `bestaudio`: Select the best quality **audio-only** format. Equivalent to `best*[vcodec=none]`
+ - `ba*`, `bestaudio*`: Select the best quality format that **contains audio**. It may also contain video. Equivalent to `best*[acodec!=none]`
+ - `w*`, `worst*`: Select the worst quality format that contains either a video or an audio
  - `w`, `worst`: Select the worst quality format that contains both video and audio. Equivalent to `worst*[vcodec!=none][acodec!=none]`
- - `bv`, `bestvideo`: Select the best quality video-only format. Equivalent to `best*[acodec=none]`
  - `wv`, `worstvideo`: Select the worst quality video-only format. Equivalent to `worst*[acodec=none]`
- - `bv*`, `bestvideo*`: Select the best quality format that contains video. It may also contain audio. Equivalent to `best*[vcodec!=none]`
  - `wv*`, `worstvideo*`: Select the worst quality format that contains video. It may also contain audio. Equivalent to `worst*[vcodec!=none]`
- - `ba`, `bestaudio`: Select the best quality audio-only format. Equivalent to `best*[vcodec=none]`
  - `wa`, `worstaudio`: Select the worst quality audio-only format. Equivalent to `worst*[vcodec=none]`
- - `ba*`, `bestaudio*`: Select the best quality format that contains audio. It may also contain video. Equivalent to `best*[acodec!=none]`
  - `wa*`, `worstaudio*`: Select the worst quality format that contains audio. It may also contain video. Equivalent to `worst*[acodec!=none]`
 
 For example, to download the worst quality video-only format you can use `-f worstvideo`. It is however recommended not to use `worst` and related options. When your format selector is `worst`, the format which is worst in all respects is selected. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-f best -S +size,+br,+res,+fps` instead of `-f worst`. See [sorting formats](#sorting-formats) for more details.
@@ -1298,12 +1327,12 @@ The available fields are:
  - `source`: Preference of the source as given by the extractor
  - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > other > `mms`/`rtsp` > unknown > `f4f`/`f4m`)
  - `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other > unknown)
- - `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `ac3` > `dts` > other > unknown)
+ - `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other > unknown)
  - `codec`: Equivalent to `vcodec,acodec`
  - `vext`: Video Extension (`mp4` > `webm` > `flv` > other > unknown). If `--prefer-free-formats` is used, `webm` is prefered.
  - `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other > unknown). If `--prefer-free-formats` is used, the order changes to `opus` > `ogg` > `webm` > `m4a` > `mp3` > `aac`.
  - `ext`: Equivalent to `vext,aext`
- - `filesize`: Exact filesize, if know in advance. This will be unavailable for mu38 and DASH formats.
+ - `filesize`: Exact filesize, if known in advance
  - `fs_approx`: Approximate filesize calculated from the manifests
  - `size`: Exact filesize if available, otherwise approximate filesize
  - `height`: Height of video
@@ -1455,7 +1484,7 @@ $ yt-dlp -S '+res:480,codec,br'
 
 # MODIFYING METADATA
 
-The metadata obtained the the extractors can be modified by using `--parse-metadata` and `--replace-in-metadata`
+The metadata obtained by the extractors can be modified by using `--parse-metadata` and `--replace-in-metadata`
 
 `--replace-in-metadata FIELDS REGEX REPLACE` is used to replace text in any metadata field using [python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax). [Backreferences](https://docs.python.org/3/library/re.html?highlight=backreferences#re.sub) can be used in the replace string for advanced use.
 
@@ -1506,6 +1535,9 @@ $ yt-dlp --parse-metadata '%(series)s S%(season_number)02dE%(episode_number)02d:
 # Set "comment" field in video metadata using description instead of webpage_url
 $ yt-dlp --parse-metadata 'description:(?s)(?P<meta_comment>.+)' --add-metadata
 
+# Remove "formats" field from the infojson by setting it to an empty string
+$ yt-dlp --parse-metadata ':(?P<formats>)' -j
+
 # Replace all spaces and "_" in title and uploader with a `-`
 $ yt-dlp --replace-in-metadata 'title,uploader' '[ _]' '-'
 
@@ -1513,27 +1545,32 @@ $ yt-dlp --replace-in-metadata 'title,uploader' '[ _]' '-'
 
 # EXTRACTOR ARGUMENTS
 
-Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) seperated string of `ARG=VAL1,VAL2`. Eg: `--extractor-args "youtube:player_client=android_agegate,web;include_live_dash" --extractor-args "funimation:version=uncut"`
+Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. Eg: `--extractor-args "youtube:player-client=android_agegate,web;include_live_dash" --extractor-args "funimation:version=uncut"`
 
 The following extractors use this feature:
-* **youtube**
-    * `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
-    * `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients
-    * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
-    * `include_live_dash`: Include live dash formats (These formats don't download properly)
-    * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side).
-    * `max_comments`: Maximum amount of comments to download (default all).
-    * `max_comment_depth`: Maximum depth for nested comments. YouTube supports depths 1 or 2 (default).
-* **youtubetab**
-  (YouTube playlists, channels, feeds, etc.)
-   * `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
 
-* **funimation**
-    * `language`: Languages to extract. Eg: `funimation:language=english,japanese`
-    * `version`: The video version to extract - `uncut` or `simulcast`
+#### youtube
+* `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
+* `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients
+* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
+* `include_live_dash`: Include live dash formats (These formats don't download properly)
+* `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
+* `max_comments`: Maximum amount of comments to download (default all)
+* `max_comment_depth`: Maximum depth for nested comments. YouTube supports depths 1 or 2 (default)
 
-* **vikiChannel**
-    * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
+#### youtubetab (YouTube playlists, channels, feeds, etc.)
+* `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
+
+#### funimation
+* `language`: Languages to extract. Eg: `funimation:language=english,japanese`
+* `version`: The video version to extract - `uncut` or `simulcast`
+
+#### crunchyroll
+* `language`: Languages to extract. Eg: `crunchyroll:language=jaJp`
+* `hardsub`: Which hard-sub versions to extract. Eg: `crunchyroll:hardsub=None,enUS`
+
+#### vikichannel
+* `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
 
 NOTE: These options may be changed/removed in the future without concern for backward compatibility
 
@@ -1561,10 +1598,10 @@ Your program should avoid parsing the normal stdout since they may change in fut
 From a Python program, you can embed yt-dlp in a more powerful fashion, like this:
 
 ```python
-import yt_dlp
+from yt_dlp import YoutubeDL
 
 ydl_opts = {}
-with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+with YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
@@ -1574,9 +1611,7 @@ Here's a more complete example of a program that outputs only errors (and a shor
 
 ```python
 import json
-
 import yt_dlp
-from yt_dlp.postprocessor.common import PostProcessor
 
 
 class MyLogger:
@@ -1598,7 +1633,7 @@ class MyLogger:
         print(msg)
 
 
-class MyCustomPP(PostProcessor):
+class MyCustomPP(yt_dlp.postprocessor.PostProcessor):
     def run(self, info):
         self.to_screen('Doing stuff')
         return [], info
@@ -1620,6 +1655,10 @@ ydl_opts = {
     'progress_hooks': [my_hook],
 }
 
+
+# Add custom headers
+yt_dlp.utils.std_headers.update({'Referer': 'https://www.google.com'})
+
 with yt_dlp.YoutubeDL(ydl_opts) as ydl:
     ydl.add_post_processor(MyCustomPP())
     info = ydl.extract_info('https://www.youtube.com/watch?v=BaW_jenozKc')

devscripts/make_supportedsites.py

@@ -29,6 +29,9 @@ def main():
                 continue
             if ie_desc is not None:
                 ie_md += ': {0}'.format(ie.IE_DESC)
+            search_key = getattr(ie, 'SEARCH_KEY', None)
+            if search_key is not None:
+                ie_md += f'; "{ie.SEARCH_KEY}:" prefix'
             if not ie.working():
                 ie_md += ' (Currently broken)'
             yield ie_md

setup.py

@@ -16,7 +16,7 @@ from distutils.spawn import spawn
 exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec'))
 
 
-DESCRIPTION = 'Command-line program to download videos from YouTube.com and many other other video platforms.'
+DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',

supportedsites.md

@@ -48,6 +48,7 @@
  - **Alura**
  - **AluraCourse**
  - **Amara**
+ - **AmazonStore**
  - **AMCNetworks**
  - **AmericasTestKitchen**
  - **AmericasTestKitchenSeason**
@@ -127,7 +128,7 @@
  - **BilibiliAudioAlbum**
  - **BilibiliChannel**
  - **BiliBiliPlayer**
- - **BiliBiliSearch**: Bilibili video search, "bilisearch" keyword
+ - **BiliBiliSearch**: Bilibili video search; "bilisearch:" prefix
  - **BiliIntl**
  - **BiliIntlSeries**
  - **BioBioChileTV**
@@ -184,7 +185,6 @@
  - **CCTV**: 央视网
  - **CDA**
  - **CeskaTelevize**
- - **CeskaTelevizePorady**
  - **CGTN**
  - **channel9**: Channel 9
  - **CharlieRose**
@@ -366,6 +366,7 @@
  - **Funk**
  - **Fusion**
  - **Fux**
+ - **Gab**
  - **GabTV**
  - **Gaia**
  - **GameInformer**
@@ -449,9 +450,11 @@
  - **Instagram**
  - **instagram:tag**: Instagram hashtag search
  - **instagram:user**: Instagram user profile
+ - **InstagramIOS**: IOS instagram:// URL
  - **Internazionale**
  - **InternetVideoArchive**
  - **IPrima**
+ - **IPrimaCNN**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ITTF**
@@ -560,6 +563,7 @@
  - **MediaKlikk**
  - **Medialaan**
  - **Mediaset**
+ - **MediasetShow**
  - **Mediasite**
  - **MediasiteCatalog**
  - **MediasiteNamedCatalog**
@@ -592,6 +596,7 @@
  - **mixcloud:user**
  - **MLB**
  - **MLBVideo**
+ - **MLSSoccer**
  - **Mnet**
  - **MNetTV**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -691,8 +696,8 @@
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **NiconicoUser**
- - **nicovideo:search**: Nico video searches
- - **nicovideo:search:date**: Nico video searches, newest first
+ - **nicovideo:search**: Nico video searches; "nicosearch:" prefix
+ - **nicovideo:search:date**: Nico video searches, newest first; "nicosearchdate:" prefix
  - **nicovideo:search_url**: Nico video search URLs
  - **Nintendo**
  - **Nitter**
@@ -801,6 +806,7 @@
  - **Pinterest**
  - **PinterestCollection**
  - **Pladform**
+ - **PlanetMarathi**
  - **Platzi**
  - **PlatziCourse**
  - **play.fm**
@@ -817,7 +823,12 @@
  - **podomatic**
  - **Pokemon**
  - **PokemonWatch**
+ - **PolsatGo**
  - **PolskieRadio**
+ - **polskieradio:kierowcow**
+ - **polskieradio:player**
+ - **polskieradio:podcast**
+ - **polskieradio:podcast:list**
  - **PolskieRadioCategory**
  - **Popcorntimes**
  - **PopcornTV**
@@ -860,6 +871,8 @@
  - **radiocanada:audiovideo**
  - **radiofrance**
  - **RadioJavan**
+ - **radiokapital**
+ - **radiokapital:show**
  - **radlive**
  - **radlive:channel**
  - **radlive:season**
@@ -867,6 +880,8 @@
  - **RaiPlay**
  - **RaiPlayLive**
  - **RaiPlayPlaylist**
+ - **RaiPlayRadio**
+ - **RaiPlayRadioPlaylist**
  - **RayWenderlich**
  - **RayWenderlichCourse**
  - **RBMARadio**
@@ -894,6 +909,7 @@
  - **RMCDecouverte**
  - **RockstarGames**
  - **RoosterTeeth**
+ - **RoosterTeethSeries**
  - **RottenTomatoes**
  - **Roxwel**
  - **Rozhlas**
@@ -936,7 +952,7 @@
  - **SBS**: sbs.com.au
  - **schooltv**
  - **ScienceChannel**
- - **screen.yahoo:search**: Yahoo screen search
+ - **screen.yahoo:search**: Yahoo screen search; "yvsearch:" prefix
  - **Screencast**
  - **ScreencastOMatic**
  - **ScrippsNetworks**
@@ -961,6 +977,7 @@
  - **Sina**
  - **sky.it**
  - **sky:news**
+ - **sky:news:story**
  - **sky:sports**
  - **sky:sports:news**
  - **skyacademy.it**
@@ -977,7 +994,7 @@
  - **SonyLIVSeries**
  - **soundcloud**
  - **soundcloud:playlist**
- - **soundcloud:search**: Soundcloud search, "scsearch" keyword
+ - **soundcloud:search**: Soundcloud search; "scsearch:" prefix
  - **soundcloud:set**
  - **soundcloud:trackstation**
  - **soundcloud:user**
@@ -1079,6 +1096,8 @@
  - **ThisAmericanLife**
  - **ThisAV**
  - **ThisOldHouse**
+ - **ThreeSpeak**
+ - **ThreeSpeakUser**
  - **TikTok**
  - **tiktok:user**
  - **tinypic**: tinypic.com videos
@@ -1095,8 +1114,8 @@
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
  - **Trovo**
- - **TrovoChannelClip**: All Clips of a trovo.live channel, "trovoclip" keyword
- - **TrovoChannelVod**: All VODs of a trovo.live channel, "trovovod" keyword
+ - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
+ - **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
  - **TrovoVod**
  - **TruNews**
  - **TruTV**
@@ -1142,6 +1161,7 @@
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
+ - **tvp:stream**
  - **TVPlayer**
  - **TVPlayHome**
  - **Tweakers**
@@ -1201,7 +1221,7 @@
  - **Viddler**
  - **Videa**
  - **video.arnes.si**: Arnes Video
- - **video.google:search**: Google Video search (Currently broken)
+ - **video.google:search**: Google Video search; "gvsearch:" prefix (Currently broken)
  - **video.sky.it**
  - **video.sky.it:live**
  - **VideoDetective**
@@ -1296,6 +1316,8 @@
  - **WistiaPlaylist**
  - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **WorldStarHipHop**
+ - **wppilot**
+ - **wppilot:channels**
  - **WSJ**: Wall Street Journal
  - **WSJArticle**
  - **WWE**
@@ -1343,19 +1365,19 @@
  - **YouPorn**
  - **YourPorn**
  - **YourUpload**
- - **youtube**: YouTube.com
- - **youtube:favorites**: YouTube.com liked videos, ":ytfav" for short (requires authentication)
- - **youtube:history**: Youtube watch history, ":ythis" for short (requires authentication)
- - **youtube:playlist**: YouTube.com playlists
- - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
- - **youtube:search**: YouTube.com searches, "ytsearch" keyword
- - **youtube:search:date**: YouTube.com searches, newest videos first, "ytsearchdate" keyword
- - **youtube:search_url**: YouTube.com search URLs
- - **youtube:subscriptions**: YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)
- - **youtube:tab**: YouTube.com tab
- - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **youtube**: YouTube
+ - **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)
+ - **youtube:history**: Youtube watch history; ":ythis" keyword (requires cookies)
+ - **youtube:playlist**: YouTube playlists
+ - **youtube:recommended**: YouTube recommended videos; ":ytrec" keyword
+ - **youtube:search**: YouTube searches; "ytsearch:" prefix
+ - **youtube:search:date**: YouTube searches, newest videos first; "ytsearchdate:" prefix
+ - **youtube:search_url**: YouTube search URLs with sorting and filter support
+ - **youtube:subscriptions**: YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)
+ - **youtube:tab**: YouTube Tabs
+ - **youtube:watchlater**: Youtube watch later list; ":ytwatchlater" keyword (requires cookies)
  - **YoutubeYtBe**: youtu.be
- - **YoutubeYtUser**: YouTube.com user videos, URL or "ytuser" keyword
+ - **YoutubeYtUser**: YouTube user videos; "ytuser:" prefix
  - **Zapiks**
  - **Zattoo**
  - **ZattooLive**

test/parameters.json

@@ -9,7 +9,7 @@
     "forcetitle": false,
     "forceurl": false,
     "force_write_download_archive": false,
-    "format": "best",
+    "format": "b/bv",
     "ignoreerrors": false,
     "listformats": null,
     "logtostderr": false,

test/test_YoutubeDL.py

@@ -656,7 +656,7 @@ class TestYoutubeDL(unittest.TestCase):
         'playlist_autonumber': 2,
         '_last_playlist_index': 100,
         'n_entries': 10,
-        'formats': [{'id': 'id1'}, {'id': 'id2'}, {'id': 'id3'}]
+        'formats': [{'id': 'id 1'}, {'id': 'id 2'}, {'id': 'id 3'}]
     }
 
     def test_prepare_outtmpl_and_filename(self):
@@ -737,6 +737,7 @@ class TestYoutubeDL(unittest.TestCase):
         test(NA_TEST_OUTTMPL, 'NA-NA-def-1234.mp4')
         test(NA_TEST_OUTTMPL, 'none-none-def-1234.mp4', outtmpl_na_placeholder='none')
         test(NA_TEST_OUTTMPL, '--def-1234.mp4', outtmpl_na_placeholder='')
+        test('%(non_existent.0)s', 'NA')
 
         # String formatting
         FMT_TEST_OUTTMPL = '%%(height)%s.%%(ext)s'
@@ -762,14 +763,15 @@ class TestYoutubeDL(unittest.TestCase):
         test('a%(width|)d', 'a', outtmpl_na_placeholder='none')
 
         FORMATS = self.outtmpl_info['formats']
-        sanitize = lambda x: x.replace(':', ' -').replace('"', "'")
+        sanitize = lambda x: x.replace(':', ' -').replace('"', "'").replace('\n', ' ')
 
         # Custom type casting
-        test('%(formats.:.id)l', 'id1, id2, id3')
-        test('%(formats.:.id)#l', ('id1\nid2\nid3', 'id1 id2 id3'))
+        test('%(formats.:.id)l', 'id 1, id 2, id 3')
+        test('%(formats.:.id)#l', ('id 1\nid 2\nid 3', 'id 1 id 2 id 3'))
         test('%(ext)l', 'mp4')
-        test('%(formats.:.id) 15l', '  id1, id2, id3')
+        test('%(formats.:.id) 18l', '  id 1, id 2, id 3')
         test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
+        test('%(formats)#j', (json.dumps(FORMATS, indent=4), sanitize(json.dumps(FORMATS, indent=4))))
         test('%(title5).3B', 'á')
         test('%(title5)U', 'áéí 𝐀')
         test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
@@ -777,8 +779,12 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
         if compat_os_name == 'nt':
             test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
+            test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))
+            test('%(formats.0.id)#q', ('"id 1"', "'id 1'"))
         else:
             test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'"))
+            test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'")
+            test('%(formats.0.id)#q', "'id 1'")
 
         # Internal formatting
         test('%(timestamp-1000>%H-%M-%S)s', '11-43-20')

test/test_jsinterp.py

@@ -112,6 +112,71 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_signature.py

@@ -14,9 +14,10 @@ import string
 
 from test.helper import FakeYDL, is_download_test
 from yt_dlp.extractor import YoutubeIE
+from yt_dlp.jsinterp import JSInterpreter
 from yt_dlp.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,17 @@ _TESTS = [
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+]
+
 
 @is_download_test
 class TestPlayerInfo(unittest.TestCase):
@@ -97,35 +109,49 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
-    assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+def t_factory(name, sig_func, url_pattern):
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
 
-    def test_func(self):
-        basename = 'player-%s.js' % test_id
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+        def test_func(self):
+            basename = f'player-{name}-{test_id}.js'
+            fn = os.path.join(self.TESTDATA_DIR, basename)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
 
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
-        with io.open(fn, encoding='utf-8') as testf:
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
-
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
+        test_func.__name__ = f'test_{name}_js_{test_id}'
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
 
 
-for test_spec in _TESTS:
-    make_tfunc(*test_spec)
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
+
+
+def n_sig(jscode, sig_input):
+    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
+
+
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
+
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':

yt_dlp/YoutubeDL.py

@@ -28,6 +28,7 @@ import traceback
 import random
 import unicodedata
 
+from enum import Enum
 from string import ascii_letters
 
 from .compat import (
@@ -55,9 +56,7 @@ from .utils import (
     DEFAULT_OUTTMPL,
     determine_ext,
     determine_protocol,
-    DOT_DESKTOP_LINK_TEMPLATE,
-    DOT_URL_LINK_TEMPLATE,
-    DOT_WEBLOC_LINK_TEMPLATE,
+    DownloadCancelled,
     DownloadError,
     encode_compat_str,
     encodeFilename,
@@ -75,12 +74,15 @@ from .utils import (
     int_or_none,
     iri_to_uri,
     ISO3166Utils,
+    join_nonempty,
     LazyList,
+    LINK_TEMPLATES,
     locked_file,
     make_dir,
     make_HTTPS_handler,
     MaxDownloadsReached,
     network_exceptions,
+    number_of_digits,
     orderedSet,
     OUTTMPL_TYPES,
     PagedList,
@@ -107,7 +109,6 @@ from .utils import (
     strftime_or_none,
     subtitles_filename,
     supports_terminal_sequences,
-    TERMINAL_SEQUENCES,
     ThrottledDownload,
     to_high_limit_path,
     traverse_obj,
@@ -123,6 +124,7 @@ from .utils import (
     YoutubeDLRedirectHandler,
 )
 from .cache import Cache
+from .minicurses import format_text
 from .extractor import (
     gen_extractor_classes,
     get_info_extractor,
@@ -213,8 +215,8 @@ class YoutubeDL(object):
     ignore_no_formats_error: Ignore "No video formats" error. Usefull for
                        extracting metadata even if the video is not actually
                        available for download (experimental)
-    format_sort:       How to sort the video formats. see "Sorting Formats"
-                       for more details.
+    format_sort:       A list of fields by which to sort the video formats.
+                       See "Sorting Formats" for more details.
     format_sort_force: Force the given format_sort. see "Sorting Formats"
                        for more details.
     allow_multiple_video_streams:   Allow multiple video streams to be merged
@@ -222,7 +224,8 @@ class YoutubeDL(object):
     allow_multiple_audio_streams:   Allow multiple audio streams to be merged
                        into a single file
     check_formats      Whether to test if the formats are downloadable.
-                       Can be True (check all), False (check none)
+                       Can be True (check all), False (check none),
+                       'selected' (check selected formats),
                        or None (check only if requested by extractor)
     paths:             Dictionary of output paths. The allowed keys are 'home'
                        'temp' and the keys of OUTTMPL_TYPES (in utils.py)
@@ -371,8 +374,7 @@ class YoutubeDL(object):
                        (with status "started" and "finished") if the processing is successful.
     merge_output_format: Extension to use when merging formats.
     final_ext:         Expected final extension; used to detect when the file was
-                       already downloaded and converted. "merge_output_format" is
-                       replaced by this extension when given
+                       already downloaded and converted
     fixup:             Automatically correct known faults of the file.
                        One of:
                        - "never": do nothing
@@ -438,7 +440,7 @@ class YoutubeDL(object):
     nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
     max_filesize, test, noresizebuffer, retries, fragment_retries, continuedl,
     noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
-    external_downloader_args.
+    external_downloader_args, concurrent_fragment_downloads.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If False, use avconv instead of ffmpeg if both are available,
@@ -524,7 +526,10 @@ class YoutubeDL(object):
 
         windows_enable_vt_mode()
         # FIXME: This will break if we ever print color to stdout
-        self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file)
+        self._allow_colors = {
+            'screen': not self.params.get('no_color') and supports_terminal_sequences(self._screen_file),
+            'err': not self.params.get('no_color') and supports_terminal_sequences(self._err_file),
+        }
 
         if sys.version_info < (3, 6):
             self.report_warning(
@@ -532,10 +537,10 @@ class YoutubeDL(object):
 
         if self.params.get('allow_unplayable_formats'):
             self.report_warning(
-                f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. '
+                f'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
                 'This is a developer option intended for debugging. \n'
                 '         If you experience any issues while using this option, '
-                f'{self._color_text("DO NOT", "red")} open a bug report')
+                f'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
 
         def check_deprecated(param, option, suggestion):
             if self.params.get(param) is not None:
@@ -554,6 +559,9 @@ class YoutubeDL(object):
         for msg in self.params.get('_warnings', []):
             self.report_warning(msg)
 
+        if 'list-formats' in self.params.get('compat_opts', []):
+            self.params['listformats_table'] = False
+
         if 'overwrites' not in self.params and self.params.get('nooverwrites') is not None:
             # nooverwrites was unnecessarily changed to overwrites
             # in 0c3d0f51778b153f65c21906031c2e091fcfb641
@@ -826,10 +834,32 @@ class YoutubeDL(object):
         self.to_stdout(
             message, skip_eol, quiet=self.params.get('quiet', False))
 
-    def _color_text(self, text, color):
-        if self.params.get('no_color'):
-            return text
-        return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}'
+    class Styles(Enum):
+        HEADERS = 'yellow'
+        EMPHASIS = 'blue'
+        ID = 'green'
+        DELIM = 'blue'
+        ERROR = 'red'
+        WARNING = 'yellow'
+
+    def __format_text(self, out, text, f, fallback=None, *, test_encoding=False):
+        assert out in ('screen', 'err')
+        if test_encoding:
+            original_text = text
+            handle = self._screen_file if out == 'screen' else self._err_file
+            encoding = self.params.get('encoding') or getattr(handle, 'encoding', 'ascii')
+            text = text.encode(encoding, 'ignore').decode(encoding)
+            if fallback is not None and text != original_text:
+                text = fallback
+        if isinstance(f, self.Styles):
+            f = f._value_
+        return format_text(text, f) if self._allow_colors[out] else text if fallback is None else fallback
+
+    def _format_screen(self, *args, **kwargs):
+        return self.__format_text('screen', *args, **kwargs)
+
+    def _format_err(self, *args, **kwargs):
+        return self.__format_text('err', *args, **kwargs)
 
     def report_warning(self, message, only_once=False):
         '''
@@ -841,14 +871,14 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once)
+            self.to_stderr(f'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once)
 
     def report_error(self, message, tb=None):
         '''
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb)
+        self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', tb)
 
     def write_debug(self, message, only_once=False):
         '''Log debug message or Print message to stderr'''
@@ -977,8 +1007,8 @@ class YoutubeDL(object):
         # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
         # of %(field)s to %(field)0Nd for backward compatibility
         field_size_compat_map = {
-            'playlist_index': len(str(info_dict.get('_last_playlist_index') or '')),
-            'playlist_autonumber': len(str(info_dict.get('n_entries') or '')),
+            'playlist_index': number_of_digits(info_dict.get('_last_playlist_index') or 0),
+            'playlist_autonumber': number_of_digits(info_dict.get('n_entries') or 0),
             'autonumber': self.params.get('autonumber_size') or 5,
         }
 
@@ -1073,22 +1103,23 @@ class YoutubeDL(object):
 
             value = default if value is None else value
 
+            flags = outer_mobj.group('conversion') or ''
             str_fmt = f'{fmt[:-1]}s'
             if fmt[-1] == 'l':  # list
-                delim = '\n' if '#' in (outer_mobj.group('conversion') or '') else ', '
+                delim = '\n' if '#' in flags else ', '
                 value, fmt = delim.join(variadic(value)), str_fmt
             elif fmt[-1] == 'j':  # json
-                value, fmt = json.dumps(value, default=_dumpjson_default), str_fmt
+                value, fmt = json.dumps(value, default=_dumpjson_default, indent=4 if '#' in flags else None), str_fmt
             elif fmt[-1] == 'q':  # quoted
-                value, fmt = compat_shlex_quote(str(value)), str_fmt
+                value = map(str, variadic(value) if '#' in flags else [value])
+                value, fmt = ' '.join(map(compat_shlex_quote, value)), str_fmt
             elif fmt[-1] == 'B':  # bytes
                 value = f'%{str_fmt}'.encode('utf-8') % str(value).encode('utf-8')
                 value, fmt = value.decode('utf-8', 'ignore'), 's'
             elif fmt[-1] == 'U':  # unicode normalized
-                opts = outer_mobj.group('conversion') or ''
                 value, fmt = unicodedata.normalize(
                     # "+" = compatibility equivalence, "#" = NFD
-                    'NF%s%s' % ('K' if '+' in opts else '', 'D' if '#' in opts else 'C'),
+                    'NF%s%s' % ('K' if '+' in flags else '', 'D' if '#' in flags else 'C'),
                     value), str_fmt
             elif fmt[-1] == 'c':
                 if value:
@@ -1139,7 +1170,7 @@ class YoutubeDL(object):
                 sub_ext = ''
                 if len(fn_groups) > 2:
                     sub_ext = fn_groups[-2]
-                filename = '.'.join(filter(None, [fn_groups[0][:trim_file_name], sub_ext, ext]))
+                filename = join_nonempty(fn_groups[0][:trim_file_name], sub_ext, ext, delim='.')
 
             return filename
         except ValueError as err:
@@ -1287,11 +1318,11 @@ class YoutubeDL(object):
                 self.report_error(msg)
             except ExtractorError as e:  # An error we somewhat expected
                 self.report_error(compat_str(e), e.format_traceback())
-            except ThrottledDownload:
+            except ThrottledDownload as e:
                 self.to_stderr('\r')
-                self.report_warning('The download speed is below throttle limit. Re-extracting data')
+                self.report_warning(f'{e}; Re-extracting data')
                 return wrapper(self, *args, **kwargs)
-            except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached, LazyList.IndexError):
+            except (DownloadCancelled, LazyList.IndexError):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors'):
@@ -1468,7 +1499,7 @@ class YoutubeDL(object):
         self.to_screen('[download] Downloading playlist: %s' % playlist)
 
         if 'entries' not in ie_result:
-            raise EntryNotInPlaylist()
+            raise EntryNotInPlaylist('There are no entries')
         incomplete_entries = bool(ie_result.get('requested_entries'))
         if incomplete_entries:
             def fill_missing_entries(entries, indexes):
@@ -1508,7 +1539,7 @@ class YoutubeDL(object):
             def get_entry(i):
                 return ie_entries[i - 1]
         else:
-            if not isinstance(ie_entries, PagedList):
+            if not isinstance(ie_entries, (PagedList, LazyList)):
                 ie_entries = LazyList(ie_entries)
 
             def get_entry(i):
@@ -1530,7 +1561,7 @@ class YoutubeDL(object):
                     raise EntryNotInPlaylist()
             except (IndexError, EntryNotInPlaylist):
                 if incomplete_entries:
-                    raise EntryNotInPlaylist()
+                    raise EntryNotInPlaylist(f'Entry {i} cannot be found')
                 elif not playlistitems:
                     break
             entries.append(entry)
@@ -1690,6 +1721,28 @@ class YoutubeDL(object):
             return op(actual_value, comparison_value)
         return _filter
 
+    def _check_formats(self, formats):
+        for f in formats:
+            self.to_screen('[info] Testing format %s' % f['format_id'])
+            temp_file = tempfile.NamedTemporaryFile(
+                suffix='.tmp', delete=False,
+                dir=self.get_output_path('temp') or None)
+            temp_file.close()
+            try:
+                success, _ = self.dl(temp_file.name, f, test=True)
+            except (DownloadError, IOError, OSError, ValueError) + network_exceptions:
+                success = False
+            finally:
+                if os.path.exists(temp_file.name):
+                    try:
+                        os.remove(temp_file.name)
+                    except OSError:
+                        self.report_warning('Unable to delete temporary file "%s"' % temp_file.name)
+            if success:
+                yield f
+            else:
+                self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
+
     def _default_format_spec(self, info_dict, download=True):
 
         def can_merge():
@@ -1729,7 +1782,7 @@ class YoutubeDL(object):
         allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False),
                                   'video': self.params.get('allow_multiple_video_streams', False)}
 
-        check_formats = self.params.get('check_formats')
+        check_formats = self.params.get('check_formats') == 'selected'
 
         def _parse_filter(tokens):
             filter_parts = []
@@ -1905,26 +1958,7 @@ class YoutubeDL(object):
             if not check_formats:
                 yield from formats
                 return
-            for f in formats:
-                self.to_screen('[info] Testing format %s' % f['format_id'])
-                temp_file = tempfile.NamedTemporaryFile(
-                    suffix='.tmp', delete=False,
-                    dir=self.get_output_path('temp') or None)
-                temp_file.close()
-                try:
-                    success, _ = self.dl(temp_file.name, f, test=True)
-                except (DownloadError, IOError, OSError, ValueError) + network_exceptions:
-                    success = False
-                finally:
-                    if os.path.exists(temp_file.name):
-                        try:
-                            os.remove(temp_file.name)
-                        except OSError:
-                            self.report_warning('Unable to delete temporary file "%s"' % temp_file.name)
-                if success:
-                    yield f
-                else:
-                    self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
+            yield from self._check_formats(formats)
 
         def _build_selector_function(selector):
             if isinstance(selector, list):  # ,
@@ -2081,42 +2115,45 @@ class YoutubeDL(object):
         self.cookiejar.add_cookie_header(pr)
         return pr.get_header('Cookie')
 
+    def _sort_thumbnails(self, thumbnails):
+        thumbnails.sort(key=lambda t: (
+            t.get('preference') if t.get('preference') is not None else -1,
+            t.get('width') if t.get('width') is not None else -1,
+            t.get('height') if t.get('height') is not None else -1,
+            t.get('id') if t.get('id') is not None else '',
+            t.get('url')))
+
     def _sanitize_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')
         if thumbnails is None:
             thumbnail = info_dict.get('thumbnail')
             if thumbnail:
                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
-        if thumbnails:
-            thumbnails.sort(key=lambda t: (
-                t.get('preference') if t.get('preference') is not None else -1,
-                t.get('width') if t.get('width') is not None else -1,
-                t.get('height') if t.get('height') is not None else -1,
-                t.get('id') if t.get('id') is not None else '',
-                t.get('url')))
+        if not thumbnails:
+            return
 
-            def thumbnail_tester():
-                def test_thumbnail(t):
-                    self.to_screen(f'[info] Testing thumbnail {t["id"]}')
-                    try:
-                        self.urlopen(HEADRequest(t['url']))
-                    except network_exceptions as err:
-                        self.to_screen(f'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
-                        return False
-                    return True
-                return test_thumbnail
+        def check_thumbnails(thumbnails):
+            for t in thumbnails:
+                self.to_screen(f'[info] Testing thumbnail {t["id"]}')
+                try:
+                    self.urlopen(HEADRequest(t['url']))
+                except network_exceptions as err:
+                    self.to_screen(f'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
+                    continue
+                yield t
 
-            for i, t in enumerate(thumbnails):
-                if t.get('id') is None:
-                    t['id'] = '%d' % i
-                if t.get('width') and t.get('height'):
-                    t['resolution'] = '%dx%d' % (t['width'], t['height'])
-                t['url'] = sanitize_url(t['url'])
+        self._sort_thumbnails(thumbnails)
+        for i, t in enumerate(thumbnails):
+            if t.get('id') is None:
+                t['id'] = '%d' % i
+            if t.get('width') and t.get('height'):
+                t['resolution'] = '%dx%d' % (t['width'], t['height'])
+            t['url'] = sanitize_url(t['url'])
 
-            if self.params.get('check_formats'):
-                info_dict['thumbnails'] = LazyList(filter(thumbnail_tester(), thumbnails[::-1])).reverse()
-            else:
-                info_dict['thumbnails'] = thumbnails
+        if self.params.get('check_formats') is True:
+            info_dict['thumbnails'] = LazyList(check_thumbnails(thumbnails[::-1])).reverse()
+        else:
+            info_dict['thumbnails'] = thumbnails
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
@@ -2222,7 +2259,6 @@ class YoutubeDL(object):
         info_dict['requested_subtitles'] = self.process_subtitles(
             info_dict['id'], subtitles, automatic_captions)
 
-        # We now pick which formats have to be downloaded
         if info_dict.get('formats') is None:
             # There's only one format available
             formats = [info_dict]
@@ -2294,6 +2330,10 @@ class YoutubeDL(object):
                 format['resolution'] = self.format_resolution(format, default=None)
             if format.get('dynamic_range') is None and format.get('vcodec') != 'none':
                 format['dynamic_range'] = 'SDR'
+            if (info_dict.get('duration') and format.get('tbr')
+                    and not format.get('filesize') and not format.get('filesize_approx')):
+                format['filesize_approx'] = info_dict['duration'] * format['tbr'] * (1024 / 8)
+
             # Add HTTP headers, so that external programs can use them from the
             # json output
             full_format_info = info_dict.copy()
@@ -2305,6 +2345,9 @@ class YoutubeDL(object):
 
         # TODO Central sorting goes here
 
+        if self.params.get('check_formats') is True:
+            formats = LazyList(self._check_formats(formats[::-1])).reverse()
+
         if not formats or formats[0] is not info_dict:
             # only set the 'formats' fields if the original info_dict list them
             # otherwise we end up with a circular reference, the first (and unique)
@@ -2622,53 +2665,41 @@ class YoutubeDL(object):
                     return
 
         # Write internet shortcut files
-        url_link = webloc_link = desktop_link = False
-        if self.params.get('writelink', False):
-            if sys.platform == "darwin":  # macOS.
-                webloc_link = True
-            elif sys.platform.startswith("linux"):
-                desktop_link = True
-            else:  # if sys.platform in ['win32', 'cygwin']:
-                url_link = True
-        if self.params.get('writeurllink', False):
-            url_link = True
-        if self.params.get('writewebloclink', False):
-            webloc_link = True
-        if self.params.get('writedesktoplink', False):
-            desktop_link = True
-
-        if url_link or webloc_link or desktop_link:
+        def _write_link_file(link_type):
             if 'webpage_url' not in info_dict:
                 self.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
-                return
-            ascii_url = iri_to_uri(info_dict['webpage_url'])
-
-        def _write_link_file(extension, template, newline, embed_filename):
-            linkfn = replace_extension(full_filename, extension, info_dict.get('ext'))
+                return False
+            linkfn = replace_extension(self.prepare_filename(info_dict, 'link'), link_type, info_dict.get('ext'))
             if self.params.get('overwrites', True) and os.path.exists(encodeFilename(linkfn)):
-                self.to_screen('[info] Internet shortcut is already present')
-            else:
-                try:
-                    self.to_screen('[info] Writing internet shortcut to: ' + linkfn)
-                    with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', newline=newline) as linkfile:
-                        template_vars = {'url': ascii_url}
-                        if embed_filename:
-                            template_vars['filename'] = linkfn[:-(len(extension) + 1)]
-                        linkfile.write(template % template_vars)
-                except (OSError, IOError):
-                    self.report_error('Cannot write internet shortcut ' + linkfn)
-                    return False
+                self.to_screen(f'[info] Internet shortcut (.{link_type}) is already present')
+                return True
+            try:
+                self.to_screen(f'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
+                with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8',
+                             newline='\r\n' if link_type == 'url' else '\n') as linkfile:
+                    template_vars = {'url': iri_to_uri(info_dict['webpage_url'])}
+                    if link_type == 'desktop':
+                        template_vars['filename'] = linkfn[:-(len(link_type) + 1)]
+                    linkfile.write(LINK_TEMPLATES[link_type] % template_vars)
+            except (OSError, IOError):
+                self.report_error(f'Cannot write internet shortcut {linkfn}')
+                return False
             return True
 
-        if url_link:
-            if not _write_link_file('url', DOT_URL_LINK_TEMPLATE, '\r\n', embed_filename=False):
-                return
-        if webloc_link:
-            if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False):
-                return
-        if desktop_link:
-            if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True):
-                return
+        write_links = {
+            'url': self.params.get('writeurllink'),
+            'webloc': self.params.get('writewebloclink'),
+            'desktop': self.params.get('writedesktoplink'),
+        }
+        if self.params.get('writelink'):
+            link_type = ('webloc' if sys.platform == 'darwin'
+                         else 'desktop' if sys.platform.startswith('linux')
+                         else 'url')
+            write_links[link_type] = True
+
+        if any(should_write and not _write_link_file(link_type)
+               for link_type, should_write in write_links.items()):
+            return
 
         try:
             info_dict, files_to_move = self.pre_process(info_dict, 'before_dl', files_to_move)
@@ -2904,8 +2935,25 @@ class YoutubeDL(object):
         if max_downloads is not None and self._num_downloads >= int(max_downloads):
             raise MaxDownloadsReached()
 
+    def __download_wrapper(self, func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                res = func(*args, **kwargs)
+            except UnavailableVideoError as e:
+                self.report_error(e)
+            except DownloadCancelled as e:
+                self.to_screen(f'[info] {e}')
+                raise
+            else:
+                if self.params.get('dump_single_json', False):
+                    self.post_extract(res)
+                    self.to_stdout(json.dumps(self.sanitize_info(res)))
+        return wrapper
+
     def download(self, url_list):
         """Download a given list of URLs."""
+        url_list = variadic(url_list)  # Passing a single URL is a common mistake
         outtmpl = self.outtmpl_dict['default']
         if (len(url_list) > 1
                 and outtmpl != '-'
@@ -2914,25 +2962,8 @@ class YoutubeDL(object):
             raise SameFileError(outtmpl)
 
         for url in url_list:
-            try:
-                # It also downloads the videos
-                res = self.extract_info(
-                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
-            except UnavailableVideoError:
-                self.report_error('unable to download video')
-            except MaxDownloadsReached:
-                self.to_screen('[info] Maximum number of downloads reached')
-                raise
-            except ExistingVideoReached:
-                self.to_screen('[info] Encountered a video that is already in the archive, stopping due to --break-on-existing')
-                raise
-            except RejectedVideoReached:
-                self.to_screen('[info] Encountered a video that did not match filter, stopping due to --break-on-reject')
-                raise
-            else:
-                if self.params.get('dump_single_json', False):
-                    self.post_extract(res)
-                    self.to_stdout(json.dumps(self.sanitize_info(res)))
+            self.__download_wrapper(self.extract_info)(
+                url, force_generic_extractor=self.params.get('force_generic_extractor', False))
 
         return self._download_retcode
 
@@ -2943,11 +2974,12 @@ class YoutubeDL(object):
             # FileInput doesn't have a read method, we can't call json.load
             info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
         try:
-            self.process_ie_result(info, download=True)
-        except (DownloadError, EntryNotInPlaylist, ThrottledDownload):
+            self.__download_wrapper(self.process_ie_result)(info, download=True)
+        except (DownloadError, EntryNotInPlaylist, ThrottledDownload) as e:
+            self.to_stderr('\r')
             webpage_url = info.get('webpage_url')
             if webpage_url is not None:
-                self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
+                self.report_warning(f'The info failed to download: {e}; trying with URL {webpage_url}')
                 return self.download([webpage_url])
             else:
                 raise
@@ -2960,7 +2992,7 @@ class YoutubeDL(object):
             return info_dict
         info_dict.setdefault('epoch', int(time.time()))
         remove_keys = {'__original_infodict'}  # Always remove this since this may contain a copy of the entire dict
-        keep_keys = ['_type'],  # Always keep this to facilitate load-info-json
+        keep_keys = ['_type']  # Always keep this to facilitate load-info-json
         if remove_private_keys:
             remove_keys |= {
                 'requested_formats', 'requested_subtitles', 'requested_entries',
@@ -3167,38 +3199,46 @@ class YoutubeDL(object):
             res += '~' + format_bytes(fdict['filesize_approx'])
         return res
 
+    def _list_format_headers(self, *headers):
+        if self.params.get('listformats_table', True) is not False:
+            return [self._format_screen(header, self.Styles.HEADERS) for header in headers]
+        return headers
+
     def list_formats(self, info_dict):
         formats = info_dict.get('formats', [info_dict])
-        new_format = (
-            'list-formats' not in self.params.get('compat_opts', [])
-            and self.params.get('listformats_table', True) is not False)
+        new_format = self.params.get('listformats_table', True) is not False
         if new_format:
+            tbr_digits = number_of_digits(max(f.get('tbr') or 0 for f in formats))
+            vbr_digits = number_of_digits(max(f.get('vbr') or 0 for f in formats))
+            abr_digits = number_of_digits(max(f.get('abr') or 0 for f in formats))
+            delim = self._format_screen('\u2502', self.Styles.DELIM, '|', test_encoding=True)
             table = [
                 [
-                    format_field(f, 'format_id'),
+                    self._format_screen(format_field(f, 'format_id'), self.Styles.ID),
                     format_field(f, 'ext'),
                     self.format_resolution(f),
-                    format_field(f, 'fps', '%d'),
+                    format_field(f, 'fps', '%3d'),
                     format_field(f, 'dynamic_range', '%s', ignore=(None, 'SDR')).replace('HDR', ''),
-                    '|',
+                    delim,
                     format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
-                    format_field(f, 'tbr', '%4dk'),
+                    format_field(f, 'tbr', f'%{tbr_digits}dk'),
                     shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
-                    '|',
+                    delim,
                     format_field(f, 'vcodec', default='unknown').replace('none', ''),
-                    format_field(f, 'vbr', '%4dk'),
+                    format_field(f, 'vbr', f'%{vbr_digits}dk'),
                     format_field(f, 'acodec', default='unknown').replace('none', ''),
-                    format_field(f, 'abr', '%3dk'),
+                    format_field(f, 'abr', f'%{abr_digits}dk'),
                     format_field(f, 'asr', '%5dHz'),
-                    ', '.join(filter(None, (
-                        'UNSUPPORTED' if f.get('ext') in ('f4f', 'f4m') else '',
+                    join_nonempty(
+                        self._format_screen('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,
                         format_field(f, 'language', '[%s]'),
                         format_field(f, 'format_note'),
                         format_field(f, 'container', ignore=(None, f.get('ext'))),
-                    ))),
+                        delim=', '),
                 ] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
-            header_line = ['ID', 'EXT', 'RESOLUTION', 'FPS', 'HDR', '|', ' FILESIZE', '  TBR', 'PROTO',
-                           '|', 'VCODEC', '  VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO']
+            header_line = self._list_format_headers(
+                'ID', 'EXT', 'RESOLUTION', 'FPS', 'HDR', delim, ' FILESIZE', '  TBR', 'PROTO',
+                delim, 'VCODEC', '  VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO')
         else:
             table = [
                 [
@@ -3213,7 +3253,10 @@ class YoutubeDL(object):
         self.to_screen(
             '[info] Available formats for %s:' % info_dict['id'])
         self.to_stdout(render_table(
-            header_line, table, delim=new_format, extraGap=(0 if new_format else 1), hideEmpty=new_format))
+            header_line, table,
+            extraGap=(0 if new_format else 1),
+            hideEmpty=new_format,
+            delim=new_format and self._format_screen('\u2500', self.Styles.DELIM, '-', test_encoding=True)))
 
     def list_thumbnails(self, info_dict):
         thumbnails = list(info_dict.get('thumbnails'))
@@ -3224,7 +3267,7 @@ class YoutubeDL(object):
         self.to_screen(
             '[info] Thumbnails for %s:' % info_dict['id'])
         self.to_stdout(render_table(
-            ['ID', 'width', 'height', 'URL'],
+            self._list_format_headers('ID', 'Width', 'Height', 'URL'),
             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
     def list_subtitles(self, video_id, subtitles, name='subtitles'):
@@ -3241,7 +3284,7 @@ class YoutubeDL(object):
             return [lang, ', '.join(names), ', '.join(exts)]
 
         self.to_stdout(render_table(
-            ['Language', 'Name', 'Formats'],
+            self._list_format_headers('Language', 'Name', 'Formats'),
             [_row(lang, formats) for lang, formats in subtitles.items()],
             hideEmpty=True))
 
@@ -3272,7 +3315,7 @@ class YoutubeDL(object):
             write_debug = lambda msg: logger.debug(f'[debug] {msg}')
             write_debug(encoding_str)
         else:
-            write_string(f'[debug] {encoding_str}', encoding=None)
+            write_string(f'[debug] {encoding_str}\n', encoding=None)
             write_debug = lambda msg: self._write_string(f'[debug] {msg}\n')
 
         source = detect_variant()
@@ -3315,7 +3358,11 @@ class YoutubeDL(object):
             platform.architecture()[0],
             platform_name()))
 
-        exe_versions = FFmpegPostProcessor.get_versions(self)
+        exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self)
+        ffmpeg_features = {key for key, val in ffmpeg_features.items() if val}
+        if ffmpeg_features:
+            exe_versions['ffmpeg'] += ' (%s)' % ','.join(ffmpeg_features)
+
         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_versions['phantomjs'] = PhantomJSwrapper._version()
         exe_str = ', '.join(
@@ -3327,13 +3374,13 @@ class YoutubeDL(object):
         from .postprocessor.embedthumbnail import has_mutagen
         from .cookies import SQLITE_AVAILABLE, KEYRING_AVAILABLE
 
-        lib_str = ', '.join(sorted(filter(None, (
+        lib_str = join_nonempty(
             compat_pycrypto_AES and compat_pycrypto_AES.__name__.split('.')[0],
-            has_websockets and 'websockets',
+            KEYRING_AVAILABLE and 'keyring',
             has_mutagen and 'mutagen',
             SQLITE_AVAILABLE and 'sqlite',
-            KEYRING_AVAILABLE and 'keyring',
-        )))) or 'none'
+            has_websockets and 'websockets',
+            delim=', ') or 'none'
         write_debug('Optional libraries: %s' % lib_str)
 
         proxy_map = {}
@@ -3526,14 +3573,15 @@ class YoutubeDL(object):
 
         for t in thumbnails[::-1]:
             thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
-            thumb_display_id = f'{label} thumbnail' + (f' {t["id"]}' if multiple else '')
+            thumb_display_id = f'{label} thumbnail {t["id"]}'
             thumb_filename = replace_extension(filename, thumb_ext, info_dict.get('ext'))
             thumb_filename_final = replace_extension(thumb_filename_base, thumb_ext, info_dict.get('ext'))
 
             if not self.params.get('overwrites', True) and os.path.exists(thumb_filename):
                 ret.append((thumb_filename, thumb_filename_final))
                 t['filepath'] = thumb_filename
-                self.to_screen(f'[info] {thumb_display_id.title()} is already present')
+                self.to_screen('[info] %s is already present' % (
+                    thumb_display_id if multiple else f'{label} thumbnail').capitalize())
             else:
                 self.to_screen(f'[info] Downloading {thumb_display_id} ...')
                 try:

yt_dlp/__init__.py

@@ -29,6 +29,8 @@ from .utils import (
     error_to_compat_str,
     ExistingVideoReached,
     expand_path,
+    float_or_none,
+    int_or_none,
     match_filter_func,
     MaxDownloadsReached,
     parse_duration,
@@ -122,10 +124,10 @@ def _real_main(argv=None):
             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
             if desc is False:
                 continue
-            if hasattr(ie, 'SEARCH_KEY'):
+            if getattr(ie, 'SEARCH_KEY', None) is not None:
                 _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                 _COUNTS = ('', '5', '10', 'all')
-                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+                desc += f'; "{ie.SEARCH_KEY}:" prefix (Example: "{ie.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(_SEARCHES)}")'
             write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
     if opts.ap_list_mso:
@@ -225,11 +227,13 @@ def _real_main(argv=None):
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
         raise ValueError('Playlist end must be greater than playlist start')
     if opts.extractaudio:
+        opts.audioformat = opts.audioformat.lower()
         if opts.audioformat not in ['best'] + list(FFmpegExtractAudioPP.SUPPORTED_EXTS):
             parser.error('invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
-        if not opts.audioquality.isdigit():
+        audioquality = int_or_none(float_or_none(opts.audioquality))  # int_or_none prevents inf, nan
+        if audioquality is None or audioquality < 0:
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
         opts.recodevideo = opts.recodevideo.replace(' ', '')
@@ -791,15 +795,15 @@ def main(argv=None):
         _real_main(argv)
     except DownloadError:
         sys.exit(1)
-    except SameFileError:
-        sys.exit('ERROR: fixed output name but more than one file to download')
+    except SameFileError as e:
+        sys.exit(f'ERROR: {e}')
     except KeyboardInterrupt:
         sys.exit('\nERROR: Interrupted by user')
-    except BrokenPipeError:
+    except BrokenPipeError as e:
         # https://docs.python.org/3/library/signal.html#note-on-sigpipe
         devnull = os.open(os.devnull, os.O_WRONLY)
         os.dup2(devnull, sys.stdout.fileno())
-        sys.exit(r'\nERROR: {err}')
+        sys.exit(f'\nERROR: {e}')
 
 
 __all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

yt_dlp/compat.py

@@ -19,6 +19,7 @@ import shlex
 import shutil
 import socket
 import struct
+import subprocess
 import sys
 import tokenize
 import urllib
@@ -162,7 +163,9 @@ except ImportError:
 def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
     if compat_os_name != 'nt':
         return
-    os.system('')
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+    subprocess.Popen('', shell=True, startupinfo=startupinfo)
 
 
 #  Deprecated

yt_dlp/cookies.py

@@ -117,7 +117,7 @@ def _extract_firefox_cookies(profile, logger):
         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
 
-    with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)
@@ -236,7 +236,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
 
     decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
 
-    with tempfile.TemporaryDirectory(prefix='youtube_dl') as tmpdir:
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)

yt_dlp/downloader/common.py

@@ -319,6 +319,10 @@ class FileDownloader(object):
                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
             else:
                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+        if s.get('fragment_index') and s.get('fragment_count'):
+            msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
+        elif s.get('fragment_index'):
+            msg_template += ' (frag %(fragment_index)s)'
         s['_default_template'] = msg_template % s
         self._report_progress_status(s)
 

yt_dlp/downloader/external.py

@@ -21,7 +21,6 @@ from ..utils import (
     encodeArgument,
     handle_youtubedl_headers,
     check_executable,
-    is_outdated_version,
     Popen,
     sanitize_open,
 )
@@ -459,7 +458,7 @@ class FFmpegFD(ExternalFD):
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4']
-                if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2', False)) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
+                if (ffpp.basename == 'ffmpeg' and ffpp._features.get('needs_adtstoasc')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
                     args += ['-bsf:a', 'aac_adtstoasc']
         elif protocol == 'rtmp':
             args += ['-f', 'flv']

yt_dlp/downloader/fragment.py

@@ -31,6 +31,10 @@ class HttpQuietDownloader(HttpFD):
     def to_screen(self, *args, **kargs):
         pass
 
+    def report_retry(self, err, count, retries):
+        super().to_screen(
+            f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
+
 
 class FragmentFD(FileDownloader):
     """
@@ -44,6 +48,7 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
                         finished
+    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
     _no_ytdl_file:      Don't use .ytdl file
 
     For each incomplete fragment download yt-dlp keeps on disk a special
@@ -167,7 +172,7 @@ class FragmentFD(FileDownloader):
             self.ydl,
             {
                 'continuedl': True,
-                'quiet': True,
+                'quiet': self.params.get('quiet'),
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
@@ -237,6 +242,7 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
+            'fragment_started': start,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -267,6 +273,9 @@ class FragmentFD(FileDownloader):
                 ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_total_bytes)
+                ctx['fragment_started'] = time.time()
                 ctx['prev_frag_downloaded_bytes'] = 0
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
@@ -275,8 +284,8 @@ class FragmentFD(FileDownloader):
                     state['eta'] = self.calc_eta(
                         start, time_now, estimated_size - resume_len,
                         state['downloaded_bytes'] - resume_len)
-                state['speed'] = s.get('speed') or ctx.get('speed')
-                ctx['speed'] = state['speed']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes)
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state, info_dict)
 

yt_dlp/extractor/adobetv.py

@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     ISO639Utils,
+    join_nonempty,
     OnDemandPagedList,
     parse_duration,
     str_or_none,
@@ -263,7 +264,7 @@ class AdobeTVVideoIE(AdobeTVBaseIE):
                 continue
             formats.append({
                 'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000),
-                'format_id': '-'.join(filter(None, [source.get('format'), source.get('label')])),
+                'format_id': join_nonempty(source.get('format'), source.get('label')),
                 'height': int_or_none(source.get('height') or None),
                 'tbr': int_or_none(source.get('bitrate') or None),
                 'width': int_or_none(source.get('width') or None),

yt_dlp/extractor/amazon.py

@@ -0,0 +1,53 @@
+# coding: utf-8
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class AmazonStoreIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)(?:www\.)?amazon\.(?:[a-z]{2,3})(?:\.[a-z]{2})?/[^/]*/?(?:dp|gp/product)/(?P<id>[^/&#$?]+)'
+
+    _TESTS = [{
+        'url': 'https://www.amazon.co.uk/dp/B098XNCHLD/',
+        'info_dict': {
+            'id': 'B098XNCHLD',
+            'title': 'md5:5f3194dbf75a8dcfc83079bd63a2abed',
+        },
+        'playlist_mincount': 1,
+        'playlist': [{
+            'info_dict': {
+                'id': 'A1F83G8C2ARO7P',
+                'ext': 'mp4',
+                'title': 'mcdodo usb c cable 100W 5a',
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+        }]
+    }, {
+        'url': 'https://www.amazon.in/Sony-WH-1000XM4-Cancelling-Headphones-Bluetooth/dp/B0863TXGM3',
+        'info_dict': {
+            'id': 'B0863TXGM3',
+            'title': 'md5:b0bde4881d3cfd40d63af19f7898b8ff',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'https://www.amazon.com/dp/B0845NXCXF/',
+        'info_dict': {
+            'id': 'B0845NXCXF',
+            'title': 'md5:2145cd4e3c7782f1ee73649a3cff1171',
+        },
+        'playlist-mincount': 1,
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        data_json = self._parse_json(self._html_search_regex(r'var\s?obj\s?=\s?jQuery\.parseJSON\(\'(.*)\'\)', webpage, 'data'), id)
+        entries = [{
+            'id': video['marketPlaceID'],
+            'url': video['url'],
+            'title': video.get('title'),
+            'thumbnail': video.get('thumbUrl') or video.get('thumb'),
+            'duration': video.get('durationSeconds'),
+            'height': int_or_none(video.get('videoHeight')),
+            'width': int_or_none(video.get('videoWidth')),
+        } for video in (data_json.get('videos') or []) if video.get('isVideo') and video.get('url')]
+        return self.playlist_result(entries, playlist_id=id, playlist_title=data_json['title'])

yt_dlp/extractor/animeondemand.py

@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     extract_attributes,
     ExtractorError,
+    join_nonempty,
     url_or_none,
     urlencode_postdata,
     urljoin,
@@ -140,15 +141,8 @@ class AnimeOnDemandIE(InfoExtractor):
                     kind = self._search_regex(
                         r'videomaterialurl/\d+/([^/]+)/',
                         playlist_url, 'media kind', default=None)
-                    format_id_list = []
-                    if lang:
-                        format_id_list.append(lang)
-                    if kind:
-                        format_id_list.append(kind)
-                    if not format_id_list and num is not None:
-                        format_id_list.append(compat_str(num))
-                    format_id = '-'.join(format_id_list)
-                    format_note = ', '.join(filter(None, (kind, lang_note)))
+                    format_id = join_nonempty(lang, kind) if lang or kind else str(num)
+                    format_note = join_nonempty(kind, lang_note, delim=', ')
                     item_id_list = []
                     if format_id:
                         item_id_list.append(format_id)
@@ -195,12 +189,10 @@ class AnimeOnDemandIE(InfoExtractor):
                         if not file_:
                             continue
                         ext = determine_ext(file_)
-                        format_id_list = [lang, kind]
-                        if ext == 'm3u8':
-                            format_id_list.append('hls')
-                        elif source.get('type') == 'video/dash' or ext == 'mpd':
-                            format_id_list.append('dash')
-                        format_id = '-'.join(filter(None, format_id_list))
+                        format_id = join_nonempty(
+                            lang, kind,
+                            'hls' if ext == 'm3u8' else None,
+                            'dash' if source.get('type') == 'video/dash' or ext == 'mpd' else None)
                         if ext == 'm3u8':
                             file_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',

yt_dlp/extractor/anvato.py

@@ -16,6 +16,7 @@ from ..utils import (
     determine_ext,
     intlist_to_bytes,
     int_or_none,
+    join_nonempty,
     strip_jsonp,
     unescapeHTML,
     unsmuggle_url,
@@ -303,13 +304,13 @@ class AnvatoIE(InfoExtractor):
             tbr = int_or_none(published_url.get('kbps'))
             a_format = {
                 'url': video_url,
-                'format_id': ('-'.join(filter(None, ['http', published_url.get('cdn_name')]))).lower(),
-                'tbr': tbr if tbr != 0 else None,
+                'format_id': join_nonempty('http', published_url.get('cdn_name')).lower(),
+                'tbr': tbr or None,
             }
 
             if media_format == 'm3u8' and tbr is not None:
                 a_format.update({
-                    'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
+                    'format_id': join_nonempty('hls', tbr),
                     'ext': 'mp4',
                 })
             elif media_format == 'm3u8-variant' or ext == 'm3u8':

yt_dlp/extractor/atresplayer.py

@@ -24,9 +24,6 @@ class AtresPlayerIE(InfoExtractor):
                 'description': 'md5:7634cdcb4d50d5381bedf93efb537fbc',
                 'duration': 3413,
             },
-            'params': {
-                'format': 'bestvideo',
-            },
             'skip': 'This video is only available for registered users'
         },
         {

yt_dlp/extractor/bandaichannel.py

@@ -21,7 +21,6 @@ class BandaiChannelIE(BrightcoveNewIE):
             'duration': 1387.733,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }]

yt_dlp/extractor/bilibili.py

@@ -376,8 +376,10 @@ class BiliBiliIE(InfoExtractor):
             replies = traverse_obj(
                 self._download_json(
                     f'https://api.bilibili.com/x/v2/reply?pn={idx}&oid={video_id}&type=1&jsonp=jsonp&sort=2&_=1567227301685',
-                    video_id, note=f'Extracting comments from page {idx}'),
-                ('data', 'replies')) or []
+                    video_id, note=f'Extracting comments from page {idx}', fatal=False),
+                ('data', 'replies'))
+            if not replies:
+                return
             for children in map(self._get_all_children, replies):
                 yield from children
 
@@ -566,7 +568,7 @@ class BilibiliCategoryIE(InfoExtractor):
 
 
 class BiliBiliSearchIE(SearchInfoExtractor):
-    IE_DESC = 'Bilibili video search, "bilisearch" keyword'
+    IE_DESC = 'Bilibili video search'
     _MAX_RESULTS = 100000
     _SEARCH_KEY = 'bilisearch'
 

yt_dlp/extractor/canvas.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import json
 
 
 from .common import InfoExtractor
@@ -41,9 +42,9 @@ class CanvasIE(InfoExtractor):
     _GEO_BYPASS = False
     _HLS_ENTRY_PROTOCOLS_MAP = {
         'HLS': 'm3u8_native',
-        'HLS_AES': 'm3u8',
+        'HLS_AES': 'm3u8_native',
     }
-    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1'
+    _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
 
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
@@ -59,16 +60,21 @@ class CanvasIE(InfoExtractor):
 
         # New API endpoint
         if not data:
+            vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
+                                             video_id, note='refreshtoken: Retrieve vrtnutoken',
+                                             errnote='refreshtoken failed')['vrtnutoken']
             headers = self.geo_verification_headers()
-            headers.update({'Content-Type': 'application/json'})
-            token = self._download_json(
+            headers.update({'Content-Type': 'application/json; charset=utf-8'})
+            vrtPlayerToken = self._download_json(
                 '%s/tokens' % self._REST_API_BASE, video_id,
-                'Downloading token', data=b'', headers=headers)['vrtPlayerToken']
+                'Downloading token', headers=headers, data=json.dumps({
+                    'identityToken': vrtnutoken
+                }).encode('utf-8'))['vrtPlayerToken']
             data = self._download_json(
                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
                 video_id, 'Downloading video JSON', query={
-                    'vrtPlayerToken': token,
-                    'client': '%s@PROD' % site_id,
+                    'vrtPlayerToken': vrtPlayerToken,
+                    'client': 'null',
                 }, expected_status=400)
             if not data.get('title'):
                 code = data.get('code')
@@ -264,7 +270,7 @@ class VrtNUIE(GigyaBaseIE):
         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
     }]
     _NETRC_MACHINE = 'vrtnu'
-    _APIKEY = '3_qhEcPa5JGFROVwu5SWKqJ4mVOIkwlFNMSKwzPDAh8QZOtHqu6L4nD5Q7lk0eXOOG'
+    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
     _CONTEXT_ID = 'R3595707040'
 
     def _real_initialize(self):
@@ -275,16 +281,13 @@ class VrtNUIE(GigyaBaseIE):
         if username is None:
             return
 
-        auth_info = self._download_json(
-            'https://accounts.vrt.be/accounts.login', None,
-            note='Login data', errnote='Could not get Login data',
-            headers={}, data=urlencode_postdata({
-                'loginID': username,
-                'password': password,
-                'sessionExpiration': '-2',
-                'APIKey': self._APIKEY,
-                'targetEnv': 'jssdk',
-            }))
+        auth_info = self._gigya_login({
+            'APIKey': self._APIKEY,
+            'targetEnv': 'jssdk',
+            'loginID': username,
+            'password': password,
+            'authMode': 'cookie',
+        })
 
         if auth_info.get('errorDetails'):
             raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
@@ -301,14 +304,15 @@ class VrtNUIE(GigyaBaseIE):
                     'UID': auth_info['UID'],
                     'UIDSignature': auth_info['UIDSignature'],
                     'signatureTimestamp': auth_info['signatureTimestamp'],
-                    'client_id': 'vrtnu-site',
                     '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
                 }
 
                 self._request_webpage(
                     'https://login.vrt.be/perform_login',
-                    None, note='Requesting a token', errnote='Could not get a token',
-                    headers={}, data=urlencode_postdata(post_data))
+                    None, note='Performing login', errnote='perform login failed',
+                    headers={}, query={
+                        'client_id': 'vrtnu-site'
+                    }, data=urlencode_postdata(post_data))
 
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:

yt_dlp/extractor/ceskatelevize.py

@@ -20,22 +20,8 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
-        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
-        'info_dict': {
-            'id': '61924494877246241',
-            'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Život v Grónsku',
-            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3350,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
         'info_dict': {
             'id': '61924494877028507',
@@ -66,12 +52,58 @@ class CeskaTelevizeIE(InfoExtractor):
     }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
         'only_matching': True,
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Hlavní město Kolumbie v doprovodu queer umělců. Vroucí svět plný vášně, sebevědomí, ale i násilí a bolesti. Připravil Peter Serge Butko',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494877311053',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 11.9,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # iframe embed
+        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-
+        parsed_url = compat_urllib_parse_urlparse(url)
         webpage = self._download_webpage(url, playlist_id)
+        site_name = self._og_search_property('site_name', webpage, fatal=False, default=None)
+        playlist_title = self._og_search_title(webpage, default=None)
+        if site_name and playlist_title:
+            playlist_title = playlist_title.replace(f' — {site_name}', '', 1)
+        playlist_description = self._og_search_description(webpage, default=None)
+        if playlist_description:
+            playlist_description = playlist_description.replace('\xa0', ' ')
+
+        if parsed_url.path.startswith('/porady/'):
+            refer_url = update_url_query(unescapeHTML(self._search_regex(
+                (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+                 r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
+                webpage, 'iframe player url', group='url')), query={'autoStart': 'true'})
+            webpage = self._download_webpage(refer_url, playlist_id)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
@@ -100,7 +132,7 @@ class CeskaTelevizeIE(InfoExtractor):
         data = {
             'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
-            'requestUrl': compat_urllib_parse_urlparse(url).path,
+            'requestUrl': parsed_url.path,
             'requestSource': 'iVysilani',
         }
 
@@ -108,7 +140,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         for user_agent in (None, USER_AGENTS['Safari']):
             req = sanitized_Request(
-                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
@@ -130,9 +162,6 @@ class CeskaTelevizeIE(InfoExtractor):
             req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
             req.add_header('Referer', url)
 
-            playlist_title = self._og_search_title(webpage, default=None)
-            playlist_description = self._og_search_description(webpage, default=None)
-
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:
                 continue
@@ -237,54 +266,3 @@ class CeskaTelevizeIE(InfoExtractor):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
-
-
-class CeskaTelevizePoradyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
-    _TESTS = [{
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # iframe embed
-        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        data_url = update_url_query(unescapeHTML(self._search_regex(
-            (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
-            webpage, 'iframe player url', group='url')), query={
-                'autoStart': 'true',
-        })
-
-        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())

yt_dlp/extractor/common.py

@@ -54,6 +54,7 @@ from ..utils import (
     GeoRestrictedError,
     GeoUtils,
     int_or_none,
+    join_nonempty,
     js_to_json,
     JSON_LD_RE,
     mimetype2ext,
@@ -74,6 +75,7 @@ from ..utils import (
     strip_or_none,
     traverse_obj,
     unescapeHTML,
+    UnsupportedError,
     unified_strdate,
     unified_timestamp,
     update_Request,
@@ -440,11 +442,11 @@ class InfoExtractor(object):
     _WORKING = True
 
     _LOGIN_HINTS = {
-        'any': 'Use --cookies, --username and --password or --netrc to provide account credentials',
+        'any': 'Use --cookies, --username and --password, or --netrc to provide account credentials',
         'cookies': (
             'Use --cookies-from-browser or --cookies for the authentication. '
             'See  https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl  for how to manually pass cookies'),
-        'password': 'Use --username and --password or --netrc to provide account credentials',
+        'password': 'Use --username and --password, or --netrc to provide account credentials',
     }
 
     def __init__(self, downloader=None):
@@ -604,10 +606,19 @@ class InfoExtractor(object):
                     if self.__maybe_fake_ip_and_retry(e.countries):
                         continue
                     raise
+        except UnsupportedError:
+            raise
         except ExtractorError as e:
-            video_id = e.video_id or self.get_temp_id(url)
-            raise ExtractorError(
-                e.msg, video_id=video_id, ie=self.IE_NAME, tb=e.traceback, expected=e.expected, cause=e.cause)
+            kwargs = {
+                'video_id': e.video_id or self.get_temp_id(url),
+                'ie': self.IE_NAME,
+                'tb': e.traceback,
+                'expected': e.expected,
+                'cause': e.cause
+            }
+            if hasattr(e, 'countries'):
+                kwargs['countries'] = e.countries
+            raise type(e)(e.msg, **kwargs)
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
@@ -1139,7 +1150,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        _name = self._downloader._color_text(name, 'blue')
+        _name = self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
 
         if mobj:
             if group is None:
@@ -1485,6 +1496,13 @@ class InfoExtractor(object):
                     break
         return dict((k, v) for k, v in info.items() if v is not None)
 
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
     @staticmethod
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
@@ -1521,7 +1539,7 @@ class InfoExtractor(object):
             'vcodec': {'type': 'ordered', 'regex': True,
                        'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
             'acodec': {'type': 'ordered', 'regex': True,
-                       'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e?a?c-?3', 'dts', '', None, 'none']},
+                       'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
             'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
                     'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
             'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
@@ -1539,8 +1557,8 @@ class InfoExtractor(object):
             'ie_pref': {'priority': True, 'type': 'extractor'},
             'hasvid': {'priority': True, 'field': 'vcodec', 'type': 'boolean', 'not_in_list': ('none',)},
             'hasaud': {'field': 'acodec', 'type': 'boolean', 'not_in_list': ('none',)},
-            'lang': {'convert': 'ignore', 'field': 'language_preference'},
-            'quality': {'convert': 'float_none', 'default': -1},
+            'lang': {'convert': 'float', 'field': 'language_preference', 'default': -1},
+            'quality': {'convert': 'float', 'default': -1},
             'filesize': {'convert': 'bytes'},
             'fs_approx': {'convert': 'bytes', 'field': 'filesize_approx'},
             'id': {'convert': 'string', 'field': 'format_id'},
@@ -1551,7 +1569,7 @@ class InfoExtractor(object):
             'vbr': {'convert': 'float_none'},
             'abr': {'convert': 'float_none'},
             'asr': {'convert': 'float_none'},
-            'source': {'convert': 'ignore', 'field': 'source_preference'},
+            'source': {'convert': 'float', 'field': 'source_preference', 'default': -1},
 
             'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
             'br': {'type': 'combined', 'field': ('tbr', 'vbr', 'abr'), 'same_limit': True},
@@ -1901,7 +1919,7 @@ class InfoExtractor(object):
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             width = int_or_none(media_el.attrib.get('width'))
             height = int_or_none(media_el.attrib.get('height'))
-            format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
+            format_id = join_nonempty(f4m_id, tbr or i)
             # If <bootstrapInfo> is present, the specified f4m is a
             # stream-level manifest, and only set-level manifests may refer to
             # external resources.  See section 11.4 and section 4 of F4M spec
@@ -1963,7 +1981,7 @@ class InfoExtractor(object):
 
     def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, quality=None, m3u8_id=None):
         return {
-            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
+            'format_id': join_nonempty(m3u8_id, 'meta'),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
@@ -2058,7 +2076,7 @@ class InfoExtractor(object):
 
         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             formats = [{
-                'format_id': '-'.join(map(str, filter(None, [m3u8_id, idx]))),
+                'format_id': join_nonempty(m3u8_id, idx),
                 'format_index': idx,
                 'url': m3u8_url,
                 'ext': ext,
@@ -2107,7 +2125,7 @@ class InfoExtractor(object):
             if media_url:
                 manifest_url = format_url(media_url)
                 formats.extend({
-                    'format_id': '-'.join(map(str, filter(None, (m3u8_id, group_id, name, idx)))),
+                    'format_id': join_nonempty(m3u8_id, group_id, name, idx),
                     'format_note': name,
                     'format_index': idx,
                     'url': manifest_url,
@@ -2164,9 +2182,9 @@ class InfoExtractor(object):
                     # format_id intact.
                     if not live:
                         stream_name = build_stream_name()
-                        format_id[1] = stream_name if stream_name else '%d' % (tbr if tbr else len(formats))
+                        format_id[1] = stream_name or '%d' % (tbr or len(formats))
                     f = {
-                        'format_id': '-'.join(map(str, filter(None, format_id))),
+                        'format_id': join_nonempty(*format_id),
                         'format_index': idx,
                         'url': manifest_url,
                         'manifest_url': m3u8_url,
@@ -2955,13 +2973,6 @@ class InfoExtractor(object):
                         })
                         fragment_ctx['time'] += fragment_ctx['duration']
 
-                format_id = []
-                if ism_id:
-                    format_id.append(ism_id)
-                if stream_name:
-                    format_id.append(stream_name)
-                format_id.append(compat_str(tbr))
-
                 if stream_type == 'text':
                     subtitles.setdefault(stream_language, []).append({
                         'ext': 'ismt',
@@ -2980,7 +2991,7 @@ class InfoExtractor(object):
                     })
                 elif stream_type in ('video', 'audio'):
                     formats.append({
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty(ism_id, stream_name, tbr),
                         'url': ism_url,
                         'manifest_url': ism_url,
                         'ext': 'ismv' if stream_type == 'video' else 'isma',
@@ -3620,9 +3631,11 @@ class SearchInfoExtractor(InfoExtractor):
     """
     Base class for paged search queries extractors.
     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
-    Instances should define _SEARCH_KEY and _MAX_RESULTS.
+    Instances should define _SEARCH_KEY and optionally _MAX_RESULTS
     """
 
+    _MAX_RESULTS = float('inf')
+
     @classmethod
     def _make_valid_url(cls):
         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY

yt_dlp/extractor/corus.py

@@ -55,7 +55,6 @@ class CorusIE(ThePlatformFeedIE):
             'timestamp': 1486392197,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
         'expected_warnings': ['Failed to parse JSON'],

yt_dlp/extractor/coub.py

@@ -57,7 +57,7 @@ class CoubIE(InfoExtractor):
 
         file_versions = coub['file_versions']
 
-        QUALITIES = ('low', 'med', 'high')
+        QUALITIES = ('low', 'med', 'high', 'higher')
 
         MOBILE = 'mobile'
         IPHONE = 'iphone'
@@ -86,6 +86,7 @@ class CoubIE(InfoExtractor):
                     'format_id': '%s-%s-%s' % (HTML5, kind, quality),
                     'filesize': int_or_none(item.get('size')),
                     'vcodec': 'none' if kind == 'audio' else None,
+                    'acodec': 'none' if kind == 'video' else None,
                     'quality': quality_key(quality),
                     'source_preference': preference_key(HTML5),
                 })

yt_dlp/extractor/crunchyroll.py

@@ -27,6 +27,7 @@ from ..utils import (
     int_or_none,
     lowercase_escape,
     merge_dicts,
+    qualities,
     remove_end,
     sanitized_Request,
     try_get,
@@ -478,19 +479,24 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             [r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', r'<div>\s*Publisher:\s*<span>\s*(.+?)\s*</span>\s*</div>'],
             webpage, 'video_uploader', default=False)
 
+        requested_languages = self._configuration_arg('language')
+        requested_hardsubs = [('' if val == 'none' else val) for val in self._configuration_arg('hardsub')]
+        language_preference = qualities((requested_languages or [language or ''])[::-1])
+        hardsub_preference = qualities((requested_hardsubs or ['', language or ''])[::-1])
+
         formats = []
         for stream in media.get('streams', []):
-            audio_lang = stream.get('audio_lang')
-            hardsub_lang = stream.get('hardsub_lang')
+            audio_lang = stream.get('audio_lang') or ''
+            hardsub_lang = stream.get('hardsub_lang') or ''
+            if (requested_languages and audio_lang.lower() not in requested_languages
+                    or requested_hardsubs and hardsub_lang.lower() not in requested_hardsubs):
+                continue
             vrv_formats = self._extract_vrv_formats(
                 stream.get('url'), video_id, stream.get('format'),
                 audio_lang, hardsub_lang)
             for f in vrv_formats:
-                f['language_preference'] = 1 if audio_lang == language else 0
-                f['quality'] = (
-                    1 if not hardsub_lang
-                    else 0 if hardsub_lang == language
-                    else -1)
+                f['language_preference'] = language_preference(audio_lang)
+                f['quality'] = hardsub_preference(hardsub_lang)
             formats.extend(vrv_formats)
         if not formats:
             available_fmts = []

yt_dlp/extractor/curiositystream.py

@@ -59,7 +59,6 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
         },
         'params': {
-            'format': 'bestvideo',
             # m3u8 download
             'skip_download': True,
         },

yt_dlp/extractor/discoverynetworks.py

@@ -19,7 +19,6 @@ class DiscoveryNetworksDeIE(DPlayIE):
             'upload_date': '20190331',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {

yt_dlp/extractor/discoveryplusindia.py

@@ -28,7 +28,6 @@ class DiscoveryPlusIndiaIE(DPlayIE):
             'creator': 'Discovery Channel',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
         'skip': 'Cookies (not necessarily logged in) are needed'

yt_dlp/extractor/disney.py

@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
-    compat_str,
     determine_ext,
+    join_nonempty,
     update_url_query,
 )
 
@@ -119,18 +119,13 @@ class DisneyIE(InfoExtractor):
                         continue
                     formats.append(f)
                 continue
-            format_id = []
-            if flavor_format:
-                format_id.append(flavor_format)
-            if tbr:
-                format_id.append(compat_str(tbr))
             ext = determine_ext(flavor_url)
             if flavor_format == 'applehttp' or ext == 'm3u8':
                 ext = 'mp4'
             width = int_or_none(flavor.get('width'))
             height = int_or_none(flavor.get('height'))
             formats.append({
-                'format_id': '-'.join(format_id),
+                'format_id': join_nonempty(flavor_format, tbr),
                 'url': flavor_url,
                 'width': width,
                 'height': height,

yt_dlp/extractor/dplay.py

@@ -46,7 +46,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 1,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -67,7 +66,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 1,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -87,7 +85,6 @@ class DPlayIE(InfoExtractor):
             'episode_number': 7,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
         'skip': 'Available for Premium users',
@@ -313,9 +310,6 @@ class HGTVDeIE(DPlayIE):
             'season_number': 3,
             'episode_number': 3,
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }]
 
     def _real_extract(self, url):
@@ -325,7 +319,7 @@ class HGTVDeIE(DPlayIE):
 
 
 class DiscoveryPlusIE(DPlayIE):
-    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video' + DPlayIE._PATH_REGEX
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/(?:\w{2}/)?video' + DPlayIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
         'info_dict': {
@@ -343,6 +337,9 @@ class DiscoveryPlusIE(DPlayIE):
             'episode_number': 1,
         },
         'skip': 'Available for Premium users',
+    }, {
+        'url': 'https://discoveryplus.com/ca/video/bering-sea-gold-discovery-ca/goldslingers',
+        'only_matching': True,
     }]
 
     _PRODUCT = 'dplus_us'

yt_dlp/extractor/dvtv.py

@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    join_nonempty,
     js_to_json,
     mimetype2ext,
     try_get,
@@ -139,13 +140,9 @@ class DVTVIE(InfoExtractor):
                     label = video.get('label')
                     height = self._search_regex(
                         r'^(\d+)[pP]', label or '', 'height', default=None)
-                    format_id = ['http']
-                    for f in (ext, label):
-                        if f:
-                            format_id.append(f)
                     formats.append({
                         'url': video_url,
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty('http', ext, label),
                         'height': int_or_none(height),
                     })
         self._sort_formats(formats)

yt_dlp/extractor/egghead.py

@@ -86,7 +86,6 @@ class EggheadLessonIE(EggheadBaseIE):
         },
         'params': {
             'skip_download': True,
-            'format': 'bestvideo',
         },
     }, {
         'url': 'https://egghead.io/api/v1/lessons/react-add-redux-to-a-react-application',

yt_dlp/extractor/extractors.py

@@ -50,6 +50,7 @@ from .animelab import (
     AnimeLabIE,
     AnimeLabShowsIE,
 )
+from .amazon import AmazonStoreIE
 from .americastestkitchen import (
     AmericasTestKitchenIE,
     AmericasTestKitchenSeasonIE,
@@ -235,10 +236,7 @@ from .ccc import (
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import (
-    CeskaTelevizeIE,
-    CeskaTelevizePoradyIE,
-)
+from .ceskatelevize import CeskaTelevizeIE
 from .cgtn import CGTNIE
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
@@ -495,7 +493,10 @@ from .funimation import (
 )
 from .funk import FunkIE
 from .fusion import FusionIE
-from .gab import GabTVIE
+from .gab import (
+    GabTVIE,
+    GabIE,
+)
 from .gaia import GaiaIE
 from .gameinformer import GameInformerIE
 from .gamespot import GameSpotIE
@@ -591,12 +592,16 @@ from .indavideo import IndavideoEmbedIE
 from .infoq import InfoQIE
 from .instagram import (
     InstagramIE,
+    InstagramIOSIE,
     InstagramUserIE,
     InstagramTagIE,
 )
 from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE
-from .iprima import IPrimaIE
+from .iprima import (
+    IPrimaIE,
+    IPrimaCNNIE
+)
 from .iqiyi import IqiyiIE
 from .ir90tv import Ir90TvIE
 from .itv import (
@@ -744,7 +749,10 @@ from .mdr import MDRIE
 from .medaltv import MedalTVIE
 from .mediaite import MediaiteIE
 from .mediaklikk import MediaKlikkIE
-from .mediaset import MediasetIE
+from .mediaset import (
+    MediasetIE,
+    MediasetShowIE,
+)
 from .mediasite import (
     MediasiteIE,
     MediasiteCatalogIE,
@@ -793,6 +801,7 @@ from .mlb import (
     MLBIE,
     MLBVideoIE,
 )
+from .mlssoccer import MLSSoccerIE
 from .mnet import MnetIE
 from .moevideo import MoeVideoIE
 from .mofosex import (
@@ -835,7 +844,10 @@ from .myvi import (
 )
 from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
-from .n1 import N1InfoIIE, N1InfoAssetIE
+from .n1 import (
+    N1InfoAssetIE,
+    N1InfoIIE,
+)
 from .nationalgeographic import (
     NationalGeographicVideoIE,
     NationalGeographicTVIE,
@@ -1071,6 +1083,7 @@ from .pinterest import (
     PinterestCollectionIE,
 )
 from .pladform import PladformIE
+from .planetmarathi import PlanetMarathiIE
 from .platzi import (
     PlatziIE,
     PlatziCourseIE,
@@ -1092,9 +1105,14 @@ from .pokemon import (
     PokemonIE,
     PokemonWatchIE,
 )
+from .polsatgo import PolsatGoIE
 from .polskieradio import (
     PolskieRadioIE,
     PolskieRadioCategoryIE,
+    PolskieRadioPlayerIE,
+    PolskieRadioPodcastIE,
+    PolskieRadioPodcastListIE,
+    PolskieRadioRadioKierowcowIE,
 )
 from .popcorntimes import PopcorntimesIE
 from .popcorntv import PopcornTVIE
@@ -1141,6 +1159,10 @@ from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
+from .radiokapital import (
+    RadioKapitalIE,
+    RadioKapitalShowIE,
+)
 from .radlive import (
     RadLiveIE,
     RadLiveChannelIE,
@@ -1151,6 +1173,8 @@ from .rai import (
     RaiPlayLiveIE,
     RaiPlayPlaylistIE,
     RaiIE,
+    RaiPlayRadioIE,
+    RaiPlayRadioPlaylistIE,
 )
 from .raywenderlich import (
     RayWenderlichIE,
@@ -1191,7 +1215,7 @@ from .rice import RICEIE
 from .rmcdecouverte import RMCDecouverteIE
 from .ro220 import Ro220IE
 from .rockstargames import RockstarGamesIE
-from .roosterteeth import RoosterTeethIE
+from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rozhlas import RozhlasIE
@@ -1289,6 +1313,7 @@ from .skynewsarabia import (
 from .skynewsau import SkyNewsAUIE
 from .sky import (
     SkyNewsIE,
+    SkyNewsStoryIE,
     SkySportsIE,
     SkySportsNewsIE,
 )
@@ -1442,6 +1467,10 @@ from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
+from .threespeak import (
+    ThreeSpeakIE,
+    ThreeSpeakUserIE,
+)
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,
@@ -1542,6 +1571,7 @@ from .tvnow import (
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
+    TVPStreamIE,
     TVPWebsiteIE,
 )
 from .tvplay import (
@@ -1759,6 +1789,10 @@ from .wistia import (
     WistiaPlaylistIE,
 )
 from .worldstarhiphop import WorldStarHipHopIE
+from .wppilot import (
+    WPPilotIE,
+    WPPilotChannelsIE,
+)
 from .wsj import (
     WSJIE,
     WSJArticleIE,

yt_dlp/extractor/fancode.py

@@ -21,7 +21,6 @@ class FancodeVodIE(InfoExtractor):
         'url': 'https://fancode.com/video/15043/match-preview-pbks-vs-mi',
         'params': {
             'skip_download': True,
-            'format': 'bestvideo'
         },
         'info_dict': {
             'id': '6249806281001',

yt_dlp/extractor/funimation.py

@@ -10,6 +10,7 @@ from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     int_or_none,
+    join_nonempty,
     js_to_json,
     orderedSet,
     qualities,
@@ -288,10 +289,11 @@ class FunimationIE(FunimationBaseIE):
                     sub_type = sub_type if sub_type != 'FULL' else None
                     current_sub = {
                         'url': text_track['src'],
-                        'name': ' '.join(filter(None, (version, text_track.get('label'), sub_type)))
+                        'name': join_nonempty(version, text_track.get('label'), sub_type, delim=' ')
                     }
-                    lang = '_'.join(filter(None, (
-                        text_track.get('language', 'und'), version if version != 'Simulcast' else None, sub_type)))
+                    lang = join_nonempty(text_track.get('language', 'und'),
+                                         version if version != 'Simulcast' else None,
+                                         sub_type, delim='_')
                     if current_sub not in subtitles.get(lang, []):
                         subtitles.setdefault(lang, []).append(current_sub)
         return subtitles

yt_dlp/extractor/gab.py

@@ -6,7 +6,11 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    int_or_none,
+    parse_codecs,
+    parse_duration,
     str_to_int,
+    unified_timestamp
 )
 
 
@@ -32,8 +36,10 @@ class GabTVIE(InfoExtractor):
         channel_name = self._search_regex(r'data-channel-name=\"(?P<channel_id>[^\"]+)', webpage, 'channel_name')
         title = self._search_regex(r'data-episode-title=\"(?P<channel_id>[^\"]+)', webpage, 'title')
         view_key = self._search_regex(r'data-view-key=\"(?P<channel_id>[^\"]+)', webpage, 'view_key')
-        description = clean_html(self._html_search_regex(self._meta_regex('description'), webpage, 'description', group='content')) or None
-        available_resolutions = re.findall(r'<a\ data-episode-id=\"%s\"\ data-resolution=\"(?P<resolution>[^\"]+)' % id, webpage)
+        description = clean_html(
+            self._html_search_regex(self._meta_regex('description'), webpage, 'description', group='content')) or None
+        available_resolutions = re.findall(r'<a\ data-episode-id=\"%s\"\ data-resolution=\"(?P<resolution>[^\"]+)' % id,
+                                           webpage)
 
         formats = []
         for resolution in available_resolutions:
@@ -62,3 +68,80 @@ class GabTVIE(InfoExtractor):
             'uploader_id': channel_id,
             'thumbnail': f'https://tv.gab.com/image/{id}',
         }
+
+
+class GabIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gab\.com/[^/]+/posts/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://gab.com/SomeBitchIKnow/posts/107163961867310434',
+        'md5': '8ca34fb00f1e1033b5c5988d79ec531d',
+        'info_dict': {
+            'id': '107163961867310434-0',
+            'ext': 'mp4',
+            'title': 'L on Gab',
+            'uploader_id': '946600',
+            'uploader': 'SomeBitchIKnow',
+            'description': 'md5:204055fafd5e1a519f5d6db953567ca3',
+            'timestamp': 1635192289,
+            'upload_date': '20211025',
+        }
+    }, {
+        'url': 'https://gab.com/TheLonelyProud/posts/107045884469287653',
+        'md5': 'f9cefcfdff6418e392611a828d47839d',
+        'info_dict': {
+            'id': '107045884469287653-0',
+            'ext': 'mp4',
+            'title': 'Jody Sadowski on Gab',
+            'uploader_id': '1390705',
+            'timestamp': 1633390571,
+            'upload_date': '20211004',
+            'uploader': 'TheLonelyProud',
+        }
+    }]
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+        json_data = self._download_json(f'https://gab.com/api/v1/statuses/{post_id}', post_id)
+
+        entries = []
+        for idx, media in enumerate(json_data['media_attachments']):
+            if media.get('type') not in ('video', 'gifv'):
+                continue
+            metadata = media['meta']
+            format_metadata = {
+                'acodec': parse_codecs(metadata.get('audio_encode')).get('acodec'),
+                'asr': int_or_none((metadata.get('audio_bitrate') or '').split(' ')[0]),
+                'fps': metadata.get('fps'),
+            }
+
+            formats = [{
+                'url': url,
+                'width': f.get('width'),
+                'height': f.get('height'),
+                'tbr': int_or_none(f.get('bitrate'), scale=1000),
+                **format_metadata,
+            } for url, f in ((media.get('url'), metadata.get('original') or {}),
+                             (media.get('source_mp4'), metadata.get('playable') or {})) if url]
+
+            self._sort_formats(formats)
+
+            author = json_data.get('account') or {}
+            entries.append({
+                'id': f'{post_id}-{idx}',
+                'title': f'{json_data["account"]["display_name"]} on Gab',
+                'timestamp': unified_timestamp(json_data.get('created_at')),
+                'formats': formats,
+                'description': clean_html(json_data.get('content')),
+                'duration': metadata.get('duration') or parse_duration(metadata.get('length')),
+                'like_count': json_data.get('favourites_count'),
+                'comment_count': json_data.get('replies_count'),
+                'repost_count': json_data.get('reblogs_count'),
+                'uploader': author.get('username'),
+                'uploader_id': author.get('id'),
+                'uploader_url': author.get('url'),
+            })
+
+        if len(entries) > 1:
+            return self.playlist_result(entries, post_id)
+
+        return entries[0]

yt_dlp/extractor/generic.py

@@ -135,6 +135,7 @@ from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
 from .simplecast import SimplecastIE
 from .wimtv import WimTVIE
+from .tvp import TVPEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -359,9 +360,6 @@ class GenericIE(InfoExtractor):
                 'formats': 'mincount:9',
                 'upload_date': '20130904',
             },
-            'params': {
-                'format': 'bestvideo',
-            },
         },
         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
         {
@@ -1188,6 +1186,21 @@ class GenericIE(InfoExtractor):
             },
             'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
         },
+        # jwplayer with only the json URL
+        {
+            'url': 'https://www.hollywoodreporter.com/news/general-news/dunkirk-team-reveals-what-christopher-nolan-said-oscar-win-meet-your-oscar-winner-1092454',
+            'info_dict': {
+                'id': 'TljWkvWH',
+                'ext': 'mp4',
+                'upload_date': '20180306',
+                'title': 'md5:91eb1862f6526415214f62c00b453936',
+                'description': 'md5:73048ae50ae953da10549d1d2fe9b3aa',
+                'timestamp': 1520367225,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Complex jwplayer
         {
             'url': 'http://www.indiedb.com/games/king-machine/videos',
@@ -2325,6 +2338,9 @@ class GenericIE(InfoExtractor):
         """Report information extraction."""
         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
 
+    def report_detected(self, name):
+        self._downloader.write_debug(f'Identified a {name}')
+
     def _extract_rss(self, url, video_id, doc):
         playlist_title = doc.find('./channel/title').text
         playlist_desc_el = doc.find('./channel/description')
@@ -2540,6 +2556,7 @@ class GenericIE(InfoExtractor):
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
+            self.report_detected('direct video link')
             format_id = compat_str(m.group('format_id'))
             subtitles = {}
             if format_id.endswith('mpegurl'):
@@ -2580,6 +2597,7 @@ class GenericIE(InfoExtractor):
 
         # Is it an M3U playlist?
         if first_bytes.startswith(b'#EXTM3U'):
+            self.report_detected('M3U playlist')
             info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
             self._sort_formats(info_dict['formats'])
             return info_dict
@@ -2610,16 +2628,20 @@ class GenericIE(InfoExtractor):
             except compat_xml_parse_error:
                 doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
+                self.report_detected('RSS feed')
                 return self._extract_rss(url, video_id, doc)
             elif doc.tag == 'SmoothStreamingMedia':
                 info_dict['formats'], info_dict['subtitles'] = self._parse_ism_formats_and_subtitles(doc, url)
+                self.report_detected('ISM manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                 smil = self._parse_smil(doc, url, video_id)
+                self.report_detected('SMIL file')
                 self._sort_formats(smil['formats'])
                 return smil
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
+                self.report_detected('XSPF playlist')
                 return self.playlist_result(
                     self._parse_xspf(
                         doc, video_id, xspf_url=url,
@@ -2630,10 +2652,12 @@ class GenericIE(InfoExtractor):
                     doc,
                     mpd_base_url=full_response.geturl().rpartition('/')[0],
                     mpd_url=url)
+                self.report_detected('DASH manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
+                self.report_detected('F4M manifest')
                 self._sort_formats(info_dict['formats'])
                 return info_dict
         except compat_xml_parse_error:
@@ -2642,6 +2666,7 @@ class GenericIE(InfoExtractor):
         # Is it a Camtasia project?
         camtasia_res = self._extract_camtasia(url, video_id, webpage)
         if camtasia_res is not None:
+            self.report_detected('Camtasia video')
             return camtasia_res
 
         # Sometimes embedded video player is hidden behind percent encoding
@@ -2692,6 +2717,8 @@ class GenericIE(InfoExtractor):
             'age_limit': age_limit,
         })
 
+        self._downloader.write_debug('Looking for video embeds')
+
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
@@ -3482,9 +3509,14 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 rumble_urls, video_id, video_title, ie=RumbleEmbedIE.ie_key())
 
+        tvp_urls = TVPEmbedIE._extract_urls(webpage)
+        if tvp_urls:
+            return self.playlist_from_matches(tvp_urls, video_id, video_title, ie=TVPEmbedIE.ie_key())
+
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
+            self.report_detected('HTML5 media')
             if len(entries) == 1:
                 entries[0].update({
                     'id': video_id,
@@ -3503,9 +3535,18 @@ class GenericIE(InfoExtractor):
         jwplayer_data = self._find_jwplayer_data(
             webpage, video_id, transform_source=js_to_json)
         if jwplayer_data:
+            if isinstance(jwplayer_data.get('playlist'), str):
+                self.report_detected('JW Player playlist')
+                return {
+                    **info_dict,
+                    '_type': 'url',
+                    'ie_key': JWPlatformIE.ie_key(),
+                    'url': jwplayer_data['playlist'],
+                }
             try:
                 info = self._parse_jwplayer_data(
                     jwplayer_data, video_id, require_title=False, base_url=url)
+                self.report_detected('JW Player data')
                 return merge_dicts(info, info_dict)
             except ExtractorError:
                 # See https://github.com/ytdl-org/youtube-dl/pull/16735
@@ -3555,15 +3596,16 @@ class GenericIE(InfoExtractor):
                         },
                     })
             if formats or subtitles:
+                self.report_detected('video.js embed')
                 self._sort_formats(formats)
                 info_dict['formats'] = formats
                 info_dict['subtitles'] = subtitles
                 return info_dict
 
         # Looking for http://schema.org/VideoObject
-        json_ld = self._search_json_ld(
-            webpage, video_id, default={}, expected_type='VideoObject')
+        json_ld = self._search_json_ld(webpage, video_id, default={})
         if json_ld.get('url'):
+            self.report_detected('JSON LD')
             return merge_dicts(json_ld, info_dict)
 
         def check_video(vurl):
@@ -3580,7 +3622,9 @@ class GenericIE(InfoExtractor):
 
         # Start with something easy: JW Player in SWFObject
         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
-        if not found:
+        if found:
+            self.report_detected('JW Player in SFWObject')
+        else:
             # Look for gorilla-vid style embedding
             found = filter_video(re.findall(r'''(?sx)
                 (?:
@@ -3590,10 +3634,13 @@ class GenericIE(InfoExtractor):
                 )
                 .*?
                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
+            if found:
+                self.report_detected('JW Player embed')
         if not found:
             # Look for generic KVS player
             found = re.search(r'<script [^>]*?src="https://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
             if found:
+                self.report_detected('KWS Player')
                 if found.group('maj_ver') not in ['4', '5']:
                     self.report_warning('Untested major version (%s) in player engine--Download may fail.' % found.group('ver'))
                 flashvars = re.search(r'(?ms)<script.*?>.*?var\s+flashvars\s*=\s*(\{.*?\});.*?</script>', webpage)
@@ -3639,10 +3686,14 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the search a little bit
             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
+            if found:
+                self.report_detected('video file')
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = filter_video(re.findall(
                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
+            if found:
+                self.report_detected('JW Player JS loader')
         if not found:
             # Flow player
             found = filter_video(re.findall(r'''(?xs)
@@ -3651,10 +3702,14 @@ class GenericIE(InfoExtractor):
                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
+            if found:
+                self.report_detected('Flow Player')
         if not found:
             # Cinerama player
             found = re.findall(
                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
+            if found:
+                self.report_detected('Cinerama player')
         if not found:
             # Try to find twitter cards info
             # twitter:player:stream should be checked before twitter:player since
@@ -3662,6 +3717,8 @@ class GenericIE(InfoExtractor):
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             found = filter_video(re.findall(
                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
+            if found:
+                self.report_detected('Twitter card')
         if not found:
             # We look for Open Graph info:
             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
@@ -3669,6 +3726,8 @@ class GenericIE(InfoExtractor):
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 found = filter_video(re.findall(r'<meta.*?property="og:(?:video|audio)".*?content="(.*?)"', webpage))
+                if found:
+                    self.report_detected('Open Graph video info')
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
@@ -3700,6 +3759,7 @@ class GenericIE(InfoExtractor):
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
             if embed_url and embed_url != url:
+                self.report_detected('twitter:player iframe')
                 return self.url_result(embed_url)
 
         if not found:

yt_dlp/extractor/imdb.py

@@ -111,7 +111,7 @@ class ImdbIE(InfoExtractor):
             'formats': formats,
             'description': info.get('videoDescription'),
             'thumbnail': url_or_none(try_get(
-                video_metadata, lambda x: x['videoSlate']['source'])),
+                info, lambda x: x['videoSlate']['source'])),
             'duration': parse_duration(info.get('videoRuntime')),
         }
 

yt_dlp/extractor/instagram.py

@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -25,9 +26,98 @@ from ..utils import (
 )
 
 
-class InstagramIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
+class InstagramBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'instagram'
+    _IS_LOGGED_IN = False
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None or self._IS_LOGGED_IN:
+            return
+
+        login_webpage = self._download_webpage(
+            'https://www.instagram.com/accounts/login/', None,
+            note='Downloading login webpage', errnote='Failed to download login webpage')
+
+        shared_data = self._parse_json(
+            self._search_regex(
+                r'window\._sharedData\s*=\s*({.+?});',
+                login_webpage, 'shared data', default='{}'),
+            None)
+
+        login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
+            'Accept': '*/*',
+            'X-IG-App-ID': '936619743392459',
+            'X-ASBD-ID': '198387',
+            'X-IG-WWW-Claim': '0',
+            'X-Requested-With': 'XMLHttpRequest',
+            'X-CSRFToken': shared_data['config']['csrf_token'],
+            'X-Instagram-AJAX': shared_data['rollout_hash'],
+            'Referer': 'https://www.instagram.com/',
+        }, data=urlencode_postdata({
+            'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
+            'username': username,
+            'queryParams': '{}',
+            'optIntoOneTap': 'false',
+            'stopDeletionNonce': '',
+            'trustedDeviceRecords': '{}',
+        }))
+
+        if not login.get('authenticated'):
+            if login.get('message'):
+                raise ExtractorError(f'Unable to login: {login["message"]}')
+            raise ExtractorError('Unable to login')
+        InstagramBaseIE._IS_LOGGED_IN = True
+
+    def _real_initialize(self):
+        self._login()
+
+
+class InstagramIOSIE(InfoExtractor):
+    IE_DESC = 'IOS instagram:// URL'
+    _VALID_URL = r'instagram://media\?id=(?P<id>[\d_]+)'
+    _TESTS = [{
+        'url': 'instagram://media?id=482584233761418119',
+        'md5': '0d2da106a9d2631273e192b372806516',
+        'info_dict': {
+            'id': 'aye83DjauH',
+            'ext': 'mp4',
+            'title': 'Video by naomipq',
+            'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 0,
+            'timestamp': 1371748545,
+            'upload_date': '20130620',
+            'uploader_id': 'naomipq',
+            'uploader': 'B E A U T Y  F O R  A S H E S',
+            'like_count': int,
+            'comment_count': int,
+            'comments': list,
+        },
+        'add_ie': ['Instagram']
+    }]
+
+    def _get_id(self, id):
+        """Source: https://stackoverflow.com/questions/24437823/getting-instagram-post-url-from-media-id"""
+        chrs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+        media_id = int(id.split('_')[0])
+        shortened_id = ''
+        while media_id > 0:
+            r = media_id % 64
+            media_id = (media_id - r) // 64
+            shortened_id = chrs[r] + shortened_id
+        return shortened_id
+
+    def _real_extract(self, url):
+        return {
+            '_type': 'url_transparent',
+            'url': f'http://instagram.com/tv/{self._get_id(self._match_id(url))}/',
+            'ie_key': 'Instagram',
+        }
+
+
+class InstagramIE(InstagramBaseIE):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
@@ -143,45 +233,6 @@ class InstagramIE(InfoExtractor):
         if mobj:
             return mobj.group('link')
 
-    def _login(self):
-        username, password = self._get_login_info()
-
-        login_webpage = self._download_webpage(
-            'https://www.instagram.com/accounts/login/', None,
-            note='Downloading login webpage', errnote='Failed to download login webpage')
-
-        shared_data = self._parse_json(
-            self._search_regex(
-                r'window\._sharedData\s*=\s*({.+?});',
-                login_webpage, 'shared data', default='{}'),
-            None)
-
-        login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
-            'Accept': '*/*',
-            'X-IG-App-ID': '936619743392459',
-            'X-ASBD-ID': '198387',
-            'X-IG-WWW-Claim': '0',
-            'X-Requested-With': 'XMLHttpRequest',
-            'X-CSRFToken': shared_data['config']['csrf_token'],
-            'X-Instagram-AJAX': shared_data['rollout_hash'],
-            'Referer': 'https://www.instagram.com/',
-        }, data=urlencode_postdata({
-            'enc_password': f'#PWD_INSTAGRAM_BROWSER:0:{int(time.time())}:{password}',
-            'username': username,
-            'queryParams': '{}',
-            'optIntoOneTap': 'false',
-            'stopDeletionNonce': '',
-            'trustedDeviceRecords': '{}',
-        }))
-
-        if not login.get('authenticated'):
-            if login.get('message'):
-                raise ExtractorError(f'Unable to login: {login["message"]}')
-            raise ExtractorError('Unable to login')
-
-    def _real_initialize(self):
-        self._login()
-
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id')
@@ -191,7 +242,7 @@ class InstagramIE(InfoExtractor):
         if 'www.instagram.com/accounts/login' in urlh.geturl().rstrip('/'):
             self.raise_login_required('You need to log in to access this content')
 
-        (media, video_url, description, thumbnail, timestamp, uploader,
+        (media, video_url, description, thumbnails, timestamp, uploader,
          uploader_id, like_count, comment_count, comments, height,
          width) = [None] * 12
 
@@ -220,17 +271,19 @@ class InstagramIE(InfoExtractor):
                     dict)
         if media:
             video_url = media.get('video_url')
-            height = int_or_none(media.get('dimensions', {}).get('height'))
-            width = int_or_none(media.get('dimensions', {}).get('width'))
+            height = int_or_none(self._html_search_meta(('og:video:height', 'video:height'), webpage)) or try_get(media, lambda x: x['dimensions']['height'])
+            width = int_or_none(self._html_search_meta(('og:video:width', 'video:width'), webpage)) or try_get(media, lambda x: x['dimensions']['width'])
             description = try_get(
                 media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
                 compat_str) or media.get('caption')
             title = media.get('title')
-            thumbnail = media.get('display_src') or media.get('display_url')
+            display_resources = media.get('display_resources')
+            if not display_resources:
+                display_resources = [{'src': media.get('display_src')}, {'src': media.get('display_url')}]
             duration = float_or_none(media.get('video_duration'))
             timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
-            uploader = media.get('owner', {}).get('full_name')
-            uploader_id = media.get('owner', {}).get('username')
+            uploader = try_get(media, lambda x: x['owner']['full_name'])
+            uploader_id = try_get(media, lambda x: x['owner']['username'])
 
             def get_count(keys, kind):
                 for key in variadic(keys):
@@ -244,6 +297,12 @@ class InstagramIE(InfoExtractor):
             comment_count = get_count(
                 ('preview_comment', 'to_comment', 'to_parent_comment'), 'comment')
 
+            thumbnails = [{
+                'url': thumbnail['src'],
+                'width': thumbnail.get('config_width'),
+                'height': thumbnail.get('config_height'),
+            } for thumbnail in display_resources if thumbnail.get('src')]
+
             comments = []
             for comment in try_get(media, lambda x: x['edge_media_to_parent_comment']['edges']):
                 comment_dict = comment.get('node', {})
@@ -292,6 +351,10 @@ class InstagramIE(InfoExtractor):
             'width': width,
             'height': height,
         }]
+        dash = try_get(media, lambda x: x['dash_info']['video_dash_manifest'])
+        if dash:
+            formats.extend(self._parse_mpd_formats(self._parse_xml(dash, video_id), mpd_id='dash'))
+        self._sort_formats(formats)
 
         if not uploader_id:
             uploader_id = self._search_regex(
@@ -304,8 +367,8 @@ class InstagramIE(InfoExtractor):
             if description is not None:
                 description = lowercase_escape(description)
 
-        if not thumbnail:
-            thumbnail = self._og_search_thumbnail(webpage)
+        if not thumbnails:
+            thumbnails = self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
@@ -314,7 +377,7 @@ class InstagramIE(InfoExtractor):
             'title': title or 'Video by %s' % uploader_id,
             'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'timestamp': timestamp,
             'uploader_id': uploader_id,
             'uploader': uploader,
@@ -327,10 +390,7 @@ class InstagramIE(InfoExtractor):
         }
 
 
-class InstagramPlaylistIE(InfoExtractor):
-    # A superclass for handling any kind of query based on GraphQL which
-    # results in a playlist.
-
+class InstagramPlaylistBaseIE(InstagramBaseIE):
     _gis_tmpl = None  # used to cache GIS request type
 
     def _parse_graphql(self, webpage, item_id):
@@ -456,11 +516,11 @@ class InstagramPlaylistIE(InfoExtractor):
             self._extract_graphql(data, url), user_or_tag, user_or_tag)
 
 
-class InstagramUserIE(InstagramPlaylistIE):
+class InstagramUserIE(InstagramPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/porsche',
         'info_dict': {
             'id': 'porsche',
@@ -472,7 +532,7 @@ class InstagramUserIE(InstagramPlaylistIE):
             'skip_download': True,
             'playlistend': 5,
         }
-    }
+    }]
 
     _QUERY_HASH = '42323d64886122307be10013ad2dcc44',
 
@@ -490,11 +550,11 @@ class InstagramUserIE(InstagramPlaylistIE):
         }
 
 
-class InstagramTagIE(InstagramPlaylistIE):
+class InstagramTagIE(InstagramPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?instagram\.com/explore/tags/(?P<id>[^/]+)'
     IE_DESC = 'Instagram hashtag search'
     IE_NAME = 'instagram:tag'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/explore/tags/lolcats',
         'info_dict': {
             'id': 'lolcats',
@@ -506,7 +566,7 @@ class InstagramTagIE(InstagramPlaylistIE):
             'skip_download': True,
             'playlistend': 50,
         }
-    }
+    }]
 
     _QUERY_HASH = 'f92f56d47dc7a55b606908374b43a314',
 

yt_dlp/extractor/internazionale.py

@@ -20,9 +20,6 @@ class InternazionaleIE(InfoExtractor):
             'upload_date': '20150219',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'https://www.internazionale.it/video/2018/08/29/telefono-stare-con-noi-stessi',
         'md5': '9db8663704cab73eb972d1cee0082c79',
@@ -36,9 +33,6 @@ class InternazionaleIE(InfoExtractor):
             'upload_date': '20180829',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/iprima.py

@@ -8,12 +8,19 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     js_to_json,
+    urlencode_postdata,
+    ExtractorError,
+    parse_qs
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
+    _NETRC_MACHINE = 'iprima'
+    _LOGIN_URL = 'https://auth.iprima.cz/oauth2/login'
+    _TOKEN_URL = 'https://auth.iprima.cz/oauth2/token'
+    access_token = None
 
     _TESTS = [{
         'url': 'https://prima.iprima.cz/particka/92-epizoda',
@@ -22,16 +29,8 @@ class IPrimaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Partička (92)',
             'description': 'md5:859d53beae4609e6dd7796413f1b6cac',
-        },
-        'params': {
-            'skip_download': True,  # m3u8 download
-        },
-    }, {
-        'url': 'https://cnn.iprima.cz/videa/70-epizoda',
-        'info_dict': {
-            'id': 'p681554',
-            'ext': 'mp4',
-            'title': 'HLAVNÍ ZPRÁVY 3.5.2020',
+            'upload_date': '20201103',
+            'timestamp': 1604437480,
         },
         'params': {
             'skip_download': True,  # m3u8 download
@@ -44,11 +43,9 @@ class IPrimaIE(InfoExtractor):
         'url': 'http://play.iprima.cz/closer-nove-pripady/closer-nove-pripady-iv-1',
         'only_matching': True,
     }, {
-        # iframe api.play-backend.iprima.cz
         'url': 'https://prima.iprima.cz/my-little-pony/mapa-znameni-2-2',
         'only_matching': True,
     }, {
-        # iframe prima.iprima.cz
         'url': 'https://prima.iprima.cz/porady/jak-se-stavi-sen/rodina-rathousova-praha',
         'only_matching': True,
     }, {
@@ -66,9 +63,127 @@ class IPrimaIE(InfoExtractor):
     }, {
         'url': 'https://love.iprima.cz/laska-az-za-hrob/slib-dany-bratrovi',
         'only_matching': True,
-    }, {
-        'url': 'https://autosalon.iprima.cz/motorsport/7-epizoda-1',
-        'only_matching': True,
+    }]
+
+    def _login(self):
+        username, password = self._get_login_info()
+
+        if username is None or password is None:
+            self.raise_login_required('Login is required to access any iPrima content', method='password')
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, note='Downloading login page',
+            errnote='Downloading login page failed')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            '_email': username,
+            '_password': password})
+
+        _, login_handle = self._download_webpage_handle(
+            self._LOGIN_URL, None, data=urlencode_postdata(login_form),
+            note='Logging in')
+
+        code = parse_qs(login_handle.geturl()).get('code')[0]
+        if not code:
+            raise ExtractorError('Login failed', expected=True)
+
+        token_request_data = {
+            'scope': 'openid+email+profile+phone+address+offline_access',
+            'client_id': 'prima_sso',
+            'grant_type': 'authorization_code',
+            'code': code,
+            'redirect_uri': 'https://auth.iprima.cz/sso/auth-check'}
+
+        token_data = self._download_json(
+            self._TOKEN_URL, None,
+            note='Downloading token', errnote='Downloading token failed',
+            data=urlencode_postdata(token_request_data))
+
+        self.access_token = token_data.get('access_token')
+        if self.access_token is None:
+            raise ExtractorError('Getting token failed', expected=True)
+
+    def _raise_access_error(self, error_code):
+        if error_code == 'PLAY_GEOIP_DENIED':
+            self.raise_geo_restricted(countries=['CZ'], metadata_available=True)
+        elif error_code is not None:
+            self.raise_no_formats('Access to stream infos forbidden', expected=True)
+
+    def _real_initialize(self):
+        if not self.access_token:
+            self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title'],
+            webpage, 'title', default=None)
+
+        video_id = self._search_regex((
+            r'productId\s*=\s*([\'"])(?P<id>p\d+)\1',
+            r'pproduct_id\s*=\s*([\'"])(?P<id>p\d+)\1'),
+            webpage, 'real id', group='id')
+
+        metadata = self._download_json(
+            f'https://api.play-backend.iprima.cz/api/v1//products/id-{video_id}/play',
+            video_id, note='Getting manifest URLs', errnote='Failed to get manifest URLs',
+            headers={'X-OTT-Access-Token': self.access_token},
+            expected_status=403)
+
+        self._raise_access_error(metadata.get('errorCode'))
+
+        stream_infos = metadata.get('streamInfos')
+        formats = []
+        if stream_infos is None:
+            self.raise_no_formats('Reading stream infos failed', expected=True)
+        else:
+            for manifest in stream_infos:
+                manifest_type = manifest.get('type')
+                manifest_url = manifest.get('url')
+                ext = determine_ext(manifest_url)
+                if manifest_type == 'HLS' or ext == 'm3u8':
+                    formats += self._extract_m3u8_formats(
+                        manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False)
+                elif manifest_type == 'DASH' or ext == 'mpd':
+                    formats += self._extract_mpd_formats(
+                        manifest_url, video_id, mpd_id='dash', fatal=False)
+            self._sort_formats(formats)
+
+        final_result = self._search_json_ld(webpage, video_id) or {}
+        final_result.update({
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._html_search_meta(
+                ['thumbnail', 'og:image', 'twitter:image'],
+                webpage, 'thumbnail', default=None),
+            'formats': formats,
+            'description': self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'],
+                webpage, 'description', default=None)})
+
+        return final_result
+
+
+class IPrimaCNNIE(InfoExtractor):
+    _VALID_URL = r'https?://cnn\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _GEO_BYPASS = False
+
+    _TESTS = [{
+        'url': 'https://cnn.iprima.cz/porady/strunc/24072020-koronaviru-mam-plne-zuby-strasit-druhou-vlnou-je-absurdni-rika-senatorka-dernerova',
+        'info_dict': {
+            'id': 'p716177',
+            'ext': 'mp4',
+            'title': 'md5:277c6b1ed0577e51b40ddd35602ff43e',
+        },
+        'params': {
+            'skip_download': 'm3u8'
+        }
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/itv.py

@@ -220,16 +220,23 @@ class ITVIE(InfoExtractor):
 
 
 class ITVBTCCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:news|btcc)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action',
         'info_dict': {
             'id': 'btcc-2019-brands-hatch-gp-race-action',
             'title': 'BTCC 2019: Brands Hatch GP race action',
         },
         'playlist_count': 12,
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
+    }, {
+        'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+        'info_dict': {
+            'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+            'title': 'md5:6ef054dd9f069330db3dcc66cb772d32'
+        },
+        'playlist_count': 4
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -240,15 +247,15 @@ class ITVBTCCIE(InfoExtractor):
             '(?s)<script[^>]+id=[\'"]__NEXT_DATA__[^>]*>([^<]+)</script>', webpage, 'json_map'), playlist_id),
             lambda x: x['props']['pageProps']['article']['body']['content']) or []
 
-        # Discard empty objects
-        video_ids = []
+        entries = []
         for video in json_map:
-            if video['data'].get('id'):
-                video_ids.append(video['data']['id'])
-
-        entries = [
-            self.url_result(
-                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
+            if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')):
+                continue
+            video_id = video['data']['id']
+            account_id = video['data']['accountId']
+            player_id = video['data']['playerId']
+            entries.append(self.url_result(
+                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id), {
                     # ITV does not like some GB IP ranges, so here are some
                     # IP blocks it accepts
                     'geo_ip_blocks': [
@@ -256,8 +263,7 @@ class ITVBTCCIE(InfoExtractor):
                     ],
                     'referrer': url,
                 }),
-                ie=BrightcoveNewIE.ie_key(), video_id=video_id)
-            for video_id in video_ids]
+                ie=BrightcoveNewIE.ie_key(), video_id=video_id))
 
         title = self._og_search_title(webpage, fatal=False)
 

yt_dlp/extractor/kinopoisk.py

@@ -23,9 +23,6 @@ class KinoPoiskIE(InfoExtractor):
             'duration': 4533,
             'age_limit': 12,
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'https://www.kinopoisk.ru/film/81041',
         'only_matching': True,

yt_dlp/extractor/la7.py

@@ -7,8 +7,9 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     float_or_none,
+    HEADRequest,
+    int_or_none,
     parse_duration,
-    smuggle_url,
     unified_strdate,
 )
 
@@ -25,19 +26,38 @@ class LA7IE(InfoExtractor):
         'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
         'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
         'info_dict': {
-            'id': '0_42j6wd36',
+            'id': 'inccool8-02-10-2015-163722',
             'ext': 'mp4',
             'title': 'Inc.Cool8',
             'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
             'thumbnail': 're:^https?://.*',
-            'uploader_id': 'kdla7pillole@iltrovatore.it',
-            'timestamp': 1443814869,
             'upload_date': '20151002',
         },
     }, {
         'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
         'only_matching': True,
     }]
+    _HOST = 'https://awsvodpkg.iltrovatore.it'
+
+    def _generate_mp4_url(self, quality, m3u8_formats):
+        for f in m3u8_formats:
+            if f['vcodec'] != 'none' and quality in f['url']:
+                http_url = '%s%s.mp4' % (self._HOST, quality)
+
+                urlh = self._request_webpage(
+                    HEADRequest(http_url), quality,
+                    note='Check filesize', fatal=False)
+                if urlh:
+                    http_f = f.copy()
+                    del http_f['manifest_url']
+                    http_f.update({
+                        'format_id': http_f['format_id'].replace('hls-', 'https-'),
+                        'url': http_url,
+                        'protocol': 'https',
+                        'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
+                    })
+                    return http_f
+                return None
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -46,22 +66,30 @@ class LA7IE(InfoExtractor):
             url = '%s//%s' % (self.http_scheme(), url)
 
         webpage = self._download_webpage(url, video_id)
+        video_path = self._search_regex(r'(/content/.*?).mp4', webpage, 'video_path')
 
-        player_data = self._search_regex(
-            [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
-            webpage, 'player data')
-        vid = self._search_regex(r'vid\s*:\s*"(.+?)",', player_data, 'vid')
+        formats = self._extract_mpd_formats(
+            f'{self._HOST}/local/dash/,{video_path}.mp4.urlset/manifest.mpd',
+            video_id, mpd_id='dash', fatal=False)
+        m3u8_formats = self._extract_m3u8_formats(
+            f'{self._HOST}/local/hls/,{video_path}.mp4.urlset/master.m3u8',
+            video_id, 'mp4', m3u8_id='hls', fatal=False)
+        formats.extend(m3u8_formats)
+
+        for q in filter(None, video_path.split(',')):
+            http_f = self._generate_mp4_url(q, m3u8_formats)
+            if http_f:
+                formats.append(http_f)
+
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'url': smuggle_url('kaltura:103:%s' % vid, {
-                'service_url': 'http://nkdam.iltrovatore.it',
-            }),
             'id': video_id,
             'title': self._og_search_title(webpage, default=None),
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
-            'ie_key': 'Kaltura',
+            'formats': formats,
+            'upload_date': unified_strdate(self._search_regex(r'datetime="(.+?)"', webpage, 'upload_date', fatal=False))
         }
 
 

yt_dlp/extractor/lego.py

@@ -8,6 +8,7 @@ from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
+    join_nonempty,
     qualities,
 )
 
@@ -102,12 +103,8 @@ class LEGOIE(InfoExtractor):
                     m3u8_id=video_source_format, fatal=False))
             else:
                 video_source_quality = video_source.get('Quality')
-                format_id = []
-                for v in (video_source_format, video_source_quality):
-                    if v:
-                        format_id.append(v)
                 f = {
-                    'format_id': '-'.join(format_id),
+                    'format_id': join_nonempty(video_source_format, video_source_quality),
                     'quality': q(video_source_quality),
                     'url': video_source_url,
                 }

yt_dlp/extractor/linkedin.py

@@ -19,6 +19,7 @@ from ..utils import (
 class LinkedInLearningBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'linkedin'
     _LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
+    _logged_in = False
 
     def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
         query = {
@@ -34,6 +35,8 @@ class LinkedInLearningBaseIE(InfoExtractor):
             })
             sub = ' %dp' % resolution
         api_url = 'https://www.linkedin.com/learning-api/detailedCourses'
+        if not self._get_cookies(api_url).get('JSESSIONID'):
+            self.raise_login_required()
         return self._download_json(
             api_url, video_slug, 'Downloading%s JSON metadata' % sub, headers={
                 'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
@@ -50,6 +53,8 @@ class LinkedInLearningBaseIE(InfoExtractor):
         return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
 
     def _real_initialize(self):
+        if self._logged_in:
+            return
         email, password = self._get_login_info()
         if email is None:
             return
@@ -72,6 +77,7 @@ class LinkedInLearningBaseIE(InfoExtractor):
             login_submit_page, 'error', default=None)
         if error:
             raise ExtractorError(error, expected=True)
+        LinkedInLearningBaseIE._logged_in = True
 
 
 class LinkedInLearningIE(LinkedInLearningBaseIE):

yt_dlp/extractor/mdr.py

@@ -2,13 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    join_nonempty,
     parse_duration,
     parse_iso8601,
     url_or_none,
@@ -148,13 +146,9 @@ class MDRIE(InfoExtractor):
                     abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
                     filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))
 
-                    format_id = [media_type]
-                    if vbr or abr:
-                        format_id.append(compat_str(vbr or abr))
-
                     f = {
                         'url': video_url,
-                        'format_id': '-'.join(format_id),
+                        'format_id': join_nonempty(media_type, vbr or abr),
                         'filesize': filesize,
                         'abr': abr,
                         'vbr': vbr,

yt_dlp/extractor/mediaset.py

@@ -1,13 +1,17 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import re
 
 from .theplatform import ThePlatformBaseIE
 from ..utils import (
     ExtractorError,
     int_or_none,
+    OnDemandPagedList,
     parse_qs,
+    try_get,
+    urljoin,
     update_url_query,
 )
 
@@ -212,3 +216,81 @@ class MediasetIE(ThePlatformBaseIE):
             'subtitles': subtitles,
         })
         return info
+
+
+class MediasetShowIE(MediasetIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        https?://
+                            (?:(?:www|static3)\.)?mediasetplay\.mediaset\.it/
+                            (?:
+                                (?:fiction|programmi-tv|serie-tv)/(?:.+?/)?
+                                    (?:[a-z]+)_SE(?P<id>\d{12})
+                                    (?:,ST(?P<st>\d{12}))?
+                                    (?:,sb(?P<sb>\d{9}))?$
+                            )
+                    )
+                    '''
+    _TESTS = [{
+        # TV Show webpage (with a single playlist)
+        'url': 'https://www.mediasetplay.mediaset.it/serie-tv/fireforce/episodi_SE000000001556',
+        'info_dict': {
+            'id': '000000001556',
+            'title': 'Fire Force',
+        },
+        'playlist_count': 1,
+    }, {
+        # TV Show webpage (with multiple playlists)
+        'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061,ST000000002763',
+        'info_dict': {
+            'id': '000000002763',
+            'title': 'Le Iene',
+        },
+        'playlist_count': 7,
+    }, {
+        # TV Show specific playlist (single page)
+        'url': 'https://www.mediasetplay.mediaset.it/serie-tv/fireforce/episodi_SE000000001556,ST000000002738,sb100013107',
+        'info_dict': {
+            'id': '100013107',
+            'title': 'Episodi',
+        },
+        'playlist_count': 4,
+    }, {
+        # TV Show specific playlist (with multiple pages)
+        'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/iservizi_SE000000000061,ST000000002763,sb100013375',
+        'info_dict': {
+            'id': '100013375',
+            'title': 'I servizi',
+        },
+        'playlist_count': 53,
+    }]
+
+    _BY_SUBBRAND = 'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2?byCustomValue={subBrandId}{%s}&sort=:publishInfo_lastPublished|desc,tvSeasonEpisodeNumber|desc&range=%d-%d'
+    _PAGE_SIZE = 25
+
+    def _fetch_page(self, sb, page):
+        lower_limit = page * self._PAGE_SIZE + 1
+        upper_limit = lower_limit + self._PAGE_SIZE - 1
+        content = self._download_json(
+            self._BY_SUBBRAND % (sb, lower_limit, upper_limit), sb)
+        for entry in content.get('entries') or []:
+            yield self.url_result(
+                'mediaset:' + entry['guid'],
+                playlist_title=entry['mediasetprogram$subBrandDescription'])
+
+    def _real_extract(self, url):
+        playlist_id, st, sb = self._match_valid_url(url).group('id', 'st', 'sb')
+        if not sb:
+            page = self._download_webpage(url, playlist_id)
+            entries = [self.url_result(urljoin('https://www.mediasetplay.mediaset.it', url))
+                       for url in re.findall(r'href="([^<>=]+SE\d{12},ST\d{12},sb\d{9})">[^<]+<', page)]
+            title = (self._html_search_regex(r'(?s)<h1[^>]*>(.+?)</h1>', page, 'title', default=None)
+                     or self._og_search_title(page))
+            return self.playlist_result(entries, st or playlist_id, title)
+
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, sb),
+            self._PAGE_SIZE)
+        title = try_get(entries, lambda x: x[0]['playlist_title'])
+
+        return self.playlist_result(entries, sb, title)

yt_dlp/extractor/mlssoccer.py

@@ -0,0 +1,118 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MLSSoccerIE(InfoExtractor):
+    _VALID_DOMAINS = r'(?:(?:cfmontreal|intermiamicf|lagalaxy|lafc|houstondynamofc|dcunited|atlutd|mlssoccer|fcdallas|columbuscrew|coloradorapids|fccincinnati|chicagofirefc|austinfc|nashvillesc|whitecapsfc|sportingkc|soundersfc|sjearthquakes|rsl|timbers|philadelphiaunion|orlandocitysc|newyorkredbulls|nycfc)\.com|(?:torontofc)\.ca|(?:revolutionsoccer)\.net)'
+    _VALID_URL = r'(?:https?://)(?:www\.)?%s/video/#?(?P<id>[^/&$#?]+)' % _VALID_DOMAINS
+
+    _TESTS = [{
+        'url': 'https://www.mlssoccer.com/video/the-octagon-can-alphonso-davies-lead-canada-to-first-world-cup-since-1986#the-octagon-can-alphonso-davies-lead-canada-to-first-world-cup-since-1986',
+        'info_dict': {
+            'id': '6276033198001',
+            'ext': 'mp4',
+            'title': 'The Octagon | Can Alphonso Davies lead Canada to first World Cup since 1986?',
+            'description': 'md5:f0a883ee33592a0221798f451a98be8f',
+            'thumbnail': 'https://cf-images.us-east-1.prod.boltdns.net/v1/static/5530036772001/1bbc44f6-c63c-4981-82fa-46b0c1f891e0/5c1ca44a-a033-4e98-b531-ff24c4947608/160x90/match/image.jpg',
+            'duration': 350.165,
+            'timestamp': 1633627291,
+            'uploader_id': '5530036772001',
+            'tags': ['club/canada'],
+            'is_live': False,
+            'duration_string': '5:50',
+            'upload_date': '20211007',
+            'filesize_approx': 255193528.83200002
+        },
+        'params': {'skip_download': True}
+    }, {
+        'url': 'https://www.whitecapsfc.com/video/highlights-san-jose-earthquakes-vs-vancouver-whitecaps-fc-october-23-2021#highlights-san-jose-earthquakes-vs-vancouver-whitecaps-fc-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.torontofc.ca/video/highlights-toronto-fc-vs-cf-montreal-october-23-2021-x6733#highlights-toronto-fc-vs-cf-montreal-october-23-2021-x6733',
+        'only_matching': True
+    }, {
+        'url': 'https://www.sportingkc.com/video/post-match-press-conference-john-pulskamp-oct-27-2021#post-match-press-conference-john-pulskamp-oct-27-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.soundersfc.com/video/highlights-seattle-sounders-fc-vs-sporting-kansas-city-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.sjearthquakes.com/video/#highlights-austin-fc-vs-san-jose-earthquakes-june-19-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.rsl.com/video/2021-u-of-u-health-mic-d-up-vs-colorado-10-16-21#2021-u-of-u-health-mic-d-up-vs-colorado-10-16-21',
+        'only_matching': True
+    }, {
+        'url': 'https://www.timbers.com/video/highlights-d-chara-asprilla-with-goals-in-portland-timbers-2-0-win-over-san-jose#highlights-d-chara-asprilla-with-goals-in-portland-timbers-2-0-win-over-san-jose',
+        'only_matching': True
+    }, {
+        'url': 'https://www.philadelphiaunion.com/video/highlights-torvphi',
+        'only_matching': True
+    }, {
+        'url': 'https://www.orlandocitysc.com/video/highlight-columbus-crew-vs-orlando-city-sc',
+        'only_matching': True
+    }, {
+        'url': 'https://www.newyorkredbulls.com/video/all-access-matchday-double-derby-week#all-access-matchday-double-derby-week',
+        'only_matching': True
+    }, {
+        'url': 'https://www.nycfc.com/video/highlights-nycfc-1-0-chicago-fire-fc#highlights-nycfc-1-0-chicago-fire-fc',
+        'only_matching': True
+    }, {
+        'url': 'https://www.revolutionsoccer.net/video/two-minute-highlights-revs-1-rapids-0-october-27-2021#two-minute-highlights-revs-1-rapids-0-october-27-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.nashvillesc.com/video/goal-c-j-sapong-nashville-sc-92nd-minute',
+        'only_matching': True
+    }, {
+        'url': 'https://www.cfmontreal.com/video/faits-saillants-tor-v-mtl#faits-saillants-orl-v-mtl-x5645',
+        'only_matching': True
+    }, {
+        'url': 'https://www.intermiamicf.com/video/all-access-victory-vs-nashville-sc-by-ukg#all-access-victory-vs-nashville-sc-by-ukg',
+        'only_matching': True
+    }, {
+        'url': 'https://www.lagalaxy.com/video/#moment-of-the-month-presented-by-san-manuel-casino-rayan-raveloson-scores-his-se',
+        'only_matching': True
+    }, {
+        'url': 'https://www.lafc.com/video/breaking-down-lafc-s-final-6-matches-of-the-2021-mls-regular-season#breaking-down-lafc-s-final-6-matches-of-the-2021-mls-regular-season',
+        'only_matching': True
+    }, {
+        'url': 'https://www.houstondynamofc.com/video/postgame-press-conference-michael-nelson-presented-by-coushatta-casino-res-x9660#postgame-press-conference-michael-nelson-presented-by-coushatta-casino-res-x9660',
+        'only_matching': True
+    }, {
+        'url': 'https://www.dcunited.com/video/tony-alfaro-my-family-pushed-me-to-believe-everything-was-possible',
+        'only_matching': True
+    }, {
+        'url': 'https://www.fcdallas.com/video/highlights-fc-dallas-vs-minnesota-united-fc-october-02-2021#highlights-fc-dallas-vs-minnesota-united-fc-october-02-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.columbuscrew.com/video/match-rewind-columbus-crew-vs-new-york-red-bulls-october-23-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.coloradorapids.com/video/postgame-reaction-robin-fraser-october-27#postgame-reaction-robin-fraser-october-27',
+        'only_matching': True
+    }, {
+        'url': 'https://www.fccincinnati.com/video/#keeping-cincy-chill-presented-by-coors-lite',
+        'only_matching': True
+    }, {
+        'url': 'https://www.chicagofirefc.com/video/all-access-fire-score-dramatic-road-win-in-cincy#all-access-fire-score-dramatic-road-win-in-cincy',
+        'only_matching': True
+    }, {
+        'url': 'https://www.austinfc.com/video/highlights-colorado-rapids-vs-austin-fc-september-29-2021#highlights-colorado-rapids-vs-austin-fc-september-29-2021',
+        'only_matching': True
+    }, {
+        'url': 'https://www.atlutd.com/video/goal-josef-martinez-scores-in-the-73rd-minute#goal-josef-martinez-scores-in-the-73rd-minute',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        data_json = self._parse_json(self._html_search_regex(r'data-options\=\"([^\"]+)\"', webpage, 'json'), id)['videoList'][0]
+        return {
+            'id': id,
+            '_type': 'url',
+            'url': 'https://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (data_json['accountId'], data_json['videoId']),
+            'ie_key': 'BrightcoveNew',
+        }

yt_dlp/extractor/mtv.py

@@ -15,6 +15,7 @@ from ..utils import (
     float_or_none,
     HEADRequest,
     int_or_none,
+    join_nonempty,
     RegexNotFoundError,
     sanitized_Request,
     strip_or_none,
@@ -99,9 +100,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
                     formats.extend([{
                         'ext': 'flv' if rtmp_video_url.startswith('rtmp') else ext,
                         'url': rtmp_video_url,
-                        'format_id': '-'.join(filter(None, [
+                        'format_id': join_nonempty(
                             'rtmp' if rtmp_video_url.startswith('rtmp') else None,
-                            rendition.get('bitrate')])),
+                            rendition.get('bitrate')),
                         'width': int(rendition.get('width')),
                         'height': int(rendition.get('height')),
                     }])
@@ -305,6 +306,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if not mgid:
             mgid = self._extract_triforce_mgid(webpage)
 
+        if not mgid:
+            mgid = self._search_regex(
+                r'"videoConfig":{"videoId":"(mgid:.*?)"', webpage, 'mgid', default=None)
+
+        if not mgid:
+            mgid = self._search_regex(
+                r'"media":{"video":{"config":{"uri":"(mgid:.*?)"', webpage, 'mgid', default=None)
+
         if not mgid:
             data = self._parse_json(self._search_regex(
                 r'__DATA__\s*=\s*({.+?});', webpage, 'data'), None)
@@ -313,10 +322,6 @@ class MTVServicesInfoExtractor(InfoExtractor):
             video_player = self._extract_child_with_type(ab_testing or main_container, 'VideoPlayer')
             mgid = video_player['props']['media']['video']['config']['uri']
 
-        if not mgid:
-            mgid = self._search_regex(
-                r'"media":{"video":{"config":{"uri":"(mgid:.*?)"', webpage, 'mgid', default=None)
-
         return mgid
 
     def _real_extract(self, url):

yt_dlp/extractor/n1.py

@@ -3,8 +3,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .youtube import YoutubeIE
-from .reddit import RedditRIE
 from .common import InfoExtractor
 from ..utils import (
     unified_timestamp,
@@ -40,7 +38,7 @@ class N1InfoAssetIE(InfoExtractor):
 
 class N1InfoIIE(InfoExtractor):
     IE_NAME = 'N1Info:article'
-    _VALID_URL = r'https?://(?:(?:ba|rs|hr)\.)?n1info\.(?:com|si)/(?:[^/]+/){1,2}(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?:(?:ba|rs|hr)\.)?n1info\.(?:com|si)|nova\.rs)/(?:[^/]+/){1,2}(?P<id>[^/]+)'
     _TESTS = [{
         # Youtube embedded
         'url': 'https://rs.n1info.com/sport-klub/tenis/kako-je-djokovic-propustio-istorijsku-priliku-video/',
@@ -90,9 +88,17 @@ class N1InfoIIE(InfoExtractor):
             'uploader': 'YouLotWhatDontStop',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
+    }, {
+        'url': 'https://nova.rs/vesti/politika/zaklina-tatalovic-ani-brnabic-pricate-lazi-video/',
+        'info_dict': {
+            'id': 'tnjganabrnabicizaklinatatalovic100danavladegp-novas-worldwide',
+            'ext': 'mp4',
+            'title': 'Žaklina Tatalović Ani Brnabić: Pričate laži (VIDEO)',
+            'upload_date': '20211102',
+            'timestamp': 1635861677,
+        },
     }, {
         'url': 'https://hr.n1info.com/vijesti/pravobraniteljica-o-ubojstvu-u-zagrebu-radi-se-o-doista-nezapamcenoj-situaciji/',
         'only_matching': True,
@@ -116,16 +122,16 @@ class N1InfoIIE(InfoExtractor):
                 'title': title,
                 'thumbnail': video_data.get('data-thumbnail'),
                 'timestamp': timestamp,
-                'ie_key': N1InfoAssetIE.ie_key()})
+                'ie_key': 'N1InfoAsset'})
 
         embedded_videos = re.findall(r'(<iframe[^>]+>)', webpage)
         for embedded_video in embedded_videos:
             video_data = extract_attributes(embedded_video)
-            url = video_data.get('src')
+            url = video_data.get('src') or ''
             if url.startswith('https://www.youtube.com'):
-                entries.append(self.url_result(url, ie=YoutubeIE.ie_key()))
+                entries.append(self.url_result(url, ie='Youtube'))
             elif url.startswith('https://www.redditmedia.com'):
-                entries.append(self.url_result(url, ie=RedditRIE.ie_key()))
+                entries.append(self.url_result(url, ie='RedditR'))
 
         return {
             '_type': 'playlist',

yt_dlp/extractor/naver.py

@@ -40,6 +40,7 @@ class NaverBaseIE(InfoExtractor):
                 formats.append({
                     'format_id': '%s_%s' % (stream.get('type') or stream_type, dict_get(encoding_option, ('name', 'id'))),
                     'url': stream_url,
+                    'ext': 'mp4',
                     'width': int_or_none(encoding_option.get('width')),
                     'height': int_or_none(encoding_option.get('height')),
                     'vbr': int_or_none(bitrate.get('video')),
@@ -174,7 +175,7 @@ class NaverLiveIE(InfoExtractor):
         'url': 'https://tv.naver.com/l/52010',
         'info_dict': {
             'id': '52010',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': '[LIVE] 뉴스특보 : "수도권 거리두기, 2주간 2단계로 조정"',
             'description': 'md5:df7f0c237a5ed5e786ce5c91efbeaab3',
             'channel_id': 'NTV-ytnnews24-0',
@@ -184,7 +185,7 @@ class NaverLiveIE(InfoExtractor):
         'url': 'https://tv.naver.com/l/51549',
         'info_dict': {
             'id': '51549',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': '연합뉴스TV - 코로나19 뉴스특보',
             'description': 'md5:c655e82091bc21e413f549c0eaccc481',
             'channel_id': 'NTV-yonhapnewstv-0',
@@ -233,7 +234,7 @@ class NaverLiveIE(InfoExtractor):
                 continue
 
             formats.extend(self._extract_m3u8_formats(
-                quality.get('url'), video_id, 'm3u8',
+                quality.get('url'), video_id, 'mp4',
                 m3u8_id=quality.get('qualityId'), live=True
             ))
         self._sort_formats(formats)

yt_dlp/extractor/newgrounds.py

@@ -6,7 +6,9 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     extract_attributes,
+    get_element_by_id,
     int_or_none,
     parse_count,
     parse_duration,
@@ -29,7 +31,8 @@ class NewgroundsIE(InfoExtractor):
             'timestamp': 1378878540,
             'upload_date': '20130911',
             'duration': 143,
-            'description': 'md5:6d885138814015dfd656c2ddb00dacfc',
+            'view_count': int,
+            'description': 'md5:b8b3c2958875189f07d8e313462e8c4f',
         },
     }, {
         'url': 'https://www.newgrounds.com/portal/view/1',
@@ -41,6 +44,7 @@ class NewgroundsIE(InfoExtractor):
             'uploader': 'Brian-Beaton',
             'timestamp': 955064100,
             'upload_date': '20000406',
+            'view_count': int,
             'description': 'Scrotum plays "catch."',
             'age_limit': 17,
         },
@@ -54,7 +58,8 @@ class NewgroundsIE(InfoExtractor):
             'uploader': 'ZONE-SAMA',
             'timestamp': 1487965140,
             'upload_date': '20170224',
-            'description': 'ZTV News Episode 8 (February 2017)',
+            'view_count': int,
+            'description': 'md5:aff9b330ec2e78ed93b1ad6d017accc6',
             'age_limit': 17,
         },
         'params': {
@@ -70,7 +75,8 @@ class NewgroundsIE(InfoExtractor):
             'uploader': 'Egoraptor',
             'timestamp': 1140663240,
             'upload_date': '20060223',
-            'description': 'Metal Gear is awesome is so is this movie.',
+            'view_count': int,
+            'description': 'md5:9246c181614e23754571995104da92e0',
             'age_limit': 13,
         }
     }, {
@@ -80,7 +86,7 @@ class NewgroundsIE(InfoExtractor):
             'id': '297383',
             'ext': 'swf',
             'title': 'Metal Gear Awesome',
-            'description': 'Metal Gear is awesome is so is this movie.',
+            'description': 'Metal Gear Awesome',
             'uploader': 'Egoraptor',
             'upload_date': '20060223',
             'timestamp': 1140663240,
@@ -145,10 +151,13 @@ class NewgroundsIE(InfoExtractor):
             (r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+</dd>\s*<dd>[^<]+)',
              r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+)'), webpage, 'timestamp',
             default=None))
+
         duration = parse_duration(self._html_search_regex(
             r'"duration"\s*:\s*["\']?(\d+)["\']?', webpage,
             'duration', default=None))
 
+        description = clean_html(get_element_by_id('author_comments', webpage)) or self._og_search_description(webpage)
+
         view_count = parse_count(self._html_search_regex(
             r'(?s)<dt>\s*(?:Views|Listens)\s*</dt>\s*<dd>([\d\.,]+)</dd>', webpage,
             'view count', default=None))
@@ -177,7 +186,7 @@ class NewgroundsIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
+            'description': description,
             'age_limit': age_limit,
             'view_count': view_count,
         }

yt_dlp/extractor/nexx.py

@@ -427,7 +427,6 @@ class NexxEmbedIE(InfoExtractor):
             'upload_date': '20140305',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {

yt_dlp/extractor/nhk.py

@@ -73,6 +73,7 @@ class NhkBaseIE(InfoExtractor):
                     m3u8_id='hls', fatal=False)
                 for f in info['formats']:
                     f['language'] = lang
+                self._sort_formats(info['formats'])
             else:
                 info.update({
                     '_type': 'url_transparent',

yt_dlp/extractor/niconico.py

@@ -704,7 +704,6 @@ class NicovideoSearchURLIE(InfoExtractor):
 
 class NicovideoSearchIE(SearchInfoExtractor, NicovideoSearchURLIE):
     IE_DESC = 'Nico video searches'
-    _MAX_RESULTS = float('inf')
     IE_NAME = NicovideoSearchIE_NAME
     _SEARCH_KEY = 'nicosearch'
     _TESTS = []

yt_dlp/extractor/nrk.py

@@ -147,7 +147,7 @@ class NRKIE(NRKBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url).split('/')[-1]
 
-        path_templ = 'playback/%s/' + video_id
+        path_templ = 'playback/%s/program/' + video_id
 
         def call_playback_api(item, query=None):
             return self._call_api(path_templ % item, video_id, item, query=query)
@@ -188,7 +188,7 @@ class NRKIE(NRKBaseIE):
         title = titles['title']
         alt_title = titles.get('subtitle')
 
-        description = preplay.get('description')
+        description = try_get(preplay, lambda x: x['description'].replace('\r', '\n'))
         duration = parse_duration(playable.get('duration')) or parse_duration(data.get('duration'))
 
         thumbnails = []

yt_dlp/extractor/nrl.py

@@ -16,7 +16,6 @@ class NRLTVIE(InfoExtractor):
         'params': {
             # m3u8 download
             'skip_download': True,
-            'format': 'bestvideo',
         },
     }
 

yt_dlp/extractor/olympics.py

@@ -2,22 +2,26 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    int_or_none,
+    try_get
+)
 
 
 class OlympicsReplayIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)(?:www\.)?olympics\.com/tokyo-2020/(?:[a-z]{2}/)?replay/(?P<id>[^/#&?]+)'
+    _VALID_URL = r'https?://(?:www\.)?olympics\.com(?:/tokyo-2020)?/[a-z]{2}/(?:replay|video)/(?P<id>[^/#&?]+)'
     _TESTS = [{
-        'url': 'https://olympics.com/tokyo-2020/en/replay/300622eb-abc0-43ea-b03b-c5f2d429ec7b/jumping-team-qualifier',
+        'url': 'https://olympics.com/fr/video/men-s-109kg-group-a-weightlifting-tokyo-2020-replays',
         'info_dict': {
-            'id': '300622eb-abc0-43ea-b03b-c5f2d429ec7b',
+            'id': 'f6a0753c-8e6f-4b7d-a435-027054a4f8e9',
             'ext': 'mp4',
-            'title': 'Jumping Team Qualifier',
-            'release_date': '20210806',
-            'upload_date': '20210713',
+            'title': '+109kg (H) Groupe A - Haltérophilie | Replay de Tokyo 2020',
+            'upload_date': '20210801',
+            'timestamp': 1627783200,
+            'description': 'md5:c66af4a5bc7429dbcc43d15845ff03b3',
         },
         'params': {
-            'format': 'bv',
+            'skip_download': True,
         },
     }, {
         'url': 'https://olympics.com/tokyo-2020/en/replay/bd242924-4b22-49a5-a846-f1d4c809250d/mens-bronze-medal-match-hun-esp',
@@ -26,31 +30,41 @@ class OlympicsReplayIE(InfoExtractor):
 
     def _real_extract(self, url):
         id = self._match_id(url)
-        # The parameters are hardcoded in the webpage, it's not necessary to download the webpage just for these parameters.
-        # If in downloading webpage serves other functions aswell, then extract these parameters from it.
-        token_url = 'https://appovptok.ovpobs.tv/api/identity/app/token?api_key=OTk5NDcxOjpvY3N3LWFwaXVzZXI%3D&api_secret=ODY4ODM2MjE3ODMwYmVjNTAxMWZlMDJiMTYxZmY0MjFiMjMwMjllMjJmNDA1YWRiYzA5ODcxYTZjZTljZDkxOTo6NTM2NWIzNjRlMTM1ZmI2YWNjNmYzMGMzOGM3NzZhZTY%3D'
-        token = self._download_webpage(token_url, id)
-        headers = {'x-obs-app-token': token}
-        data_json = self._download_json(f'https://appocswtok.ovpobs.tv/api/schedule-sessions/{id}?include=stream',
-                                        id, headers=headers)
-        meta_data = data_json['data']['attributes']
-        for t_dict in data_json['included']:
-            if t_dict.get('type') == 'Stream':
-                stream_data = t_dict['attributes']
+
+        webpage = self._download_webpage(url, id)
+        title = self._html_search_meta(('title', 'og:title', 'twitter:title'), webpage)
+        uuid = self._html_search_meta('episode_uid', webpage)
+        m3u8_url = self._html_search_meta('video_url', webpage)
+        json_ld = self._search_json_ld(webpage, uuid)
+        thumbnails_list = json_ld.get('image')
+        if not thumbnails_list:
+            thumbnails_list = self._html_search_regex(
+                r'["\']image["\']:\s*["\']([^"\']+)["\']', webpage, 'images', default='')
+            thumbnails_list = thumbnails_list.replace('[', '').replace(']', '').split(',')
+            thumbnails_list = [thumbnail.strip() for thumbnail in thumbnails_list]
+        thumbnails = []
+        for thumbnail in thumbnails_list:
+            width_a, height_a, width = self._search_regex(
+                r'/images/image/private/t_(?P<width_a>\d+)-(?P<height_a>\d+)_(?P<width>\d+)/primary/[\W\w\d]+',
+                thumbnail, 'thumb', group=(1, 2, 3), default=(None, None, None))
+            width_a, height_a, width = int_or_none(width_a), int_or_none(height_a), int_or_none(width)
+            thumbnails.append({
+                'url': thumbnail,
+                'width': width,
+                'height': int_or_none(try_get(width, lambda x: x * height_a / width_a))
+            })
         m3u8_url = self._download_json(
-            'https://meteringtok.ovpobs.tv/api/playback-sessions', id, headers=headers, query={
-                'alias': stream_data['alias'],
-                'stream': stream_data['stream'],
-                'type': 'vod'
-            })['data']['attributes']['url']
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, id)
+            f'https://olympics.com/tokenGenerator?url={m3u8_url}', uuid, note='Downloading m3u8 url')
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, uuid, m3u8_id='hls')
         self._sort_formats(formats)
 
         return {
-            'id': id,
-            'title': meta_data['title'],
-            'release_date': unified_strdate(meta_data.get('start') or meta_data.get('broadcastPublished')),
-            'upload_date': unified_strdate(meta_data.get('publishedAt')),
+            'id': uuid,
+            'title': title,
+            'timestamp': json_ld.get('timestamp'),
+            'description': json_ld.get('description'),
+            'thumbnails': thumbnails,
+            'duration': json_ld.get('duration'),
             'formats': formats,
             'subtitles': subtitles,
         }

yt_dlp/extractor/orf.py

@@ -11,6 +11,7 @@ from ..utils import (
     float_or_none,
     HEADRequest,
     int_or_none,
+    join_nonempty,
     orderedSet,
     remove_end,
     str_or_none,
@@ -82,12 +83,7 @@ class ORFTVthekIE(InfoExtractor):
                 src = url_or_none(fd.get('src'))
                 if not src:
                     continue
-                format_id_list = []
-                for key in ('delivery', 'quality', 'quality_string'):
-                    value = fd.get(key)
-                    if value:
-                        format_id_list.append(value)
-                format_id = '-'.join(format_id_list)
+                format_id = join_nonempty('delivery', 'quality', 'quality_string', from_dict=fd)
                 ext = determine_ext(src)
                 if ext == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(

yt_dlp/extractor/paramountplus.py

@@ -60,7 +60,6 @@ class ParamountPlusIE(CBSBaseIE):
         },
         'params': {
             'skip_download': 'm3u8',
-            'format': 'bestvideo',
         },
         'expected_warnings': ['Ignoring subtitle tracks'],  # TODO: Investigate this
     }, {
@@ -76,7 +75,6 @@ class ParamountPlusIE(CBSBaseIE):
         },
         'params': {
             'skip_download': 'm3u8',
-            'format': 'bestvideo',
         },
         'expected_warnings': ['Ignoring subtitle tracks'],
     }, {

yt_dlp/extractor/parliamentliveuk.py

@@ -25,9 +25,6 @@ class ParliamentLiveUKIE(InfoExtractor):
             'timestamp': 1395153872,
             'upload_date': '20140318',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'http://parliamentlive.tv/event/index/3f24936f-130f-40bf-9a5d-b3d6479da6a4',
         'only_matching': True,

yt_dlp/extractor/peloton.py

@@ -203,7 +203,6 @@ class PelotonLiveIE(InfoExtractor):
             'chapters': 'count:3'
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': 'm3u8',
         },
         '_skip': 'Account needed'

yt_dlp/extractor/picarto.py

@@ -111,7 +111,7 @@ class PicartoVodIE(InfoExtractor):
         vod_info = self._parse_json(
             self._search_regex(
                 r'(?s)#vod-player["\']\s*,\s*(\{.+?\})\s*\)', webpage,
-                video_id),
+                'vod player'),
             video_id, transform_source=js_to_json)
 
         formats = self._extract_m3u8_formats(

yt_dlp/extractor/piksel.py

@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     dict_get,
     ExtractorError,
     int_or_none,
+    join_nonempty,
     parse_iso8601,
     try_get,
     unescapeHTML,
@@ -116,12 +116,8 @@ class PikselIE(InfoExtractor):
             elif asset_type == 'audio':
                 tbr = abr
 
-            format_id = ['http']
-            if tbr:
-                format_id.append(compat_str(tbr))
-
             formats.append({
-                'format_id': '-'.join(format_id),
+                'format_id': join_nonempty('http', tbr),
                 'url': unescapeHTML(http_url),
                 'vbr': vbr,
                 'abr': abr,
@@ -167,7 +163,7 @@ class PikselIE(InfoExtractor):
                 re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
                 transform_source=transform_source, fatal=False))
 
-        self._sort_formats(formats)
+        self._sort_formats(formats, ('tbr', ))  # Incomplete resolution information
 
         subtitles = {}
         for caption in video_data.get('captions', []):

yt_dlp/extractor/planetmarathi.py

@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    try_get,
+    unified_strdate,
+)
+
+
+class PlanetMarathiIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)(?:www\.)?planetmarathi\.com/titles/(?P<id>[^/#&?$]+)'
+    _TESTS = [{
+        'url': 'https://www.planetmarathi.com/titles/ek-unad-divas',
+        'playlist_mincount': 2,
+        'info_dict': {
+            'id': 'ek-unad-divas',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'ASSETS-MOVIE-ASSET-01_ek-unad-divas',
+                'ext': 'mp4',
+                'title': 'ek unad divas',
+                'alt_title': 'चित्रपट',
+                'description': 'md5:41c7ed6b041c2fea9820a3f3125bd881',
+                'season_number': None,
+                'episode_number': 1,
+                'duration': 5539,
+                'upload_date': '20210829',
+            },
+        }]  # Trailer skipped
+    }, {
+        'url': 'https://www.planetmarathi.com/titles/baap-beep-baap-season-1',
+        'playlist_mincount': 10,
+        'info_dict': {
+            'id': 'baap-beep-baap-season-1',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'ASSETS-CHARACTER-PROFILE-SEASON-01-ASSET-01_baap-beep-baap-season-1',
+                'ext': 'mp4',
+                'title': 'Manohar Kanhere',
+                'alt_title': 'मनोहर कान्हेरे',
+                'description': 'md5:285ed45d5c0ab5522cac9a043354ebc6',
+                'season_number': 1,
+                'episode_number': 1,
+                'duration': 29,
+                'upload_date': '20210829',
+            },
+        }]  # Trailers, Episodes, other Character profiles skipped
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        entries = []
+        json_data = self._download_json(f'https://www.planetmarathi.com/api/v1/titles/{id}/assets', id)['assets']
+        for asset in json_data:
+            asset_title = asset['mediaAssetName']['en']
+            if asset_title == 'Movie':
+                asset_title = id.replace('-', ' ')
+            asset_id = f'{asset["sk"]}_{id}'.replace('#', '-')
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(asset['mediaAssetURL'], asset_id)
+            self._sort_formats(formats)
+            entries.append({
+                'id': asset_id,
+                'title': asset_title,
+                'alt_title': try_get(asset, lambda x: x['mediaAssetName']['mr']),
+                'description': try_get(asset, lambda x: x['mediaAssetDescription']['en']),
+                'season_number': asset.get('mediaAssetSeason'),
+                'episode_number': asset.get('mediaAssetIndexForAssetType'),
+                'duration': asset.get('mediaAssetDurationInSeconds'),
+                'upload_date': unified_strdate(asset.get('created')),
+                'formats': formats,
+                'subtitles': subtitles,
+            })
+        return self.playlist_result(entries, playlist_id=id)

yt_dlp/extractor/polsatgo.py

@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from uuid import uuid4
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    try_get,
+    url_or_none,
+    ExtractorError,
+)
+
+
+class PolsatGoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?polsat(?:box)?go\.pl/.+/(?P<id>[0-9a-fA-F]+)(?:[/#?]|$)'
+    _TESTS = [{
+        'url': 'https://polsatgo.pl/wideo/seriale/swiat-wedlug-kiepskich/5024045/sezon-1/5028300/swiat-wedlug-kiepskich-odcinek-88/4121',
+        'info_dict': {
+            'id': '4121',
+            'ext': 'mp4',
+            'title': 'Świat według Kiepskich - Odcinek 88',
+            'age_limit': 12,
+        },
+    }]
+
+    def _extract_formats(self, sources, video_id):
+        for source in sources or []:
+            if not source.get('id'):
+                continue
+            url = url_or_none(self._call_api(
+                'drm', video_id, 'getPseudoLicense',
+                {'mediaId': video_id, 'sourceId': source['id']}).get('url'))
+            if not url:
+                continue
+            yield {
+                'url': url,
+                'height': int_or_none(try_get(source, lambda x: x['quality'][:-1]))
+            }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        media = self._call_api('navigation', video_id, 'prePlayData', {'mediaId': video_id})['mediaItem']
+
+        formats = list(self._extract_formats(
+            try_get(media, lambda x: x['playback']['mediaSources']), video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': media['displayInfo']['title'],
+            'formats': formats,
+            'age_limit': int_or_none(media['displayInfo']['ageGroup'])
+        }
+
+    def _call_api(self, endpoint, media_id, method, params):
+        rand_uuid = str(uuid4())
+        res = self._download_json(
+            f'https://b2c-mobile.redefine.pl/rpc/{endpoint}/', media_id,
+            note=f'Downloading {method} JSON metadata',
+            data=json.dumps({
+                'method': method,
+                'id': '2137',
+                'jsonrpc': '2.0',
+                'params': {
+                    **params,
+                    'userAgentData': {
+                        'deviceType': 'mobile',
+                        'application': 'native',
+                        'os': 'android',
+                        'build': 10003,
+                        'widevine': False,
+                        'portal': 'pg',
+                        'player': 'cpplayer',
+                    },
+                    'deviceId': {
+                        'type': 'other',
+                        'value': rand_uuid,
+                    },
+                    'clientId': rand_uuid,
+                    'cpid': 1,
+                },
+            }).encode('utf-8'),
+            headers={'Content-type': 'application/json'})
+        if not res.get('result'):
+            if res['error']['code'] == 13404:
+                raise ExtractorError('This video is either unavailable in your region or is DRM protected', expected=True)
+            raise ExtractorError(f'Solorz said: {res["error"]["message"]} - {res["error"]["data"]["userMessage"]}')
+        return res['result']

yt_dlp/extractor/polskieradio.py

@@ -2,6 +2,8 @@
 from __future__ import unicode_literals
 
 import itertools
+import json
+import math
 import re
 
 from .common import InfoExtractor
@@ -12,15 +14,45 @@ from ..compat import (
 )
 from ..utils import (
     extract_attributes,
+    ExtractorError,
+    InAdvancePagedList,
     int_or_none,
+    js_to_json,
+    parse_iso8601,
     strip_or_none,
     unified_timestamp,
     unescapeHTML,
+    url_or_none,
 )
 
 
-class PolskieRadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
+class PolskieRadioBaseExtractor(InfoExtractor):
+    def _extract_webpage_player_entries(self, webpage, playlist_id, base_data):
+        media_urls = set()
+
+        for data_media in re.findall(r'<[^>]+data-media="?({[^>]+})"?', webpage):
+            media = self._parse_json(data_media, playlist_id, transform_source=unescapeHTML, fatal=False)
+            if not media.get('file') or not media.get('desc'):
+                continue
+            media_url = self._proto_relative_url(media['file'])
+            if media_url in media_urls:
+                continue
+            media_urls.add(media_url)
+            entry = base_data.copy()
+            entry.update({
+                'id': compat_str(media['id']),
+                'url': media_url,
+                'duration': int_or_none(media.get('length')),
+                'vcodec': 'none' if media.get('provider') == 'audio' else None,
+            })
+            entry_title = compat_urllib_parse_unquote(media['desc'])
+            if entry_title:
+                entry['title'] = entry_title
+            yield entry
+
+
+class PolskieRadioIE(PolskieRadioBaseExtractor):
+    _VALID_URL = r'https?://(?:www\.)?polskieradio(?:24)?\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
     _TESTS = [{  # Old-style single broadcast.
         'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
         'info_dict': {
@@ -59,22 +91,14 @@ class PolskieRadioIE(InfoExtractor):
                 'thumbnail': r're:^https?://static\.prsa\.pl/images/.*\.jpg$'
             },
         }],
-    }, {  # Old-style multiple broadcast playlist.
-        'url': 'https://www.polskieradio.pl/8/4346/Artykul/2487823,Marek-Kondrat-czyta-Mistrza-i-Malgorzate',
+    }, {
+        # PR4 audition - other frontend
+        'url': 'https://www.polskieradio.pl/10/6071/Artykul/2610977,Poglos-29-pazdziernika-godz-2301',
         'info_dict': {
-            'id': '2487823',
-            'title': 'Marek Kondrat czyta "Mistrza i Małgorzatę"',
-            'description': 'md5:8422a95cc83834f2aaeff9d82e9c8f39',
+            'id': '2610977',
+            'ext': 'mp3',
+            'title': 'Pogłos 29 października godz. 23:01',
         },
-        'playlist_mincount': 50,
-    }, {  # New-style multiple broadcast playlist.
-        'url': 'https://www.polskieradio.pl/8/4346/Artykul/2541317,Czytamy-Kalendarz-i-klepsydre-Tadeusza-Konwickiego',
-        'info_dict': {
-            'id': '2541317',
-            'title': 'Czytamy "Kalendarz i klepsydrę" Tadeusza Konwickiego',
-            'description': 'md5:0baeaa46d877f1351fb2eeed3e871f9f',
-        },
-        'playlist_mincount': 15,
     }, {
         'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
         'only_matching': True,
@@ -85,6 +109,9 @@ class PolskieRadioIE(InfoExtractor):
         # with mp4 video
         'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
         'only_matching': True,
+    }, {
+        'url': 'https://polskieradio24.pl/130/4503/Artykul/2621876,Narusza-nasza-suwerennosc-Publicysci-o-uzaleznieniu-funduszy-UE-od-praworzadnosci',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -94,40 +121,38 @@ class PolskieRadioIE(InfoExtractor):
 
         content = self._search_regex(
             r'(?s)<div[^>]+class="\s*this-article\s*"[^>]*>(.+?)<div[^>]+class="tags"[^>]*>',
-            webpage, 'content')
+            webpage, 'content', default=None)
 
         timestamp = unified_timestamp(self._html_search_regex(
             r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
-            webpage, 'timestamp', fatal=False))
+            webpage, 'timestamp', default=None))
 
-        thumbnail_url = self._og_search_thumbnail(webpage)
-
-        entries = []
-
-        media_urls = set()
-
-        for data_media in re.findall(r'<[^>]+data-media="?({[^>]+})"?', content):
-            media = self._parse_json(data_media, playlist_id, transform_source=unescapeHTML, fatal=False)
-            if not media.get('file') or not media.get('desc'):
-                continue
-            media_url = self._proto_relative_url(media['file'], 'http:')
-            if media_url in media_urls:
-                continue
-            media_urls.add(media_url)
-            entries.append({
-                'id': compat_str(media['id']),
-                'url': media_url,
-                'title': compat_urllib_parse_unquote(media['desc']),
-                'duration': int_or_none(media.get('length')),
-                'vcodec': 'none' if media.get('provider') == 'audio' else None,
-                'timestamp': timestamp,
-                'thumbnail': thumbnail_url
-            })
+        thumbnail_url = self._og_search_thumbnail(webpage, default=None)
 
         title = self._og_search_title(webpage).strip()
-        description = strip_or_none(self._og_search_description(webpage))
+
+        description = strip_or_none(self._og_search_description(webpage, default=None))
         description = description.replace('\xa0', ' ') if description is not None else None
 
+        if not content:
+            return {
+                'id': playlist_id,
+                'url': self._proto_relative_url(
+                    self._search_regex(
+                        r"source:\s*'(//static\.prsa\.pl/[^']+)'",
+                        webpage, 'audition record url')),
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+                'thumbnail': thumbnail_url,
+            }
+
+        entries = self._extract_webpage_player_entries(content, playlist_id, {
+            'title': title,
+            'timestamp': timestamp,
+            'thumbnail': thumbnail_url,
+        })
+
         return self.playlist_result(entries, playlist_id, title, description)
 
 
@@ -207,3 +232,201 @@ class PolskieRadioCategoryIE(InfoExtractor):
         return self.playlist_result(
             self._entries(url, webpage, category_id),
             category_id, title)
+
+
+class PolskieRadioPlayerIE(InfoExtractor):
+    IE_NAME = 'polskieradio:player'
+    _VALID_URL = r'https?://player\.polskieradio\.pl/anteny/(?P<id>[^/]+)'
+
+    _BASE_URL = 'https://player.polskieradio.pl'
+    _PLAYER_URL = 'https://player.polskieradio.pl/main.bundle.js'
+    _STATIONS_API_URL = 'https://apipr.polskieradio.pl/api/stacje'
+
+    _TESTS = [{
+        'url': 'https://player.polskieradio.pl/anteny/trojka',
+        'info_dict': {
+            'id': '3',
+            'ext': 'm4a',
+            'title': 'Trójka',
+        },
+        'params': {
+            'format': 'bestaudio',
+            'skip_download': 'endless stream',
+        },
+    }]
+
+    def _get_channel_list(self, channel_url='no_channel'):
+        player_code = self._download_webpage(
+            self._PLAYER_URL, channel_url,
+            note='Downloading js player')
+        channel_list = js_to_json(self._search_regex(
+            r';var r="anteny",a=(\[.+?\])},', player_code, 'channel list'))
+        return self._parse_json(channel_list, channel_url)
+
+    def _real_extract(self, url):
+        channel_url = self._match_id(url)
+        channel_list = self._get_channel_list(channel_url)
+
+        channel = next((c for c in channel_list if c.get('url') == channel_url), None)
+
+        if not channel:
+            raise ExtractorError('Channel not found')
+
+        station_list = self._download_json(self._STATIONS_API_URL, channel_url,
+                                           note='Downloading stream url list',
+                                           headers={
+                                               'Accept': 'application/json',
+                                               'Referer': url,
+                                               'Origin': self._BASE_URL,
+                                           })
+        station = next((s for s in station_list
+                        if s.get('Name') == (channel.get('streamName') or channel.get('name'))), None)
+        if not station:
+            raise ExtractorError('Station not found even though we extracted channel')
+
+        formats = []
+        for stream_url in station['Streams']:
+            stream_url = self._proto_relative_url(stream_url)
+            if stream_url.endswith('/playlist.m3u8'):
+                formats.extend(self._extract_m3u8_formats(stream_url, channel_url, live=True))
+            elif stream_url.endswith('/manifest.f4m'):
+                formats.extend(self._extract_mpd_formats(stream_url, channel_url))
+            elif stream_url.endswith('/Manifest'):
+                formats.extend(self._extract_ism_formats(stream_url, channel_url))
+            else:
+                formats.append({
+                    'url': stream_url,
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': compat_str(channel['id']),
+            'formats': formats,
+            'title': channel.get('name') or channel.get('streamName'),
+            'display_id': channel_url,
+            'thumbnail': f'{self._BASE_URL}/images/{channel_url}-color-logo.png',
+            'is_live': True,
+        }
+
+
+class PolskieRadioPodcastBaseExtractor(InfoExtractor):
+    _API_BASE = 'https://apipodcasts.polskieradio.pl/api'
+
+    def _parse_episode(self, data):
+        return {
+            'id': data['guid'],
+            'formats': [{
+                'url': data['url'],
+                'filesize': int_or_none(data.get('fileSize')),
+            }],
+            'title': data['title'],
+            'description': data.get('description'),
+            'duration': int_or_none(data.get('length')),
+            'timestamp': parse_iso8601(data.get('publishDate')),
+            'thumbnail': url_or_none(data.get('image')),
+            'series': data.get('podcastTitle'),
+            'episode': data['title'],
+        }
+
+
+class PolskieRadioPodcastListIE(PolskieRadioPodcastBaseExtractor):
+    IE_NAME = 'polskieradio:podcast:list'
+    _VALID_URL = r'https?://podcasty\.polskieradio\.pl/podcast/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://podcasty.polskieradio.pl/podcast/8/',
+        'info_dict': {
+            'id': '8',
+            'title': 'Śniadanie w Trójce',
+            'description': 'md5:57abcc27bc4c6a6b25baa3061975b9ef',
+            'uploader': 'Beata Michniewicz',
+        },
+        'playlist_mincount': 714,
+    }]
+    _PAGE_SIZE = 10
+
+    def _call_api(self, podcast_id, page):
+        return self._download_json(
+            f'{self._API_BASE}/Podcasts/{podcast_id}/?pageSize={self._PAGE_SIZE}&page={page}',
+            podcast_id, f'Downloading page {page}')
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+        data = self._call_api(podcast_id, 1)
+
+        def get_page(page_num):
+            page_data = self._call_api(podcast_id, page_num + 1) if page_num else data
+            yield from (self._parse_episode(ep) for ep in page_data['items'])
+
+        return {
+            '_type': 'playlist',
+            'entries': InAdvancePagedList(
+                get_page, math.ceil(data['itemCount'] / self._PAGE_SIZE), self._PAGE_SIZE),
+            'id': str(data['id']),
+            'title': data['title'],
+            'description': data.get('description'),
+            'uploader': data.get('announcer'),
+        }
+
+
+class PolskieRadioPodcastIE(PolskieRadioPodcastBaseExtractor):
+    IE_NAME = 'polskieradio:podcast'
+    _VALID_URL = r'https?://podcasty\.polskieradio\.pl/track/(?P<id>[a-f\d]{8}(?:-[a-f\d]{4}){4}[a-f\d]{8})'
+    _TESTS = [{
+        'url': 'https://podcasty.polskieradio.pl/track/6eafe403-cb8f-4756-b896-4455c3713c32',
+        'info_dict': {
+            'id': '6eafe403-cb8f-4756-b896-4455c3713c32',
+            'ext': 'mp3',
+            'title': 'Theresa May rezygnuje. Co dalej z brexitem?',
+            'description': 'md5:e41c409a29d022b70ef0faa61dbded60',
+        },
+    }]
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+        data = self._download_json(
+            f'{self._API_BASE}/audio',
+            podcast_id, 'Downloading podcast metadata',
+            data=json.dumps({
+                'guids': [podcast_id],
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
+        return self._parse_episode(data[0])
+
+
+class PolskieRadioRadioKierowcowIE(PolskieRadioBaseExtractor):
+    _VALID_URL = r'https?://(?:www\.)?radiokierowcow\.pl/artykul/(?P<id>[0-9]+)'
+    IE_NAME = 'polskieradio:kierowcow'
+
+    _TESTS = [{
+        'url': 'https://radiokierowcow.pl/artykul/2694529',
+        'info_dict': {
+            'id': '2694529',
+            'title': 'Zielona fala reliktem przeszłości?',
+            'description': 'md5:343950a8717c9818fdfd4bd2b8ca9ff2',
+        },
+        'playlist_count': 3,
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        webpage = self._download_webpage(url, media_id)
+        nextjs_build = self._search_nextjs_data(webpage, media_id)['buildId']
+        article = self._download_json(
+            f'https://radiokierowcow.pl/_next/data/{nextjs_build}/artykul/{media_id}.json?articleId={media_id}',
+            media_id)
+        data = article['pageProps']['data']
+        title = data['title']
+        entries = self._extract_webpage_player_entries(data['content'], media_id, {
+            'title': title,
+        })
+
+        return {
+            '_type': 'playlist',
+            'id': media_id,
+            'entries': entries,
+            'title': title,
+            'description': data.get('lead'),
+        }

yt_dlp/extractor/pornflip.py

@@ -29,7 +29,6 @@ class PornFlipIE(InfoExtractor):
                 'age_limit': 18,
             },
             'params': {
-                'format': 'bestvideo',
                 'skip_download': True,
             },
         },

yt_dlp/extractor/radiokapital.py

@@ -0,0 +1,99 @@
+# coding: utf-8
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    traverse_obj,
+    unescapeHTML,
+)
+
+import itertools
+from urllib.parse import urlencode
+
+
+class RadioKapitalBaseIE(InfoExtractor):
+    def _call_api(self, resource, video_id, note='Downloading JSON metadata', qs={}):
+        return self._download_json(
+            f'https://www.radiokapital.pl/wp-json/kapital/v1/{resource}?{urlencode(qs)}',
+            video_id, note=note)
+
+    def _parse_episode(self, data):
+        release = '%s%s%s' % (data['published'][6:11], data['published'][3:6], data['published'][:3])
+        return {
+            '_type': 'url_transparent',
+            'url': data['mixcloud_url'],
+            'ie_key': 'Mixcloud',
+            'title': unescapeHTML(data['title']),
+            'description': clean_html(data.get('content')),
+            'tags': traverse_obj(data, ('tags', ..., 'name')),
+            'release_date': release,
+            'series': traverse_obj(data, ('show', 'title')),
+        }
+
+
+class RadioKapitalIE(RadioKapitalBaseIE):
+    IE_NAME = 'radiokapital'
+    _VALID_URL = r'https?://(?:www\.)?radiokapital\.pl/shows/[a-z\d-]+/(?P<id>[a-z\d-]+)'
+
+    _TESTS = [{
+        'url': 'https://radiokapital.pl/shows/tutaj-sa-smoki/5-its-okay-to-be-immaterial',
+        'info_dict': {
+            'id': 'radiokapital_radio-kapitał-tutaj-są-smoki-5-its-okay-to-be-immaterial-2021-05-20',
+            'ext': 'm4a',
+            'title': '#5: It’s okay to\xa0be\xa0immaterial',
+            'description': 'md5:2499da5fbfb0e88333b7d37ec8e9e4c4',
+            'uploader': 'Radio Kapitał',
+            'uploader_id': 'radiokapital',
+            'timestamp': 1621640164,
+            'upload_date': '20210521',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        episode = self._call_api('episodes/%s' % video_id, video_id)
+        return self._parse_episode(episode)
+
+
+class RadioKapitalShowIE(RadioKapitalBaseIE):
+    IE_NAME = 'radiokapital:show'
+    _VALID_URL = r'https?://(?:www\.)?radiokapital\.pl/shows/(?P<id>[a-z\d-]+)/?(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://radiokapital.pl/shows/wesz',
+        'info_dict': {
+            'id': '100',
+            'title': 'WĘSZ',
+            'description': 'md5:3a557a1e0f31af612b0dcc85b1e0ca5c',
+        },
+        'playlist_mincount': 17,
+    }]
+
+    def _get_episode_list(self, series_id, page_no):
+        return self._call_api(
+            'episodes', series_id,
+            f'Downloading episode list page #{page_no}', qs={
+                'show': series_id,
+                'page': page_no,
+            })
+
+    def _entries(self, series_id):
+        for page_no in itertools.count(1):
+            episode_list = self._get_episode_list(series_id, page_no)
+            yield from (self._parse_episode(ep) for ep in episode_list['items'])
+            if episode_list['next'] is None:
+                break
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+
+        show = self._call_api(f'shows/{series_id}', series_id, 'Downloading show metadata')
+        entries = self._entries(series_id)
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': str(show['id']),
+            'title': show.get('title'),
+            'description': clean_html(show.get('content')),
+        }

yt_dlp/extractor/rai.py

@@ -14,12 +14,15 @@ from ..utils import (
     find_xpath_attr,
     fix_xml_ampersands,
     GeoRestrictedError,
+    get_element_by_class,
     HEADRequest,
     int_or_none,
     parse_duration,
+    parse_list,
     remove_start,
     strip_or_none,
     try_get,
+    unescapeHTML,
     unified_strdate,
     unified_timestamp,
     update_url_query,
@@ -585,3 +588,84 @@ class RaiIE(RaiBaseIE):
         info.update(relinker_info)
 
         return info
+
+
+class RaiPlayRadioBaseIE(InfoExtractor):
+    _BASE = 'https://www.raiplayradio.it'
+
+    def get_playlist_iter(self, url, uid):
+        webpage = self._download_webpage(url, uid)
+        for attrs in parse_list(webpage):
+            title = attrs['data-title'].strip()
+            audio_url = urljoin(url, attrs['data-mediapolis'])
+            entry = {
+                'url': audio_url,
+                'id': attrs['data-uniquename'].lstrip('ContentItem-'),
+                'title': title,
+                'ext': 'mp3',
+                'language': 'it',
+            }
+            if 'data-image' in attrs:
+                entry['thumbnail'] = urljoin(url, attrs['data-image'])
+            yield entry
+
+
+class RaiPlayRadioIE(RaiPlayRadioBaseIE):
+    _VALID_URL = r'%s/audio/.+?-(?P<id>%s)\.html' % (
+        RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE)
+    _TEST = {
+        'url': 'https://www.raiplayradio.it/audio/2019/07/RADIO3---LEZIONI-DI-MUSICA-36b099ff-4123-4443-9bf9-38e43ef5e025.html',
+        'info_dict': {
+            'id': '36b099ff-4123-4443-9bf9-38e43ef5e025',
+            'ext': 'mp3',
+            'title': 'Dal "Chiaro di luna" al  "Clair de lune", prima parte con Giovanni Bietti',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'language': 'it',
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        list_url = url.replace('.html', '-list.html')
+        return next(entry for entry in self.get_playlist_iter(list_url, audio_id) if entry['id'] == audio_id)
+
+
+class RaiPlayRadioPlaylistIE(RaiPlayRadioBaseIE):
+    _VALID_URL = r'%s/playlist/.+?-(?P<id>%s)\.html' % (
+        RaiPlayRadioBaseIE._BASE, RaiBaseIE._UUID_RE)
+    _TEST = {
+        'url': 'https://www.raiplayradio.it/playlist/2017/12/Alice-nel-paese-delle-meraviglie-72371d3c-d998-49f3-8860-d168cfdf4966.html',
+        'info_dict': {
+            'id': '72371d3c-d998-49f3-8860-d168cfdf4966',
+            'title': "Alice nel paese delle meraviglie",
+            'description': "di Lewis Carrol letto da Aldo Busi",
+        },
+        'playlist_count': 11,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        playlist_webpage = self._download_webpage(url, playlist_id)
+        playlist_title = unescapeHTML(self._html_search_regex(
+            r'data-playlist-title="(.+?)"', playlist_webpage, 'title'))
+        playlist_creator = self._html_search_meta(
+            'nomeProgramma', playlist_webpage)
+        playlist_description = get_element_by_class(
+            'textDescriptionProgramma', playlist_webpage)
+
+        player_href = self._html_search_regex(
+            r'data-player-href="(.+?)"', playlist_webpage, 'href')
+        list_url = urljoin(url, player_href)
+
+        entries = list(self.get_playlist_iter(list_url, playlist_id))
+        for index, entry in enumerate(entries, start=1):
+            entry.update({
+                'track': entry['title'],
+                'track_number': index,
+                'artist': playlist_creator,
+                'album': playlist_title
+            })
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description,
+            creator=playlist_creator)

yt_dlp/extractor/rcti.py

@@ -85,9 +85,6 @@ class RCTIPlusIE(RCTIPlusBaseIE):
             'series': 'iNews Malam',
             'channel': 'INews',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {  # Missed event/replay
         'url': 'https://www.rctiplus.com/missed-event/2507/mou-signing-ceremony-27-juli-2021-1400-wib',
         'md5': '649c5f27250faed1452ca8b91e06922d',
@@ -132,7 +129,6 @@ class RCTIPlusIE(RCTIPlusBaseIE):
         },
         'params': {
             'skip_download': True,
-            'format': 'bestvideo',
         },
     }]
     _CONVIVA_JSON_TEMPLATE = {
@@ -329,7 +325,6 @@ class RCTIPlusTVIE(RCTIPlusBaseIE):
         },
         'params': {
             'skip_download': True,
-            'format': 'bestvideo',
         }
     }, {
         # Returned video will always change

yt_dlp/extractor/reddit.py

@@ -22,9 +22,6 @@ class RedditIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'zv89llsvexdz',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }
 
     def _real_extract(self, url):
@@ -67,7 +64,6 @@ class RedditRIE(InfoExtractor):
             'age_limit': 0,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {

yt_dlp/extractor/rmcdecouverte.py

@@ -26,7 +26,6 @@ class RMCDecouverteIE(InfoExtractor):
             'upload_date': '20210428',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {

yt_dlp/extractor/roosterteeth.py

@@ -1,74 +1,31 @@
 # coding: utf-8
-from __future__ import unicode_literals
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
+    join_nonempty,
+    LazyList,
+    parse_qs,
     str_or_none,
+    traverse_obj,
+    url_or_none,
     urlencode_postdata,
+    urljoin,
 )
 
 
-class RoosterTeethIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:episode|watch)/(?P<id>[^/?#&]+)'
+class RoosterTeethBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'roosterteeth'
-    _TESTS = [{
-        'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
-        'md5': 'e2bd7764732d785ef797700a2489f212',
-        'info_dict': {
-            'id': '9156',
-            'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement',
-            'ext': 'mp4',
-            'title': 'Million Dollars, But... The Game Announcement',
-            'description': 'md5:168a54b40e228e79f4ddb141e89fe4f5',
-            'thumbnail': r're:^https?://.*\.png$',
-            'series': 'Million Dollars, But...',
-            'episode': 'Million Dollars, But... The Game Announcement',
-        },
-    }, {
-        'url': 'https://roosterteeth.com/watch/rwby-bonus-25',
-        'md5': 'fe8d9d976b272c18a24fe7f1f5830084',
-        'info_dict': {
-            'id': '31',
-            'display_id': 'rwby-bonus-25',
-            'title': 'Volume 2, World of Remnant 3',
-            'description': 'md5:8d58d3270292ea11da00ea712bbfb009',
-            'episode': 'Volume 2, World of Remnant 3',
-            'channel_id': 'fab60c1c-29cb-43bc-9383-5c3538d9e246',
-            'thumbnail': r're:^https?://.*\.(png|jpe?g)$',
-            'ext': 'mp4',
-        },
-    }, {
-        'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
-        'only_matching': True,
-    }, {
-        'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts',
-        'only_matching': True,
-    }, {
-        'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow',
-        'only_matching': True,
-    }, {
-        'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
-        'only_matching': True,
-    }, {
-        # only available for FIRST members
-        'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one',
-        'only_matching': True,
-    }, {
-        'url': 'https://roosterteeth.com/watch/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
-        'only_matching': True,
-    }]
-    _EPISODE_BASE_URL = 'https://svod-be.roosterteeth.com/api/v1/watch/'
+    _API_BASE = 'https://svod-be.roosterteeth.com'
+    _API_BASE_URL = f'{_API_BASE}/api/v1'
 
     def _login(self):
         username, password = self._get_login_info()
         if username is None:
             return
+        if self._get_cookies(self._API_BASE_URL).get('rt_access_token'):
+            return
 
         try:
             self._download_json(
@@ -90,13 +47,95 @@ class RoosterTeethIE(InfoExtractor):
             self.report_warning(msg)
 
     def _real_initialize(self):
-        if self._get_cookies(self._EPISODE_BASE_URL).get('rt_access_token'):
-            return
         self._login()
 
+    def _extract_video_info(self, data):
+        thumbnails = []
+        for image in traverse_obj(data, ('included', 'images')):
+            if image.get('type') not in ('episode_image', 'bonus_feature_image'):
+                continue
+            thumbnails.extend([{
+                'id': name,
+                'url': url,
+            } for name, url in (image.get('attributes') or {}).items() if url_or_none(url)])
+
+        attributes = data.get('attributes') or {}
+        title = traverse_obj(attributes, 'title', 'display_title')
+        sub_only = attributes.get('is_sponsors_only')
+
+        return {
+            'id': str(data.get('id')),
+            'display_id': attributes.get('slug'),
+            'title': title,
+            'description': traverse_obj(attributes, 'description', 'caption'),
+            'series': attributes.get('show_title'),
+            'season_number': int_or_none(attributes.get('season_number')),
+            'season_id': attributes.get('season_id'),
+            'episode': title,
+            'episode_number': int_or_none(attributes.get('number')),
+            'episode_id': str_or_none(data.get('uuid')),
+            'channel_id': attributes.get('channel_id'),
+            'duration': int_or_none(attributes.get('length')),
+            'thumbnails': thumbnails,
+            'availability': self._availability(
+                needs_premium=sub_only, needs_subscription=sub_only, needs_auth=sub_only,
+                is_private=False, is_unlisted=False),
+            'tags': attributes.get('genres')
+        }
+
+
+class RoosterTeethIE(RoosterTeethBaseIE):
+    _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:episode|watch)/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
+        'info_dict': {
+            'id': '9156',
+            'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement',
+            'ext': 'mp4',
+            'title': 'Million Dollars, But... The Game Announcement',
+            'description': 'md5:168a54b40e228e79f4ddb141e89fe4f5',
+            'thumbnail': r're:^https?://.*\.png$',
+            'series': 'Million Dollars, But...',
+            'episode': 'Million Dollars, But... The Game Announcement',
+        },
+        'skip_download': 'm3u8',
+    }, {
+        'url': 'https://roosterteeth.com/watch/rwby-bonus-25',
+        'info_dict': {
+            'id': '40432',
+            'display_id': 'rwby-bonus-25',
+            'title': 'Grimm',
+            'description': 'md5:f30ff570741213418a8d2c19868b93ab',
+            'episode': 'Grimm',
+            'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1',
+            'thumbnail': r're:^https?://.*\.(png|jpe?g)$',
+            'ext': 'mp4',
+        },
+        'skip_download': 'm3u8',
+    }, {
+        'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
+        'only_matching': True,
+    }, {
+        'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts',
+        'only_matching': True,
+    }, {
+        'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow',
+        'only_matching': True,
+    }, {
+        'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
+        'only_matching': True,
+    }, {
+        # only available for FIRST members
+        'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one',
+        'only_matching': True,
+    }, {
+        'url': 'https://roosterteeth.com/watch/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
+        'only_matching': True,
+    }]
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        api_episode_url = self._EPISODE_BASE_URL + display_id
+        api_episode_url = f'{self._API_BASE_URL}/watch/{display_id}'
 
         try:
             video_data = self._download_json(
@@ -118,36 +157,55 @@ class RoosterTeethIE(InfoExtractor):
         episode = self._download_json(
             api_episode_url, display_id,
             'Downloading episode JSON metadata')['data'][0]
-        attributes = episode['attributes']
-        title = attributes.get('title') or attributes['display_title']
-        video_id = compat_str(episode['id'])
-
-        thumbnails = []
-        for image in episode.get('included', {}).get('images', []):
-            if image.get('type') in ('episode_image', 'bonus_feature_image'):
-                img_attributes = image.get('attributes') or {}
-                for k in ('thumb', 'small', 'medium', 'large'):
-                    img_url = img_attributes.get(k)
-                    if img_url:
-                        thumbnails.append({
-                            'id': k,
-                            'url': img_url,
-                        })
 
         return {
-            'id': video_id,
             'display_id': display_id,
-            'title': title,
-            'description': attributes.get('description') or attributes.get('caption'),
-            'thumbnails': thumbnails,
-            'series': attributes.get('show_title'),
-            'season_number': int_or_none(attributes.get('season_number')),
-            'season_id': attributes.get('season_id'),
-            'episode': title,
-            'episode_number': int_or_none(attributes.get('number')),
-            'episode_id': str_or_none(episode.get('uuid')),
             'formats': formats,
-            'channel_id': attributes.get('channel_id'),
-            'duration': int_or_none(attributes.get('length')),
-            'subtitles': subtitles
+            'subtitles': subtitles,
+            **self._extract_video_info(episode)
         }
+
+
+class RoosterTeethSeriesIE(RoosterTeethBaseIE):
+    _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/series/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://roosterteeth.com/series/rwby?season=7',
+        'playlist_count': 13,
+        'info_dict': {
+            'id': 'rwby-7',
+            'title': 'RWBY - Season 7',
+        }
+    }, {
+        'url': 'https://roosterteeth.com/series/role-initiative',
+        'playlist_mincount': 16,
+        'info_dict': {
+            'id': 'role-initiative',
+            'title': 'Role Initiative',
+        }
+    }]
+
+    def _entries(self, series_id, season_number):
+        display_id = join_nonempty(series_id, season_number)
+        # TODO: extract bonus material
+        for data in self._download_json(
+                f'{self._API_BASE_URL}/shows/{series_id}/seasons?order=asc&order_by', display_id)['data']:
+            idx = traverse_obj(data, ('attributes', 'number'))
+            if season_number and idx != season_number:
+                continue
+            season_url = urljoin(self._API_BASE, data['links']['episodes'])
+            season = self._download_json(season_url, display_id, f'Downloading season {idx} JSON metadata')['data']
+            for episode in season:
+                yield self.url_result(
+                    f'https://www.roosterteeth.com{episode["canonical_links"]["self"]}',
+                    RoosterTeethIE.ie_key(),
+                    **self._extract_video_info(episode))
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        season_number = traverse_obj(parse_qs(url), ('season', 0), expected_type=int_or_none)
+
+        entries = LazyList(self._entries(series_id, season_number))
+        return self.playlist_result(
+            entries,
+            join_nonempty(series_id, season_number),
+            join_nonempty(entries[0].get('series'), season_number, delim=' - Season '))

yt_dlp/extractor/sevenplus.py

@@ -35,7 +35,6 @@ class SevenPlusIE(BrightcoveNewIE):
             'episode': 'Wind Surf',
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         }
     }, {

yt_dlp/extractor/sky.py

@@ -105,6 +105,34 @@ class SkyNewsIE(SkyBaseIE):
     }
 
 
+class SkyNewsStoryIE(SkyBaseIE):
+    IE_NAME = 'sky:news:story'
+    _VALID_URL = r'https?://news\.sky\.com/story/[0-9a-z-]+-(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://news.sky.com/story/budget-2021-chancellor-rishi-sunak-vows-address-will-deliver-strong-economy-fit-for-a-new-age-of-optimism-12445425',
+        'info_dict': {
+            'id': 'ref:0714acb9-123d-42c8-91b8-5c1bc6c73f20',
+            'title': 'md5:e408dd7aad63f31a1817bbe40c7d276f',
+            'description': 'md5:a881e12f49212f92be2befe4a09d288a',
+            'ext': 'mp4',
+            'upload_date': '20211027',
+            'timestamp': 1635317494,
+            'uploader_id': '6058004172001',
+        }
+    }
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+
+        entries = [self._process_ooyala_element(webpage, sdc_el, url)
+                   for sdc_el in re.findall(self._SDC_EL_REGEX, webpage)]
+
+        return self.playlist_result(
+            entries, article_id, self._og_search_title(webpage),
+            self._html_search_meta(['og:description', 'description'], webpage))
+
+
 class SkySportsNewsIE(SkyBaseIE):
     IE_NAME = 'sky:sports:news'
     _VALID_URL = r'https?://(?:www\.)?skysports\.com/([^/]+/)*news/\d+/(?P<id>\d+)'

yt_dlp/extractor/slideslive.py

@@ -35,9 +35,6 @@ class SlidesLiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         # video_service_name = youtube
         'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend',

yt_dlp/extractor/soundcloud.py

@@ -855,8 +855,8 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
 
 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     IE_NAME = 'soundcloud:search'
-    IE_DESC = 'Soundcloud search, "scsearch" keyword'
-    _MAX_RESULTS = float('inf')
+    IE_DESC = 'Soundcloud search'
+    _SEARCH_KEY = 'scsearch'
     _TESTS = [{
         'url': 'scsearch15:post-avant jazzcore',
         'info_dict': {
@@ -865,7 +865,6 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
         'playlist_count': 15,
     }]
 
-    _SEARCH_KEY = 'scsearch'
     _MAX_RESULTS_PER_PAGE = 200
     _DEFAULT_RESULTS_PER_PAGE = 50
 

yt_dlp/extractor/srgssr.py

@@ -7,6 +7,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    join_nonempty,
     parse_iso8601,
     qualities,
     try_get,
@@ -94,11 +95,7 @@ class SRGSSRIE(InfoExtractor):
                 continue
             protocol = source.get('protocol')
             quality = source.get('quality')
-            format_id = []
-            for e in (protocol, source.get('encoding'), quality):
-                if e:
-                    format_id.append(e)
-            format_id = '-'.join(format_id)
+            format_id = join_nonempty(protocol, source.get('encoding'), quality)
 
             if protocol in ('HDS', 'HLS'):
                 if source.get('tokenType') == 'AKAMAI':

yt_dlp/extractor/svt.py

@@ -168,7 +168,6 @@ class SVTPlayIE(SVTPlayBaseIE):
             },
         },
         'params': {
-            'format': 'bestvideo',
             # skip for now due to download test asserts that segment is > 10000 bytes and svt uses
             # init segments that are smaller
             # AssertionError: Expected test_SVTPlay_jNwpV9P.mp4 to be at least 9.77KiB, but it's only 864.00B

yt_dlp/extractor/telemundo.py

@@ -1,4 +1,4 @@
-# coding=utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

yt_dlp/extractor/telequebec.py

@@ -43,9 +43,6 @@ class TeleQuebecIE(TeleQuebecBaseIE):
             'uploader_id': '6150020952001',
             'upload_date': '20200512',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
         'add_ie': ['BrightcoveNew'],
     }, {
         'url': 'https://zonevideo.telequebec.tv/media/55267/le-soleil/passe-partout',
@@ -58,9 +55,6 @@ class TeleQuebecIE(TeleQuebecBaseIE):
             'upload_date': '20200625',
             'timestamp': 1593090307,
         },
-        'params': {
-            'format': 'bestvideo',
-        },
         'add_ie': ['BrightcoveNew'],
     }, {
         # no description
@@ -157,9 +151,6 @@ class TeleQuebecEmissionIE(InfoExtractor):
             'timestamp': 1588713424,
             'uploader_id': '6150020952001',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'http://bancpublic.telequebec.tv/emissions/emission-49/31986/jeunes-meres-sous-pression',
         'only_matching': True,
@@ -220,9 +211,6 @@ class TeleQuebecVideoIE(TeleQuebecBaseIE):
             'timestamp': 1603115930,
             'uploader_id': '6101674910001',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         'url': 'https://video.telequebec.tv/player-live/28527',
         'only_matching': True,

yt_dlp/extractor/tf1.py

@@ -29,7 +29,6 @@ class TF1IE(InfoExtractor):
         'params': {
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
-            'format': 'bestvideo',
         },
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',