Release 2021.03.24.1

[ard] improve clip id extraction
https://github.com/ytdl-org/youtube-dl/commit/8562218350a79d4709da8593bb0c538aa0824acf

This was merged incorrectly

.github/ISSUE_TEMPLATE/1_broken_site.md

@@ -21,7 +21,7 @@ assignees: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
-- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.07. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.24. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in https://github.com/yt-dlp/yt-dlp.
 - Search the bugtracker for similar issues: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
@@ -29,7 +29,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running yt-dlp version **2021.03.07**
+- [ ] I've verified that I'm running yt-dlp version **2021.03.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -44,7 +44,7 @@ Add the `-v` flag to your command line you run yt-dlp with (`yt-dlp -v <your com
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] yt-dlp version 2021.03.07
+ [debug] yt-dlp version 2021.03.24
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}

.github/ISSUE_TEMPLATE/2_site_support_request.md

@@ -21,7 +21,7 @@ assignees: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
-- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.07. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.24. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://github.com/yt-dlp/yt-dlp. yt-dlp does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
@@ -29,7 +29,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running yt-dlp version **2021.03.07**
+- [ ] I've verified that I'm running yt-dlp version **2021.03.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones

.github/ISSUE_TEMPLATE/3_site_feature_request.md

@@ -21,13 +21,13 @@ assignees: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
-- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.07. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.24. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
 - Finally, put x into all relevant boxes like this [x] (Dont forget to delete the empty space)
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running yt-dlp version **2021.03.07**
+- [ ] I've verified that I'm running yt-dlp version **2021.03.24**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 

.github/ISSUE_TEMPLATE/4_bug_report.md

@@ -21,7 +21,7 @@ assignees: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
-- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.07. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.24. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in https://github.com/yt-dlp/yt-dlp.
 - Search the bugtracker for similar issues: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
@@ -30,7 +30,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running yt-dlp version **2021.03.07**
+- [ ] I've verified that I'm running yt-dlp version **2021.03.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -46,7 +46,7 @@ Add the `-v` flag to your command line you run yt-dlp with (`yt-dlp -v <your com
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] yt-dlp version 2021.03.07
+ [debug] yt-dlp version 2021.03.24
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}

.github/ISSUE_TEMPLATE/5_feature_request.md

@@ -21,13 +21,13 @@ assignees: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
-- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.07. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.03.24. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
 - Finally, put x into all relevant boxes like this [x] (Dont forget to delete the empty space)
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running yt-dlp version **2021.03.07**
+- [ ] I've verified that I'm running yt-dlp version **2021.03.24**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 

CONTRIBUTORS

@@ -1,5 +1,6 @@
 pukkandan (owner)
 shirt-dev (collaborator)
+colethedj (collaborator)
 h-h-h-h
 pauldubois98
 nixxo
@@ -23,10 +24,14 @@ tsukumi
 bbepis
 animelover1984
 Pccode66
-Ashish
+Ashish0804
 RobinD42
 hseg
-colethedj
 DennyDai
 codeasashu
 teesid
+kevinoconnor7
+damianoamatruda
+2ShedsJackson
+CXwudi
+xtkoba

Changelog.md

@@ -17,6 +17,51 @@
 -->
 
 
+### 2021.03.24.1
+* Revert [commit/8562218](https://github.com/ytdl-org/youtube-dl/commit/8562218350a79d4709da8593bb0c538aa0824acf)
+
+
+### 2021.03.24
+* Merge youtube-dl: Upto [commit/8562218](https://github.com/ytdl-org/youtube-dl/commit/8562218350a79d4709da8593bb0c538aa0824acf)
+* Parse metadata from multiple fields using `--parse-metadata`
+* Ability to load playlist infojson using `--load-info-json`
+* Write current epoch to infojson when using `--no-clean-infojson`
+* [youtube_live_chat] fix bug when trying to set cookies
+* [niconico] Fix for when logged in by: @CXwudi and @xtkoba
+* [linuxacadamy] Fix login
+
+
+### 2021.03.21
+* Merge youtube-dl: Upto [commit/7e79ba7](https://github.com/ytdl-org/youtube-dl/commit/7e79ba7dd6e6649dd2ce3a74004b2044f2182881)
+* Option `--no-clean-infojson` to keep private keys in the infojson
+* [aria2c] Support retry/abort unavailable fragments by [damianoamatruda](https://github.com/damianoamatruda)
+* [aria2c] Better default arguments
+* [movefiles] Fix bugs and make more robust
+* [formatSort] Fix `quality` being ignored
+* [splitchapters] Fix for older ffmpeg
+* [sponskrub] Pass proxy to sponskrub
+* Make sure `post_hook` gets the final filename
+* Recursively remove any private keys from infojson
+* Embed video URL metadata inside `mp4` by [damianoamatruda](https://github.com/damianoamatruda) and [pukkandan](https://github.com/pukkandan)
+* Merge `webm` formats into `mkv` if thumbnails are to be embedded by [damianoamatruda](https://github.com/damianoamatruda)
+* Use headers and cookies when downloading subtitles by [damianoamatruda](https://github.com/damianoamatruda)
+* Parse resolution in info dictionary by [damianoamatruda](https://github.com/damianoamatruda)
+* More consistent warning messages by [damianoamatruda](https://github.com/damianoamatruda) and [pukkandan](https://github.com/pukkandan)
+* [documentation] Add deprecated options and aliases in readme
+* [documentation] Fix some minor mistakes
+
+* [niconico] Partial fix adapted from [animelover1984/youtube-dl@b5eff52](https://github.com/animelover1984/youtube-dl/commit/b5eff52dd9ed5565672ea1694b38c9296db3fade) (login and smile formats still don't work)
+* [niconico] Add user extractor by [animelover1984](https://github.com/animelover1984)
+* [bilibili] Add anthology support by [animelover1984](https://github.com/animelover1984)
+* [amcnetworks] Fix extractor by [2ShedsJackson](https://github.com/2ShedsJackson)
+* [stitcher] Merge from youtube-dl by [nixxo](https://github.com/nixxo)
+* [rcs] Improved extraction by [nixxo](https://github.com/nixxo)
+* [linuxacadamy] Improve regex
+* [youtube] Show if video is `private`, `unlisted` etc in info (`availability`) by [colethedj](https://github.com/colethedj) and [pukkandan](https://github.com/pukkandan)
+* [youtube] bugfix for channel playlist extraction
+* [nbc] Improve metadata extraction by [2ShedsJackson](https://github.com/2ShedsJackson)
+
+
 ### 2021.03.15
 * **Split video by chapters**: using option `--split-chapters`
     * The output file of the split files can be set with `-o`/`-P` using the prefix `chapter:`
@@ -24,13 +69,12 @@
 * **Parallel fragment downloads** by [shirt](https://github.com/shirt-dev)
     * Use option `--concurrent-fragments` (`-N`) to set the number of threads (default 1)
 * Merge youtube-dl: Upto [commit/3be0980](https://github.com/ytdl-org/youtube-dl/commit/3be098010f667b14075e3dfad1e74e5e2becc8ea)
-* [Zee5] Add Show Extractor by [Ashish](https://github.com/Ashish) and [pukkandan](https://github.com/pukkandan)
+* [zee5] Add Show Extractor by [Ashish0804](https://github.com/Ashish0804) and [pukkandan](https://github.com/pukkandan)
 * [rai] fix drm check [nixxo](https://github.com/nixxo)
-* [zee5] Support zee5originals
 * [wimtv] Add extractor by [nixxo](https://github.com/nixxo)
 * [mtv] Add mtv.it and extract series metadata by [nixxo](https://github.com/nixxo)
 * [pluto.tv] Add extractor by [kevinoconnor7](https://github.com/kevinoconnor7)
-* [Youtube] Rewrite comment extraction by [colethedj](https://github.com/colethedj)
+* [youtube] Rewrite comment extraction by [colethedj](https://github.com/colethedj)
 * [embedthumbnail] Set mtime correctly
 * Refactor some postprocessor/downloader code by [pukkandan](https://github.com/pukkandan) and [shirt](https://github.com/shirt-dev)
 
@@ -40,7 +84,7 @@
 * [youtube] Fix private feeds/playlists on multi-channel accounts by [colethedj](https://github.com/colethedj)
 * [youtube] Extract alerts from continuation by [colethedj](https://github.com/colethedj)
 * [cbs] Add support for ParamountPlus by [shirt](https://github.com/shirt-dev)
-* [mxplayer] Rewrite extractor with show support by [pukkandan](https://github.com/pukkandan) and [Ashish](https://github.com/Ashish)
+* [mxplayer] Rewrite extractor with show support by [pukkandan](https://github.com/pukkandan) and [Ashish0804](https://github.com/Ashish0804)
 * [gedi] Improvements from youtube-dl by [nixxo](https://github.com/nixxo)
 * [vimeo] Fix videos with password by [teesid](https://github.com/teesid)
 * [lbry] Support `lbry://` url by [nixxo](https://github.com/nixxo)
@@ -50,7 +94,7 @@
 * [downloader] Fix bug for `ffmpeg`/`httpie`
 * [update] Fix updater removing the executable bit on some UNIX distros
 * [update] Fix current build hash for UNIX
-* [documentation] Include wget/curl/aria2c install instructions for Unix by [Ashish](https://github.com/Ashish)
+* [documentation] Include wget/curl/aria2c install instructions for Unix by [Ashish0804](https://github.com/Ashish0804)
 * Fix some videos downloading with `m3u8` extension
 * Remove "fixup is ignored" warning when fixup wasn't passed by user
 
@@ -100,7 +144,7 @@
 * Merge youtube-dl: Upto [commit/4460329](https://github.com/ytdl-org/youtube-dl/commit/44603290e5002153f3ebad6230cc73aef42cc2cd) (except tmz, gedi)
 * [Readthedocs](https://yt-dlp.readthedocs.io) support by [shirt](https://github.com/shirt-dev)
 * [youtube] Show if video was a live stream in info (`was_live`)
-* [Zee5] Add new extractor by [Ashish](https://github.com/Ashish) and [pukkandan](https://github.com/pukkandan)
+* [Zee5] Add new extractor by [Ashish0804](https://github.com/Ashish0804) and [pukkandan](https://github.com/pukkandan)
 * [jwplatform] Add support for `hyland.com`
 * [tennistv] Fix extractor
 * [hls] Support media initialization by [shirt](https://github.com/shirt-dev)
@@ -296,7 +340,7 @@
 * [archive.org] Fix extractor and add support for audio and playlists by [wporr](https://github.com/wporr)
 * [Animelab] Added by [mariuszskon](https://github.com/mariuszskon)
 * [youtube:search] Fix view_count by [ohnonot](https://github.com/ohnonot)
-* [youtube] Show if video is embeddable in info
+* [youtube] Show if video is embeddable in info (`playable_in_embed`)
 * Update version badge automatically in README
 * Enable `test_youtube_search_matching`
 * Create `to_screen` and similar functions in postprocessor/common

README.md

@@ -3,7 +3,7 @@
 [![Release version](https://img.shields.io/github/v/release/yt-dlp/yt-dlp?color=brightgreen&label=Release)](https://github.com/yt-dlp/yt-dlp/releases/latest)
 [![License: Unlicense](https://img.shields.io/badge/License-Unlicense-blue.svg)](LICENSE)
 [![CI Status](https://github.com/yt-dlp/yt-dlp/workflows/Core%20Tests/badge.svg?branch=master)](https://github.com/yt-dlp/yt-dlp/actions)
-[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&label=discord&logo=discord)](https://discord.gg/S75JaBna)  
+[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&label=discord&logo=discord)](https://discord.gg/H5MNcFW63r)  
 
 [![Commits](https://img.shields.io/github/commit-activity/m/yt-dlp/yt-dlp?label=commits)](https://github.com/yt-dlp/yt-dlp/commits)
 [![Last Commit](https://img.shields.io/github/last-commit/yt-dlp/yt-dlp/master)](https://github.com/yt-dlp/yt-dlp/commits)
@@ -13,7 +13,7 @@
 
 A command-line program to download videos from youtube.com and many other [video platforms](supportedsites.md)
 
-This is a fork of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) which is inturn a fork of [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+This is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on the now inactive [youtube-dlc](https://github.com/blackjack4494/yt-dlc). The main focus of this project is adding new features and patches while also keeping up to date with the original project
 
 * [NEW FEATURES](#new-features)
 * [INSTALLATION](#installation)
@@ -47,6 +47,7 @@ This is a fork of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) which i
     * [Sorting Formats](#sorting-formats)
     * [Format Selection examples](#format-selection-examples)
 * [PLUGINS](#plugins)
+* [DEPRECATED OPTIONS](#deprecated-options)
 * [MORE](#more)
 
 
@@ -57,7 +58,7 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **[Format Sorting](#sorting-formats)**: The default format sorting options have been changed so that higher resolution and better codecs will be now preferred instead of simply using larger bitrate. Furthermore, you can now specify the sort order using `-S`. This allows for much easier format selection that what is possible by simply using `--format` ([examples](#format-selection-examples))
 
-* **Merged with youtube-dl v2021.03.14**: You get all the latest features and patches of [youtube-dl](https://github.com/ytdl-org/youtube-dl) in addition to all the features of [youtube-dlc](https://github.com/blackjack4494/yt-dlc)
+* **Merged with youtube-dl v2021.03.25**: You get all the latest features and patches of [youtube-dl](https://github.com/ytdl-org/youtube-dl) in addition to all the features of [youtube-dlc](https://github.com/blackjack4494/yt-dlc)
 
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--get-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, Playlist infojson etc. Note that the NicoNico improvements are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
@@ -74,9 +75,9 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/
 
 * **New extractors**: AnimeLab, Philo MSO, Rcs, Gedi, bitwave.tv, mildom, audius, zee5, mtv.it, wimtv, pluto.tv
 
-* **Fixed extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, tiktok, akamai, ina, rumble, tennistv
+* **Fixed extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, tiktok, akamai, ina, rumble, tennistv, amcnetworks
 
-* **Plugin support**: Extractors can be loaded from an external file. See [plugins](#plugins) for details
+* **Plugin extractors**: Extractors can be loaded from an external file. See [plugins](#plugins) for details
 
 * **Multiple paths and output templates**: You can give different [output templates](#output-template) and download paths for different types of files. You can also set a temporary path where intermediary files are downloaded to using `--paths` (`-P`)
 
@@ -423,6 +424,11 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      --write-description etc. (default)
     --no-write-playlist-metafiles    Do not write playlist metadata when using
                                      --write-info-json, --write-description etc.
+    --clean-infojson                 Remove some private fields such as
+                                     filenames from the infojson. Note that it
+                                     could still contain some personal
+                                     information (default)
+    --no-clean-infojson              Write all fields to the infojson
     --get-comments                   Retrieve video comments to be placed in the
                                      .info.json file. The comments are fetched
                                      even without this option if the extraction
@@ -466,7 +472,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --no-warnings                    Ignore warnings
     -s, --simulate                   Do not download the video and do not write
                                      anything to disk
-    --skip-download                  Do not download the video
+    --skip-download                  Do not download the video but write all
+                                     related files (Alias: --no-download)
     -g, --get-url                    Simulate, quiet but print URL
     -e, --get-title                  Simulate, quiet but print title
     --get-id                         Simulate, quiet but print id
@@ -503,7 +510,7 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve
-                                     information about the video. (Currently
+                                     information about the video (Currently
                                      supported only for YouTube)
     --user-agent UA                  Specify a custom user agent
     --referer URL                    Specify a custom referer, use if the video
@@ -517,15 +524,11 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --sleep-requests SECONDS         Number of seconds to sleep between requests
                                      during data extraction
     --sleep-interval SECONDS         Number of seconds to sleep before each
-                                     download when used alone or a lower bound
+                                     download. This is the minimum time to sleep
-                                     of a range for randomized sleep before each
+                                     when used along with --max-sleep-interval
-                                     download (minimum possible number of
+                                     (Alias: --min-sleep-interval)
-                                     seconds to sleep) when used along with
+    --max-sleep-interval SECONDS     Maximum number of seconds to sleep. Can
-                                     --max-sleep-interval
+                                     only be used along with --min-sleep-interval
-    --max-sleep-interval SECONDS     Upper bound of a range for randomized sleep
-                                     before each download (maximum possible
-                                     number of seconds to sleep). Must only be
-                                     used along with --min-sleep-interval
     --sleep-subtitles SECONDS        Number of seconds to sleep before each
                                      subtitle download
 
@@ -575,16 +578,16 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --write-subs                     Write subtitle file
     --no-write-subs                  Do not write subtitle file (default)
     --write-auto-subs                Write automatically generated subtitle file
-                                     (YouTube only)
+                                     (Alias: --write-automatic-subs)
-    --no-write-auto-subs             Do not write automatically generated
+    --no-write-auto-subs             Do not write auto-generated subtitles
-                                     subtitle file (default)
+                                     (default) (Alias: --no-write-automatic-subs)
     --all-subs                       Download all the available subtitles of the
                                      video
     --list-subs                      List all available subtitles for the video
     --sub-format FORMAT              Subtitle format, accepts formats
                                      preference, for example: "srt" or
                                      "ass/srt/best"
-    --sub-lang LANGS                 Languages of the subtitles to download
+    --sub-langs LANGS                Languages of the subtitles to download
                                      (optional) separated by commas, use --list-
                                      subs for available language tags
 
@@ -638,19 +641,19 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      ExtractAudio, VideoRemuxer, VideoConvertor,
                                      EmbedSubtitle, Metadata, Merger,
                                      FixupStretched, FixupM4a, FixupM3u8,
-                                     SubtitlesConvertor and EmbedThumbnail. The
+                                     SubtitlesConvertor, EmbedThumbnail and
-                                     supported executables are: SponSkrub,
+                                     SplitChapters. The supported executables
-                                     FFmpeg, FFprobe, and AtomicParsley. You can
+                                     are: SponSkrub, FFmpeg, FFprobe, and
-                                     also specify "PP+EXE:ARGS" to give the
+                                     AtomicParsley. You can also specify
-                                     arguments to the specified executable only
+                                     "PP+EXE:ARGS" to give the arguments to the
-                                     when being used by the specified
+                                     specified executable only when being used
-                                     postprocessor. Additionally, for
+                                     by the specified postprocessor.
-                                     ffmpeg/ffprobe, "_i"/"_o" can be appended
+                                     Additionally, for ffmpeg/ffprobe, "_i"/"_o"
-                                     to the prefix optionally followed by a
+                                     can be appended to the prefix optionally
-                                     number to pass the argument before the
+                                     followed by a number to pass the argument
-                                     specified input/output file. Eg: --ppa
+                                     before the specified input/output file. Eg:
-                                     "Merger+ffmpeg_i1:-v quiet". You can use
+                                     --ppa "Merger+ffmpeg_i1:-v quiet". You can
-                                     this option multiple times to give
+                                     use this option multiple times to give
                                      different arguments to different
                                      postprocessors. (Alias: --ppa)
     -k, --keep-video                 Keep the intermediate video file on disk
@@ -667,18 +670,24 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --add-metadata                   Write metadata to the video file
     --no-add-metadata                Do not write metadata (default)
     --parse-metadata FIELD:FORMAT    Parse additional metadata like title/artist
-                                     from other fields. Give field name to
+                                     from other fields. Give a template or field
-                                     extract data from, and format of the field
+                                     name to extract data from and the format to
-                                     seperated by a ":". Either regular
+                                     interpret it as, seperated by a ":". Either
-                                     expression with named capture groups or a
+                                     regular expression with named capture
-                                     similar syntax to the output template can
+                                     groups or a similar syntax to the output
-                                     also be used. The parsed parameters replace
+                                     template can be used for the FORMAT.
-                                     any existing values and can be use in
+                                     Similarly, the syntax for output template
-                                     output template. This option can be used
+                                     can be used for FIELD to parse the data
-                                     multiple times. Example: --parse-metadata
+                                     from multiple fields. The parsed parameters
-                                     "title:%(artist)s - %(title)s" matches a
+                                     replace any existing values and can be used
-                                     title like "Coldplay - Paradise". Example
+                                     in output templates. This option can be
-                                     (regex): --parse-metadata
+                                     used multiple times. Example: --parse-
+                                     metadata "title:%(artist)s - %(title)s"
+                                     matches a title like "Coldplay - Paradise".
+                                     Example: --parse-metadata "%(series)s
+                                     %(episode_number)s:%(title)s" sets the
+                                     title using series and episode number.
+                                     Example (regex): --parse-metadata
                                      "description:Artist - (?P<artist>.+?)"
     --xattrs                         Write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
@@ -694,8 +703,9 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      downloading and post-processing, similar to
                                      find's -exec syntax. Example: --exec 'adb
                                      push {} /sdcard/Music/ && rm {}'
-    --convert-subs FORMAT            Convert the subtitles to other format
+    --convert-subs FORMAT            Convert the subtitles to another format
                                      (currently supported: srt|ass|vtt|lrc)
+                                     (Alias: --convert-subtitles)
     --split-chapters                 Split video into multiple files based on
                                      internal chapters. The "chapter:" prefix
                                      can be used with "--paths" and "--output"
@@ -853,6 +863,7 @@ The available fields are:
  - `is_live` (boolean): Whether this video is a live stream or a fixed-length video
  - `was_live` (boolean): Whether this video was originally a live stream
  - `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
+ - `availability` (string): Whether the video is 'private', 'premium_only', 'subscriber_only', 'needs_auth', 'unlisted' or 'public'
  - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
  - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
  - `format` (string): A human-readable description of the format
@@ -1198,5 +1209,39 @@ Plugins are loaded from `<root-dir>/ytdlp_plugins/<type>/__init__.py`. Currently
 
 **Note**: `<root-dir>` is the directory of the binary (`<root-dir>/yt-dlp`), or the root directory of the module if you are running directly from source-code (`<root dir>/yt_dlp/__main__.py`)
 
+# DEPRECATED OPTIONS
+
+These are all the deprecated options and the current alternative to achieve the same effect
+
+    --cn-verification-proxy URL      --geo-verification-proxy URL
+    --id                             -o "%(id)s.%(ext)s"
+    -A, --auto-number                -o "%(autonumber)s-%(id)s.%(ext)s"
+    -t, --title                      -o "%(title)s-%(id)s.%(ext)s"
+    -l, --literal                    -o accepts literal names
+    --autonumber-size NUMBER         Use string formatting. Eg: %(autonumber)03d
+    --metadata-from-title FORMAT     --parse-metadata "title:FORMAT"
+    --prefer-avconv                  avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
+    --prefer-ffmpeg                  Default (Alias: --no-prefer-avconv)
+    --avconv-location                avconv is no longer officially supported
+    -C, --call-home                  Not implemented
+    --no-call-home                   Default
+    --write-srt                      --write-subs
+    --no-write-srt                   --no-write-subs
+    --srt-lang LANGS                 --sub-langs LANGS
+    --prefer-unsecure                --prefer-insecure
+    --rate-limit RATE                --limit-rate RATE
+    --force-write-download-archive   --force-write-archive
+    --dump-intermediate-pages        --dump-pages
+    --dump-headers                   --print-traffic
+    --youtube-print-sig-code         No longer supported
+    --trim-file-names LENGTH         --trim-filenames LENGTH
+    --yes-overwrites                 --force-overwrites
+    --load-info                      --load-info-json
+    --split-tracks                   --split-chapters
+    --no-split-tracks                --no-split-chapters
+    --sponskrub-args ARGS            --ppa "sponskrub:ARGS"
+    --test                           Only used for testing extractors
+
+
 # MORE
 For FAQ, Developer Instructions etc., see the [original README](https://github.com/ytdl-org/youtube-dl#faq)

supportedsites.md

@@ -97,7 +97,8 @@
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
- - **bbc.co.uk:iplayer:playlist**
+ - **bbc.co.uk:iplayer:episodes**
+ - **bbc.co.uk:iplayer:group**
  - **bbc.co.uk:playlist**
  - **BBVTV**
  - **Beatport**
@@ -543,6 +544,7 @@
  - **mixcloud:playlist**
  - **mixcloud:user**
  - **MLB**
+ - **MLBVideo**
  - **Mnet**
  - **MNetTV**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -634,6 +636,7 @@
  - **nicknight**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
+ - **NiconicoUser**
  - **Nintendo**
  - **Nitter**
  - **njoy**: N-JOY
@@ -917,6 +920,7 @@
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **Stitcher**
+ - **StitcherShow**
  - **StoryFire**
  - **StoryFireSeries**
  - **StoryFireUser**
@@ -1248,5 +1252,6 @@
  - **zee5:series**
  - **Zhihu**
  - **zingmp3**: mp3.zing.vn
+ - **zingmp3:album**
  - **zoom**
  - **Zype**

yt_dlp/YoutubeDL.py

@@ -60,12 +60,14 @@ from .utils import (
     encode_compat_str,
     encodeFilename,
     error_to_compat_str,
+    EntryNotInPlaylist,
     ExistingVideoReached,
     expand_path,
     ExtractorError,
     float_or_none,
     format_bytes,
     format_field,
+    FORMAT_RE,
     formatSeconds,
     GeoRestrictedError,
     int_or_none,
@@ -216,6 +218,7 @@ class YoutubeDL(object):
     logtostderr:       Log messages to stderr instead of stdout.
     writedescription:  Write the video description to a .description file
     writeinfojson:     Write the video description to a .info.json file
+    clean_infojson:    Remove private fields from the infojson
     writecomments:     Extract video comments. This will not be written to disk
                        unless writeinfojson is also given
     writeannotations:  Write the video annotations to a .annotations.xml file
@@ -770,95 +773,93 @@ class YoutubeDL(object):
                     'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
         return outtmpl_dict
 
+    def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
+        """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
+        template_dict = dict(info_dict)
+
+        # duration_string
+        template_dict['duration_string'] = (  # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
+            formatSeconds(info_dict['duration'], '-')
+            if info_dict.get('duration', None) is not None
+            else None)
+
+        # epoch
+        template_dict['epoch'] = int(time.time())
+
+        # autonumber
+        autonumber_size = self.params.get('autonumber_size')
+        if autonumber_size is None:
+            autonumber_size = 5
+        template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
+
+        # resolution if not defined
+        if template_dict.get('resolution') is None:
+            if template_dict.get('width') and template_dict.get('height'):
+                template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
+            elif template_dict.get('height'):
+                template_dict['resolution'] = '%sp' % template_dict['height']
+            elif template_dict.get('width'):
+                template_dict['resolution'] = '%dx?' % template_dict['width']
+
+        if sanitize is None:
+            sanitize = lambda k, v: v
+        template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
+                             for k, v in template_dict.items()
+                             if v is not None and not isinstance(v, (list, tuple, dict)))
+        na = self.params.get('outtmpl_na_placeholder', 'NA')
+        template_dict = collections.defaultdict(lambda: na, template_dict)
+
+        # For fields playlist_index and autonumber convert all occurrences
+        # of %(field)s to %(field)0Nd for backward compatibility
+        field_size_compat_map = {
+            'playlist_index': len(str(template_dict['n_entries'])),
+            'autonumber': autonumber_size,
+        }
+        FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
+        mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
+        if mobj:
+            outtmpl = re.sub(
+                FIELD_SIZE_COMPAT_RE,
+                r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
+                outtmpl)
+
+        numeric_fields = list(self._NUMERIC_FIELDS)
+
+        # Format date
+        FORMAT_DATE_RE = FORMAT_RE.format(r'(?P<key>(?P<field>\w+)>(?P<format>.+?))')
+        for mobj in re.finditer(FORMAT_DATE_RE, outtmpl):
+            conv_type, field, frmt, key = mobj.group('type', 'field', 'format', 'key')
+            if key in template_dict:
+                continue
+            value = strftime_or_none(template_dict.get(field), frmt, na)
+            if conv_type in 'crs':  # string
+                value = sanitize(field, value)
+            else:  # number
+                numeric_fields.append(key)
+                value = float_or_none(value, default=None)
+            if value is not None:
+                template_dict[key] = value
+
+        # Missing numeric fields used together with integer presentation types
+        # in format specification will break the argument substitution since
+        # string NA placeholder is returned for missing fields. We will patch
+        # output template for missing fields to meet string presentation type.
+        for numeric_field in numeric_fields:
+            if numeric_field not in template_dict:
+                outtmpl = re.sub(
+                    FORMAT_RE.format(re.escape(numeric_field)),
+                    r'%({0})s'.format(numeric_field), outtmpl)
+
+        return outtmpl, template_dict
+
     def _prepare_filename(self, info_dict, tmpl_type='default'):
         try:
-            template_dict = dict(info_dict)
-
-            template_dict['duration_string'] = (  # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
-                formatSeconds(info_dict['duration'], '-')
-                if info_dict.get('duration', None) is not None
-                else None)
-
-            template_dict['epoch'] = int(time.time())
-            autonumber_size = self.params.get('autonumber_size')
-            if autonumber_size is None:
-                autonumber_size = 5
-            template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
-            if template_dict.get('resolution') is None:
-                if template_dict.get('width') and template_dict.get('height'):
-                    template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
-                elif template_dict.get('height'):
-                    template_dict['resolution'] = '%sp' % template_dict['height']
-                elif template_dict.get('width'):
-                    template_dict['resolution'] = '%dx?' % template_dict['width']
-
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
                 restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id' or k.endswith('_id')))
-            template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
-                                 for k, v in template_dict.items()
-                                 if v is not None and not isinstance(v, (list, tuple, dict)))
-            na = self.params.get('outtmpl_na_placeholder', 'NA')
-            template_dict = collections.defaultdict(lambda: na, template_dict)
-
             outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
-            force_ext = OUTTMPL_TYPES.get(tmpl_type)
+            outtmpl, template_dict = self.prepare_outtmpl(outtmpl, info_dict, sanitize)
-
-            # For fields playlist_index and autonumber convert all occurrences
-            # of %(field)s to %(field)0Nd for backward compatibility
-            field_size_compat_map = {
-                'playlist_index': len(str(template_dict['n_entries'])),
-                'autonumber': autonumber_size,
-            }
-            FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
-            mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
-            if mobj:
-                outtmpl = re.sub(
-                    FIELD_SIZE_COMPAT_RE,
-                    r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
-                    outtmpl)
-
-            # As of [1] format syntax is:
-            #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
-            # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
-            FORMAT_RE = r'''(?x)
-                (?<!%)
-                %
-                \({0}\)  # mapping key
-                (?:[#0\-+ ]+)?  # conversion flags (optional)
-                (?:\d+)?  # minimum field width (optional)
-                (?:\.\d+)?  # precision (optional)
-                [hlL]?  # length modifier (optional)
-                (?P<type>[diouxXeEfFgGcrs%])  # conversion type
-            '''
-
-            numeric_fields = list(self._NUMERIC_FIELDS)
-
-            # Format date
-            FORMAT_DATE_RE = FORMAT_RE.format(r'(?P<key>(?P<field>\w+)>(?P<format>.+?))')
-            for mobj in re.finditer(FORMAT_DATE_RE, outtmpl):
-                conv_type, field, frmt, key = mobj.group('type', 'field', 'format', 'key')
-                if key in template_dict:
-                    continue
-                value = strftime_or_none(template_dict.get(field), frmt, na)
-                if conv_type in 'crs':  # string
-                    value = sanitize(field, value)
-                else:  # number
-                    numeric_fields.append(key)
-                    value = float_or_none(value, default=None)
-                if value is not None:
-                    template_dict[key] = value
-
-            # Missing numeric fields used together with integer presentation types
-            # in format specification will break the argument substitution since
-            # string NA placeholder is returned for missing fields. We will patch
-            # output template for missing fields to meet string presentation type.
-            for numeric_field in numeric_fields:
-                if numeric_field not in template_dict:
-                    outtmpl = re.sub(
-                        FORMAT_RE.format(re.escape(numeric_field)),
-                        r'%({0})s'.format(numeric_field), outtmpl)
 
             # expand_path translates '%%' into '%' and '$$' into '$'
             # correspondingly that is not what we want since we need to keep
@@ -873,6 +874,7 @@ class YoutubeDL(object):
             # title "Hello $PATH", we don't want `$PATH` to be expanded.
             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 
+            force_ext = OUTTMPL_TYPES.get(tmpl_type)
             if force_ext is not None:
                 filename = replace_extension(filename, force_ext, template_dict.get('ext'))
 
@@ -1179,49 +1181,16 @@ class YoutubeDL(object):
         playlist = ie_result.get('title') or ie_result.get('id')
         self.to_screen('[download] Downloading playlist: %s' % playlist)
 
-        if self.params.get('allow_playlist_files', True):
+        if 'entries' not in ie_result:
-            ie_copy = {
+            raise EntryNotInPlaylist()
-                'playlist': playlist,
+        incomplete_entries = bool(ie_result.get('requested_entries'))
-                'playlist_id': ie_result.get('id'),
+        if incomplete_entries:
-                'playlist_title': ie_result.get('title'),
+            def fill_missing_entries(entries, indexes):
-                'playlist_uploader': ie_result.get('uploader'),
+                ret = [None] * max(*indexes)
-                'playlist_uploader_id': ie_result.get('uploader_id'),
+                for i, entry in zip(indexes, entries):
-                'playlist_index': 0
+                    ret[i - 1] = entry
-            }
+                return ret
-            ie_copy.update(dict(ie_result))
+            ie_result['entries'] = fill_missing_entries(ie_result['entries'], ie_result['requested_entries'])
-
-            if self.params.get('writeinfojson', False):
-                infofn = self.prepare_filename(ie_copy, 'pl_infojson')
-                if not self._ensure_dir_exists(encodeFilename(infofn)):
-                    return
-                if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
-                    self.to_screen('[info] Playlist metadata is already present')
-                else:
-                    playlist_info = dict(ie_result)
-                    # playlist_info['entries'] = list(playlist_info['entries'])  # Entries is a generator which shouldnot be resolved here
-                    del playlist_info['entries']
-                    self.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn)
-                    try:
-                        write_json_file(self.filter_requested_info(playlist_info), infofn)
-                    except (OSError, IOError):
-                        self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
-
-            if self.params.get('writedescription', False):
-                descfn = self.prepare_filename(ie_copy, 'pl_description')
-                if not self._ensure_dir_exists(encodeFilename(descfn)):
-                    return
-                if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
-                    self.to_screen('[info] Playlist description is already present')
-                elif ie_result.get('description') is None:
-                    self.report_warning('There\'s no playlist description to write.')
-                else:
-                    try:
-                        self.to_screen('[info] Writing playlist description to: ' + descfn)
-                        with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
-                            descfile.write(ie_result['description'])
-                    except (OSError, IOError):
-                        self.report_error('Cannot write playlist description file ' + descfn)
-                        return
 
         playlist_results = []
 
@@ -1248,25 +1217,20 @@ class YoutubeDL(object):
 
         def make_playlistitems_entries(list_ie_entries):
             num_entries = len(list_ie_entries)
-            return [
+            for i in playlistitems:
-                list_ie_entries[i - 1] for i in playlistitems
+                if -num_entries < i <= num_entries:
-                if -num_entries <= i - 1 < num_entries]
+                    yield list_ie_entries[i - 1]
-
+                elif incomplete_entries:
-        def report_download(num_entries):
+                    raise EntryNotInPlaylist()
-            self.to_screen(
-                '[%s] playlist %s: Downloading %d videos' %
-                (ie_result['extractor'], playlist, num_entries))
 
         if isinstance(ie_entries, list):
             n_all_entries = len(ie_entries)
             if playlistitems:
-                entries = make_playlistitems_entries(ie_entries)
+                entries = list(make_playlistitems_entries(ie_entries))
             else:
                 entries = ie_entries[playliststart:playlistend]
             n_entries = len(entries)
-            self.to_screen(
+            msg = 'Collected %d videos; downloading %d of them' % (n_all_entries, n_entries)
-                '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
-                (ie_result['extractor'], playlist, n_all_entries, n_entries))
         elif isinstance(ie_entries, PagedList):
             if playlistitems:
                 entries = []
@@ -1278,25 +1242,73 @@ class YoutubeDL(object):
                 entries = ie_entries.getslice(
                     playliststart, playlistend)
             n_entries = len(entries)
-            report_download(n_entries)
+            msg = 'Downloading %d videos' % n_entries
         else:  # iterable
             if playlistitems:
-                entries = make_playlistitems_entries(list(itertools.islice(
+                entries = list(make_playlistitems_entries(list(itertools.islice(
-                    ie_entries, 0, max(playlistitems))))
+                    ie_entries, 0, max(playlistitems)))))
             else:
                 entries = list(itertools.islice(
                     ie_entries, playliststart, playlistend))
             n_entries = len(entries)
-            report_download(n_entries)
+            msg = 'Downloading %d videos' % n_entries
+
+        if any((entry is None for entry in entries)):
+            raise EntryNotInPlaylist()
+        if not playlistitems and (playliststart or playlistend):
+            playlistitems = list(range(1 + playliststart, 1 + playliststart + len(entries)))
+        ie_result['entries'] = entries
+        ie_result['requested_entries'] = playlistitems
+
+        if self.params.get('allow_playlist_files', True):
+            ie_copy = {
+                'playlist': playlist,
+                'playlist_id': ie_result.get('id'),
+                'playlist_title': ie_result.get('title'),
+                'playlist_uploader': ie_result.get('uploader'),
+                'playlist_uploader_id': ie_result.get('uploader_id'),
+                'playlist_index': 0
+            }
+            ie_copy.update(dict(ie_result))
+
+            if self.params.get('writeinfojson', False):
+                infofn = self.prepare_filename(ie_copy, 'pl_infojson')
+                if not self._ensure_dir_exists(encodeFilename(infofn)):
+                    return
+                if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
+                    self.to_screen('[info] Playlist metadata is already present')
+                else:
+                    self.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn)
+                    try:
+                        write_json_file(self.filter_requested_info(ie_result, self.params.get('clean_infojson', True)), infofn)
+                    except (OSError, IOError):
+                        self.report_error('Cannot write playlist metadata to JSON file ' + infofn)
+
+            if self.params.get('writedescription', False):
+                descfn = self.prepare_filename(ie_copy, 'pl_description')
+                if not self._ensure_dir_exists(encodeFilename(descfn)):
+                    return
+                if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
+                    self.to_screen('[info] Playlist description is already present')
+                elif ie_result.get('description') is None:
+                    self.report_warning('There\'s no playlist description to write.')
+                else:
+                    try:
+                        self.to_screen('[info] Writing playlist description to: ' + descfn)
+                        with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+                            descfile.write(ie_result['description'])
+                    except (OSError, IOError):
+                        self.report_error('Cannot write playlist description file ' + descfn)
+                        return
 
         if self.params.get('playlistreverse', False):
             entries = entries[::-1]
-
         if self.params.get('playlistrandom', False):
             random.shuffle(entries)
 
         x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 
+        self.to_screen('[%s] playlist %s: %s' % (ie_result['extractor'], playlist, msg))
         for i, entry in enumerate(entries, 1):
             self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
             # This __x_forwarded_for_ip thing is a bit ugly but requires
@@ -1310,7 +1322,7 @@ class YoutubeDL(object):
                 'playlist_title': ie_result.get('title'),
                 'playlist_uploader': ie_result.get('uploader'),
                 'playlist_uploader_id': ie_result.get('uploader_id'),
-                'playlist_index': playlistitems[i - 1] if playlistitems else i + playliststart,
+                'playlist_index': playlistitems[i - 1] if playlistitems else i,
                 'extractor': ie_result['extractor'],
                 'webpage_url': ie_result['webpage_url'],
                 'webpage_url_basename': url_basename(ie_result['webpage_url']),
@@ -1644,7 +1656,7 @@ class YoutubeDL(object):
                         new_dict.update({
                             'width': the_only_video.get('width'),
                             'height': the_only_video.get('height'),
-                            'resolution': the_only_video.get('resolution'),
+                            'resolution': the_only_video.get('resolution') or self.format_resolution(the_only_video),
                             'fps': the_only_video.get('fps'),
                             'vcodec': the_only_video.get('vcodec'),
                             'vbr': the_only_video.get('vbr'),
@@ -2047,7 +2059,7 @@ class YoutubeDL(object):
         print_mandatory('format')
         if self.params.get('forcejson', False):
             self.post_extract(info_dict)
-            self.to_stdout(json.dumps(info_dict))
+            self.to_stdout(json.dumps(info_dict, default=repr))
 
     def process_info(self, info_dict):
         """Process a single resolved IE result."""
@@ -2075,6 +2087,7 @@ class YoutubeDL(object):
 
         info_dict = self.pre_process(info_dict)
 
+        # info_dict['_filename'] needs to be set for backward compatibility
         info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
         temp_filename = self.prepare_filename(info_dict, 'temp')
         files_to_move = {}
@@ -2140,7 +2153,10 @@ class YoutubeDL(object):
                 fd.add_progress_hook(ph)
             if self.params.get('verbose'):
                 self.to_screen('[debug] Invoking downloader on %r' % info.get('url'))
-            return fd.download(name, info, subtitle)
+            new_info = dict(info)
+            if new_info.get('http_headers') is None:
+                new_info['http_headers'] = self._calc_headers(new_info)
+            return fd.download(name, new_info, subtitle)
 
         subtitles_are_requested = any([self.params.get('writesubtitles', False),
                                        self.params.get('writeautomaticsub')])
@@ -2159,6 +2175,7 @@ class YoutubeDL(object):
                 sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext'))
                 if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
                     self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
+                    sub_info['filepath'] = sub_filename
                     files_to_move[sub_filename] = sub_filename_final
                 else:
                     self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@@ -2168,13 +2185,15 @@ class YoutubeDL(object):
                             # See https://github.com/ytdl-org/youtube-dl/issues/10268
                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                                 subfile.write(sub_info['data'])
+                            sub_info['filepath'] = sub_filename
                             files_to_move[sub_filename] = sub_filename_final
                         except (OSError, IOError):
                             self.report_error('Cannot write subtitles file ' + sub_filename)
                             return
                     else:
                         try:
-                            dl(sub_filename, sub_info, subtitle=True)
+                            dl(sub_filename, sub_info.copy(), subtitle=True)
+                            sub_info['filepath'] = sub_filename
                             files_to_move[sub_filename] = sub_filename_final
                         except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                             self.report_warning('Unable to download subtitle for "%s": %s' %
@@ -2212,7 +2231,7 @@ class YoutubeDL(object):
             else:
                 self.to_screen('[info] Writing video metadata as JSON to: ' + infofn)
                 try:
-                    write_json_file(self.filter_requested_info(info_dict), infofn)
+                    write_json_file(self.filter_requested_info(info_dict, self.params.get('clean_infojson', True)), infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write video metadata to JSON file ' + infofn)
                     return
@@ -2223,7 +2242,7 @@ class YoutubeDL(object):
         for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp):
             thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext'))
             thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext'))
-            files_to_move[thumb_filename_temp] = info_dict['__thumbnail_filename'] = thumb_filename
+            files_to_move[thumb_filename_temp] = thumb_filename
 
         # Write internet shortcut files
         url_link = webloc_link = desktop_link = False
@@ -2336,10 +2355,17 @@ class YoutubeDL(object):
 
                     requested_formats = info_dict['requested_formats']
                     old_ext = info_dict['ext']
-                    if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
+                    if self.params.get('merge_output_format') is None:
-                        info_dict['ext'] = 'mkv'
+                        if not compatible_formats(requested_formats):
-                        self.report_warning(
+                            info_dict['ext'] = 'mkv'
-                            'Requested formats are incompatible for merge and will be merged into mkv.')
+                            self.report_warning(
+                                'Requested formats are incompatible for merge and will be merged into mkv.')
+                        if (info_dict['ext'] == 'webm'
+                                and self.params.get('writethumbnail', False)
+                                and info_dict.get('thumbnails')):
+                            info_dict['ext'] = 'mkv'
+                            self.report_warning(
+                                'webm doesn\'t support embedding a thumbnail, mkv will be used.')
 
                     def correct_ext(filename):
                         filename_real_ext = os.path.splitext(filename)[1][1:]
@@ -2455,13 +2481,13 @@ class YoutubeDL(object):
                         assert fixup_policy in ('ignore', 'never')
 
                 try:
-                    self.post_process(dl_filename, info_dict, files_to_move)
+                    info_dict = self.post_process(dl_filename, info_dict, files_to_move)
                 except PostProcessingError as err:
                     self.report_error('Postprocessing: %s' % str(err))
                     return
                 try:
                     for ph in self._post_hooks:
-                        ph(full_filename)
+                        ph(info_dict['filepath'])
                 except Exception as err:
                     self.report_error('post hooks: %s' % str(err))
                     return
@@ -2501,7 +2527,7 @@ class YoutubeDL(object):
             else:
                 if self.params.get('dump_single_json', False):
                     self.post_extract(res)
-                    self.to_stdout(json.dumps(res))
+                    self.to_stdout(json.dumps(res, default=repr))
 
         return self._download_retcode
 
@@ -2510,10 +2536,10 @@ class YoutubeDL(object):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
-            info = self.filter_requested_info(json.loads('\n'.join(f)))
+            info = self.filter_requested_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True))
         try:
             self.process_ie_result(info, download=True)
-        except DownloadError:
+        except (DownloadError, EntryNotInPlaylist):
             webpage_url = info.get('webpage_url')
             if webpage_url is not None:
                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
@@ -2523,21 +2549,32 @@ class YoutubeDL(object):
         return self._download_retcode
 
     @staticmethod
-    def filter_requested_info(info_dict):
+    def filter_requested_info(info_dict, actually_filter=True):
-        fields_to_remove = ('requested_formats', 'requested_subtitles')
+        if not actually_filter:
-        return dict(
+            info_dict['epoch'] = int(time.time())
-            (k, v) for k, v in info_dict.items()
+            return info_dict
-            if (k[0] != '_' or k == '_type') and k not in fields_to_remove)
+        exceptions = {
+            'remove': ['requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries'],
+            'keep': ['_type'],
+        }
+        keep_key = lambda k: k in exceptions['keep'] or not (k.startswith('_') or k in exceptions['remove'])
+        filter_fn = lambda obj: (
+            list(map(filter_fn, obj)) if isinstance(obj, (list, tuple))
+            else obj if not isinstance(obj, dict)
+            else dict((k, filter_fn(v)) for k, v in obj.items() if keep_key(k)))
+        return filter_fn(info_dict)
 
-    def run_pp(self, pp, infodict, files_to_move={}):
+    def run_pp(self, pp, infodict):
         files_to_delete = []
+        if '__files_to_move' not in infodict:
+            infodict['__files_to_move'] = {}
         files_to_delete, infodict = pp.run(infodict)
         if not files_to_delete:
-            return files_to_move, infodict
+            return infodict
 
         if self.params.get('keepvideo', False):
             for f in files_to_delete:
-                files_to_move.setdefault(f, '')
+                infodict['__files_to_move'].setdefault(f, '')
         else:
             for old_filename in set(files_to_delete):
                 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
@@ -2545,9 +2582,9 @@ class YoutubeDL(object):
                     os.remove(encodeFilename(old_filename))
                 except (IOError, OSError):
                     self.report_warning('Unable to remove downloaded original file')
-                if old_filename in files_to_move:
+                if old_filename in infodict['__files_to_move']:
-                    del files_to_move[old_filename]
+                    del infodict['__files_to_move'][old_filename]
-        return files_to_move, infodict
+        return infodict
 
     @staticmethod
     def post_extract(info_dict):
@@ -2570,20 +2607,22 @@ class YoutubeDL(object):
     def pre_process(self, ie_info):
         info = dict(ie_info)
         for pp in self._pps['beforedl']:
-            info = self.run_pp(pp, info)[1]
+            info = self.run_pp(pp, info)
         return info
 
-    def post_process(self, filename, ie_info, files_to_move={}):
+    def post_process(self, filename, ie_info, files_to_move=None):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        info['__files_to_move'] = {}
+        info['__files_to_move'] = files_to_move or {}
 
         for pp in ie_info.get('__postprocessors', []) + self._pps['normal']:
-            files_to_move, info = self.run_pp(pp, info, files_to_move)
+            info = self.run_pp(pp, info)
-        info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info)[1]
+        info = self.run_pp(MoveFilesAfterDownloadPP(self), info)
+        del info['__files_to_move']
         for pp in self._pps['aftermove']:
-            info = self.run_pp(pp, info, {})[1]
+            info = self.run_pp(pp, info)
+        return info
 
     def _make_archive_id(self, info_dict):
         video_id = info_dict.get('id')
@@ -2632,12 +2671,11 @@ class YoutubeDL(object):
             return 'audio only'
         if format.get('resolution') is not None:
             return format['resolution']
-        if format.get('height') is not None:
+        if format.get('width') and format.get('height'):
-            if format.get('width') is not None:
+            res = '%dx%d' % (format['width'], format['height'])
-                res = '%sx%s' % (format['width'], format['height'])
+        elif format.get('height'):
-            else:
+            res = '%sp' % format['height']
-                res = '%sp' % format['height']
+        elif format.get('width'):
-        elif format.get('width') is not None:
             res = '%dx?' % format['width']
         else:
             res = default
@@ -2951,7 +2989,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '%s.' % t['id'] if multiple else ''
             thumb_display_id = '%s ' % t['id'] if multiple else ''
-            t['filename'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
+            t['filepath'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
 
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
                 ret.append(suffix + thumb_ext)

yt_dlp/__init__.py

@@ -491,6 +491,7 @@ def _real_main(argv=None):
         'writeannotations': opts.writeannotations,
         'writeinfojson': opts.writeinfojson,
         'allow_playlist_files': opts.allow_playlist_files,
+        'clean_infojson': opts.clean_infojson,
         'getcomments': opts.getcomments,
         'writethumbnail': opts.writethumbnail,
         'write_all_thumbnails': opts.write_all_thumbnails,

yt_dlp/downloader/dash.py

@@ -63,6 +63,8 @@ class DashSegmentsFD(FragmentFD):
             })
 
         if real_downloader:
+            self.to_screen(
+                '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
             info_copy = info_dict.copy()
             info_copy['fragments'] = fragments_to_download
             fd = real_downloader(self.ydl, self.params)
@@ -110,7 +112,7 @@ class DashSegmentsFD(FragmentFD):
                 if count > fragment_retries:
                     if not fatal:
                         return False, frag_index
-                    self.report_error('giving up after %s fragment retries' % fragment_retries)
+                    self.report_error('Giving up after %s fragment retries' % fragment_retries)
                     return False, frag_index
 
                 return frag_content, frag_index

yt_dlp/downloader/external.py

@@ -24,7 +24,6 @@ from ..utils import (
     cli_bool_option,
     cli_configuration_args,
     encodeFilename,
-    error_to_compat_str,
     encodeArgument,
     handle_youtubedl_headers,
     check_executable,
@@ -117,19 +116,42 @@ class ExternalFD(FileDownloader):
 
         self._debug_cmd(cmd)
 
-        p = subprocess.Popen(
-            cmd, stderr=subprocess.PIPE)
-        _, stderr = process_communicate_or_kill(p)
-        if p.returncode != 0:
-            self.to_stderr(stderr.decode('utf-8', 'replace'))
-
         if 'fragments' in info_dict:
-            file_list = []
+            fragment_retries = self.params.get('fragment_retries', 0)
+            skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
+
+            count = 0
+            while count <= fragment_retries:
+                p = subprocess.Popen(
+                    cmd, stderr=subprocess.PIPE)
+                _, stderr = process_communicate_or_kill(p)
+                if p.returncode == 0:
+                    break
+                # TODO: Decide whether to retry based on error code
+                # https://aria2.github.io/manual/en/html/aria2c.html#exit-status
+                self.to_stderr(stderr.decode('utf-8', 'replace'))
+                count += 1
+                if count <= fragment_retries:
+                    self.to_screen(
+                        '[%s] Got error. Retrying fragments (attempt %d of %s)...'
+                        % (self.get_basename(), count, self.format_retries(fragment_retries)))
+            if count > fragment_retries:
+                if not skip_unavailable_fragments:
+                    self.report_error('Giving up after %s fragment retries' % fragment_retries)
+                    return -1
+
             dest, _ = sanitize_open(tmpfilename, 'wb')
-            for i, fragment in enumerate(info_dict['fragments']):
+            for frag_index, fragment in enumerate(info_dict['fragments']):
-                file = '%s-Frag%d' % (tmpfilename, i)
+                fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index)
+                try:
+                    src, _ = sanitize_open(fragment_filename, 'rb')
+                except IOError:
+                    if skip_unavailable_fragments and frag_index > 1:
+                        self.to_screen('[%s] Skipping fragment %d ...' % (self.get_basename(), frag_index))
+                        continue
+                    self.report_error('Unable to open fragment %d' % frag_index)
+                    return -1
                 decrypt_info = fragment.get('decrypt_info')
-                src, _ = sanitize_open(file, 'rb')
                 if decrypt_info:
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV')
@@ -146,20 +168,16 @@ class ExternalFD(FileDownloader):
                     fragment_data = src.read()
                     dest.write(fragment_data)
                 src.close()
-                file_list.append(file)
+                if not self.params.get('keep_fragments', False):
+                    os.remove(encodeFilename(fragment_filename))
             dest.close()
-            if not self.params.get('keep_fragments', False):
+            os.remove(encodeFilename('%s.frag.urls' % tmpfilename))
-                for file_path in file_list:
+        else:
-                    try:
+            p = subprocess.Popen(
-                        os.remove(file_path)
+                cmd, stderr=subprocess.PIPE)
-                    except OSError as ose:
+            _, stderr = process_communicate_or_kill(p)
-                        self.report_error("Unable to delete file %s; %s" % (file_path, error_to_compat_str(ose)))
+            if p.returncode != 0:
-                try:
+                self.to_stderr(stderr.decode('utf-8', 'replace'))
-                    file_path = '%s.frag.urls' % tmpfilename
-                    os.remove(file_path)
-                except OSError as ose:
-                    self.report_error("Unable to delete file %s; %s" % (file_path, error_to_compat_str(ose)))
-
         return p.returncode
 
     def _prepare_url(self, info_dict, url):
@@ -253,14 +271,12 @@ class Aria2cFD(ExternalFD):
         return all(check_results)
 
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-c']
+        cmd = [self.exe, '-c',
-        dn = os.path.dirname(tmpfilename)
+               '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
-        if 'fragments' not in info_dict:
+               '--file-allocation=none', '-x16', '-j16', '-s16']
-            cmd += ['--out', os.path.basename(tmpfilename)]
+        if 'fragments' in info_dict:
-        verbose_level_args = ['--console-log-level=warn', '--summary-interval=0']
+            cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
-        cmd += self._configuration_args(['--file-allocation=none', '-x16', '-j16', '-s16'] + verbose_level_args)
+
-        if dn:
-            cmd += ['--dir', dn]
         if info_dict.get('http_headers') is not None:
             for key, val in info_dict['http_headers'].items():
                 cmd += ['--header', '%s: %s' % (key, val)]
@@ -268,19 +284,25 @@ class Aria2cFD(ExternalFD):
         cmd += self._option('--all-proxy', 'proxy')
         cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
         cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
+        cmd += self._configuration_args()
+
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            cmd += ['--dir', dn]
+        if 'fragments' not in info_dict:
+            cmd += ['--out', os.path.basename(tmpfilename)]
         cmd += ['--auto-file-renaming=false']
+
         if 'fragments' in info_dict:
-            cmd += verbose_level_args
+            cmd += ['--file-allocation=none', '--uri-selector=inorder']
-            cmd += ['--uri-selector', 'inorder', '--download-result=hide']
             url_list_file = '%s.frag.urls' % tmpfilename
             url_list = []
-            for i, fragment in enumerate(info_dict['fragments']):
+            for frag_index, fragment in enumerate(info_dict['fragments']):
-                tmpsegmentname = '%s-Frag%d' % (os.path.basename(tmpfilename), i)
+                fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
-                url_list.append('%s\n\tout=%s' % (fragment['url'], tmpsegmentname))
+                url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename))
             stream, _ = sanitize_open(url_list_file, 'wb')
             stream.write('\n'.join(url_list).encode('utf-8'))
             stream.close()
-
             cmd += ['-i', url_list_file]
         else:
             cmd += ['--', info_dict['url']]

yt_dlp/downloader/hls.py

@@ -83,14 +83,13 @@ class HlsFD(FragmentFD):
 
         if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')):
             if info_dict.get('extra_param_to_segment_url') or info_dict.get('_decryption_key_url'):
-                self.report_error('pycryptodome not found. Please install it.')
+                self.report_error('pycryptodome not found. Please install')
                 return False
             if self.can_download(s, info_dict, with_crypto=True):
-                self.report_warning('pycryptodome is needed to download this file with hlsnative')
+                self.report_warning('pycryptodome is needed to download this file natively')
-            self.report_warning(
-                'hlsnative has detected features it does not support, '
-                'extraction will be delegated to ffmpeg')
             fd = FFmpegFD(self.ydl, self.params)
+            self.report_warning(
+                '%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
             # TODO: Make progress updates work without hooking twice
             # for ph in self._progress_hooks:
             #     fd.add_progress_hook(ph)
@@ -99,6 +98,9 @@ class HlsFD(FragmentFD):
         real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
         if real_downloader and not real_downloader.supports_manifest(s):
             real_downloader = None
+        if real_downloader:
+            self.to_screen(
+                '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
 
         def is_ad_fragment_start(s):
             return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
@@ -186,7 +188,7 @@ class HlsFD(FragmentFD):
                         continue
                     if frag_index > 0:
                         self.report_error(
-                            'initialization fragment found after media fragments, unable to download')
+                            'Initialization fragment found after media fragments, unable to download')
                         return False
                     frag_index += 1
                     map_info = parse_m3u8_attributes(line[11:])
@@ -289,6 +291,7 @@ class HlsFD(FragmentFD):
                         if count <= fragment_retries:
                             self.report_retry_fragment(err, frag_index, count, fragment_retries)
                 if count > fragment_retries:
+                    self.report_error('Giving up after %s fragment retries' % fragment_retries)
                     return False, frag_index
 
                 if decrypt_info['METHOD'] == 'AES-128':

yt_dlp/downloader/rtmp.py

@@ -117,7 +117,7 @@ class RtmpFD(FileDownloader):
 
         # Check for rtmpdump first
         if not check_executable('rtmpdump', ['-h']):
-            self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install it.')
+            self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install')
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when

yt_dlp/downloader/rtsp.py

@@ -24,7 +24,7 @@ class RtspFD(FileDownloader):
             args = [
                 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
         else:
-            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install one')
             return False
 
         self._debug_cmd(args)

yt_dlp/downloader/youtube_live_chat.py

@@ -79,8 +79,7 @@ class YoutubeLiveChatReplayFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        success, raw_fragment = dl_fragment(
+        success, raw_fragment = dl_fragment(info_dict['url'])
-            'https://www.youtube.com/watch?v={}'.format(video_id))
         if not success:
             return False
         try:

yt_dlp/extractor/amcnetworks.py

@@ -65,15 +65,35 @@ class AMCNetworksIE(ThePlatformIE):
     def _real_extract(self, url):
         site, display_id = re.match(self._VALID_URL, url).groups()
         requestor_id = self._REQUESTOR_ID_MAP[site]
-        properties = self._download_json(
+        page_data = self._download_json(
-            'https://content-delivery-gw.svc.ds.amcn.com/api/v2/content/amcn/%s/url/%s' % (requestor_id.lower(), display_id),
+            'https://content-delivery-gw.svc.ds.amcn.com/api/v2/content/amcn/%s/url/%s'
-            display_id)['data']['properties']
+            % (requestor_id.lower(), display_id), display_id)['data']
+        properties = page_data.get('properties') or {}
         query = {
             'mbr': 'true',
             'manifest': 'm3u',
         }
-        tp_path = 'M_UwQC/media/' + properties['videoPid']
+
-        media_url = 'https://link.theplatform.com/s/' + tp_path
+        video_player_count = 0
+        try:
+            for v in page_data['children']:
+                if v.get('type') == 'video-player':
+                    releasePid = v['properties']['currentVideo']['meta']['releasePid']
+                    tp_path = 'M_UwQC/' + releasePid
+                    media_url = 'https://link.theplatform.com/s/' + tp_path
+                    video_player_count += 1
+        except KeyError:
+            pass
+        if video_player_count > 1:
+            self.report_warning(
+                'The JSON data has %d video players. Only one will be extracted' % video_player_count)
+
+        # Fall back to videoPid if releasePid not found.
+        # TODO: Fall back to videoPid if releasePid manifest uses DRM.
+        if not video_player_count:
+            tp_path = 'M_UwQC/media/' + properties['videoPid']
+            media_url = 'https://link.theplatform.com/s/' + tp_path
+
         theplatform_metadata = self._download_theplatform_metadata(tp_path, display_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)
         video_id = theplatform_metadata['pid']
@@ -90,30 +110,41 @@ class AMCNetworksIE(ThePlatformIE):
         formats, subtitles = self._extract_theplatform_smil(
             media_url, video_id)
         self._sort_formats(formats)
+
+        thumbnails = []
+        thumbnail_urls = [properties.get('imageDesktop')]
+        if 'thumbnail' in info:
+            thumbnail_urls.append(info.pop('thumbnail'))
+        for thumbnail_url in thumbnail_urls:
+            if not thumbnail_url:
+                continue
+            mobj = re.search(r'(\d+)x(\d+)', thumbnail_url)
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int(mobj.group(1)) if mobj else None,
+                'height': int(mobj.group(2)) if mobj else None,
+            })
+
         info.update({
+            'age_limit': parse_age_limit(rating),
+            'formats': formats,
             'id': video_id,
             'subtitles': subtitles,
-            'formats': formats,
+            'thumbnails': thumbnails,
-            'age_limit': parse_age_limit(parse_age_limit(rating)),
         })
         ns_keys = theplatform_metadata.get('$xmlns', {}).keys()
         if ns_keys:
             ns = list(ns_keys)[0]
-            series = theplatform_metadata.get(ns + '$show')
+            episode = theplatform_metadata.get(ns + '$episodeTitle') or None
-            season_number = int_or_none(
-                theplatform_metadata.get(ns + '$season'))
-            episode = theplatform_metadata.get(ns + '$episodeTitle')
             episode_number = int_or_none(
                 theplatform_metadata.get(ns + '$episode'))
-            if season_number:
+            season_number = int_or_none(
-                title = 'Season %d - %s' % (season_number, title)
+                theplatform_metadata.get(ns + '$season'))
-            if series:
+            series = theplatform_metadata.get(ns + '$show') or None
-                title = '%s - %s' % (series, title)
             info.update({
-                'title': title,
-                'series': series,
-                'season_number': season_number,
                 'episode': episode,
                 'episode_number': episode_number,
+                'season_number': season_number,
+                'series': series,
             })
         return info

yt_dlp/extractor/bbc.py

@@ -1,17 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import itertools
+import json
 import re
 
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_Element,
     compat_HTTPError,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    OnDemandPagedList,
     clean_html,
     dict_get,
     float_or_none,
@@ -811,7 +816,7 @@ class BBCIE(BBCCoUkIE):
 
     @classmethod
     def suitable(cls, url):
-        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
+        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerEpisodesIE, BBCCoUkIPlayerGroupIE, BBCCoUkPlaylistIE)
         return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
                 else super(BBCIE, cls).suitable(url))
 
@@ -1338,21 +1343,149 @@ class BBCCoUkPlaylistBaseIE(InfoExtractor):
             playlist_id, title, description)
 
 
-class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
+class BBCCoUkIPlayerPlaylistBaseIE(InfoExtractor):
-    IE_NAME = 'bbc.co.uk:iplayer:playlist'
+    _VALID_URL_TMPL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/%%s/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/(?:episodes|group)/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
+
-    _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
+    @staticmethod
-    _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
+    def _get_default(episode, key, default_key='default'):
+        return try_get(episode, lambda x: x[key][default_key])
+
+    def _get_description(self, data):
+        synopsis = data.get(self._DESCRIPTION_KEY) or {}
+        return dict_get(synopsis, ('large', 'medium', 'small'))
+
+    def _fetch_page(self, programme_id, per_page, series_id, page):
+        elements = self._get_elements(self._call_api(
+            programme_id, per_page, page + 1, series_id))
+        for element in elements:
+            episode = self._get_episode(element)
+            episode_id = episode.get('id')
+            if not episode_id:
+                continue
+            thumbnail = None
+            image = self._get_episode_image(episode)
+            if image:
+                thumbnail = image.replace('{recipe}', 'raw')
+            category = self._get_default(episode, 'labels', 'category')
+            yield {
+                '_type': 'url',
+                'id': episode_id,
+                'title': self._get_episode_field(episode, 'subtitle'),
+                'url': 'https://www.bbc.co.uk/iplayer/episode/' + episode_id,
+                'thumbnail': thumbnail,
+                'description': self._get_description(episode),
+                'categories': [category] if category else None,
+                'series': self._get_episode_field(episode, 'title'),
+                'ie_key': BBCCoUkIE.ie_key(),
+            }
+
+    def _real_extract(self, url):
+        pid = self._match_id(url)
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        series_id = qs.get('seriesId', [None])[0]
+        page = qs.get('page', [None])[0]
+        per_page = 36 if page else self._PAGE_SIZE
+        fetch_page = functools.partial(self._fetch_page, pid, per_page, series_id)
+        entries = fetch_page(int(page) - 1) if page else OnDemandPagedList(fetch_page, self._PAGE_SIZE)
+        playlist_data = self._get_playlist_data(self._call_api(pid, 1))
+        return self.playlist_result(
+            entries, pid, self._get_playlist_title(playlist_data),
+            self._get_description(playlist_data))
+
+
+class BBCCoUkIPlayerEpisodesIE(BBCCoUkIPlayerPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:iplayer:episodes'
+    _VALID_URL = BBCCoUkIPlayerPlaylistBaseIE._VALID_URL_TMPL % 'episodes'
     _TESTS = [{
         'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
         'info_dict': {
             'id': 'b05rcz9v',
             'title': 'The Disappearance',
-            'description': 'French thriller serial about a missing teenager.',
+            'description': 'md5:58eb101aee3116bad4da05f91179c0cb',
         },
-        'playlist_mincount': 6,
+        'playlist_mincount': 8,
-        'skip': 'This programme is not currently available on BBC iPlayer',
     }, {
+        # all seasons
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/b094m5t9/doctor-foster',
+        'info_dict': {
+            'id': 'b094m5t9',
+            'title': 'Doctor Foster',
+            'description': 'md5:5aa9195fad900e8e14b52acd765a9fd6',
+        },
+        'playlist_mincount': 10,
+    }, {
+        # explicit season
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/b094m5t9/doctor-foster?seriesId=b094m6nv',
+        'info_dict': {
+            'id': 'b094m5t9',
+            'title': 'Doctor Foster',
+            'description': 'md5:5aa9195fad900e8e14b52acd765a9fd6',
+        },
+        'playlist_mincount': 5,
+    }, {
+        # all pages
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/m0004c4v/beechgrove',
+        'info_dict': {
+            'id': 'm0004c4v',
+            'title': 'Beechgrove',
+            'description': 'Gardening show that celebrates Scottish horticulture and growing conditions.',
+        },
+        'playlist_mincount': 37,
+    }, {
+        # explicit page
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/m0004c4v/beechgrove?page=2',
+        'info_dict': {
+            'id': 'm0004c4v',
+            'title': 'Beechgrove',
+            'description': 'Gardening show that celebrates Scottish horticulture and growing conditions.',
+        },
+        'playlist_mincount': 1,
+    }]
+    _PAGE_SIZE = 100
+    _DESCRIPTION_KEY = 'synopsis'
+
+    def _get_episode_image(self, episode):
+        return self._get_default(episode, 'image')
+
+    def _get_episode_field(self, episode, field):
+        return self._get_default(episode, field)
+
+    @staticmethod
+    def _get_elements(data):
+        return data['entities']['results']
+
+    @staticmethod
+    def _get_episode(element):
+        return element.get('episode') or {}
+
+    def _call_api(self, pid, per_page, page=1, series_id=None):
+        variables = {
+            'id': pid,
+            'page': page,
+            'perPage': per_page,
+        }
+        if series_id:
+            variables['sliceId'] = series_id
+        return self._download_json(
+            'https://graph.ibl.api.bbc.co.uk/', pid, headers={
+                'Content-Type': 'application/json'
+            }, data=json.dumps({
+                'id': '5692d93d5aac8d796a0305e895e61551',
+                'variables': variables,
+            }).encode('utf-8'))['data']['programme']
+
+    @staticmethod
+    def _get_playlist_data(data):
+        return data
+
+    def _get_playlist_title(self, data):
+        return self._get_default(data, 'title')
+
+
+class BBCCoUkIPlayerGroupIE(BBCCoUkIPlayerPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:iplayer:group'
+    _VALID_URL = BBCCoUkIPlayerPlaylistBaseIE._VALID_URL_TMPL % 'group'
+    _TESTS = [{
         # Available for over a year unlike 30 days for most other programmes
         'url': 'http://www.bbc.co.uk/iplayer/group/p02tcc32',
         'info_dict': {
@@ -1361,14 +1494,56 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
         },
         'playlist_mincount': 10,
+    }, {
+        # all pages
+        'url': 'https://www.bbc.co.uk/iplayer/group/p081d7j7',
+        'info_dict': {
+            'id': 'p081d7j7',
+            'title': 'Music in Scotland',
+            'description': 'Perfomances in Scotland and programmes featuring Scottish acts.',
+        },
+        'playlist_mincount': 47,
+    }, {
+        # explicit page
+        'url': 'https://www.bbc.co.uk/iplayer/group/p081d7j7?page=2',
+        'info_dict': {
+            'id': 'p081d7j7',
+            'title': 'Music in Scotland',
+            'description': 'Perfomances in Scotland and programmes featuring Scottish acts.',
+        },
+        'playlist_mincount': 11,
     }]
+    _PAGE_SIZE = 200
+    _DESCRIPTION_KEY = 'synopses'
 
-    def _extract_title_and_description(self, webpage):
+    def _get_episode_image(self, episode):
-        title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
+        return self._get_default(episode, 'images', 'standard')
-        description = self._search_regex(
+
-            r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
+    def _get_episode_field(self, episode, field):
-            webpage, 'description', fatal=False, group='value')
+        return episode.get(field)
-        return title, description
+
+    @staticmethod
+    def _get_elements(data):
+        return data['elements']
+
+    @staticmethod
+    def _get_episode(element):
+        return element
+
+    def _call_api(self, pid, per_page, page=1, series_id=None):
+        return self._download_json(
+            'http://ibl.api.bbc.co.uk/ibl/v1/groups/%s/episodes' % pid,
+            pid, query={
+                'page': page,
+                'per_page': per_page,
+            })['group_episodes']
+
+    @staticmethod
+    def _get_playlist_data(data):
+        return data['group']
+
+    def _get_playlist_title(self, data):
+        return data.get('title')
 
 
 class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):

yt_dlp/extractor/bilibili.py

@@ -7,6 +7,7 @@ import re
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
+    compat_str,
     compat_parse_qs,
     compat_urlparse,
 )
@@ -15,6 +16,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_iso8601,
+    try_get,
     smuggle_url,
     str_or_none,
     strip_jsonp,
@@ -113,6 +115,13 @@ class BiliBiliIE(InfoExtractor):
         # new BV video id format
         'url': 'https://www.bilibili.com/video/BV1JE411F741',
         'only_matching': True,
+    }, {
+        # Anthology
+        'url': 'https://www.bilibili.com/video/BV1bK411W797',
+        'info_dict': {
+            'id': 'BV1bK411W797',
+        },
+        'playlist_count': 17,
     }]
 
     _APP_KEY = 'iVGUTjsxvpLeuDCf'
@@ -139,9 +148,19 @@ class BiliBiliIE(InfoExtractor):
         page_id = mobj.group('page')
         webpage = self._download_webpage(url, video_id)
 
+        # Bilibili anthologies are similar to playlists but all videos share the same video ID as the anthology itself.
+        # If the video has no page argument, check to see if it's an anthology
+        if page_id is None:
+            if not self._downloader.params.get('noplaylist'):
+                r = self._extract_anthology_entries(bv_id, video_id, webpage)
+                if r is not None:
+                    self.to_screen('Downloading anthology %s - add --no-playlist to just download video' % video_id)
+                    return r
+            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+
         if 'anime/' not in url:
             cid = self._search_regex(
-                r'\bcid(?:["\']:|=)(\d+),["\']page(?:["\']:|=)' + str(page_id), webpage, 'cid',
+                r'\bcid(?:["\']:|=)(\d+),["\']page(?:["\']:|=)' + compat_str(page_id), webpage, 'cid',
                 default=None
             ) or self._search_regex(
                 r'\bcid(?:["\']:|=)(\d+)', webpage, 'cid',
@@ -224,7 +243,18 @@ class BiliBiliIE(InfoExtractor):
         title = self._html_search_regex(
             (r'<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
              r'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
-            group='title') + ('_p' + str(page_id) if page_id is not None else '')
+            group='title')
+
+        # Get part title for anthologies
+        if page_id is not None:
+            # TODO: The json is already downloaded by _extract_anthology_entries. Don't redownload for each video
+            part_title = try_get(
+                self._download_json(
+                    "https://api.bilibili.com/x/player/pagelist?bvid=%s&jsonp=jsonp" % bv_id,
+                    video_id, note='Extracting videos in anthology'),
+                lambda x: x['data'][int(page_id) - 1]['part'])
+            title = part_title or title
+
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time',
@@ -234,7 +264,7 @@ class BiliBiliIE(InfoExtractor):
 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {
-            'id': str(video_id) if page_id is None else '%s_p%s' % (video_id, page_id),
+            'id': compat_str(video_id) if page_id is None else '%s_p%s' % (video_id, page_id),
             'cid': cid,
             'title': title,
             'description': description,
@@ -300,7 +330,7 @@ class BiliBiliIE(InfoExtractor):
 
             global_info = {
                 '_type': 'multi_video',
-                'id': video_id,
+                'id': compat_str(video_id),
                 'bv_id': bv_id,
                 'title': title,
                 'description': description,
@@ -312,6 +342,20 @@ class BiliBiliIE(InfoExtractor):
 
             return global_info
 
+    def _extract_anthology_entries(self, bv_id, video_id, webpage):
+        title = self._html_search_regex(
+            (r'<h1[^>]+\btitle=(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'(?s)<h1[^>]*>(?P<title>.+?)</h1>'), webpage, 'title',
+            group='title')
+        json_data = self._download_json(
+            "https://api.bilibili.com/x/player/pagelist?bvid=%s&jsonp=jsonp" % bv_id,
+            video_id, note='Extracting videos in anthology')
+
+        if len(json_data['data']) > 1:
+            return self.playlist_from_matches(
+                json_data['data'], bv_id, title, ie=BiliBiliIE.ie_key(),
+                getter=lambda entry: 'https://www.bilibili.com/video/%s?p=%d' % (bv_id, entry['page']))
+
     def _get_video_id_set(self, id, is_bv):
         query = {'bvid': id} if is_bv else {'aid': id}
         response = self._download_json(
@@ -506,7 +550,7 @@ class BiliBiliSearchIE(SearchInfoExtractor):
 
             videos = data['result']
             for video in videos:
-                e = self.url_result(video['arcurl'], 'BiliBili', str(video['aid']))
+                e = self.url_result(video['arcurl'], 'BiliBili', compat_str(video['aid']))
                 entries.append(e)
 
             if(len(entries) >= n or len(videos) >= BiliBiliSearchIE.MAX_NUMBER_OF_RESULTS):

yt_dlp/extractor/common.py

@@ -252,8 +252,8 @@ class InfoExtractor(object):
                         * "data": The subtitles file contents
                         * "url": A URL pointing to the subtitles file
                     "ext" will be calculated from URL if missing
-    automatic_captions: Like 'subtitles', used by the YoutubeIE for
+    automatic_captions: Like 'subtitles'; contains automatically generated
-                    automatically generated captions
+                    captions instead of normal subtitles
     duration:       Length of the video in seconds, as an integer or float.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
@@ -301,7 +301,11 @@ class InfoExtractor(object):
     playable_in_embed: Whether this video is allowed to play in embedded
                     players on other sites. Can be True (=always allowed),
                     False (=never allowed), None (=unknown), or a string
-                    specifying the criteria for embedability (Eg: 'whitelist').
+                    specifying the criteria for embedability (Eg: 'whitelist')
+    availability:   Under what condition the video is available. One of
+                    'private', 'premium_only', 'subscriber_only', 'needs_auth',
+                    'unlisted' or 'public'. Use 'InfoExtractor._availability'
+                    to set it
     __post_extractor: A function to be called just before the metadata is
                     written to either disk, logger or console. The function
                     must return a dict which will be added to the info_dict.
@@ -1417,8 +1421,8 @@ class InfoExtractor(object):
             'ie_pref': {'priority': True, 'type': 'extractor'},
             'hasvid': {'priority': True, 'field': 'vcodec', 'type': 'boolean', 'not_in_list': ('none',)},
             'hasaud': {'field': 'acodec', 'type': 'boolean', 'not_in_list': ('none',)},
-            'lang': {'priority': True, 'convert': 'ignore', 'type': 'extractor', 'field': 'language_preference'},
+            'lang': {'priority': True, 'convert': 'ignore', 'field': 'language_preference'},
-            'quality': {'convert': 'float_none', 'type': 'extractor'},
+            'quality': {'convert': 'float_none'},
             'filesize': {'convert': 'bytes'},
             'fs_approx': {'convert': 'bytes', 'field': 'filesize_approx'},
             'id': {'convert': 'string', 'field': 'format_id'},
@@ -1429,7 +1433,7 @@ class InfoExtractor(object):
             'vbr': {'convert': 'float_none'},
             'abr': {'convert': 'float_none'},
             'asr': {'convert': 'float_none'},
-            'source': {'convert': 'ignore', 'type': 'extractor', 'field': 'source_preference'},
+            'source': {'convert': 'ignore', 'field': 'source_preference'},
 
             'codec': {'type': 'combined', 'field': ('vcodec', 'acodec')},
             'br': {'type': 'combined', 'field': ('tbr', 'vbr', 'abr'), 'same_limit': True},
@@ -3332,6 +3336,20 @@ class InfoExtractor(object):
     def _generic_title(self, url):
         return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
 
+    @staticmethod
+    def _availability(is_private, needs_premium, needs_subscription, needs_auth, is_unlisted):
+        all_known = all(map(
+            lambda x: x is not None,
+            (is_private, needs_premium, needs_subscription, needs_auth, is_unlisted)))
+        return (
+            'private' if is_private
+            else 'premium_only' if needs_premium
+            else 'subscriber_only' if needs_subscription
+            else 'needs_auth' if needs_auth
+            else 'unlisted' if is_unlisted
+            else 'public' if all_known
+            else None)
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

yt_dlp/extractor/extractors.py

@@ -108,7 +108,8 @@ from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,
-    BBCCoUkIPlayerPlaylistIE,
+    BBCCoUkIPlayerEpisodesIE,
+    BBCCoUkIPlayerGroupIE,
     BBCCoUkPlaylistIE,
     BBCIE,
 )
@@ -708,7 +709,10 @@ from .mixcloud import (
     MixcloudUserIE,
     MixcloudPlaylistIE,
 )
-from .mlb import MLBIE
+from .mlb import (
+    MLBIE,
+    MLBVideoIE,
+)
 from .mnet import MnetIE
 from .moevideo import MoeVideoIE
 from .mofosex import (
@@ -822,7 +826,7 @@ from .nick import (
     NickNightIE,
     NickRuIE,
 )
-from .niconico import NiconicoIE, NiconicoPlaylistIE
+from .niconico import NiconicoIE, NiconicoPlaylistIE, NiconicoUserIE
 from .ninecninemedia import NineCNineMediaIE
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
@@ -1179,7 +1183,10 @@ from .spike import (
     BellatorIE,
     ParamountNetworkIE,
 )
-from .stitcher import StitcherIE
+from .stitcher import (
+    StitcherIE,
+    StitcherShowIE,
+)
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
@@ -1667,9 +1674,14 @@ from .zattoo import (
     ZattooLiveIE,
 )
 from .zdf import ZDFIE, ZDFChannelIE
+from .zee5 import (
+    Zee5IE,
+    Zee5SeriesIE,
+)
 from .zhihu import ZhihuIE
-from .zingmp3 import ZingMp3IE
+from .zingmp3 import (
-from .zee5 import Zee5IE
+    ZingMp3IE,
-from .zee5 import Zee5SeriesIE
+    ZingMp3AlbumIE,
+)
 from .zoom import ZoomIE
 from .zype import ZypeIE

yt_dlp/extractor/generic.py

@@ -2965,7 +2965,7 @@ class GenericIE(InfoExtractor):
             webpage)
         if not mobj:
             mobj = re.search(
-                r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
+                r'data-video-link=["\'](?P<url>http://m\.mlb\.com/video/[^"\']+)',
                 webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')

yt_dlp/extractor/ivi.py

@@ -146,7 +146,7 @@ class IviIE(InfoExtractor):
                         expected=True)
                 elif not pycryptodomex_found:
                     raise ExtractorError(
-                        'pycryptodomex not found. Please install it.',
+                        'pycryptodomex not found. Please install',
                         expected=True)
                 elif message:
                     extractor_msg += ': ' + message

yt_dlp/extractor/linuxacademy.py

@@ -38,8 +38,8 @@ class LinuxAcademyIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'What Is Data Science',
             'description': 'md5:c574a3c20607144fb36cb65bdde76c99',
-            'timestamp': 1607387907,
+            'timestamp': int,  # The timestamp and upload date changes
-            'upload_date': '20201208',
+            'upload_date': r're:\d+',
             'duration': 304,
         },
         'params': {
@@ -59,6 +59,16 @@ class LinuxAcademyIE(InfoExtractor):
         },
         'playlist_count': 41,
         'skip': 'Requires Linux Academy account credentials',
+    }, {
+        'url': 'https://linuxacademy.com/cp/modules/view/id/39',
+        'info_dict': {
+            'id': '39',
+            'title': 'Red Hat Certified Systems Administrator - RHCSA (EX200) Exam Prep  (legacy)',
+            'description': 'md5:0f1d3369e90c3fb14a79813b863c902f',
+            'duration': 89280,
+        },
+        'playlist_count': 73,
+        'skip': 'Requires Linux Academy account credentials',
     }]
 
     _AUTHORIZE_URL = 'https://login.linuxacademy.com/authorize'
@@ -102,7 +112,7 @@ class LinuxAcademyIE(InfoExtractor):
             'client_id': self._CLIENT_ID,
             'redirect_uri': self._ORIGIN_URL,
             'tenant': 'lacausers',
-            'connection': 'Username-Password-Authentication',
+            'connection': 'Username-Password-ACG-Proxy',
             'username': username,
             'password': password,
             'sso': 'true',
@@ -162,7 +172,7 @@ class LinuxAcademyIE(InfoExtractor):
         if course_id:
             module = self._parse_json(
                 self._search_regex(
-                    r'window\.module\s*=\s*({.+?})\s*;', webpage, 'module'),
+                    r'window\.module\s*=\s*({(?:(?!};)[^"]|"([^"]|\\")*")+})\s*;', webpage, 'module'),
                 item_id)
             entries = []
             chapter_number = None

yt_dlp/extractor/mlb.py

@@ -1,15 +1,91 @@
 from __future__ import unicode_literals
 
-from .nhl import NHLBaseIE
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    try_get,
+)
 
 
-class MLBIE(NHLBaseIE):
+class MLBBaseIE(InfoExtractor):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        video = self._download_video_data(display_id)
+        video_id = video['id']
+        title = video['title']
+        feed = self._get_feed(video)
+
+        formats = []
+        for playback in (feed.get('playbacks') or []):
+            playback_url = playback.get('url')
+            if not playback_url:
+                continue
+            name = playback.get('name')
+            ext = determine_ext(playback_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    playback_url, video_id, 'mp4',
+                    'm3u8_native', m3u8_id=name, fatal=False))
+            else:
+                f = {
+                    'format_id': name,
+                    'url': playback_url,
+                }
+                mobj = re.search(r'_(\d+)K_(\d+)X(\d+)', name)
+                if mobj:
+                    f.update({
+                        'height': int(mobj.group(3)),
+                        'tbr': int(mobj.group(1)),
+                        'width': int(mobj.group(2)),
+                    })
+                mobj = re.search(r'_(\d+)x(\d+)_(\d+)_(\d+)K\.mp4', playback_url)
+                if mobj:
+                    f.update({
+                        'fps': int(mobj.group(3)),
+                        'height': int(mobj.group(2)),
+                        'tbr': int(mobj.group(4)),
+                        'width': int(mobj.group(1)),
+                    })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for cut in (try_get(feed, lambda x: x['image']['cuts'], list) or []):
+            src = cut.get('src')
+            if not src:
+                continue
+            thumbnails.append({
+                'height': int_or_none(cut.get('height')),
+                'url': src,
+                'width': int_or_none(cut.get('width')),
+            })
+
+        language = (video.get('language') or 'EN').lower()
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': video.get('description'),
+            'duration': parse_duration(feed.get('duration')),
+            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(video.get(self._TIMESTAMP_KEY)),
+            'subtitles': self._extract_mlb_subtitles(feed, language),
+        }
+
+
+class MLBIE(MLBBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:[\da-z_-]+\.)*(?P<site>mlb)\.com/
+                        (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
-                                (?:[^/]+/)*c-|
+                                (?:[^/]+/)*video/[^/]+/c-|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
                                     (?:[^/]+/)+(?:play|index)\.jsp|
@@ -18,7 +94,6 @@ class MLBIE(NHLBaseIE):
                             (?P<id>\d+)
                         )
                     '''
-    _CONTENT_DOMAIN = 'content.mlb.com'
     _TESTS = [
         {
             'url': 'https://www.mlb.com/mariners/video/ackleys-spectacular-catch/c-34698933',
@@ -76,18 +151,6 @@ class MLBIE(NHLBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
-        {
-            'url': 'https://www.mlb.com/news/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer/c-118550098',
-            'md5': 'e09e37b552351fddbf4d9e699c924d68',
-            'info_dict': {
-                'id': '75609783',
-                'ext': 'mp4',
-                'title': 'Must C: Pillar climbs for catch',
-                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
-                'timestamp': 1429139220,
-                'upload_date': '20150415',
-            }
-        },
         {
             'url': 'https://www.mlb.com/video/hargrove-homers-off-caldwell/c-1352023483?tid=67793694',
             'only_matching': True,
@@ -113,8 +176,92 @@ class MLBIE(NHLBaseIE):
             'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
             'only_matching': True,
         },
-        {
-            'url': 'https://www.mlb.com/cut4/carlos-gomez-borrowed-sunglasses-from-an-as-fan/c-278912842',
-            'only_matching': True,
-        }
     ]
+    _TIMESTAMP_KEY = 'date'
+
+    @staticmethod
+    def _get_feed(video):
+        return video
+
+    @staticmethod
+    def _extract_mlb_subtitles(feed, language):
+        subtitles = {}
+        for keyword in (feed.get('keywordsAll') or []):
+            keyword_type = keyword.get('type')
+            if keyword_type and keyword_type.startswith('closed_captions_location_'):
+                cc_location = keyword.get('value')
+                if cc_location:
+                    subtitles.setdefault(language, []).append({
+                        'url': cc_location,
+                    })
+        return subtitles
+
+    def _download_video_data(self, display_id):
+        return self._download_json(
+            'http://content.mlb.com/mlb/item/id/v1/%s/details/web-v1.json' % display_id,
+            display_id)
+
+
+class MLBVideoIE(MLBBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?mlb\.com/(?:[^/]+/)*video/(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://www.mlb.com/mariners/video/ackley-s-spectacular-catch-c34698933',
+        'md5': '632358dacfceec06bad823b83d21df2d',
+        'info_dict': {
+            'id': 'c04a8863-f569-42e6-9f87-992393657614',
+            'ext': 'mp4',
+            'title': "Ackley's spectacular catch",
+            'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
+            'duration': 66,
+            'timestamp': 1405995000,
+            'upload_date': '20140722',
+            'thumbnail': r're:^https?://.+',
+        },
+    }
+    _TIMESTAMP_KEY = 'timestamp'
+
+    @classmethod
+    def suitable(cls, url):
+        return False if MLBIE.suitable(url) else super(MLBVideoIE, cls).suitable(url)
+
+    @staticmethod
+    def _get_feed(video):
+        return video['feeds'][0]
+
+    @staticmethod
+    def _extract_mlb_subtitles(feed, language):
+        subtitles = {}
+        for cc_location in (feed.get('closedCaptions') or []):
+            subtitles.setdefault(language, []).append({
+                'url': cc_location,
+            })
+
+    def _download_video_data(self, display_id):
+        # https://www.mlb.com/data-service/en/videos/[SLUG]
+        return self._download_json(
+            'https://fastball-gateway.mlb.com/graphql',
+            display_id, query={
+                'query': '''{
+  mediaPlayback(ids: "%s") {
+    description
+    feeds(types: CMS) {
+      closedCaptions
+      duration
+      image {
+        cuts {
+          width
+          height
+          src
+        }
+      }
+      playbacks {
+        name
+        url
+      }
+    }
+    id
+    timestamp
+    title
+  }
+}''' % display_id,
+            })['data']['mediaPlayback'][0]

yt_dlp/extractor/mtv.py

@@ -340,7 +340,7 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media.mtvnservices.com/embed/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media\.mtvnservices\.com/embed/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

yt_dlp/extractor/nbc.py

@@ -10,6 +10,7 @@ from .adobepass import AdobePassIE
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
+    parse_age_limit,
     parse_duration,
     smuggle_url,
     try_get,
@@ -18,7 +19,7 @@ from ..utils import (
 )
 
 
-class NBCIE(AdobePassIE):
+class NBCIE(ThePlatformIE):
     _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
@@ -132,7 +133,9 @@ class NBCIE(AdobePassIE):
             'manifest': 'm3u',
         }
         video_id = video_data['mpxGuid']
-        title = video_data['secondaryTitle']
+        tp_path = 'NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id)
+        tpm = self._download_theplatform_metadata(tp_path, video_id)
+        title = tpm.get('title') or video_data.get('secondaryTitle')
         if video_data.get('locked'):
             resource = self._get_mvpd_resource(
                 video_data.get('resourceId') or 'nbcentertainment',
@@ -142,18 +145,40 @@ class NBCIE(AdobePassIE):
         theplatform_url = smuggle_url(update_url_query(
             'http://link.theplatform.com/s/NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id),
             query), {'force_smil_url': True})
+
+        # Empty string or 0 can be valid values for these. So the check must be `is None`
+        description = video_data.get('description')
+        if description is None:
+            description = tpm.get('description')
+        episode_number = int_or_none(video_data.get('episodeNumber'))
+        if episode_number is None:
+            episode_number = int_or_none(tpm.get('nbcu$airOrder'))
+        rating = video_data.get('rating')
+        if rating is None:
+            try_get(tpm, lambda x: x['ratings'][0]['rating'])
+        season_number = int_or_none(video_data.get('seasonNumber'))
+        if season_number is None:
+            season_number = int_or_none(tpm.get('nbcu$seasonNumber'))
+        series = video_data.get('seriesShortTitle')
+        if series is None:
+            series = tpm.get('nbcu$seriesShortTitle')
+        tags = video_data.get('keywords')
+        if tags is None or len(tags) == 0:
+            tags = tpm.get('keywords')
+
         return {
             '_type': 'url_transparent',
+            'age_limit': parse_age_limit(rating),
+            'description': description,
+            'episode': title,
+            'episode_number': episode_number,
             'id': video_id,
+            'ie_key': 'ThePlatform',
+            'season_number': season_number,
+            'series': series,
+            'tags': tags,
             'title': title,
             'url': theplatform_url,
-            'description': video_data.get('description'),
-            'tags': video_data.get('keywords'),
-            'season_number': int_or_none(video_data.get('seasonNumber')),
-            'episode_number': int_or_none(video_data.get('episodeNumber')),
-            'episode': title,
-            'series': video_data.get('seriesShortTitle'),
-            'ie_key': 'ThePlatform',
         }
 
 

yt_dlp/extractor/niconico.py

@@ -8,6 +8,7 @@ import datetime
 from .common import InfoExtractor
 from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..compat import (
+    compat_str,
     compat_parse_qs,
     compat_urllib_parse_urlparse,
 )
@@ -20,6 +21,7 @@ from ..utils import (
     parse_duration,
     parse_iso8601,
     PostProcessingError,
+    str_or_none,
     remove_start,
     try_get,
     unified_timestamp,
@@ -34,7 +36,7 @@ class NiconicoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
-        'md5': 'd1a75c0823e2f629128c43e1212760f9',
+        'md5': 'a5bad06f1347452102953f323c69da34s',
         'info_dict': {
             'id': 'sm22312215',
             'ext': 'mp4',
@@ -203,7 +205,7 @@ class NiconicoIE(InfoExtractor):
             'data-api-data="([^"]+)"', webpage,
             'API data', default='{}'), video_id)
 
-        session_api_data = try_get(api_data, lambda x: x['video']['dmcInfo']['session_api'])
+        session_api_data = try_get(api_data, lambda x: x['media']['delivery']['movie']['session'])
         session_api_endpoint = try_get(session_api_data, lambda x: x['urls'][0])
 
         # ping
@@ -220,7 +222,7 @@ class NiconicoIE(InfoExtractor):
         yesno = lambda x: 'yes' if x else 'no'
 
         # m3u8 (encryption)
-        if 'encryption' in try_get(api_data, lambda x: x['video']['dmcInfo']) or {}:
+        if 'encryption' in (try_get(api_data, lambda x: x['media']['delivery']['movie']) or {}):
             protocol = 'm3u8'
             session_api_http_parameters = {
                 'parameters': {
@@ -244,8 +246,8 @@ class NiconicoIE(InfoExtractor):
             session_api_http_parameters = {
                 'parameters': {
                     'http_output_download_parameters': {
-                        'use_ssl': yesno(session_api_endpoint['is_ssl']),
+                        'use_ssl': yesno(session_api_endpoint['isSsl']),
-                        'use_well_known_port': yesno(session_api_endpoint['is_well_known_port']),
+                        'use_well_known_port': yesno(session_api_endpoint['isWellKnownPort']),
                     }
                 }
             }
@@ -258,15 +260,15 @@ class NiconicoIE(InfoExtractor):
             data=json.dumps({
                 'session': {
                     'client_info': {
-                        'player_id': session_api_data.get('player_id'),
+                        'player_id': session_api_data.get('playerId'),
                     },
                     'content_auth': {
-                        'auth_type': try_get(session_api_data, lambda x: x['auth_types'][session_api_data['protocols'][0]]),
+                        'auth_type': try_get(session_api_data, lambda x: x['authTypes'][session_api_data['protocols'][0]]),
-                        'content_key_timeout': session_api_data.get('content_key_timeout'),
+                        'content_key_timeout': session_api_data.get('contentKeyTimeout'),
                         'service_id': 'nicovideo',
-                        'service_user_id': session_api_data.get('service_user_id')
+                        'service_user_id': session_api_data.get('serviceUserId')
                     },
-                    'content_id': session_api_data.get('content_id'),
+                    'content_id': session_api_data.get('contentId'),
                     'content_src_id_sets': [{
                         'content_src_ids': [{
                             'src_id_to_mux': {
@@ -279,7 +281,7 @@ class NiconicoIE(InfoExtractor):
                     'content_uri': '',
                     'keep_method': {
                         'heartbeat': {
-                            'lifetime': session_api_data.get('heartbeat_lifetime')
+                            'lifetime': session_api_data.get('heartbeatLifetime')
                         }
                     },
                     'priority': session_api_data.get('priority'),
@@ -289,7 +291,7 @@ class NiconicoIE(InfoExtractor):
                             'http_parameters': session_api_http_parameters
                         }
                     },
-                    'recipe_id': session_api_data.get('recipe_id'),
+                    'recipe_id': session_api_data.get('recipeId'),
                     'session_operation_auth': {
                         'session_operation_auth_by_signature': {
                             'signature': session_api_data.get('signature'),
@@ -308,7 +310,7 @@ class NiconicoIE(InfoExtractor):
             'url': session_api_endpoint['url'] + '/' + session_response['data']['session']['id'] + '?_format=json&_method=PUT',
             'data': json.dumps(session_response['data']),
             # interval, convert milliseconds to seconds, then halve to make a buffer.
-            'interval': float_or_none(session_api_data.get('heartbeat_lifetime'), scale=2000),
+            'interval': float_or_none(session_api_data.get('heartbeatLifetime'), scale=2000),
         }
 
         return info_dict, heartbeat_info_dict
@@ -327,15 +329,17 @@ class NiconicoIE(InfoExtractor):
         format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality]))
         vdict = parse_format_id(video_quality['id'])
         adict = parse_format_id(audio_quality['id'])
-        resolution = video_quality.get('resolution', {'height': vdict.get('res')})
+        resolution = try_get(video_quality, lambda x: x['metadata']['resolution'], dict) or {'height': vdict.get('res')}
+        vbr = try_get(video_quality, lambda x: x['metadata']['bitrate'], float)
 
         return {
             'url': '%s:%s/%s/%s' % (protocol, video_id, video_quality['id'], audio_quality['id']),
             'format_id': format_id,
+            'format_note': 'DMC %s' % try_get(video_quality, lambda x: x['metadata']['label'], compat_str),
             'ext': 'mp4',  # Session API are used in HTML5, which always serves mp4
             'vcodec': vdict.get('codec'),
             'acodec': adict.get('codec'),
-            'vbr': float_or_none(video_quality.get('bitrate'), 1000) or float_or_none(vdict.get('br')),
+            'vbr': float_or_none(vbr, 1000) or float_or_none(vdict.get('br')),
             'abr': float_or_none(audio_quality.get('bitrate'), 1000) or float_or_none(adict.get('br')),
             'height': int_or_none(resolution.get('height', vdict.get('res'))),
             'width': int_or_none(resolution.get('width')),
@@ -394,92 +398,93 @@ class NiconicoIE(InfoExtractor):
         formats = []
 
         # Get HTML5 videos info
-        try:
+        quality_info = try_get(api_data, lambda x: x['media']['delivery']['movie'])
-            dmc_info = api_data['video']['dmcInfo']
+        if not quality_info:
-        except KeyError:
+            raise ExtractorError('The video can\'t downloaded.', expected=True)
-            raise ExtractorError('The video can\'t downloaded.',
-                                 expected=True)
 
-        quality_info = dmc_info.get('quality')
         for audio_quality in quality_info.get('audios') or {}:
             for video_quality in quality_info.get('videos') or {}:
-                if not audio_quality.get('available') or not video_quality.get('available'):
+                if not audio_quality.get('isAvailable') or not video_quality.get('isAvailable'):
                     continue
                 formats.append(self._extract_format_for_quality(
                     api_data, video_id, audio_quality, video_quality))
 
         # Get flv/swf info
+        timestamp = None
         video_real_url = try_get(api_data, lambda x: x['video']['smileInfo']['url'])
-        is_economy = video_real_url.endswith('low')
+        if not video_real_url:
+            self.report_warning('Unable to obtain smile video information')
+        else:
+            is_economy = video_real_url.endswith('low')
 
-        if is_economy:
+            if is_economy:
-            self.report_warning('Site is currently in economy mode! You will only have access to lower quality streams')
+                self.report_warning('Site is currently in economy mode! You will only have access to lower quality streams')
 
-        # Invoking ffprobe to determine resolution
+            # Invoking ffprobe to determine resolution
-        pp = FFmpegPostProcessor(self._downloader)
+            pp = FFmpegPostProcessor(self._downloader)
-        cookies = self._get_cookies('https://nicovideo.jp').output(header='', sep='; path=/; domain=nicovideo.jp;\n')
+            cookies = self._get_cookies('https://nicovideo.jp').output(header='', sep='; path=/; domain=nicovideo.jp;\n')
 
-        self.to_screen('%s: %s' % (video_id, 'Checking smile format with ffprobe'))
+            self.to_screen('%s: %s' % (video_id, 'Checking smile format with ffprobe'))
 
-        try:
+            try:
-            metadata = pp.get_metadata_object(video_real_url, ['-cookies', cookies])
+                metadata = pp.get_metadata_object(video_real_url, ['-cookies', cookies])
-        except PostProcessingError as err:
+            except PostProcessingError as err:
-            raise ExtractorError(err.msg, expected=True)
+                raise ExtractorError(err.msg, expected=True)
 
-        v_stream = a_stream = {}
+            v_stream = a_stream = {}
 
-        # Some complex swf files doesn't have video stream (e.g. nm4809023)
+            # Some complex swf files doesn't have video stream (e.g. nm4809023)
-        for stream in metadata['streams']:
+            for stream in metadata['streams']:
-            if stream['codec_type'] == 'video':
+                if stream['codec_type'] == 'video':
-                v_stream = stream
+                    v_stream = stream
-            elif stream['codec_type'] == 'audio':
+                elif stream['codec_type'] == 'audio':
-                a_stream = stream
+                    a_stream = stream
 
-        # Community restricted videos seem to have issues with the thumb API not returning anything at all
+            # Community restricted videos seem to have issues with the thumb API not returning anything at all
-        filesize = int(
+            filesize = int(
-            (get_video_info_xml('size_high') if not is_economy else get_video_info_xml('size_low'))
+                (get_video_info_xml('size_high') if not is_economy else get_video_info_xml('size_low'))
-            or metadata['format']['size']
+                or metadata['format']['size']
-        )
+            )
-        extension = (
+            extension = (
-            get_video_info_xml('movie_type')
+                get_video_info_xml('movie_type')
-            or 'mp4' if 'mp4' in metadata['format']['format_name'] else metadata['format']['format_name']
+                or 'mp4' if 'mp4' in metadata['format']['format_name'] else metadata['format']['format_name']
-        )
+            )
 
-        # 'creation_time' tag on video stream of re-encoded SMILEVIDEO mp4 files are '1970-01-01T00:00:00.000000Z'.
+            # 'creation_time' tag on video stream of re-encoded SMILEVIDEO mp4 files are '1970-01-01T00:00:00.000000Z'.
-        timestamp = (
+            timestamp = (
-            parse_iso8601(get_video_info_web('first_retrieve'))
+                parse_iso8601(get_video_info_web('first_retrieve'))
-            or unified_timestamp(get_video_info_web('postedDateTime'))
+                or unified_timestamp(get_video_info_web('postedDateTime'))
-        )
+            )
-        metadata_timestamp = (
+            metadata_timestamp = (
-            parse_iso8601(try_get(v_stream, lambda x: x['tags']['creation_time']))
+                parse_iso8601(try_get(v_stream, lambda x: x['tags']['creation_time']))
-            or timestamp if extension != 'mp4' else 0
+                or timestamp if extension != 'mp4' else 0
-        )
+            )
 
-        # According to compconf, smile videos from pre-2017 are always better quality than their DMC counterparts
+            # According to compconf, smile videos from pre-2017 are always better quality than their DMC counterparts
-        smile_threshold_timestamp = parse_iso8601('2016-12-08T00:00:00+09:00')
+            smile_threshold_timestamp = parse_iso8601('2016-12-08T00:00:00+09:00')
 
-        is_source = timestamp < smile_threshold_timestamp or metadata_timestamp > 0
+            is_source = timestamp < smile_threshold_timestamp or metadata_timestamp > 0
 
-        # If movie file size is unstable, old server movie is not source movie.
+            # If movie file size is unstable, old server movie is not source movie.
-        if filesize > 1:
+            if filesize > 1:
-            formats.append({
+                formats.append({
-                'url': video_real_url,
+                    'url': video_real_url,
-                'format_id': 'smile' if not is_economy else 'smile_low',
+                    'format_id': 'smile' if not is_economy else 'smile_low',
-                'format_note': 'SMILEVIDEO source' if not is_economy else 'SMILEVIDEO low quality',
+                    'format_note': 'SMILEVIDEO source' if not is_economy else 'SMILEVIDEO low quality',
-                'ext': extension,
+                    'ext': extension,
-                'container': extension,
+                    'container': extension,
-                'vcodec': v_stream.get('codec_name'),
+                    'vcodec': v_stream.get('codec_name'),
-                'acodec': a_stream.get('codec_name'),
+                    'acodec': a_stream.get('codec_name'),
-                # Some complex swf files doesn't have total bit rate metadata (e.g. nm6049209)
+                    # Some complex swf files doesn't have total bit rate metadata (e.g. nm6049209)
-                'tbr': int_or_none(metadata['format'].get('bit_rate'), scale=1000),
+                    'tbr': int_or_none(metadata['format'].get('bit_rate'), scale=1000),
-                'vbr': int_or_none(v_stream.get('bit_rate'), scale=1000),
+                    'vbr': int_or_none(v_stream.get('bit_rate'), scale=1000),
-                'abr': int_or_none(a_stream.get('bit_rate'), scale=1000),
+                    'abr': int_or_none(a_stream.get('bit_rate'), scale=1000),
-                'height': int_or_none(v_stream.get('height')),
+                    'height': int_or_none(v_stream.get('height')),
-                'width': int_or_none(v_stream.get('width')),
+                    'width': int_or_none(v_stream.get('width')),
-                'source_preference': 5 if not is_economy else -2,
+                    'source_preference': 5 if not is_economy else -2,
-                'quality': 5 if is_source and not is_economy else None,
+                    'quality': 5 if is_source and not is_economy else None,
-                'filesize': filesize
+                    'filesize': filesize
-            })
+                })
 
         if len(formats) == 0:
             raise ExtractorError('Unable to find video info.')
@@ -487,13 +492,12 @@ class NiconicoIE(InfoExtractor):
         self._sort_formats(formats)
 
         # Start extracting information
-        title = get_video_info_web('originalTitle')
+        title = (
-        if not title:
+            get_video_info_web(['originalTitle', 'title'])
-            title = self._og_search_title(webpage, default=None)
+            or self._og_search_title(webpage, default=None)
-        if not title:
+            or self._html_search_regex(
-            title = self._html_search_regex(
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
-                webpage, 'video title')
+                webpage, 'video title'))
 
         watch_api_data_string = self._html_search_regex(
             r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
@@ -517,6 +521,7 @@ class NiconicoIE(InfoExtractor):
             timestamp = parse_iso8601(
                 video_detail['postedAt'].replace('/', '-'),
                 delimiter=' ', timezone=datetime.timedelta(hours=9))
+        timestamp = timestamp or try_get(api_data, lambda x: parse_iso8601(x['video']['registeredAt']))
 
         view_count = int_or_none(get_video_info_web(['view_counter', 'viewCount']))
         if not view_count:
@@ -525,11 +530,16 @@ class NiconicoIE(InfoExtractor):
                 webpage, 'view count', default=None)
             if match:
                 view_count = int_or_none(match.replace(',', ''))
-        view_count = view_count or video_detail.get('viewCount')
+        view_count = (
+            view_count
+            or video_detail.get('viewCount')
+            or try_get(api_data, lambda x: x['video']['count']['view']))
+
+        comment_count = (
+            int_or_none(get_video_info_web('comment_num'))
+            or video_detail.get('commentCount')
+            or try_get(api_data, lambda x: x['video']['count']['comment']))
 
-        comment_count = (int_or_none(get_video_info_web('comment_num'))
-                         or video_detail.get('commentCount')
-                         or try_get(api_data, lambda x: x['thread']['commentCount']))
         if not comment_count:
             match = self._html_search_regex(
                 r'>Comments: <strong[^>]*>([^<]+)</strong>',
@@ -559,7 +569,7 @@ class NiconicoIE(InfoExtractor):
         # Note: cannot use api_data.get('owner', {}) because owner may be set to "null"
         # in the JSON, which will cause None to be returned instead of {}.
         owner = try_get(api_data, lambda x: x.get('owner'), dict) or {}
-        uploader_id = (
+        uploader_id = str_or_none(
             get_video_info_web(['ch_id', 'user_id'])
             or owner.get('id')
             or channel_id
@@ -589,7 +599,7 @@ class NiconicoIE(InfoExtractor):
 
 
 class NiconicoPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/(?:user/\d+/)?mylist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/(?:user/\d+/|my/)?mylist/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.nicovideo.jp/mylist/27411728',
@@ -647,3 +657,40 @@ class NiconicoPlaylistIE(InfoExtractor):
             'uploader_id': uploader_id,
             'entries': OnDemandPagedList(pagefunc, 25),
         }
+
+
+class NiconicoUserIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/user/(?P<id>\d+)/?(?:$|[#?])'
+    _TEST = {
+        'url': 'https://www.nicovideo.jp/user/419948',
+        'info_dict': {
+            'id': '419948',
+        },
+        'playlist_mincount': 101,
+    }
+    _API_URL = "https://nvapi.nicovideo.jp/v1/users/%s/videos?sortKey=registeredAt&sortOrder=desc&pageSize=%s&page=%s"
+    _api_headers = {
+        'X-Frontend-ID': '6',
+        'X-Frontend-Version': '0',
+        'X-Niconico-Language': 'en-us'
+    }
+    _PAGE_SIZE = 100
+
+    def _entries(self, list_id, ):
+        total_count = 1
+        count = page_num = 0
+        while count < total_count:
+            json_parsed = self._download_json(
+                self._API_URL % (list_id, self._PAGE_SIZE, page_num + 1), list_id,
+                headers=self._api_headers,
+                note='Downloading JSON metadata%s' % (' page %d' % page_num if page_num else ''))
+            if not page_num:
+                total_count = int_or_none(json_parsed['data'].get('totalCount'))
+            for entry in json_parsed["data"]["items"]:
+                count += 1
+                yield self.url_result('https://www.nicovideo.jp/watch/%s' % entry['id'])
+            page_num += 1
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        return self.playlist_result(self._entries(list_id), list_id, ie=NiconicoIE.ie_key())

yt_dlp/extractor/rcs.py

@@ -15,6 +15,9 @@ from ..utils import (
 
 
 class RCSBaseIE(InfoExtractor):
+    # based on VideoPlayerLoader.prototype.getVideoSrc
+    # and VideoPlayerLoader.prototype.transformSrc from
+    # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
     _ALL_REPLACE = {
         'media2vam.corriere.it.edgesuite.net':
             'media2vam-corriere-it.akamaized.net',
@@ -191,10 +194,10 @@ class RCSBaseIE(InfoExtractor):
             urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
             m3u8_id='hls', fatal=False)
 
-        if not formats:
+        if urls.get('mp4'):
             formats.append({
                 'format_id': 'http-mp4',
-                'url': urls.get('mp4')
+                'url': urls['mp4']
             })
         self._sort_formats(formats)
         return formats
@@ -216,10 +219,12 @@ class RCSBaseIE(InfoExtractor):
         video_data = None
         # look for json video data url
         json = self._search_regex(
-            r'''(?x)var url\s*=\s*["']((?:https?:)?
+            r'''(?x)url\s*=\s*(["'])
-                //video\.rcs\.it
+            (?P<url>
-                /fragment-includes/video-includes/.+?\.json)["'];''',
+                (?:https?:)?//video\.rcs\.it
-            page, video_id, default=None)
+                /fragment-includes/video-includes/.+?\.json
+            )\1;''',
+            page, video_id, group='url', default=None)
         if json:
             if json.startswith('//'):
                 json = 'https:%s' % json
@@ -227,13 +232,16 @@ class RCSBaseIE(InfoExtractor):
 
         # if json url not found, look for json video data directly in the page
         else:
+            # RCS normal pages and most of the embeds
             json = self._search_regex(
                 r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)',
                 page, video_id, default=None)
-            if json:
+            if not json and 'video-embed' in url:
-                video_data = self._parse_json(
+                page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id)
-                    json, video_id, transform_source=js_to_json)
+                json = self._search_regex(
-            else:
+                    r'##start-video##({[\s\S]+?})##end-video##',
+                    page, video_id, default=None)
+            if not json:
                 # if no video data found try search for iframes
                 emb = RCSEmbedsIE._extract_url(page)
                 if emb:
@@ -242,6 +250,9 @@ class RCSBaseIE(InfoExtractor):
                         'url': emb,
                         'ie_key': RCSEmbedsIE.ie_key()
                     }
+            if json:
+                video_data = self._parse_json(
+                    json, video_id, transform_source=js_to_json)
 
         if not video_data:
             raise ExtractorError('Video data not found in the page')
@@ -250,7 +261,8 @@ class RCSBaseIE(InfoExtractor):
             self._get_video_src(video_data), video_id)
 
         description = (video_data.get('description')
-                       or clean_html(video_data.get('htmlDescription')))
+                       or clean_html(video_data.get('htmlDescription'))
+                       or self._html_search_meta('description', page))
         uploader = video_data.get('provider') or mobj.group('cdn')
 
         return {
@@ -283,6 +295,7 @@ class RCSEmbedsIE(RCSBaseIE):
             'uploader': 'rcs.it',
         }
     }, {
+        # redownload the page changing 'video-embed' in 'video-json'
         'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
         'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
         'info_dict': {
@@ -359,6 +372,7 @@ class RCSIE(RCSBaseIE):
             'uploader': 'Corriere Tv',
         }
     }, {
+        # video data inside iframe
         'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
         'md5': 'da378e4918d2afbf7d61c35abb948d4c',
         'info_dict': {
@@ -389,15 +403,15 @@ class RCSVariousIE(RCSBaseIE):
                     (?P<cdn>
                         leitv\.it|
                         youreporter\.it
-                    )/(?:video/)?(?P<id>[^/]+?)(?:$|\?|/)'''
+                    )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)'''
     _TESTS = [{
-        'url': 'https://www.leitv.it/video/marmellata-di-ciliegie-fatta-in-casa/',
+        'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/',
-        'md5': '618aaabac32152199c1af86784d4d554',
+        'md5': '92b4e63667b8f95acb0a04da25ae28a1',
         'info_dict': {
-            'id': 'marmellata-di-ciliegie-fatta-in-casa',
+            'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa',
             'ext': 'mp4',
-            'title': 'Marmellata di ciliegie fatta in casa',
+            'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
-            'description': 'md5:89133864d6aad456dbcf6e7a29f86263',
+            'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
             'uploader': 'leitv.it',
         }
     }, {

yt_dlp/extractor/stitcher.py

@@ -1,19 +1,61 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     clean_html,
+    clean_podcast_url,
     ExtractorError,
     int_or_none,
     str_or_none,
     try_get,
+    url_or_none,
 )
 
 
-class StitcherIE(InfoExtractor):
+class StitcherBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/(?:[^/]+/)+e(?:pisode)?/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+    _VALID_URL_BASE = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/'
+
+    def _call_api(self, path, video_id, query):
+        resp = self._download_json(
+            'https://api.prod.stitcher.com/' + path,
+            video_id, query=query)
+        error_massage = try_get(resp, lambda x: x['errors'][0]['message'])
+        if error_massage:
+            raise ExtractorError(error_massage, expected=True)
+        return resp['data']
+
+    def _extract_description(self, data):
+        return clean_html(data.get('html_description') or data.get('description'))
+
+    def _extract_audio_url(self, episode):
+        return url_or_none(episode.get('audio_url') or episode.get('guid'))
+
+    def _extract_show_info(self, show):
+        return {
+            'thumbnail': show.get('image_base_url'),
+            'series': show.get('title'),
+        }
+
+    def _extract_episode(self, episode, audio_url, show_info):
+        info = {
+            'id': compat_str(episode['id']),
+            'display_id': episode.get('slug'),
+            'title': episode['title'].strip(),
+            'description': self._extract_description(episode),
+            'duration': int_or_none(episode.get('duration')),
+            'url': clean_podcast_url(audio_url),
+            'vcodec': 'none',
+            'timestamp': int_or_none(episode.get('date_published')),
+            'season_number': int_or_none(episode.get('season')),
+            'season_id': str_or_none(episode.get('season_id')),
+        }
+        info.update(show_info)
+        return info
+
+
+class StitcherIE(StitcherBaseIE):
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?:[^/]+/)+e(?:pisode)?/(?:[^/#?&]+-)?(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
         'md5': 'e9635098e0da10b21a0e2b85585530f6',
@@ -24,8 +66,9 @@ class StitcherIE(InfoExtractor):
             'description': 'md5:547adb4081864be114ae3831b4c2b42f',
             'duration': 1604,
             'thumbnail': r're:^https?://.*\.jpg',
-            'upload_date': '20180126',
+            'upload_date': '20151008',
-            'timestamp': 1516989316,
+            'timestamp': 1444285800,
+            'series': 'Talking Machines',
         },
     }, {
         'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
@@ -55,33 +98,47 @@ class StitcherIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id, audio_id = re.match(self._VALID_URL, url).groups()
+        audio_id = self._match_id(url)
+        data = self._call_api(
+            'shows/episodes', audio_id, {'episode_ids': audio_id})
+        episode = data['episodes'][0]
+        audio_url = self._extract_audio_url(episode)
+        if not audio_url:
+            self.raise_login_required()
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        return self._extract_episode(
+            episode, audio_url, self._extract_show_info(show))
 
-        resp = self._download_json(
-            'https://api.prod.stitcher.com/episode/' + audio_id,
-            display_id or audio_id)
-        episode = try_get(resp, lambda x: x['data']['episodes'][0], dict)
-        if not episode:
-            raise ExtractorError(resp['errors'][0]['message'], expected=True)
 
-        title = episode['title'].strip()
+class StitcherShowIE(StitcherBaseIE):
-        audio_url = episode['audio_url']
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?P<id>[^/#?&]+)/?(?:[?#&]|$)'
+    _TESTS = [{
+        'url': 'http://www.stitcher.com/podcast/the-talking-machines',
+        'info_dict': {
+            'id': 'the-talking-machines',
+            'title': 'Talking Machines',
+            'description': 'md5:831f0995e40f26c10231af39cf1ebf0b',
+        },
+        'playlist_mincount': 106,
+    }, {
+        'url': 'https://www.stitcher.com/show/the-talking-machines',
+        'only_matching': True,
+    }]
 
-        thumbnail = None
+    def _real_extract(self, url):
-        show_id = episode.get('show_id')
+        show_slug = self._match_id(url)
-        if show_id and episode.get('classic_id') != -1:
+        data = self._call_api(
-            thumbnail = 'https://stitcher-classic.imgix.net/feedimages/%s.jpg' % show_id
+            'search/show/%s/allEpisodes' % show_slug, show_slug, {'count': 10000})
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        show_info = self._extract_show_info(show)
 
-        return {
+        entries = []
-            'id': audio_id,
+        for episode in (data.get('episodes') or []):
-            'display_id': display_id,
+            audio_url = self._extract_audio_url(episode)
-            'title': title,
+            if not audio_url:
-            'description': clean_html(episode.get('html_description') or episode.get('description')),
+                continue
-            'duration': int_or_none(episode.get('duration')),
+            entries.append(self._extract_episode(episode, audio_url, show_info))
-            'thumbnail': thumbnail,
+
-            'url': audio_url,
+        return self.playlist_result(
-            'vcodec': 'none',
+            entries, show_slug, show.get('title'),
-            'timestamp': int_or_none(episode.get('date_created')),
+            self._extract_description(show))
-            'season_number': int_or_none(episode.get('season')),
-            'season_id': str_or_none(episode.get('season_id')),
-        }

yt_dlp/extractor/svt.py

@@ -146,18 +146,19 @@ class SVTPlayIE(SVTPlayBaseIE):
                         )
                         (?P<svt_id>[^/?#&]+)|
                         https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)
+                        (?:.*?modalId=(?P<modal_id>[\da-zA-Z-]+))?
                     )
                     '''
     _TESTS = [{
-        'url': 'https://www.svtplay.se/video/26194546/det-har-ar-himlen',
+        'url': 'https://www.svtplay.se/video/30479064',
         'md5': '2382036fd6f8c994856c323fe51c426e',
         'info_dict': {
-            'id': 'jNwpV9P',
+            'id': '8zVbDPA',
             'ext': 'mp4',
-            'title': 'Det här är himlen',
+            'title': 'Designdrömmar i Stenungsund',
-            'timestamp': 1586044800,
+            'timestamp': 1615770000,
-            'upload_date': '20200405',
+            'upload_date': '20210315',
-            'duration': 3515,
+            'duration': 3519,
             'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
             'age_limit': 0,
             'subtitles': {
@@ -173,6 +174,9 @@ class SVTPlayIE(SVTPlayBaseIE):
             # AssertionError: Expected test_SVTPlay_jNwpV9P.mp4 to be at least 9.77KiB, but it's only 864.00B
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
+        'only_matching': True,
     }, {
         # geo restricted to Sweden
         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
@@ -219,7 +223,8 @@ class SVTPlayIE(SVTPlayBaseIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id, svt_id = mobj.group('id', 'svt_id')
+        video_id = mobj.group('id')
+        svt_id = mobj.group('svt_id') or mobj.group('modal_id')
 
         if svt_id:
             return self._extract_by_video_id(svt_id)
@@ -254,6 +259,7 @@ class SVTPlayIE(SVTPlayBaseIE):
         if not svt_id:
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
+                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\bmodalId=([\da-zA-Z-]+)' % re.escape(video_id),
                  r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
                  r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
                  r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',

yt_dlp/extractor/tiktok.py

@@ -143,7 +143,10 @@ class TikTokIE(TikTokBaseIE):
         props_data = try_get(json_data, lambda x: x['props'], expected_type=dict)
 
         # Chech statusCode for success
-        if props_data.get('pageProps').get('statusCode') == 0:
+        status = props_data.get('pageProps').get('statusCode')
+        if status == 0:
             return self._extract_aweme(props_data, webpage, url)
+        elif status == 10216:
+            raise ExtractorError('This video is private', expected=True)
 
         raise ExtractorError('Video not available', video_id=video_id)

yt_dlp/extractor/vgtv.py

@@ -23,6 +23,8 @@ class VGTVIE(XstreamIE):
         'fvn.no/fvntv': 'fvntv',
         'aftenposten.no/webtv': 'aptv',
         'ap.vgtv.no/webtv': 'aptv',
+        'tv.aftonbladet.se': 'abtv',
+        # obsolete URL schemas, kept in order to save one HTTP redirect
         'tv.aftonbladet.se/abtv': 'abtv',
         'www.aftonbladet.se/tv': 'abtv',
     }
@@ -140,6 +142,10 @@ class VGTVIE(XstreamIE):
             'url': 'http://www.vgtv.no/#!/video/127205/inside-the-mind-of-favela-funk',
             'only_matching': True,
         },
+        {
+            'url': 'https://tv.aftonbladet.se/video/36015/vulkanutbrott-i-rymden-nu-slapper-nasa-bilderna',
+            'only_matching': True,
+        },
         {
             'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
             'only_matching': True,

yt_dlp/extractor/vimeo.py

@@ -991,11 +991,15 @@ class VimeoAlbumIE(VimeoBaseInfoExtractor):
         }
         if hashed_pass:
             query['_hashed_pass'] = hashed_pass
-        videos = self._download_json(
+        try:
-            'https://api.vimeo.com/albums/%s/videos' % album_id,
+            videos = self._download_json(
-            album_id, 'Downloading page %d' % api_page, query=query, headers={
+                'https://api.vimeo.com/albums/%s/videos' % album_id,
-                'Authorization': 'jwt ' + authorization,
+                album_id, 'Downloading page %d' % api_page, query=query, headers={
-            })['data']
+                    'Authorization': 'jwt ' + authorization,
+                })['data']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                return
         for video in videos:
             link = video.get('link')
             if not link:

yt_dlp/extractor/vvvvid.py

@@ -182,17 +182,20 @@ class VVVVIDIE(InfoExtractor):
             if not embed_code:
                 continue
             embed_code = ds(embed_code)
-            if video_type in ('video/rcs', 'video/kenc'):
+            if video_type == 'video/kenc':
-                if video_type == 'video/kenc':
+                embed_code = re.sub(r'https?(://[^/]+)/z/', r'https\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
-                    kenc = self._download_json(
+                kenc = self._download_json(
-                        'https://www.vvvvid.it/kenc', video_id, query={
+                    'https://www.vvvvid.it/kenc', video_id, query={
-                            'action': 'kt',
+                        'action': 'kt',
-                            'conn_id': self._conn_id,
+                        'conn_id': self._conn_id,
-                            'url': embed_code,
+                        'url': embed_code,
-                        }, fatal=False) or {}
+                    }, fatal=False) or {}
-                    kenc_message = kenc.get('message')
+                kenc_message = kenc.get('message')
-                    if kenc_message:
+                if kenc_message:
-                        embed_code += '?' + ds(kenc_message)
+                    embed_code += '?' + ds(kenc_message)
+                formats.extend(self._extract_m3u8_formats(
+                    embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
+            elif video_type == 'video/rcs':
                 formats.extend(self._extract_akamai_formats(embed_code, video_id))
             elif video_type == 'video/youtube':
                 info.update({

yt_dlp/extractor/youtube.py

@@ -25,6 +25,7 @@ from ..compat import (
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
+    bool_or_none,
     clean_html,
     dict_get,
     ExtractorError,
@@ -1946,7 +1947,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     f['format_id'] = itag
                 formats.append(f)
 
-        if self._downloader.params.get('youtube_include_dash_manifest'):
+        if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_manifest_url = streaming_data.get('dashManifestUrl')
             if dash_manifest_url:
                 for f in self._extract_mpd_formats(
@@ -2066,7 +2067,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'tags': keywords,
             'is_live': is_live,
             'playable_in_embed': playability_status.get('playableInEmbed'),
-            'was_live': video_details.get('isLiveContent')
+            'was_live': video_details.get('isLiveContent'),
         }
 
         pctr = try_get(
@@ -2149,6 +2150,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # This will error if there is no livechat
                 initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
                 info['subtitles']['live_chat'] = [{
+                    'url': 'https://www.youtube.com/watch?v=%s' % video_id,  # url is needed to set cookies
                     'video_id': video_id,
                     'ext': 'json',
                     'protocol': 'youtube_live_chat_replay',
@@ -2283,6 +2285,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if v:
                 info[d_k] = v
 
+        is_private = bool_or_none(video_details.get('isPrivate'))
+        is_unlisted = bool_or_none(microformat.get('isUnlisted'))
+        is_membersonly = None
+        if initial_data and is_private is not None:
+            is_membersonly = False
+            contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list)
+            for content in contents or []:
+                badges = try_get(content, lambda x: x['videoPrimaryInfoRenderer']['badges'], list)
+                for badge in badges or []:
+                    label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label']) or ''
+                    if label.lower() == 'members only':
+                        is_membersonly = True
+                        break
+                if is_membersonly:
+                    break
+
+        # TODO: Add this for playlists
+        info['availability'] = self._availability(
+            is_private=is_private,
+            needs_premium=False,  # Youtube no longer have premium-only videos?
+            needs_subscription=is_membersonly,
+            needs_auth=info['age_limit'] >= 18,
+            is_unlisted=None if is_private is None else is_unlisted)
+
         # get xsrf for annotations or comments
         get_annotations = self._downloader.params.get('writeannotations', False)
         get_comments = self._downloader.params.get('getcomments', False)
@@ -2666,7 +2692,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     def _extract_basic_item_renderer(item):
         # Modified from _extract_grid_item_renderer
         known_renderers = (
-            'playlistRenderer', 'videoRenderer', 'channelRenderer'
+            'playlistRenderer', 'videoRenderer', 'channelRenderer',
             'gridPlaylistRenderer', 'gridVideoRenderer', 'gridChannelRenderer'
         )
         for key, renderer in item.items():

yt_dlp/extractor/zingmp3.py

@@ -1,93 +1,94 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    update_url_query,
 )
 
 
-class ZingMp3BaseInfoExtractor(InfoExtractor):
+class ZingMp3BaseIE(InfoExtractor):
+    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?:%s)/[^/]+/(?P<id>\w+)\.html'
+    _GEO_COUNTRIES = ['VN']
 
-    def _extract_item(self, item, page_type, fatal=True):
+    def _extract_item(self, item, fatal):
-        error_message = item.get('msg')
+        item_id = item['id']
-        if error_message:
+        title = item.get('name') or item['title']
-            if not fatal:
-                return
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_message),
-                expected=True)
 
         formats = []
-        for quality, source_url in zip(item.get('qualities') or item.get('quality', []), item.get('source_list') or item.get('source', [])):
+        for k, v in (item.get('source') or {}).items():
-            if not source_url or source_url == 'require vip':
+            if not v:
                 continue
-            if not re.match(r'https?://', source_url):
+            if k in ('mp4', 'hls'):
-                source_url = '//' + source_url
+                for res, video_url in v.items():
-            source_url = self._proto_relative_url(source_url, 'http:')
+                    if not video_url:
-            quality_num = int_or_none(quality)
+                        continue
-            f = {
+                    if k == 'hls':
-                'format_id': quality,
+                        formats.extend(self._extract_m3u8_formats(
-                'url': source_url,
+                            video_url, item_id, 'mp4',
-            }
+                            'm3u8_native', m3u8_id=k, fatal=False))
-            if page_type == 'video':
+                    elif k == 'mp4':
-                f.update({
+                        formats.append({
-                    'height': quality_num,
+                            'format_id': 'mp4-' + res,
-                    'ext': 'mp4',
+                            'url': video_url,
-                })
+                            'height': int_or_none(self._search_regex(
+                                r'^(\d+)p', res, 'resolution', default=None)),
+                        })
             else:
-                f.update({
+                formats.append({
-                    'abr': quality_num,
                     'ext': 'mp3',
+                    'format_id': k,
+                    'tbr': int_or_none(k),
+                    'url': self._proto_relative_url(v),
+                    'vcodec': 'none',
                 })
-            formats.append(f)
+        if not formats:
+            if not fatal:
+                return
+            msg = item['msg']
+            if msg == 'Sorry, this content is not available in your country.':
+                self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+            raise ExtractorError(msg, expected=True)
+        self._sort_formats(formats)
 
-        cover = item.get('cover')
+        subtitles = None
+        lyric = item.get('lyric')
+        if lyric:
+            subtitles = {
+                'origin': [{
+                    'url': lyric,
+                }],
+            }
+
+        album = item.get('album') or {}
 
         return {
-            'title': (item.get('name') or item.get('title')).strip(),
+            'id': item_id,
+            'title': title,
             'formats': formats,
-            'thumbnail': 'http:/' + cover if cover else None,
+            'thumbnail': item.get('thumbnail'),
-            'artist': item.get('artist'),
+            'subtitles': subtitles,
+            'duration': int_or_none(item.get('duration')),
+            'track': title,
+            'artist': item.get('artists_names'),
+            'album': album.get('name') or album.get('title'),
+            'album_artist': album.get('artists_names'),
         }
 
-    def _extract_player_json(self, player_json_url, id, page_type, playlist_title=None):
+    def _real_extract(self, url):
-        player_json = self._download_json(player_json_url, id, 'Downloading Player JSON')
+        page_id = self._match_id(url)
-        items = player_json['data']
+        webpage = self._download_webpage(
-        if 'item' in items:
+            url.replace('://zingmp3.vn/', '://mp3.zing.vn/'),
-            items = items['item']
+            page_id, query={'play_song': 1})
-
+        data_path = self._search_regex(
-        if len(items) == 1:
+            r'data-xml="([^"]+)', webpage, 'data path')
-            # one single song
+        return self._process_data(self._download_json(
-            data = self._extract_item(items[0], page_type)
+            'https://mp3.zing.vn/xhr' + data_path, page_id)['data'])
-            data['id'] = id
-
-            return data
-        else:
-            # playlist of songs
-            entries = []
-
-            for i, item in enumerate(items, 1):
-                entry = self._extract_item(item, page_type, fatal=False)
-                if not entry:
-                    continue
-                entry['id'] = '%s-%d' % (id, i)
-                entries.append(entry)
-
-            return {
-                '_type': 'playlist',
-                'id': id,
-                'title': playlist_title,
-                'entries': entries,
-            }
 
 
-class ZingMp3IE(ZingMp3BaseInfoExtractor):
+class ZingMp3IE(ZingMp3BaseIE):
-    _VALID_URL = r'https?://mp3\.zing\.vn/(?:bai-hat|album|playlist|video-clip)/[^/]+/(?P<id>\w+)\.html'
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip'
     _TESTS = [{
         'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'md5': 'ead7ae13693b3205cbc89536a077daed',
@@ -95,49 +96,66 @@ class ZingMp3IE(ZingMp3BaseInfoExtractor):
             'id': 'ZWZB9WAB',
             'title': 'Xa Mãi Xa',
             'ext': 'mp3',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'subtitles': {
+                'origin': [{
+                    'ext': 'lrc',
+                }]
+            },
+            'duration': 255,
+            'track': 'Xa Mãi Xa',
+            'artist': 'Bảo Thy',
+            'album': 'Special Album',
+            'album_artist': 'Bảo Thy',
         },
     }, {
-        'url': 'http://mp3.zing.vn/video-clip/Let-It-Go-Frozen-OST-Sungha-Jung/ZW6BAEA0.html',
+        'url': 'https://mp3.zing.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html',
-        'md5': '870295a9cd8045c0e15663565902618d',
+        'md5': 'e9c972b693aa88301ef981c8151c4343',
         'info_dict': {
-            'id': 'ZW6BAEA0',
+            'id': 'ZO8ZF7C7',
-            'title': 'Let It Go (Frozen OST)',
+            'title': 'Sương Hoa Đưa Lối',
             'ext': 'mp4',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'duration': 207,
+            'track': 'Sương Hoa Đưa Lối',
+            'artist': 'K-ICM, RYO',
         },
     }, {
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'url': 'https://zingmp3.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
-        'info_dict': {
-            '_type': 'playlist',
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều,Minh Tuyết | Album 320 lossless',
-        },
-        'playlist_count': 10,
-        'skip': 'removed at the request of the owner',
-    }, {
-        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
     }]
     IE_NAME = 'zingmp3'
     IE_DESC = 'mp3.zing.vn'
 
-    def _real_extract(self, url):
+    def _process_data(self, data):
-        page_id = self._match_id(url)
+        return self._extract_item(data, True)
 
-        webpage = self._download_webpage(url, page_id)
 
-        player_json_url = self._search_regex([
+class ZingMp3AlbumIE(ZingMp3BaseIE):
-            r'data-xml="([^"]+)',
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'album|playlist'
-            r'&amp;xmlURL=([^&]+)&'
+    _TESTS = [{
-        ], webpage, 'player xml url')
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': 'Lâu Đài Tình Ái',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://zingmp3.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'only_matching': True,
+    }]
+    IE_NAME = 'zingmp3:album'
 
-        playlist_title = None
+    def _process_data(self, data):
-        page_type = self._search_regex(r'/(?:html5)?xml/([^/-]+)', player_json_url, 'page type')
+        def entries():
-        if page_type == 'video':
+            for item in (data.get('items') or []):
-            player_json_url = update_url_query(player_json_url, {'format': 'json'})
+                entry = self._extract_item(item, False)
-        else:
+                if entry:
-            player_json_url = player_json_url.replace('/xml/', '/html5xml/')
+                    yield entry
-            if page_type == 'album':
+        info = data.get('info') or {}
-                playlist_title = self._og_search_title(webpage)
+        return self.playlist_result(
-
+            entries(), info.get('id'), info.get('name') or info.get('title'))
-        return self._extract_player_json(player_json_url, page_id, page_type, playlist_title)

yt_dlp/extractor/zoom.py

@@ -1,82 +1,68 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    url_or_none,
+    js_to_json,
     parse_filesize,
-    urlencode_postdata
+    urlencode_postdata,
 )
 
 
 class ZoomIE(InfoExtractor):
     IE_NAME = 'zoom'
-    _VALID_URL = r'https://(?:.*).?zoom.us/rec(?:ording)?/(play|share)/(?P<id>[A-Za-z0-9\-_.]+)'
+    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom.us/)rec(?:ording)?/(?:play|share)/(?P<id>[A-Za-z0-9_.-]+)'
-
     _TEST = {
-        'url': 'https://zoom.us/recording/play/SILVuCL4bFtRwWTtOCFQQxAsBQsJljFtm9e4Z_bvo-A8B-nzUSYZRNuPl3qW5IGK',
+        'url': 'https://economist.zoom.us/rec/play/dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
+        'md5': 'ab445e8c911fddc4f9adc842c2c5d434',
         'info_dict': {
-            'md5': '031a5b379f1547a8b29c5c4c837dccf2',
+            'id': 'dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
-            'title': "GAZ Transformational Tuesdays W/ Landon & Stapes",
+            'ext': 'mp4',
-            'id': "SILVuCL4bFtRwWTtOCFQQxAsBQsJljFtm9e4Z_bvo-A8B-nzUSYZRNuPl3qW5IGK",
+            'title': 'China\'s "two sessions" and the new five-year plan',
-            'ext': "mp4"
         }
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        base_url, play_id = re.match(self._VALID_URL, url).groups()
-        webpage = self._download_webpage(url, display_id)
+        webpage = self._download_webpage(url, play_id)
 
-        password_protected = self._search_regex(r'<form[^>]+?id="(password_form)"', webpage, 'password field', fatal=False, default=None)
+        try:
-        if password_protected is not None:
+            form = self._form_hidden_inputs('password_form', webpage)
-            self._verify_video_password(url, display_id, webpage)
+        except ExtractorError:
-            webpage = self._download_webpage(url, display_id)
+            form = None
+        if form:
+            password = self._downloader.params.get('videopassword')
+            if not password:
+                raise ExtractorError(
+                    'This video is protected by a passcode, use the --video-password option', expected=True)
+            is_meeting = form.get('useWhichPasswd') == 'meeting'
+            validation = self._download_json(
+                base_url + 'rec/validate%s_passwd' % ('_meet' if is_meeting else ''),
+                play_id, 'Validating passcode', 'Wrong passcode', data=urlencode_postdata({
+                    'id': form[('meet' if is_meeting else 'file') + 'Id'],
+                    'passwd': password,
+                    'action': form.get('action'),
+                }))
+            if not validation.get('status'):
+                raise ExtractorError(validation['errorMessage'], expected=True)
+            webpage = self._download_webpage(url, play_id)
 
-        video_url = self._search_regex(r"viewMp4Url: \'(.*)\'", webpage, 'video url')
+        data = self._parse_json(self._search_regex(
-        title = self._html_search_regex([r"topic: \"(.*)\",", r"<title>(.*) - Zoom</title>"], webpage, 'title')
+            r'(?s)window\.__data__\s*=\s*({.+?});',
-        viewResolvtionsWidth = self._search_regex(r"viewResolvtionsWidth: (\d*)", webpage, 'res width', fatal=False)
+            webpage, 'data'), play_id, js_to_json)
-        viewResolvtionsHeight = self._search_regex(r"viewResolvtionsHeight: (\d*)", webpage, 'res height', fatal=False)
-        fileSize = parse_filesize(self._search_regex(r"fileSize: \'(.+)\'", webpage, 'fileSize', fatal=False))
-
-        urlprefix = url.split("zoom.us")[0] + "zoom.us/"
-
-        formats = []
-        formats.append({
-            'url': url_or_none(video_url),
-            'width': int_or_none(viewResolvtionsWidth),
-            'height': int_or_none(viewResolvtionsHeight),
-            'http_headers': {'Accept': 'video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5',
-                             'Referer': urlprefix},
-            'ext': "mp4",
-            'filesize_approx': int_or_none(fileSize)
-        })
-        self._sort_formats(formats)
 
         return {
-            'id': display_id,
+            'id': play_id,
-            'title': title,
+            'title': data['topic'],
-            'formats': formats
+            'url': data['viewMp4Url'],
+            'width': int_or_none(data.get('viewResolvtionsWidth')),
+            'height': int_or_none(data.get('viewResolvtionsHeight')),
+            'http_headers': {
+                'Referer': base_url,
+            },
+            'filesize_approx': parse_filesize(data.get('fileSize')),
         }
-
-    def _verify_video_password(self, url, video_id, webpage):
-        password = self._downloader.params.get('videopassword')
-        if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        meetId = self._search_regex(r'<input[^>]+?id="meetId" value="([^\"]+)"', webpage, 'meetId')
-        data = urlencode_postdata({
-            'id': meetId,
-            'passwd': password,
-            'action': "viewdetailedpage",
-            'recaptcha': ""
-        })
-        validation_url = url.split("zoom.us")[0] + "zoom.us/rec/validate_meet_passwd"
-        validation_response = self._download_json(
-            validation_url, video_id,
-            note='Validating Password...',
-            errnote='Wrong password?',
-            data=data)
-
-        if validation_response['errorCode'] != 0:
-            raise ExtractorError('Login failed, %s said: %r' % (self.IE_NAME, validation_response['errorMessage']))

yt_dlp/options.py

@@ -533,11 +533,11 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--write-auto-subs', '--write-automatic-subs',
         action='store_true', dest='writeautomaticsub', default=False,
-        help='Write automatically generated subtitle file (YouTube only)')
+        help='Write automatically generated subtitle file (Alias: --write-automatic-subs)')
     subtitles.add_option(
         '--no-write-auto-subs', '--no-write-automatic-subs',
         action='store_false', dest='writeautomaticsub', default=False,
-        help='Do not write automatically generated subtitle file (default)')
+        help='Do not write auto-generated subtitles (default) (Alias: --no-write-automatic-subs)')
     subtitles.add_option(
         '--all-subs',
         action='store_true', dest='allsubtitles', default=False,
@@ -551,7 +551,7 @@ def parseOpts(overrideArguments=None):
         action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
         help='Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"')
     subtitles.add_option(
-        '--sub-lang', '--sub-langs', '--srt-lang',
+        '--sub-langs', '--srt-langs',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
         default=[], callback=_comma_separated_values_options_callback,
         help='Languages of the subtitles to download (optional) separated by commas, use --list-subs for available language tags')
@@ -679,7 +679,7 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--prefer-insecure', '--prefer-unsecure',
         action='store_true', dest='prefer_insecure',
-        help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
+        help='Use an unencrypted connection to retrieve information about the video (Currently supported only for YouTube)')
     workarounds.add_option(
         '--user-agent',
         metavar='UA', dest='user_agent',
@@ -707,17 +707,13 @@ def parseOpts(overrideArguments=None):
         '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
         dest='sleep_interval', type=float,
         help=(
-            'Number of seconds to sleep before each download when used alone '
+            'Number of seconds to sleep before each download. '
-            'or a lower bound of a range for randomized sleep before each download '
+            'This is the minimum time to sleep when used along with --max-sleep-interval '
-            '(minimum possible number of seconds to sleep) when used along with '
+            '(Alias: --min-sleep-interval)'))
-            '--max-sleep-interval'))
     workarounds.add_option(
         '--max-sleep-interval', metavar='SECONDS',
         dest='max_sleep_interval', type=float,
-        help=(
+        help='Maximum number of seconds to sleep. Can only be used along with --min-sleep-interval')
-            'Upper bound of a range for randomized sleep before each download '
-            '(maximum possible number of seconds to sleep). Must only be used '
-            'along with --min-sleep-interval'))
     workarounds.add_option(
         '--sleep-subtitles', metavar='SECONDS',
         dest='sleep_interval_subtitles', default=0, type=int,
@@ -739,7 +735,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--skip-download', '--no-download',
         action='store_true', dest='skip_download', default=False,
-        help='Do not download the video')
+        help='Do not download the video but write all related files (Alias: --no-download)')
     verbosity.add_option(
         '-g', '--get-url',
         action='store_true', dest='geturl', default=False,
@@ -982,9 +978,17 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--no-write-playlist-metafiles',
         action='store_false', dest='allow_playlist_files',
+        help='Do not write playlist metadata when using --write-info-json, --write-description etc.')
+    filesystem.add_option(
+        '--clean-infojson',
+        action='store_true', dest='clean_infojson', default=True,
         help=(
-            'Do not write playlist metadata when using '
+            'Remove some private fields such as filenames from the infojson. '
-            '--write-info-json, --write-description etc.'))
+            'Note that it could still contain some personal information (default)'))
+    filesystem.add_option(
+        '--no-clean-infojson',
+        action='store_false', dest='clean_infojson',
+        help='Write all fields to the infojson')
     filesystem.add_option(
         '--get-comments',
         action='store_true', dest='getcomments', default=False,
@@ -1086,7 +1090,7 @@ def parseOpts(overrideArguments=None):
             'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
             'to give the argument to the specified postprocessor/executable. Supported postprocessors are: '
             'SponSkrub, ExtractAudio, VideoRemuxer, VideoConvertor, EmbedSubtitle, Metadata, Merger, '
-            'FixupStretched, FixupM4a, FixupM3u8, SubtitlesConvertor and EmbedThumbnail. '
+            'FixupStretched, FixupM4a, FixupM3u8, SubtitlesConvertor, EmbedThumbnail and SplitChapters. '
             'The supported executables are: SponSkrub, FFmpeg, FFprobe, and AtomicParsley. '
             'You can also specify "PP+EXE:ARGS" to give the arguments to the specified executable '
             'only when being used by the specified postprocessor. Additionally, for ffmpeg/ffprobe, '
@@ -1143,13 +1147,18 @@ def parseOpts(overrideArguments=None):
         metavar='FIELD:FORMAT', dest='metafromfield', action='append',
         help=(
             'Parse additional metadata like title/artist from other fields. '
-            'Give field name to extract data from, and format of the field seperated by a ":". '
+            'Give a template or field name to extract data from and the '
+            'format to interpret it as, seperated by a ":". '
             'Either regular expression with named capture groups or a '
-            'similar syntax to the output template can also be used. '
+            'similar syntax to the output template can be used for the FORMAT. '
-            'The parsed parameters replace any existing values and can be use in output template. '
+            'Similarly, the syntax for output template can be used for FIELD '
+            'to parse the data from multiple fields. '
+            'The parsed parameters replace any existing values and can be used in output templates. '
             'This option can be used multiple times. '
             'Example: --parse-metadata "title:%(artist)s - %(title)s" matches a title like '
             '"Coldplay - Paradise". '
+            'Example: --parse-metadata "%(series)s %(episode_number)s:%(title)s" '
+            'sets the title using series and episode number. '
             'Example (regex): --parse-metadata "description:Artist - (?P<artist>.+?)"'))
     postproc.add_option(
         '--xattrs',
@@ -1179,9 +1188,9 @@ def parseOpts(overrideArguments=None):
         metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading and post-processing, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
     postproc.add_option(
-        '--convert-subs', '--convert-subtitles',
+        '--convert-subs', '--convert-sub', '--convert-subtitles',
         metavar='FORMAT', dest='convertsubtitles', default=None,
-        help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)')
+        help='Convert the subtitles to another format (currently supported: srt|ass|vtt|lrc) (Alias: --convert-subtitles)')
     postproc.add_option(
         '--split-chapters', '--split-tracks',
         dest='split_chapters', action='store_true', default=False,

yt_dlp/postprocessor/embedthumbnail.py

@@ -47,7 +47,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             self.to_screen('There aren\'t any thumbnails to embed')
             return [], info
 
-        original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filename']
+        initial_thumbnail = original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filepath']
 
         if not os.path.exists(encodeFilename(thumbnail_filename)):
             self.report_warning('Skipping embedding the thumbnail because the file is missing.')
@@ -65,6 +65,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if thumbnail_ext != 'webp' and is_webp(thumbnail_filename):
                 self.to_screen('Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename)
                 thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp')
+                if os.path.exists(thumbnail_webp_filename):
+                    os.remove(thumbnail_webp_filename)
                 os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename))
                 original_thumbnail = thumbnail_filename = thumbnail_webp_filename
                 thumbnail_ext = 'webp'
@@ -133,7 +135,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                     x for x in ['AtomicParsley', 'atomicparsley']
                     if check_executable(x, ['-v'])), None)
                 if atomicparsley is None:
-                    raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
+                    raise EmbedThumbnailPPError('AtomicParsley was not found. Please install')
 
                 cmd = [encodeFilename(atomicparsley, True),
                        encodeFilename(filename, True),
@@ -194,7 +196,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
         files_to_delete = [thumbnail_filename]
         if self._already_have_thumbnail:
             info['__files_to_move'][original_thumbnail] = replace_extension(
-                info['__thumbnail_filename'], os.path.splitext(original_thumbnail)[1][1:])
+                info['__files_to_move'][initial_thumbnail],
+                os.path.splitext(original_thumbnail)[1][1:])
             if original_thumbnail == thumbnail_filename:
                 files_to_delete = []
         elif original_thumbnail != thumbnail_filename:

yt_dlp/postprocessor/ffmpeg.py

@@ -19,7 +19,6 @@ from ..utils import (
     PostProcessingError,
     prepend_extension,
     shell_quote,
-    subtitles_filename,
     dfxp2srt,
     ISO639Utils,
     process_communicate_or_kill,
@@ -62,7 +61,7 @@ class FFmpegPostProcessor(PostProcessor):
 
     def check_version(self):
         if not self.available:
-            raise FFmpegPostProcessorError('ffmpeg not found. Please install')
+            raise FFmpegPostProcessorError('ffmpeg not found. Please install or provide the path using --ffmpeg-location')
 
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
@@ -166,7 +165,7 @@ class FFmpegPostProcessor(PostProcessor):
 
     def get_audio_codec(self, path):
         if not self.probe_available and not self.available:
-            raise PostProcessingError('ffprobe and ffmpeg not found. Please install')
+            raise PostProcessingError('ffprobe and ffmpeg not found. Please install or provide the path using --ffmpeg-location')
         try:
             if self.probe_available:
                 cmd = [
@@ -208,7 +207,7 @@ class FFmpegPostProcessor(PostProcessor):
         if self.probe_basename != 'ffprobe':
             if self.probe_available:
                 self.report_warning('Only ffprobe is supported for metadata extraction')
-            raise PostProcessingError('ffprobe not found. Please install.')
+            raise PostProcessingError('ffprobe not found. Please install or provide the path using --ffmpeg-location')
         self.check_version()
 
         cmd = [
@@ -486,7 +485,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
                 self.report_warning('JSON subtitles cannot be embedded')
             elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
                 sub_langs.append(lang)
-                sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext))
+                sub_filenames.append(sub_info['filepath'])
             else:
                 if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
                     webm_vtt_warn = True
@@ -551,8 +550,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         add('title', ('track', 'title'))
         add('date', 'upload_date')
-        add(('description', 'comment'), 'description')
+        add(('description', 'synopsis'), 'description')
-        add('purl', 'webpage_url')
+        add(('purl', 'comment'), 'webpage_url')
         add('track', 'track_number')
         add('artist', ('artist', 'creator', 'uploader', 'uploader_id'))
         add('genre')
@@ -712,7 +711,6 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
     def run(self, info):
         subs = info.get('requested_subtitles')
-        filename = info['filepath']
         new_ext = self.format
         new_format = new_ext
         if new_format == 'vtt':
@@ -732,9 +730,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'You have requested to convert json subtitles into another format, '
                     'which is currently not possible')
                 continue
-            old_file = subtitles_filename(filename, lang, ext, info.get('ext'))
+            old_file = sub['filepath']
             sub_filenames.append(old_file)
-            new_file = subtitles_filename(filename, lang, new_ext, info.get('ext'))
+            new_file = replace_extension(old_file, new_ext)
 
             if ext in ('dfxp', 'ttml', 'tt'):
                 self.report_warning(
@@ -742,7 +740,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'which results in style information loss')
 
                 dfxp_file = old_file
-                srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext'))
+                srt_file = replace_extension(old_file, 'srt')
 
                 with open(dfxp_file, 'rb') as f:
                     srt_data = dfxp2srt(f.read())
@@ -753,7 +751,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 subs[lang] = {
                     'ext': 'srt',
-                    'data': srt_data
+                    'data': srt_data,
+                    'filepath': srt_file,
                 }
 
                 if new_ext == 'srt':
@@ -767,8 +766,12 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 subs[lang] = {
                     'ext': new_ext,
                     'data': f.read(),
+                    'filepath': new_file,
                 }
 
+            info['__files_to_move'][new_file] = replace_extension(
+                info['__files_to_move'][old_file], new_ext)
+
         return sub_filenames, info
 
 
@@ -789,17 +792,17 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor):
         if not self._downloader._ensure_dir_exists(encodeFilename(destination)):
             return
 
-        chapter['_filename'] = destination
+        chapter['filepath'] = destination
         self.to_screen('Chapter %03d; Destination: %s' % (number, destination))
         return (
             destination,
             ['-ss', compat_str(chapter['start_time']),
-             '-to', compat_str(chapter['end_time'])])
+             '-t', compat_str(chapter['end_time'] - chapter['start_time'])])
 
     def run(self, info):
         chapters = info.get('chapters') or []
         if not chapters:
-            self.report_warning('There are no tracks to extract')
+            self.report_warning('Chapter information is unavailable')
             return [], info
 
         self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters))

yt_dlp/postprocessor/metadatafromfield.py

@@ -4,11 +4,10 @@ import re
 
 from .common import PostProcessor
 from ..compat import compat_str
-from ..utils import str_or_none
 
 
 class MetadataFromFieldPP(PostProcessor):
-    regex = r'(?P<field>\w+):(?P<format>.+)$'
+    regex = r'(?P<in>.+):(?P<out>.+)$'
 
     def __init__(self, downloader, formats):
         PostProcessor.__init__(self, downloader)
@@ -19,11 +18,20 @@ class MetadataFromFieldPP(PostProcessor):
             match = re.match(self.regex, f)
             assert match is not None
             self._data.append({
-                'field': match.group('field'),
+                'in': match.group('in'),
-                'format': match.group('format'),
+                'out': match.group('out'),
-                'regex': self.format_to_regex(match.group('format'))})
+                'tmpl': self.field_to_template(match.group('in')),
+                'regex': self.format_to_regex(match.group('out')),
+            })
 
-    def format_to_regex(self, fmt):
+    @staticmethod
+    def field_to_template(tmpl):
+        if re.match(r'\w+$', tmpl):
+            return '%%(%s)s' % tmpl
+        return tmpl
+
+    @staticmethod
+    def format_to_regex(fmt):
         r"""
         Converts a string like
            '%(title)s - %(artist)s'
@@ -37,7 +45,7 @@ class MetadataFromFieldPP(PostProcessor):
         # replace %(..)s with regex group and escape other string parts
         for match in re.finditer(r'%\((\w+)\)s', fmt):
             regex += re.escape(fmt[lastpos:match.start()])
-            regex += r'(?P<' + match.group(1) + r'>[^\r\n]+)'
+            regex += r'(?P<%s>[^\r\n]+)' % match.group(1)
             lastpos = match.end()
         if lastpos < len(fmt):
             regex += re.escape(fmt[lastpos:])
@@ -45,22 +53,16 @@ class MetadataFromFieldPP(PostProcessor):
 
     def run(self, info):
         for dictn in self._data:
-            field, regex = dictn['field'], dictn['regex']
+            tmpl, info_copy = self._downloader.prepare_outtmpl(dictn['tmpl'], info)
-            if field not in info:
+            data_to_parse = tmpl % info_copy
-                self.report_warning('Video doesnot have a %s' % field)
+            self.write_debug('Searching for r"%s" in %s' % (dictn['regex'], tmpl))
-                continue
+            match = re.search(dictn['regex'], data_to_parse)
-            data_to_parse = str_or_none(info[field])
-            if data_to_parse is None:
-                self.report_warning('Field %s cannot be parsed' % field)
-                continue
-            self.write_debug('Searching for r"%s" in %s' % (regex, field))
-            match = re.search(regex, data_to_parse)
             if match is None:
-                self.report_warning('Could not interpret video %s as "%s"' % (field, dictn['format']))
+                self.report_warning('Could not interpret video %s as "%s"' % (dictn['in'], dictn['out']))
                 continue
             for attribute, value in match.groupdict().items():
                 info[attribute] = value
-                self.to_screen('parsed %s from %s: %s' % (attribute, field, value if value is not None else 'NA'))
+                self.to_screen('parsed %s from "%s": %s' % (attribute, dictn['in'], value if value is not None else 'NA'))
         return [], info
 
 

yt_dlp/postprocessor/movefilesafterdownload.py

@@ -13,10 +13,6 @@ from ..utils import (
 
 class MoveFilesAfterDownloadPP(PostProcessor):
 
-    def __init__(self, downloader, files_to_move):
-        PostProcessor.__init__(self, downloader)
-        self.files_to_move = files_to_move
-
     @classmethod
     def pp_key(cls):
         return 'MoveFiles'
@@ -25,11 +21,10 @@ class MoveFilesAfterDownloadPP(PostProcessor):
         dl_path, dl_name = os.path.split(encodeFilename(info['filepath']))
         finaldir = info.get('__finaldir', dl_path)
         finalpath = os.path.join(finaldir, dl_name)
-        self.files_to_move.update(info['__files_to_move'])
+        info['__files_to_move'][info['filepath']] = decodeFilename(finalpath)
-        self.files_to_move[info['filepath']] = decodeFilename(finalpath)
 
         make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old))))
-        for oldfile, newfile in self.files_to_move.items():
+        for oldfile, newfile in info['__files_to_move'].items():
             if not newfile:
                 newfile = make_newfilename(oldfile)
             if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):

yt_dlp/postprocessor/sponskrub.py

@@ -6,6 +6,7 @@ from .common import PostProcessor
 from ..compat import compat_shlex_split
 from ..utils import (
     check_executable,
+    cli_option,
     encodeArgument,
     encodeFilename,
     shell_quote,
@@ -31,7 +32,7 @@ class SponSkrubPP(PostProcessor):
             if path:
                 raise PostProcessingError('sponskrub not found in "%s"' % path)
             else:
-                raise PostProcessingError('sponskrub not found. Please install or provide the path using --sponskrub-path.')
+                raise PostProcessingError('sponskrub not found. Please install or provide the path using --sponskrub-path')
 
     def get_exe(self, path=''):
         if not path or not check_executable(path, ['-h']):
@@ -70,6 +71,7 @@ class SponSkrubPP(PostProcessor):
         cmd = [self.path]
         if not self.cutout:
             cmd += ['-chapter']
+        cmd += cli_option(self._downloader.params, '-proxy', 'proxy')
         cmd += compat_shlex_split(self.args)  # For backward compatibility
         cmd += self._configuration_args(self._exe_name, use_compat=False)
         cmd += ['--', information['id'], filename, temp_filename]

yt_dlp/utils.py

@@ -1836,7 +1836,7 @@ def write_json_file(obj, fn):
 
     try:
         with tf:
-            json.dump(obj, tf)
+            json.dump(obj, tf, default=repr)
         if sys.platform == 'win32':
             # Need to remove existing file on Windows, else os.rename raises
             # WindowsError or FileExistsError.
@@ -2423,6 +2423,15 @@ class DownloadError(YoutubeDLError):
         self.exc_info = exc_info
 
 
+class EntryNotInPlaylist(YoutubeDLError):
+    """Entry not in playlist exception.
+
+    This exception will be thrown by YoutubeDL when a requested entry
+    is not found in the playlist info_dict
+    """
+    pass
+
+
 class SameFileError(YoutubeDLError):
     """Same File exception.
 
@@ -4106,6 +4115,7 @@ def parse_age_limit(s):
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
     if m:
         return int(m.group('age'))
+    s = s.upper()
     if s in US_RATINGS:
         return US_RATINGS[s]
     m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
@@ -4195,6 +4205,20 @@ OUTTMPL_TYPES = {
     'pl_infojson': 'info.json',
 }
 
+# As of [1] format syntax is:
+#  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
+# 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
+FORMAT_RE = r'''(?x)
+    (?<!%)
+    %
+    \({0}\)  # mapping key
+    (?:[#0\-+ ]+)?  # conversion flags (optional)
+    (?:\d+)?  # minimum field width (optional)
+    (?:\.\d+)?  # precision (optional)
+    [hlL]?  # length modifier (optional)
+    (?P<type>[diouxXeEfFgGcrs%])  # conversion type
+'''
+
 
 def limit_length(s, length):
     """ Add ellipses to overly long strings """

yt_dlp/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.07'
+__version__ = '2021.03.24'