mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2026-08-09 05:38:38 +03:00
Compare commits
117
Commits
2021.12.01
...
2021.12.27
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1e5c83b26b | ||
|
|
6223f67a8c | ||
|
|
6a34813a0d | ||
|
|
f59f5ef8b6 | ||
|
|
f44afb54ef | ||
|
|
77cee0f188 | ||
|
|
6a17677577 | ||
|
|
ee7b9bdf5d | ||
|
|
185bf31070 | ||
|
|
0b77924a38 | ||
|
|
8126298c1b | ||
|
|
6da22e7d4f | ||
|
|
c62ecf0d90 | ||
|
|
3774f4f427 | ||
|
|
9980d3d213 | ||
|
|
8eb4b1bb8e | ||
|
|
332da56f52 | ||
|
|
459aea84c3 | ||
|
|
87e0499624 | ||
|
|
0f86a1cd59 | ||
|
|
d80d98e7d4 | ||
|
|
352d5da812 | ||
|
|
d43de6821c | ||
|
|
070f6a85ea | ||
|
|
4b4b7f746c | ||
|
|
e9efb99f66 | ||
|
|
a709d87335 | ||
|
|
774a46c53d | ||
|
|
c8b80b9643 | ||
|
|
4e260d1a56 | ||
|
|
4f3fa23e5a | ||
|
|
b28bac93ab | ||
|
|
37893bb0c9 | ||
|
|
c25de59cf7 | ||
|
|
205a0654c0 | ||
|
|
663949f825 | ||
|
|
b69fd25c25 | ||
|
|
e0fd95737d | ||
|
|
4ac5b94807 | ||
|
|
4273cc776d | ||
|
|
fa9f30b802 | ||
|
|
1cefca9e44 | ||
|
|
5edb8dfec2 | ||
|
|
0fcba15d57 | ||
|
|
adbc4ec4bb | ||
|
|
c031b0414c | ||
|
|
f3aa3c3f98 | ||
|
|
ae43a4b986 | ||
|
|
ca5db158ae | ||
|
|
5f549d4959 | ||
|
|
6839d02cb6 | ||
|
|
2aae2c91ff | ||
|
|
c2dedf12e8 | ||
|
|
e75bb0d6c3 | ||
|
|
dd0228ce1f | ||
|
|
37e57a9fd4 | ||
|
|
940a67a3e2 | ||
|
|
e6ae51c123 | ||
|
|
75ad33572b | ||
|
|
aab41cdd33 | ||
|
|
b3a5115ff1 | ||
|
|
d76d15a669 | ||
|
|
e978789f0f | ||
|
|
ec2e44fc57 | ||
|
|
375d9360bf | ||
|
|
d5c3254889 | ||
|
|
fed1309651 | ||
|
|
fe69f52e5c | ||
|
|
3116be32b4 | ||
|
|
a8549f19e7 | ||
|
|
39ca3b5c7f | ||
|
|
46383212b3 | ||
|
|
0bb322b9c0 | ||
|
|
ff9f925b63 | ||
|
|
5bfc8bee5a | ||
|
|
19188702ef | ||
|
|
d984a98def | ||
|
|
069c6ccf02 | ||
|
|
53dad39e30 | ||
|
|
db77c49c84 | ||
|
|
abc07b554c | ||
|
|
86f3d52f8c | ||
|
|
8b688881ba | ||
|
|
13debc86e7 | ||
|
|
b5f94e4fa1 | ||
|
|
61882afdc5 | ||
|
|
aa4b054512 | ||
|
|
487c5b3389 | ||
|
|
8157a09d22 | ||
|
|
b1aaf1c07f | ||
|
|
5f9aaac8c2 | ||
|
|
54c2521ca6 | ||
|
|
2814f12ba4 | ||
|
|
1619836cb7 | ||
|
|
e3c7d49571 | ||
|
|
ddd24c9949 | ||
|
|
443b21dc4e | ||
|
|
66f4c04e50 | ||
|
|
93864403ea | ||
|
|
b5475f1145 | ||
|
|
38d79fd16c | ||
|
|
acc0d6a411 | ||
|
|
146cc4114a | ||
|
|
818faa3a86 | ||
|
|
aa5ecf082c | ||
|
|
d2b2fca53f | ||
|
|
63ccf4ff1a | ||
|
|
43b2290658 | ||
|
|
99148c6a33 | ||
|
|
9bdd99cf39 | ||
|
|
2c4aaaddc9 | ||
|
|
5f7cb91ae9 | ||
|
|
3efb96a6d1 | ||
|
|
3262f8abf2 | ||
|
|
bdbafb3913 | ||
|
|
a804f6d89c | ||
|
|
814dfb7e25 |
@@ -1,6 +1,6 @@
|
||||
name: Broken site support
|
||||
description: Report broken or misfunctioning site
|
||||
labels: [triage, extractor-bug]
|
||||
labels: [triage, site-bug]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
@@ -11,7 +11,7 @@ body:
|
||||
options:
|
||||
- label: I'm reporting a broken site
|
||||
required: true
|
||||
- label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
- label: I've verified that I'm running yt-dlp version **2021.12.27**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
required: true
|
||||
- label: I've checked that all provided URLs are alive and playable in a browser
|
||||
required: true
|
||||
@@ -51,12 +51,12 @@ body:
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.11.10.1 (exe)
|
||||
[debug] yt-dlp version 2021.12.27 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.11.10.1)
|
||||
yt-dlp is up to date (2021.12.27)
|
||||
<more lines>
|
||||
render: shell
|
||||
validations:
|
||||
|
||||
@@ -11,7 +11,7 @@ body:
|
||||
options:
|
||||
- label: I'm reporting a new site support request
|
||||
required: true
|
||||
- label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
- label: I've verified that I'm running yt-dlp version **2021.12.27**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
required: true
|
||||
- label: I've checked that all provided URLs are alive and playable in a browser
|
||||
required: true
|
||||
@@ -34,7 +34,7 @@ body:
|
||||
label: Example URLs
|
||||
description: |
|
||||
Provide all kinds of example URLs for which support should be added
|
||||
value: |
|
||||
placeholder: |
|
||||
- Single video: https://www.youtube.com/watch?v=BaW_jenozKc
|
||||
- Single video: https://youtu.be/BaW_jenozKc
|
||||
- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
|
||||
@@ -62,12 +62,12 @@ body:
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.11.10.1 (exe)
|
||||
[debug] yt-dlp version 2021.12.27 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.11.10.1)
|
||||
yt-dlp is up to date (2021.12.27)
|
||||
<more lines>
|
||||
render: shell
|
||||
validations:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Site feature request
|
||||
description: Request a new functionality for a site
|
||||
description: Request a new functionality for a supported site
|
||||
labels: [triage, site-enhancement]
|
||||
body:
|
||||
- type: checkboxes
|
||||
@@ -11,7 +11,7 @@ body:
|
||||
options:
|
||||
- label: I'm reporting a site feature request
|
||||
required: true
|
||||
- label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
- label: I've verified that I'm running yt-dlp version **2021.12.27**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
required: true
|
||||
- label: I've checked that all provided URLs are alive and playable in a browser
|
||||
required: true
|
||||
@@ -47,3 +47,26 @@ body:
|
||||
placeholder: WRITE DESCRIPTION HERE
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: Verbose log
|
||||
description: |
|
||||
Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
|
||||
Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
|
||||
It should look similar to this:
|
||||
placeholder: |
|
||||
[debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.12.27 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.12.27)
|
||||
<more lines>
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: Bug report
|
||||
description: Report a bug unrelated to any particular site or extractor
|
||||
labels: [triage,bug]
|
||||
labels: [triage, bug]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
@@ -11,7 +11,7 @@ body:
|
||||
options:
|
||||
- label: I'm reporting a bug unrelated to a specific site
|
||||
required: true
|
||||
- label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
- label: I've verified that I'm running yt-dlp version **2021.12.27**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
required: true
|
||||
- label: I've checked that all provided URLs are alive and playable in a browser
|
||||
required: true
|
||||
@@ -45,12 +45,12 @@ body:
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.11.10.1 (exe)
|
||||
[debug] yt-dlp version 2021.12.27 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.11.10.1)
|
||||
yt-dlp is up to date (2021.12.27)
|
||||
<more lines>
|
||||
render: shell
|
||||
validations:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Feature request request
|
||||
name: Feature request
|
||||
description: Request a new functionality unrelated to any particular site or extractor
|
||||
labels: [triage, enhancement]
|
||||
body:
|
||||
@@ -11,7 +11,7 @@ body:
|
||||
options:
|
||||
- label: I'm reporting a feature request
|
||||
required: true
|
||||
- label: I've verified that I'm running yt-dlp version **2021.11.10.1**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
- label: I've verified that I'm running yt-dlp version **2021.12.27**. ([update instructions](https://github.com/yt-dlp/yt-dlp#update))
|
||||
required: true
|
||||
- label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
|
||||
required: true
|
||||
|
||||
@@ -9,7 +9,7 @@ body:
|
||||
description: |
|
||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
|
||||
options:
|
||||
- label: I'm asking a question and not reporting a bug/feature request
|
||||
- label: I'm asking a question and **not** reporting a bug/feature request
|
||||
required: true
|
||||
- label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
|
||||
required: true
|
||||
@@ -24,7 +24,29 @@ body:
|
||||
description: |
|
||||
Ask your question in an arbitrary form.
|
||||
Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
|
||||
Provide any additional information and as much context and examples as possible
|
||||
Provide any additional information and as much context and examples as possible.
|
||||
If your question contains "isn't working" or "can you add", this is most likely the wrong template
|
||||
placeholder: WRITE QUESTION HERE
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: Verbose log
|
||||
description: |
|
||||
If your question involes a yt-dlp command, provide the complete verbose output of that command.
|
||||
Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
|
||||
It should look similar to this:
|
||||
placeholder: |
|
||||
[debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.12.01 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.12.01)
|
||||
<more lines>
|
||||
render: shell
|
||||
|
||||
@@ -2,4 +2,4 @@ blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Get help from the community on Discord
|
||||
url: https://discord.gg/H5MNcFW63r
|
||||
about: Join the yt-dlp Discord for community-powered support!
|
||||
about: Join the yt-dlp Discord for community-powered support!
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: Broken site support
|
||||
description: Report broken or misfunctioning site
|
||||
labels: [triage, extractor-bug]
|
||||
labels: [triage, site-bug]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
|
||||
@@ -34,7 +34,7 @@ body:
|
||||
label: Example URLs
|
||||
description: |
|
||||
Provide all kinds of example URLs for which support should be added
|
||||
value: |
|
||||
placeholder: |
|
||||
- Single video: https://www.youtube.com/watch?v=BaW_jenozKc
|
||||
- Single video: https://youtu.be/BaW_jenozKc
|
||||
- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Site feature request
|
||||
description: Request a new functionality for a site
|
||||
description: Request a new functionality for a supported site
|
||||
labels: [triage, site-enhancement]
|
||||
body:
|
||||
- type: checkboxes
|
||||
@@ -47,3 +47,26 @@ body:
|
||||
placeholder: WRITE DESCRIPTION HERE
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: Verbose log
|
||||
description: |
|
||||
Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
|
||||
Add the `-Uv` flag to your command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
|
||||
It should look similar to this:
|
||||
placeholder: |
|
||||
[debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version %(version)s (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (%(version)s)
|
||||
<more lines>
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: Bug report
|
||||
description: Report a bug unrelated to any particular site or extractor
|
||||
labels: [triage,bug]
|
||||
labels: [triage, bug]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Feature request request
|
||||
name: Feature request
|
||||
description: Request a new functionality unrelated to any particular site or extractor
|
||||
labels: [triage, enhancement]
|
||||
body:
|
||||
|
||||
@@ -9,7 +9,7 @@ body:
|
||||
description: |
|
||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
|
||||
options:
|
||||
- label: I'm asking a question and not reporting a bug/feature request
|
||||
- label: I'm asking a question and **not** reporting a bug/feature request
|
||||
required: true
|
||||
- label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
|
||||
required: true
|
||||
@@ -24,7 +24,29 @@ body:
|
||||
description: |
|
||||
Ask your question in an arbitrary form.
|
||||
Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
|
||||
Provide any additional information and as much context and examples as possible
|
||||
Provide any additional information and as much context and examples as possible.
|
||||
If your question contains "isn't working" or "can you add", this is most likely the wrong template
|
||||
placeholder: WRITE QUESTION HERE
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: Verbose log
|
||||
description: |
|
||||
If your question involes a yt-dlp command, provide the complete verbose output of that command.
|
||||
Add the `-Uv` flag to **your** command line you run yt-dlp with (`yt-dlp -Uv <your command line>`), copy the WHOLE output and insert it below.
|
||||
It should look similar to this:
|
||||
placeholder: |
|
||||
[debug] Command-line config: ['-Uv', 'http://www.youtube.com/watch?v=BaW_jenozKc']
|
||||
[debug] Portable config file: yt-dlp.conf
|
||||
[debug] Portable config: ['-i']
|
||||
[debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
|
||||
[debug] yt-dlp version 2021.12.01 (exe)
|
||||
[debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
|
||||
[debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
|
||||
[debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
|
||||
[debug] Proxy map: {}
|
||||
yt-dlp is up to date (2021.12.01)
|
||||
<more lines>
|
||||
render: shell
|
||||
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
with:
|
||||
tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
|
||||
release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
|
||||
commitish: ${{ steps.push_update.outputs.head_sha }}
|
||||
commitish: ${{ steps.push_release.outputs.head_sha }}
|
||||
body: |
|
||||
#### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
|
||||
|
||||
|
||||
+13
-14
@@ -1,27 +1,32 @@
|
||||
# Config
|
||||
*.conf
|
||||
*.spec
|
||||
cookies
|
||||
*cookies.txt
|
||||
.netrc
|
||||
|
||||
# Downloaded
|
||||
*.3gp
|
||||
*.annotations.xml
|
||||
*.ape
|
||||
*.aria2
|
||||
*.avi
|
||||
*.description
|
||||
*.desktop
|
||||
*.dump
|
||||
*.flac
|
||||
*.flv
|
||||
*.frag
|
||||
*.frag.aria2
|
||||
*.frag.urls
|
||||
*.info.json
|
||||
*.live_chat.json
|
||||
*.part*
|
||||
*.unknown_video
|
||||
*.ytdl
|
||||
.cache/
|
||||
|
||||
*.3gp
|
||||
*.ape
|
||||
*.avi
|
||||
*.desktop
|
||||
*.flac
|
||||
*.flv
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.live_chat.json
|
||||
*.m4a
|
||||
*.m4v
|
||||
*.mhtml
|
||||
@@ -31,23 +36,18 @@ cookies
|
||||
*.mp4
|
||||
*.ogg
|
||||
*.opus
|
||||
*.part
|
||||
*.part-*
|
||||
*.png
|
||||
*.sbv
|
||||
*.srt
|
||||
*.swf
|
||||
*.swp
|
||||
*.ttml
|
||||
*.unknown_video
|
||||
*.url
|
||||
*.vtt
|
||||
*.wav
|
||||
*.webloc
|
||||
*.webm
|
||||
*.webp
|
||||
*.ytdl
|
||||
.cache/
|
||||
|
||||
# Allow config/media files in testdata
|
||||
!test/**
|
||||
@@ -86,7 +86,6 @@ README.txt
|
||||
*.1
|
||||
*.bash-completion
|
||||
*.fish
|
||||
*.exe
|
||||
*.tar.gz
|
||||
*.zsh
|
||||
*.spec
|
||||
|
||||
+10
-2
@@ -10,6 +10,7 @@
|
||||
- [Does the issue involve one problem, and one problem only?](#does-the-issue-involve-one-problem-and-one-problem-only)
|
||||
- [Is anyone going to need the feature?](#is-anyone-going-to-need-the-feature)
|
||||
- [Is your question about yt-dlp?](#is-your-question-about-yt-dlp)
|
||||
- [Are you willing to share account details if needed?](#are-you-willing-to-share-account-details-if-needed)
|
||||
- [DEVELOPER INSTRUCTIONS](#developer-instructions)
|
||||
- [Adding new feature or making overarching changes](#adding-new-feature-or-making-overarching-changes)
|
||||
- [Adding support for a new site](#adding-support-for-a-new-site)
|
||||
@@ -215,7 +216,7 @@ After you have ensured this site is distributing its content legally, you can fo
|
||||
|
||||
$ flake8 yt_dlp/extractor/yourextractor.py
|
||||
|
||||
1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.6 and above. Backward compatability is not required for even older versions of Python.
|
||||
1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.6 and above. Backward compatibility is not required for even older versions of Python.
|
||||
1. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files, [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
|
||||
|
||||
$ git add yt_dlp/extractor/extractors.py
|
||||
@@ -227,6 +228,13 @@ After you have ensured this site is distributing its content legally, you can fo
|
||||
|
||||
In any case, thank you very much for your contributions!
|
||||
|
||||
**Tip:** To test extractors that require login information, create a file `test/local_parameters.json` and add `"usenetrc": true` or your username and password in it:
|
||||
```json
|
||||
{
|
||||
"username": "your user name",
|
||||
"password": "your password"
|
||||
}
|
||||
```
|
||||
|
||||
## yt-dlp coding conventions
|
||||
|
||||
@@ -243,7 +251,7 @@ For extraction to work yt-dlp relies on metadata your extractor extracts and pro
|
||||
- `title` (media title)
|
||||
- `url` (media download URL) or `formats`
|
||||
|
||||
The aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken. While, in fact, only `id` is technically mandatory, due to compatability reasons, yt-dlp also treats `title` as mandatory. The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - Eg: when the video is a live stream that has not started yet.
|
||||
The aforementioned metafields are the critical data that the extraction does not make any sense without and if any of them fail to be extracted then the extractor is considered completely broken. While, in fact, only `id` is technically mandatory, due to compatibility reasons, yt-dlp also treats `title` as mandatory. The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - Eg: when the video is a live stream that has not started yet.
|
||||
|
||||
[Any field](yt_dlp/extractor/common.py#219-L426) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
|
||||
|
||||
|
||||
@@ -155,3 +155,26 @@ staubichsauger
|
||||
xenova
|
||||
Yakabuff
|
||||
zulaport
|
||||
ehoogeveen-medweb
|
||||
PilzAdam
|
||||
zmousm
|
||||
iw0nderhow
|
||||
unit193
|
||||
TwoThousandHedgehogs
|
||||
Jertzukka
|
||||
cypheron
|
||||
Hyeeji
|
||||
bwildenhain
|
||||
C0D3D3V
|
||||
kebianizao
|
||||
Lapin0t
|
||||
abdullah-if
|
||||
DavidSkrundz
|
||||
mkubecek
|
||||
raleeper
|
||||
YuenSzeHong
|
||||
Sematre
|
||||
jaller94
|
||||
r5d
|
||||
julien-hadleyjack
|
||||
git-anony-mouse
|
||||
|
||||
+121
-3
@@ -10,6 +10,124 @@
|
||||
* Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
|
||||
-->
|
||||
|
||||
|
||||
### 2021.12.27
|
||||
|
||||
* Avoid recursion error when re-extracting info
|
||||
* [ffmpeg] Fix position of `--ppa`
|
||||
* [aria2c] Don't show progress when `--no-progress`
|
||||
* [cookies] Support other keyrings by [mbway](https://github.com/mbway)
|
||||
* [EmbedThumbnail] Prefer AtomicParsley over ffmpeg if available
|
||||
* [generic] Fix HTTP KVS Player by [git-anony-mouse](https://github.com/git-anony-mouse)
|
||||
* [ThumbnailsConvertor] Fix for when there are no thumbnails
|
||||
* [docs] Add examples for using `TYPES:` in `-P`/`-o`
|
||||
* [PixivSketch] Add extractors by [nao20010128nao](https://github.com/nao20010128nao)
|
||||
* [tiktok] Add music, sticker and tag IEs by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [BiliIntl] Fix extractor by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [CBC] Fix URL regex
|
||||
* [tiktok] Fix `extractor_key` used in archive
|
||||
* [youtube] **End `live-from-start` properly when stream ends with 403**
|
||||
* [Zee5] Fix VALID_URL for tv-shows by [Ashish0804](https://github.com/Ashish0804)
|
||||
|
||||
### 2021.12.25
|
||||
|
||||
* [dash,youtube] **Download live from start to end** by [nao20010128nao](https://github.com/nao20010128nao), [pukkandan](https://github.com/pukkandan)
|
||||
* Add option `--live-from-start` to enable downloading live videos from start
|
||||
* Add key `is_from_start` in formats to identify formats (of live videos) that downloads from start
|
||||
* [dash] Create protocol `http_dash_segments_generator` that allows a function to be passed instead of fragments
|
||||
* [fragment] Allow multiple live dash formats to download simultaneously
|
||||
* [youtube] Implement fragment re-fetching for the live dash formats
|
||||
* [youtube] Re-extract dash manifest every 5 hours (manifest expires in 6hrs)
|
||||
* [postprocessor/ffmpeg] Add `FFmpegFixupDuplicateMoovPP` to fixup duplicated moov atoms
|
||||
* Known issues:
|
||||
* Ctrl+C doesn't work on Windows when downloading multiple formats
|
||||
* If video becomes private, download hangs
|
||||
* [SponsorBlock] Add `Filler` and `Highlight` categories by [nihil-admirari](https://github.com/nihil-admirari), [pukkandan](https://github.com/pukkandan)
|
||||
* Change `--sponsorblock-cut all` to `--sponsorblock-cut default` if you do not want filler sections to be removed
|
||||
* Add field `webpage_url_domain`
|
||||
* Add interactive format selection with `-f -`
|
||||
* Add option `--file-access-retries` by [ehoogeveen-medweb](https://github.com/ehoogeveen-medweb)
|
||||
* [outtmpl] Add alternate forms `S`, `D` and improve `id` detection
|
||||
* [outtmpl] Add operator `&` for replacement text by [PilzAdam](https://github.com/PilzAdam)
|
||||
* [EmbedSubtitle] Disable duration check temporarily
|
||||
* [extractor] Add `_search_nuxt_data` by [nao20010128nao](https://github.com/nao20010128nao)
|
||||
* [extractor] Ignore errors in comment extraction when `-i` is given
|
||||
* [extractor] Standardize `_live_title`
|
||||
* [FormatSort] Prevent incorrect deprecation warning
|
||||
* [generic] Extract m3u8 formats from JSON-LD
|
||||
* [postprocessor/ffmpeg] Always add `faststart`
|
||||
* [utils] Fix parsing `YYYYMMDD` dates in Nov/Dec by [wlritchi](https://github.com/wlritchi)
|
||||
* [utils] Improve `parse_count`
|
||||
* [utils] Update `std_headers` by [kikuyan](https://github.com/kikuyan), [fstirlitz](https://github.com/fstirlitz)
|
||||
* [lazy_extractors] Fix for search IEs
|
||||
* [extractor] Support default implicit graph in JSON-LD by [zmousm](https://github.com/zmousm)
|
||||
* Allow `--no-write-thumbnail` to override `--write-all-thumbnail`
|
||||
* Fix `--throttled-rate`
|
||||
* Fix control characters being printed to `--console-title`
|
||||
* Fix PostProcessor hooks not registered for some PPs
|
||||
* Pre-process when using `--flat-playlist`
|
||||
* Remove known invalid thumbnails from `info_dict`
|
||||
* Add warning when using `-f best`
|
||||
* Use `parse_duration` for `--wait-for-video` and some minor fix
|
||||
* [test/download] Add more fields
|
||||
* [test/download] Ignore field `webpage_url_domain` by [std-move](https://github.com/std-move)
|
||||
* [compat] Suppress errors in enabling VT mode
|
||||
* [docs] Improve manpage format by [iw0nderhow](https://github.com/iw0nderhow), [pukkandan](https://github.com/pukkandan)
|
||||
* [docs,cleanup] Minor fixes and cleanup
|
||||
* [cleanup] Fix some typos by [unit193](https://github.com/unit193)
|
||||
* [ABC:iview] Add show extractor by [pabs3](https://github.com/pabs3)
|
||||
* [dropout] Add extractor by [TwoThousandHedgehogs](https://github.com/TwoThousandHedgehogs), [pukkandan](https://github.com/pukkandan)
|
||||
* [GameJolt] Add extractors by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [gofile] Add extractor by [Jertzukka](https://github.com/Jertzukka), [Ashish0804](https://github.com/Ashish0804)
|
||||
* [hse] Add extractors by [cypheron](https://github.com/cypheron), [pukkandan](https://github.com/pukkandan)
|
||||
* [NateTV] Add NateIE and NateProgramIE by [Ashish0804](https://github.com/Ashish0804), [Hyeeji](https://github.com/Hyeeji)
|
||||
* [OpenCast] Add extractors by [bwildenhain](https://github.com/bwildenhain), [C0D3D3V](https://github.com/C0D3D3V)
|
||||
* [rtve] Add `RTVEAudioIE` by [kebianizao](https://github.com/kebianizao)
|
||||
* [Rutube] Add RutubeChannelIE by [Ashish0804](https://github.com/Ashish0804)
|
||||
* [skeb] Add extractor by [nao20010128nao](https://github.com/nao20010128nao)
|
||||
* [soundcloud] Add related tracks extractor by [Lapin0t](https://github.com/Lapin0t)
|
||||
* [toggo] Add extractor by [nyuszika7h](https://github.com/nyuszika7h)
|
||||
* [TrueID] Add extractor by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [audiomack] Update album and song VALID_URL by [abdullah-if](https://github.com/abdullah-if), [dirkf](https://github.com/dirkf)
|
||||
* [CBC Gem] Extract 1080p formats by [DavidSkrundz](https://github.com/DavidSkrundz)
|
||||
* [ceskatelevize] Fetch iframe from nextJS data by [mkubecek](https://github.com/mkubecek)
|
||||
* [crackle] Look for non-DRM formats by [raleeper](https://github.com/raleeper)
|
||||
* [dplay] Temporary fix for `discoveryplus.com/it`
|
||||
* [DiscoveryPlusShowBaseIE] yield actual video id by [Ashish0804](https://github.com/Ashish0804)
|
||||
* [Facebook] Handle redirect URLs
|
||||
* [fujitv] Extract 1080p from `tv_android` m3u8 by [YuenSzeHong](https://github.com/YuenSzeHong)
|
||||
* [gronkh] Support new URL pattern by [Sematre](https://github.com/Sematre)
|
||||
* [instagram] Expand valid URL by [u-spec-png](https://github.com/u-spec-png)
|
||||
* [Instagram] Try bypassing login wall with embed page by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [Jamendo] Fix use of `_VALID_URL_RE` by [jaller94](https://github.com/jaller94)
|
||||
* [LBRY] Support livestreams by [Ashish0804](https://github.com/Ashish0804), [pukkandan](https://github.com/pukkandan)
|
||||
* [NJPWWorld] Extract formats from m3u8 by [aarubui](https://github.com/aarubui)
|
||||
* [NovaEmbed] update player regex by [std-move](https://github.com/std-move)
|
||||
* [npr] Make SMIL extraction non-fatal by [r5d](https://github.com/r5d)
|
||||
* [ntvcojp] Extract NUXT data by [nao20010128nao](https://github.com/nao20010128nao)
|
||||
* [ok.ru] add mobile fallback by [nao20010128nao](https://github.com/nao20010128nao)
|
||||
* [olympics] Add uploader and cleanup by [u-spec-png](https://github.com/u-spec-png)
|
||||
* [ondemandkorea] Update `jw_config` regex by [julien-hadleyjack](https://github.com/julien-hadleyjack)
|
||||
* [PlutoTV] Expand `_VALID_URL`
|
||||
* [RaiNews] Fix extractor by [nixxo](https://github.com/nixxo)
|
||||
* [RCTIPlusSeries] Lazy extraction and video type selection by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [redtube] Handle formats delivered inside a JSON by [dirkf](https://github.com/dirkf), [nixxo](https://github.com/nixxo)
|
||||
* [SonyLiv] Add OTP login support by [Ashish0804](https://github.com/Ashish0804)
|
||||
* [Steam] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
|
||||
* [TikTok] Pass cookies to mobile API by [MinePlayersPE](https://github.com/MinePlayersPE)
|
||||
* [trovo] Fix inheritance of `TrovoChannelBaseIE`
|
||||
* [TVer] Extract better thumbnails by [YuenSzeHong](https://github.com/YuenSzeHong)
|
||||
* [vimeo] Extract chapters
|
||||
* [web.archive:youtube] Improve metadata extraction by [coletdjnz](https://github.com/coletdjnz)
|
||||
* [youtube:comments] Add more options for limiting number of comments extracted by [coletdjnz](https://github.com/coletdjnz)
|
||||
* [youtube:tab] Extract more metadata from feeds/channels/playlists by [coletdjnz](https://github.com/coletdjnz)
|
||||
* [youtube:tab] Extract video thumbnails from playlist by [coletdjnz](https://github.com/coletdjnz), [pukkandan](https://github.com/pukkandan)
|
||||
* [youtube:tab] Ignore query when redirecting channel to playlist and cleanup of related code
|
||||
* [youtube] Fix `ytsearchdate`
|
||||
* [zdf] Support videos with different ptmd location by [iw0nderhow](https://github.com/iw0nderhow)
|
||||
* [zee5] Support /episodes in URL
|
||||
|
||||
|
||||
### 2021.12.01
|
||||
|
||||
* **Add option `--wait-for-video` to wait for scheduled streams**
|
||||
@@ -38,7 +156,7 @@
|
||||
* Ensure directory exists when checking formats
|
||||
* Ensure path for link files exists by [Zirro](https://github.com/Zirro)
|
||||
* Ensure same config file is not loaded multiple times
|
||||
* Fix 'postprocessor_hooks`
|
||||
* Fix `postprocessor_hooks`
|
||||
* Fix `--break-on-archive` when pre-checking
|
||||
* Fix `--check-formats` for `mhtml`
|
||||
* Fix `--load-info-json` of playlists with failed entries
|
||||
@@ -110,10 +228,9 @@
|
||||
* [youtube] Decrypt n-sig for URLs with `ratebypass`
|
||||
* [youtube] Minor improvement to format sorting
|
||||
* [cleanup] Add deprecation warnings
|
||||
* [cleanup] Minor cleanup
|
||||
* [cleanup] Misc cleanup
|
||||
* [cleanup] Refactor `JSInterpreter._seperate`
|
||||
* [Cleanup] Remove some unnecessary groups in regexes by [Ashish0804](https://github.com/Ashish0804)
|
||||
* [cleanup] Misc cleanup
|
||||
|
||||
|
||||
### 2021.11.10.1
|
||||
@@ -1476,6 +1593,7 @@
|
||||
|
||||
**Note**: All uncredited changes above this point are authored by [pukkandan](https://github.com/pukkandan)
|
||||
|
||||
### Unreleased changes in [blackjack4494/yt-dlc](https://github.com/blackjack4494/yt-dlc)
|
||||
* Updated to youtube-dl release 2020.11.26 by [pukkandan](https://github.com/pukkandan)
|
||||
* Youtube improvements by [pukkandan](https://github.com/pukkandan)
|
||||
* Implemented all Youtube Feeds (ytfav, ytwatchlater, ytsubs, ythistory, ytrec) and SearchURL
|
||||
|
||||
@@ -28,6 +28,7 @@ You can also find lists of all [contributors of yt-dlp](CONTRIBUTORS) and [autho
|
||||
[](https://github.com/sponsors/coletdjnz)
|
||||
|
||||
* YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
|
||||
* Added support for downloading YoutubeWebArchive videos
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -13,10 +13,10 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites com
|
||||
.PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites
|
||||
|
||||
clean-test:
|
||||
rm -rf *.3gp *.annotations.xml *.ape *.avi *.description *.dump *.flac *.flv *.frag *.frag.aria2 *.frag.urls \
|
||||
*.info.json *.jpeg *.jpg *.live_chat.json *.m4a *.m4v *.mkv *.mp3 *.mp4 *.ogg *.opus *.part* *.png *.sbv *.srt \
|
||||
*.swf *.swp *.ttml *.vtt *.wav *.webm *.webp *.mhtml *.mov *.unknown_video *.desktop *.url *.webloc *.ytdl \
|
||||
test/testdata/player-*.js tmp/
|
||||
rm -rf test/testdata/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
|
||||
*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.part* *.unknown_video *.ytdl \
|
||||
*.3gp *.ape *.avi *.desktop *.flac *.flv *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
|
||||
*.mp4 *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
|
||||
clean-dist:
|
||||
rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
|
||||
yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
<div align="center">
|
||||
|
||||
[](#readme)
|
||||
@@ -15,9 +16,13 @@
|
||||
[](https://pypi.org/project/yt-dlp)
|
||||
|
||||
</div>
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on the now inactive [youtube-dlc](https://github.com/blackjack4494/yt-dlc). The main focus of this project is adding new features and patches while also keeping up to date with the original project
|
||||
|
||||
<!-- MANPAGE: MOVE "USAGE AND OPTIONS" SECTION HERE -->
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
* [NEW FEATURES](#new-features)
|
||||
* [Differences in default behavior](#differences-in-default-behavior)
|
||||
* [INSTALLATION](#installation)
|
||||
@@ -61,11 +66,12 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
|
||||
* [Opening an Issue](CONTRIBUTING.md#opening-an-issue)
|
||||
* [Developer Instructions](CONTRIBUTING.md#developer-instructions)
|
||||
* [MORE](#more)
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
|
||||
# NEW FEATURES
|
||||
|
||||
* Based on **youtube-dl 2021.06.06 [commit/379f52a](https://github.com/ytdl-org/youtube-dl/commit/379f52a4954013767219d25099cce9e0f9401961)** and **youtube-dlc 2020.11.11-3 [commit/98e248f](https://github.com/blackjack4494/yt-dlc/commit/98e248faa49e69d795abc60f7cdefcf91e2612aa)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
|
||||
* Based on **youtube-dl 2021.12.17 [commit/5014bd6](https://github.com/ytdl-org/youtube-dl/commit/5014bd67c22b421207b2650d4dc874b95b36dda1)** and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
|
||||
|
||||
* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
|
||||
|
||||
@@ -82,8 +88,9 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
|
||||
* Redirect channel's home URL automatically to `/video` to preserve the old behaviour
|
||||
* `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
|
||||
* Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
|
||||
* Download livestreams from the start using `--live-from-start`
|
||||
|
||||
* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[:PROFILE]`
|
||||
* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE]`
|
||||
|
||||
* **Split video by chapters**: Videos can be split into multiple files based on chapters using `--split-chapters`
|
||||
|
||||
@@ -91,7 +98,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
|
||||
|
||||
* **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats
|
||||
|
||||
* **New and fixed extractors**: Many new extractors have been added and a lot of exisiting ones have been fixed. See the [changelog](Changelog.md) or the [list of supported sites](supportedsites.md)
|
||||
* **New and fixed extractors**: Many new extractors have been added and a lot of existing ones have been fixed. See the [changelog](Changelog.md) or the [list of supported sites](supportedsites.md)
|
||||
|
||||
* **New MSOs**: Philo, Spectrum, SlingTV, Cablevision, RCN
|
||||
|
||||
@@ -118,10 +125,10 @@ See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/comm
|
||||
Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
|
||||
|
||||
* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details
|
||||
* `avconv` is not supported as as an alternative to `ffmpeg`
|
||||
* `avconv` is not supported as an alternative to `ffmpeg`
|
||||
* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename`
|
||||
* The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order
|
||||
* The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be prefered. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
|
||||
* The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this
|
||||
* Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both
|
||||
* `--ignore-errors` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead
|
||||
* When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files
|
||||
@@ -136,7 +143,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
|
||||
* If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
|
||||
* Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
|
||||
* Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
|
||||
* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the seperate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
|
||||
* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
|
||||
|
||||
For ease of use, a few more compat options are available:
|
||||
* `--compat-options all`: Use all compat options
|
||||
@@ -165,7 +172,7 @@ sudo chmod a+rx /usr/local/bin/yt-dlp
|
||||
```
|
||||
|
||||
```
|
||||
sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
|
||||
sudo aria2c https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp --dir /usr/local/bin -o yt-dlp
|
||||
sudo chmod a+rx /usr/local/bin/yt-dlp
|
||||
```
|
||||
|
||||
@@ -205,6 +212,7 @@ If you [installed with pip](#with-pip), simply re-run the same command that was
|
||||
|
||||
If you [installed using Homebrew](#with-homebrew), run `brew upgrade yt-dlp/taps/yt-dlp`
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
## RELEASE FILES
|
||||
|
||||
#### Recommended
|
||||
@@ -231,6 +239,7 @@ File|Description
|
||||
[yt-dlp.tar.gz](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz)|Source tarball. Also contains manpages, completions, etc
|
||||
[SHA2-512SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-512SUMS)|GNU-style SHA512 sums
|
||||
[SHA2-256SUMS](https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS)|GNU-style SHA256 sums
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
## DEPENDENCIES
|
||||
Python versions 3.6+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly.
|
||||
@@ -240,12 +249,13 @@ Python versions 3.6+ (CPython and PyPy) are supported. Other versions and implem
|
||||
On windows, [Microsoft Visual C++ 2010 SP1 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe) is also necessary to run yt-dlp. You probably already have this, but if the executable throws an error due to missing `MSVCR100.dll` you need to install it manually.
|
||||
-->
|
||||
|
||||
While all the other dependancies are optional, `ffmpeg` and `ffprobe` are highly recommended
|
||||
* [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging seperate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. Licence [depends on the build](https://www.ffmpeg.org/legal.html)
|
||||
While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly recommended
|
||||
|
||||
* [**ffmpeg** and **ffprobe**](https://www.ffmpeg.org) - Required for [merging separate video and audio files](#format-selection) as well as for various [post-processing](#post-processing-options) tasks. License [depends on the build](https://www.ffmpeg.org/legal.html)
|
||||
* [**mutagen**](https://github.com/quodlibet/mutagen) - For embedding thumbnail in certain formats. Licensed under [GPLv2+](https://github.com/quodlibet/mutagen/blob/master/COPYING)
|
||||
* [**pycryptodomex**](https://github.com/Legrandin/pycryptodome) - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD2](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
|
||||
* [**websockets**](https://github.com/aaugustin/websockets) - For downloading over websocket. Licensed under [BSD3](https://github.com/aaugustin/websockets/blob/main/LICENSE)
|
||||
* [**keyring**](https://github.com/jaraco/keyring) - For decrypting cookies of chromium-based browsers on Linux. Licensed under [MIT](https://github.com/jaraco/keyring/blob/main/LICENSE)
|
||||
* [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
|
||||
* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen is not present. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
|
||||
* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
|
||||
* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
|
||||
@@ -257,7 +267,7 @@ To use or redistribute the dependencies, you must agree to their respective lice
|
||||
|
||||
The Windows and MacOS standalone release binaries are already built with the python interpreter, mutagen, pycryptodomex and websockets included.
|
||||
|
||||
**Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependancy, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds/wiki/Latest#latest-autobuilds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specifc issues solved by these builds
|
||||
**Note**: There are some regressions in newer ffmpeg versions that causes various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds/wiki/Latest#latest-autobuilds) with patches for these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds
|
||||
|
||||
|
||||
## COMPILE
|
||||
@@ -281,11 +291,13 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
|
||||
# USAGE AND OPTIONS
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
yt-dlp [OPTIONS] [--] URL [URL...]
|
||||
|
||||
`Ctrl+F` is your friend :D
|
||||
<!-- Auto generated -->
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
<!-- Auto generated -->
|
||||
## General Options:
|
||||
-h, --help Print this help text and exit
|
||||
--version Print program version and exit
|
||||
@@ -329,6 +341,10 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
--flat-playlist Do not extract the videos of a playlist,
|
||||
only list them
|
||||
--no-flat-playlist Extract the videos of a playlist
|
||||
--live-from-start Download livestreams from the start.
|
||||
Currently only supported for YouTube
|
||||
--no-live-from-start Download livestreams from the current time
|
||||
(default)
|
||||
--wait-for-video MIN[-MAX] Wait for scheduled streams to become
|
||||
available. Pass the minimum number of
|
||||
seconds (or range) to wait between retries
|
||||
@@ -438,8 +454,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
|
||||
## Download Options:
|
||||
-N, --concurrent-fragments N Number of fragments of a dash/hlsnative
|
||||
video that should be download concurrently
|
||||
(default is 1)
|
||||
video that should be downloaded
|
||||
concurrently (default is 1)
|
||||
-r, --limit-rate RATE Maximum download rate in bytes per second
|
||||
(e.g. 50K or 4.2M)
|
||||
--throttled-rate RATE Minimum download rate in bytes per second
|
||||
@@ -447,6 +463,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
video data is re-extracted (e.g. 100K)
|
||||
-R, --retries RETRIES Number of retries (default is 10), or
|
||||
"infinite"
|
||||
--file-access-retries RETRIES Number of times to retry on file access
|
||||
error (default is 10), or "infinite"
|
||||
--fragment-retries RETRIES Number of retries for a fragment (default
|
||||
is 10), or "infinite" (DASH, hlsnative and
|
||||
ISM)
|
||||
@@ -509,15 +527,15 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
(Alias: --external-downloader-args)
|
||||
|
||||
## Filesystem Options:
|
||||
-a, --batch-file FILE File containing URLs to download ('-' for
|
||||
-a, --batch-file FILE File containing URLs to download ("-" for
|
||||
stdin), one URL per line. Lines starting
|
||||
with '#', ';' or ']' are considered as
|
||||
with "#", ";" or "]" are considered as
|
||||
comments and ignored
|
||||
--no-batch-file Do not read URLs from batch file (default)
|
||||
-P, --paths [TYPES:]PATH The paths where the files should be
|
||||
downloaded. Specify the type of file and
|
||||
the path separated by a colon ":". All the
|
||||
same types as --output are supported.
|
||||
same TYPES as --output are supported.
|
||||
Additionally, you can also provide "home"
|
||||
(default) and "temp" paths. All
|
||||
intermediary files are first downloaded to
|
||||
@@ -589,16 +607,19 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
from and dump cookie jar in
|
||||
--no-cookies Do not read/dump cookies from/to file
|
||||
(default)
|
||||
--cookies-from-browser BROWSER[:PROFILE]
|
||||
Load cookies from a user profile of the
|
||||
given web browser. Currently supported
|
||||
browsers are: brave, chrome, chromium,
|
||||
edge, firefox, opera, safari, vivaldi. You
|
||||
can specify the user profile name or
|
||||
directory using "BROWSER:PROFILE_NAME" or
|
||||
"BROWSER:PROFILE_PATH". If no profile is
|
||||
given, the most recently accessed one is
|
||||
used
|
||||
--cookies-from-browser BROWSER[+KEYRING][:PROFILE]
|
||||
The name of the browser and (optionally)
|
||||
the name/path of the profile to load
|
||||
cookies from, separated by a ":". Currently
|
||||
supported browsers are: brave, chrome,
|
||||
chromium, edge, firefox, opera, safari,
|
||||
vivaldi. By default, the most recently
|
||||
accessed profile is used. The keyring used
|
||||
for decrypting Chromium cookies on Linux
|
||||
can be (optionally) specified after the
|
||||
browser name separated by a "+". Currently
|
||||
supported keyrings are: basictext,
|
||||
gnomekeyring, kwallet
|
||||
--no-cookies-from-browser Do not load cookies from browser (default)
|
||||
--cache-dir DIR Location in the filesystem where youtube-dl
|
||||
can store some downloaded information (such
|
||||
@@ -869,7 +890,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
emit a warning), detect_or_warn (the
|
||||
default; fix file if we can, warn
|
||||
otherwise), force (try fixing even if file
|
||||
already exists
|
||||
already exists)
|
||||
--ffmpeg-location PATH Location of the ffmpeg binary; either the
|
||||
path to the binary or its containing
|
||||
directory
|
||||
@@ -913,7 +934,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
(default)
|
||||
--force-keyframes-at-cuts Force keyframes around the chapters before
|
||||
removing/splitting them. Requires a
|
||||
reencode and thus is very slow, but the
|
||||
re-encode and thus is very slow, but the
|
||||
resulting video may have fewer artifacts
|
||||
around the cuts
|
||||
--no-force-keyframes-at-cuts Do not force keyframes around the chapters
|
||||
@@ -921,7 +942,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
|
||||
--use-postprocessor NAME[:ARGS] The (case sensitive) name of plugin
|
||||
postprocessors to be enabled, and
|
||||
(optionally) arguments to be passed to it,
|
||||
seperated by a colon ":". ARGS are a
|
||||
separated by a colon ":". ARGS are a
|
||||
semicolon ";" delimited list of NAME=VALUE.
|
||||
The "when" argument determines when the
|
||||
postprocessor is invoked. It can be one of
|
||||
@@ -940,19 +961,21 @@ Make chapter entries for, or remove various segments (sponsor,
|
||||
|
||||
--sponsorblock-mark CATS SponsorBlock categories to create chapters
|
||||
for, separated by commas. Available
|
||||
categories are all, sponsor, intro, outro,
|
||||
selfpromo, interaction, preview,
|
||||
music_offtopic. You can prefix the category
|
||||
with a "-" to exempt it. See
|
||||
https://wiki.sponsor.ajay.app/index.php/Segment_Categories
|
||||
for description of the categories. Eg:
|
||||
--sponsorblock-query all,-preview
|
||||
categories are all, default(=all), sponsor,
|
||||
intro, outro, selfpromo, preview, filler,
|
||||
interaction, music_offtopic, poi_highlight.
|
||||
You can prefix the category with a "-" to
|
||||
exempt it. See [1] for description of the
|
||||
categories. Eg: --sponsorblock-mark all,-preview
|
||||
[1] https://wiki.sponsor.ajay.app/w/Segment_Categories
|
||||
--sponsorblock-remove CATS SponsorBlock categories to be removed from
|
||||
the video file, separated by commas. If a
|
||||
category is present in both mark and
|
||||
remove, remove takes precedence. The syntax
|
||||
and available categories are the same as
|
||||
for --sponsorblock-mark
|
||||
for --sponsorblock-mark except that
|
||||
"default" refers to "all,-filler" and
|
||||
poi_highlight is not available
|
||||
--sponsorblock-chapter-title TEMPLATE
|
||||
The title template for SponsorBlock
|
||||
chapters created by --sponsorblock-mark.
|
||||
@@ -998,7 +1021,7 @@ You can configure yt-dlp by placing any supported command line option to a confi
|
||||
* `~/yt-dlp.conf`
|
||||
* `~/yt-dlp.conf.txt`
|
||||
|
||||
`%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `%APPDATA%` generally points to (`C:\Users\<user name>\AppData\Roaming`) and `~` points to `%HOME%` if present, `%USERPROFILE%` (generally `C:\Users\<user name>`), or `%HOMEDRIVE%%HOMEPATH%`
|
||||
`%XDG_CONFIG_HOME%` defaults to `~/.config` if undefined. On windows, `%APPDATA%` generally points to `C:\Users\<user name>\AppData\Roaming` and `~` points to `%HOME%` if present, `%USERPROFILE%` (generally `C:\Users\<user name>`), or `%HOMEDRIVE%%HOMEPATH%`
|
||||
1. **System Configuration**: `/etc/yt-dlp.conf`
|
||||
|
||||
For example, with the following configuration file yt-dlp will always extract the audio, not copy the mtime, use a proxy and save all videos under `YouTube` directory in your home directory:
|
||||
@@ -1046,7 +1069,9 @@ The default location of the .netrc file is `$HOME` (`~`) in UNIX. On Windows, it
|
||||
|
||||
The `-o` option is used to indicate a template for the output file names while `-P` option is used to specify the path each type of file should be saved to.
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
**tl;dr:** [navigate me to examples](#output-template-examples).
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
The simplest usage of `-o` is not to set any template arguments when downloading a single file, like in `yt-dlp -o funny_video.flv "https://some/video"` (hard-coding file extension like this is _not_ recommended and could break some post-processing).
|
||||
|
||||
@@ -1054,19 +1079,27 @@ It may however also contain special sequences that will be replaced when downloa
|
||||
|
||||
The field names themselves (the part inside the parenthesis) can also have some special formatting:
|
||||
1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)s`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. `%()s` refers to the entire infodict. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
|
||||
|
||||
1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. Eg: `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
|
||||
|
||||
1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
|
||||
1. **Alternatives**: Alternate fields can be specified seperated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
|
||||
1. **Default**: A literal default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
|
||||
1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma seperated **l**ist (flag `#` for `\n` newline-seperated) and a string **q**uoted for the terminal (flag `#` to split a list into different arguments), respectively
|
||||
|
||||
1. **Alternatives**: Alternate fields can be specified separated with a `,`. Eg: `%(release_date>%Y,upload_date>%Y|Unknown)s`
|
||||
|
||||
1. **Replacement**: A replacement value can specified using a `&` separator. If the field is *not* empty, this replacement value will be used instead of the actual field content. This is done after alternate fields are considered; thus the replacement is used if *any* of the alternative fields is *not* empty.
|
||||
|
||||
1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
|
||||
|
||||
1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q`, `D`, `S` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma separated **l**ist (flag `#` for `\n` newline-separated), a string **q**uoted for the terminal (flag `#` to split a list into different arguments), to add **D**ecimal suffixes (Eg: 10M), and to **S**anitize as filename (flag `#` for restricted), respectively
|
||||
|
||||
1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
|
||||
|
||||
To summarize, the general syntax for a field is:
|
||||
```
|
||||
%(name[.keys][addition][>strf][,alternate][|default])[flags][width][.precision][length]type
|
||||
%(name[.keys][addition][>strf][,alternate][&replacement][|default])[flags][width][.precision][length]type
|
||||
```
|
||||
|
||||
Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'` will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
|
||||
Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"` will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video.
|
||||
|
||||
The available fields are:
|
||||
|
||||
@@ -1097,11 +1130,11 @@ The available fields are:
|
||||
- `average_rating` (numeric): Average rating give by users, the scale used depends on the webpage
|
||||
- `comment_count` (numeric): Number of comments on the video (For some extractors, comments are only downloaded at the end, and so this field cannot be used)
|
||||
- `age_limit` (numeric): Age restriction for the video (years)
|
||||
- `live_status` (string): One of 'is_live', 'was_live', 'is_upcoming', 'not_live'
|
||||
- `live_status` (string): One of "is_live", "was_live", "is_upcoming", "not_live"
|
||||
- `is_live` (boolean): Whether this video is a live stream or a fixed-length video
|
||||
- `was_live` (boolean): Whether this video was originally a live stream
|
||||
- `playable_in_embed` (string): Whether this video is allowed to play in embedded players on other sites
|
||||
- `availability` (string): Whether the video is 'private', 'premium_only', 'subscriber_only', 'needs_auth', 'unlisted' or 'public'
|
||||
- `availability` (string): Whether the video is "private", "premium_only", "subscriber_only", "needs_auth", "unlisted" or "public"
|
||||
- `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
|
||||
- `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
|
||||
- `format` (string): A human-readable description of the format
|
||||
@@ -1135,6 +1168,8 @@ The available fields are:
|
||||
- `playlist_uploader` (string): Full name of the playlist uploader
|
||||
- `playlist_uploader_id` (string): Nickname or id of the playlist uploader
|
||||
- `webpage_url` (string): A URL to the video webpage which if given to yt-dlp should allow to get the same result again
|
||||
- `webpage_url_basename` (string): The basename of the webpage URL
|
||||
- `webpage_url_domain` (string): The domain of the webpage URL
|
||||
- `original_url` (string): The URL given by the user (or same as `webpage_url` for playlist entries)
|
||||
|
||||
Available for the video that belongs to some logical chapter or section:
|
||||
@@ -1195,7 +1230,7 @@ Note that some of the sequences are not guaranteed to be present since they depe
|
||||
|
||||
For numeric sequences you can use [numeric related formatting](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting), for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
|
||||
|
||||
Output templates can also contain arbitrary hierarchical path, e.g. `-o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s'` which will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
|
||||
Output templates can also contain arbitrary hierarchical path, e.g. `-o "%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s"` which will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
|
||||
|
||||
To use percent literals in an output template use `%%`. To output to stdout use `-o -`.
|
||||
|
||||
@@ -1203,38 +1238,48 @@ The current default template is `%(title)s [%(id)s].%(ext)s`.
|
||||
|
||||
In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title.
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
#### Output template and Windows batch files
|
||||
|
||||
If you are using an output template inside a Windows batch file then you must escape plain percent characters (`%`) by doubling, so that `-o "%(title)s-%(id)s.%(ext)s"` should become `-o "%%(title)s-%%(id)s.%%(ext)s"`. However you should not touch `%`'s that are not plain characters, e.g. environment variables for expansion should stay intact: `-o "C:\%HOMEPATH%\Desktop\%%(title)s.%%(ext)s"`.
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
#### Output template examples
|
||||
|
||||
Note that on Windows you need to use double quotes instead of single.
|
||||
|
||||
```bash
|
||||
$ yt-dlp --get-filename -o 'test video.%(ext)s' BaW_jenozKc
|
||||
$ yt-dlp --get-filename -o "test video.%(ext)s" BaW_jenozKc
|
||||
test video.webm # Literal name with correct extension
|
||||
|
||||
$ yt-dlp --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
|
||||
$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
|
||||
youtube-dl test video ''_ä↭𝕐.webm # All kinds of weird characters
|
||||
|
||||
$ yt-dlp --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames
|
||||
$ yt-dlp --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
|
||||
youtube-dl_test_video_.webm # Restricted file name
|
||||
|
||||
# Download YouTube playlist videos in separate directory indexed by video order in a playlist
|
||||
$ yt-dlp -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
|
||||
$ yt-dlp -o "%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s" "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
|
||||
|
||||
# Download YouTube playlist videos in separate directories according to their uploaded year
|
||||
$ yt-dlp -o '%(upload_date>%Y)s/%(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
|
||||
$ yt-dlp -o "%(upload_date>%Y)s/%(title)s.%(ext)s" "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
|
||||
|
||||
# Prefix playlist index with " - " separator, but only if it is available
|
||||
$ yt-dlp -o '%(playlist_index|)s%(playlist_index& - |)s%(title)s.%(ext)s' BaW_jenozKc "https://www.youtube.com/user/TheLinuxFoundation/playlists"
|
||||
|
||||
# Download all playlists of YouTube channel/user keeping each playlist in separate directory:
|
||||
$ yt-dlp -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/user/TheLinuxFoundation/playlists
|
||||
$ yt-dlp -o "%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s" "https://www.youtube.com/user/TheLinuxFoundation/playlists"
|
||||
|
||||
# Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home
|
||||
$ yt-dlp -u user -p password -P '~/MyVideos' -o '%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
|
||||
$ yt-dlp -u user -p password -P "~/MyVideos" -o "%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s" "https://www.udemy.com/java-tutorial"
|
||||
|
||||
# Download entire series season keeping each series and each season in separate directory under C:/MyVideos
|
||||
$ yt-dlp -P "C:/MyVideos" -o "%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" https://videomore.ru/kino_v_detalayah/5_sezon/367617
|
||||
$ yt-dlp -P "C:/MyVideos" -o "%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" "https://videomore.ru/kino_v_detalayah/5_sezon/367617"
|
||||
|
||||
# Download video as "C:\MyVideos\uploader\title.ext", subtitles as "C:\MyVideos\subs\uploader\title.ext"
|
||||
# and put all temporary files in "C:\MyVideos\tmp"
|
||||
$ yt-dlp -P "C:/MyVideos" -P "temp:tmp" -P "subtitle:subs" -o "%(uploader)s/%(title)s.%(ext)s" BaW_jenoz --write-subs
|
||||
|
||||
# Download video as "C:\MyVideos\uploader\title.ext" and subtitles as "C:\MyVideos\uploader\subs\title.ext"
|
||||
$ yt-dlp -P "C:/MyVideos" -o "%(uploader)s/%(title)s.%(ext)s" -o "subtitle:%(uploader)s/subs/%(title)s.%(ext)s" BaW_jenozKc --write-subs
|
||||
|
||||
# Stream the video being downloaded to stdout
|
||||
$ yt-dlp -o - BaW_jenozKc
|
||||
@@ -1249,12 +1294,16 @@ This is generally equivalent to using `-f bestvideo*+bestaudio/best`. However, i
|
||||
|
||||
The general syntax for format selection is `-f FORMAT` (or `--format FORMAT`) where `FORMAT` is a *selector expression*, i.e. an expression that describes format or formats you would like to download.
|
||||
|
||||
<!-- MANPAGE: BEGIN EXCLUDED SECTION -->
|
||||
**tl;dr:** [navigate me to examples](#format-selection-examples).
|
||||
<!-- MANPAGE: END EXCLUDED SECTION -->
|
||||
|
||||
The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific.
|
||||
|
||||
You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download the best quality format of a particular file extension served as a single file, e.g. `-f webm` will download the best quality format with the `webm` extension served as a single file.
|
||||
|
||||
You can use `-f -` to interactively provide the format selector *for each video*
|
||||
|
||||
You can also use special names to select particular edge case formats:
|
||||
|
||||
- `all`: Select **all formats** separately
|
||||
@@ -1317,7 +1366,7 @@ Note that none of the aforementioned meta fields are guaranteed to be present si
|
||||
|
||||
Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height<=?720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. You can also use the filters with `all` to download all formats that satisfy the filter. For example, `-f "all[vcodec=none]"` selects all audio-only formats.
|
||||
|
||||
Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
|
||||
Format selectors can also be grouped using parentheses, for example if you want to download the best pre-merged mp4 and webm formats with a height lower than 480 you can use `-f "(mp4,webm)[height<480]"`.
|
||||
|
||||
## Sorting Formats
|
||||
|
||||
@@ -1331,12 +1380,12 @@ The available fields are:
|
||||
- `lang`: Language preference as given by the extractor
|
||||
- `quality`: The quality of the format as given by the extractor
|
||||
- `source`: Preference of the source as given by the extractor
|
||||
- `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > other > `mms`/`rtsp` > unknown > `f4f`/`f4m`)
|
||||
- `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other > unknown)
|
||||
- `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other > unknown)
|
||||
- `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8_native`/`m3u8` > `http_dash_segments`> `websocket_frag` > `mms`/`rtsp` > `f4f`/`f4m`)
|
||||
- `vcodec`: Video Codec (`av01` > `vp9.2` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other)
|
||||
- `acodec`: Audio Codec (`flac`/`alac` > `wav`/`aiff` > `opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `eac3` > `ac3` > `dts` > other)
|
||||
- `codec`: Equivalent to `vcodec,acodec`
|
||||
- `vext`: Video Extension (`mp4` > `webm` > `flv` > other > unknown). If `--prefer-free-formats` is used, `webm` is prefered.
|
||||
- `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other > unknown). If `--prefer-free-formats` is used, the order changes to `opus` > `ogg` > `webm` > `m4a` > `mp3` > `aac`.
|
||||
- `vext`: Video Extension (`mp4` > `webm` > `flv` > other). If `--prefer-free-formats` is used, `webm` is preferred.
|
||||
- `aext`: Audio Extension (`m4a` > `aac` > `mp3` > `ogg` > `opus` > `webm` > other). If `--prefer-free-formats` is used, the order changes to `opus` > `ogg` > `webm` > `m4a` > `mp3` > `aac`.
|
||||
- `ext`: Equivalent to `vext,aext`
|
||||
- `filesize`: Exact filesize, if known in advance
|
||||
- `fs_approx`: Approximate filesize calculated from the manifests
|
||||
@@ -1358,7 +1407,7 @@ All fields, unless specified otherwise, are sorted in descending order. To rever
|
||||
|
||||
The fields `hasvid` and `ie_pref` are always given highest priority in sorting, irrespective of the user-defined order. This behaviour can be changed by using `--format-sort-force`. Apart from these, the default order used is: `lang,quality,res,fps,hdr:12,codec:vp9.2,size,br,asr,proto,ext,hasaud,source,id`. The extractors may override this default order, but they cannot override the user-provided order.
|
||||
|
||||
Note that the default has `codec:vp9.2`; i.e. `av1` is not prefered. Similarly, the default for hdr is `hdr:12`; i.e. dolby vision is not prefered. These choices are made since DV and AV1 formats are not yet fully compatible with most devices. This may be changed in the future as more devices become capable of smoothly playing back these formats.
|
||||
Note that the default has `codec:vp9.2`; i.e. `av1` is not preferred. Similarly, the default for hdr is `hdr:12`; i.e. dolby vision is not preferred. These choices are made since DV and AV1 formats are not yet fully compatible with most devices. This may be changed in the future as more devices become capable of smoothly playing back these formats.
|
||||
|
||||
If your format selector is `worst`, the last item is selected after sorting. This means it will select the format that is worst in all respects. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-f best -S +size,+br,+res,+fps`.
|
||||
|
||||
@@ -1366,16 +1415,14 @@ If your format selector is `worst`, the last item is selected after sorting. Thi
|
||||
|
||||
## Format Selection examples
|
||||
|
||||
Note that on Windows you may need to use double quotes instead of single.
|
||||
|
||||
```bash
|
||||
# Download and merge the best video-only format and the best audio-only format,
|
||||
# or download the best combined format if video-only format is not available
|
||||
$ yt-dlp -f 'bv+ba/b'
|
||||
$ yt-dlp -f "bv+ba/b"
|
||||
|
||||
# Download best format that contains video,
|
||||
# and if it doesn't already have an audio stream, merge it with best audio-only format
|
||||
$ yt-dlp -f 'bv*+ba/b'
|
||||
$ yt-dlp -f "bv*+ba/b"
|
||||
|
||||
# Same as above
|
||||
$ yt-dlp
|
||||
@@ -1383,89 +1430,89 @@ $ yt-dlp
|
||||
# Download the best video-only format and the best audio-only format without merging them
|
||||
# For this case, an output template should be used since
|
||||
# by default, bestvideo and bestaudio will have the same file name.
|
||||
$ yt-dlp -f 'bv,ba' -o '%(title)s.f%(format_id)s.%(ext)s'
|
||||
$ yt-dlp -f "bv,ba" -o "%(title)s.f%(format_id)s.%(ext)s"
|
||||
|
||||
# Download and merge the best format that has a video stream,
|
||||
# and all audio-only formats into one file
|
||||
$ yt-dlp -f 'bv*+mergeall[vcodec=none]' --audio-multistreams
|
||||
$ yt-dlp -f "bv*+mergeall[vcodec=none]" --audio-multistreams
|
||||
|
||||
# Download and merge the best format that has a video stream,
|
||||
# and the best 2 audio-only formats into one file
|
||||
$ yt-dlp -f 'bv*+ba+ba.2' --audio-multistreams
|
||||
$ yt-dlp -f "bv*+ba+ba.2" --audio-multistreams
|
||||
|
||||
|
||||
# The following examples show the old method (without -S) of format selection
|
||||
# and how to use -S to achieve a similar but (generally) better result
|
||||
|
||||
# Download the worst video available (old method)
|
||||
$ yt-dlp -f 'wv*+wa/w'
|
||||
$ yt-dlp -f "wv*+wa/w"
|
||||
|
||||
# Download the best video available but with the smallest resolution
|
||||
$ yt-dlp -S '+res'
|
||||
$ yt-dlp -S "+res"
|
||||
|
||||
# Download the smallest video available
|
||||
$ yt-dlp -S '+size,+br'
|
||||
$ yt-dlp -S "+size,+br"
|
||||
|
||||
|
||||
|
||||
# Download the best mp4 video available, or the best video if no mp4 available
|
||||
$ yt-dlp -f 'bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4] / bv*+ba/b'
|
||||
$ yt-dlp -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4] / bv*+ba/b"
|
||||
|
||||
# Download the best video with the best extension
|
||||
# (For video, mp4 > webm > flv. For audio, m4a > aac > mp3 ...)
|
||||
$ yt-dlp -S 'ext'
|
||||
$ yt-dlp -S "ext"
|
||||
|
||||
|
||||
|
||||
# Download the best video available but no better than 480p,
|
||||
# or the worst video if there is no video under 480p
|
||||
$ yt-dlp -f 'bv*[height<=480]+ba/b[height<=480] / wv*+ba/w'
|
||||
$ yt-dlp -f "bv*[height<=480]+ba/b[height<=480] / wv*+ba/w"
|
||||
|
||||
# Download the best video available with the largest height but no better than 480p,
|
||||
# or the best video with the smallest resolution if there is no video under 480p
|
||||
$ yt-dlp -S 'height:480'
|
||||
$ yt-dlp -S "height:480"
|
||||
|
||||
# Download the best video available with the largest resolution but no better than 480p,
|
||||
# or the best video with the smallest resolution if there is no video under 480p
|
||||
# Resolution is determined by using the smallest dimension.
|
||||
# So this works correctly for vertical videos as well
|
||||
$ yt-dlp -S 'res:480'
|
||||
$ yt-dlp -S "res:480"
|
||||
|
||||
|
||||
|
||||
# Download the best video (that also has audio) but no bigger than 50 MB,
|
||||
# or the worst video (that also has audio) if there is no video under 50 MB
|
||||
$ yt-dlp -f 'b[filesize<50M] / w'
|
||||
$ yt-dlp -f "b[filesize<50M] / w"
|
||||
|
||||
# Download largest video (that also has audio) but no bigger than 50 MB,
|
||||
# or the smallest video (that also has audio) if there is no video under 50 MB
|
||||
$ yt-dlp -f 'b' -S 'filesize:50M'
|
||||
$ yt-dlp -f "b" -S "filesize:50M"
|
||||
|
||||
# Download best video (that also has audio) that is closest in size to 50 MB
|
||||
$ yt-dlp -f 'b' -S 'filesize~50M'
|
||||
$ yt-dlp -f "b" -S "filesize~50M"
|
||||
|
||||
|
||||
|
||||
# Download best video available via direct link over HTTP/HTTPS protocol,
|
||||
# or the best video available via any protocol if there is no such video
|
||||
$ yt-dlp -f '(bv*+ba/b)[protocol^=http][protocol!*=dash] / (bv*+ba/b)'
|
||||
$ yt-dlp -f "(bv*+ba/b)[protocol^=http][protocol!*=dash] / (bv*+ba/b)"
|
||||
|
||||
# Download best video available via the best protocol
|
||||
# (https/ftps > http/ftp > m3u8_native > m3u8 > http_dash_segments ...)
|
||||
$ yt-dlp -S 'proto'
|
||||
$ yt-dlp -S "proto"
|
||||
|
||||
|
||||
|
||||
# Download the best video with h264 codec, or the best video if there is no such video
|
||||
$ yt-dlp -f '(bv*+ba/b)[vcodec^=avc1] / (bv*+ba/b)'
|
||||
$ yt-dlp -f "(bv*+ba/b)[vcodec^=avc1] / (bv*+ba/b)"
|
||||
|
||||
# Download the best video with best codec no better than h264,
|
||||
# or the best video with worst codec if there is no such video
|
||||
$ yt-dlp -S 'codec:h264'
|
||||
$ yt-dlp -S "codec:h264"
|
||||
|
||||
# Download the best video with worst codec no worse than h264,
|
||||
# or the best video with best codec if there is no such video
|
||||
$ yt-dlp -S '+codec:h264'
|
||||
$ yt-dlp -S "+codec:h264"
|
||||
|
||||
|
||||
|
||||
@@ -1473,19 +1520,19 @@ $ yt-dlp -S '+codec:h264'
|
||||
|
||||
# Download the best video no better than 720p preferring framerate greater than 30,
|
||||
# or the worst video (still preferring framerate greater than 30) if there is no such video
|
||||
$ yt-dlp -f '((bv*[fps>30]/bv*)[height<=720]/(wv*[fps>30]/wv*)) + ba / (b[fps>30]/b)[height<=720]/(w[fps>30]/w)'
|
||||
$ yt-dlp -f "((bv*[fps>30]/bv*)[height<=720]/(wv*[fps>30]/wv*)) + ba / (b[fps>30]/b)[height<=720]/(w[fps>30]/w)"
|
||||
|
||||
# Download the video with the largest resolution no better than 720p,
|
||||
# or the video with the smallest resolution available if there is no such video,
|
||||
# preferring larger framerate for formats with the same resolution
|
||||
$ yt-dlp -S 'res:720,fps'
|
||||
$ yt-dlp -S "res:720,fps"
|
||||
|
||||
|
||||
|
||||
# Download the video with smallest resolution no worse than 480p,
|
||||
# or the video with the largest resolution available if there is no such video,
|
||||
# preferring better codec and then larger total bitrate for the same resolution
|
||||
$ yt-dlp -S '+res:480,codec,br'
|
||||
$ yt-dlp -S "+res:480,codec,br"
|
||||
```
|
||||
|
||||
# MODIFYING METADATA
|
||||
@@ -1526,26 +1573,24 @@ Metadata fields|From
|
||||
|
||||
## Modifying metadata examples
|
||||
|
||||
Note that on Windows you may need to use double quotes instead of single.
|
||||
|
||||
```bash
|
||||
# Interpret the title as "Artist - Title"
|
||||
$ yt-dlp --parse-metadata 'title:%(artist)s - %(title)s'
|
||||
$ yt-dlp --parse-metadata "title:%(artist)s - %(title)s"
|
||||
|
||||
# Regex example
|
||||
$ yt-dlp --parse-metadata 'description:Artist - (?P<artist>.+)'
|
||||
$ yt-dlp --parse-metadata "description:Artist - (?P<artist>.+)"
|
||||
|
||||
# Set title as "Series name S01E05"
|
||||
$ yt-dlp --parse-metadata '%(series)s S%(season_number)02dE%(episode_number)02d:%(title)s'
|
||||
$ yt-dlp --parse-metadata "%(series)s S%(season_number)02dE%(episode_number)02d:%(title)s"
|
||||
|
||||
# Set "comment" field in video metadata using description instead of webpage_url
|
||||
$ yt-dlp --parse-metadata 'description:(?s)(?P<meta_comment>.+)' --add-metadata
|
||||
$ yt-dlp --parse-metadata "description:(?s)(?P<meta_comment>.+)" --add-metadata
|
||||
|
||||
# Remove "formats" field from the infojson by setting it to an empty string
|
||||
$ yt-dlp --parse-metadata ':(?P<formats>)' -j
|
||||
$ yt-dlp --parse-metadata ":(?P<formats>)" -j
|
||||
|
||||
# Replace all spaces and "_" in title and uploader with a `-`
|
||||
$ yt-dlp --replace-in-metadata 'title,uploader' '[ _]' '-'
|
||||
$ yt-dlp --replace-in-metadata "title,uploader" "[ _]" "-"
|
||||
|
||||
```
|
||||
|
||||
@@ -1559,10 +1604,12 @@ The following extractors use this feature:
|
||||
* `skip`: `hls` or `dash` (or both) to skip download of the respective manifests
|
||||
* `player_client`: Clients to extract video data from. The main clients are `web`, `android`, `ios`, `mweb`. These also have `_music`, `_embedded`, `_agegate`, and `_creator` variants (Eg: `web_embedded`) (`mweb` has only `_agegate`). By default, `android,web` is used, but the agegate and creator variants are added as required for age-gated videos. Similarly the music variants are added for `music.youtube.com` urls. You can also use `all` to use all the clients, and `default` for the default clients.
|
||||
* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
|
||||
* `include_live_dash`: Include live dash formats (These formats don't download properly)
|
||||
* `include_live_dash`: Include live dash formats even without `--live-from-start` (These formats don't download properly)
|
||||
* `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)
|
||||
* `max_comments`: Maximum amount of comments to download (default all)
|
||||
* `max_comment_depth`: Maximum depth for nested comments. YouTube supports depths 1 or 2 (default)
|
||||
* `max_comments`: Limit the amount of comments to gather. Comma-separated list of integers representing `max-comments,max-parents,max-replies,max-replies-per-thread`. Default is `all,all,all,all`.
|
||||
* E.g. `all,all,1000,10` will get a maximum of 1000 replies total, with up to 10 replies per thread. `1000,all,100` will get a maximum of 1000 comments, with a maximum of 100 replies total.
|
||||
* `max_comment_depth` Maximum depth for nested comments. YouTube supports depths 1 or 2 (default)
|
||||
* **Deprecated**: Set `max-replies` to `0` or `all` in `max_comments` instead (e.g. `max_comments=all,all,0` to get no replies)
|
||||
|
||||
#### youtubetab (YouTube playlists, channels, feeds, etc.)
|
||||
* `skip`: One or more of `webpage` (skip initial webpage download), `authcheck` (allow the download of playlists requiring authentication when no initial webpage is downloaded. This may cause unwanted behavior, see [#1122](https://github.com/yt-dlp/yt-dlp/pull/1122) for more details)
|
||||
@@ -1578,8 +1625,16 @@ The following extractors use this feature:
|
||||
#### vikichannel
|
||||
* `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
|
||||
|
||||
#### youtubewebarchive
|
||||
* `check_all`: Try to check more at the cost of more requests. One or more of `thumbnails`, `captures`
|
||||
|
||||
#### gamejolt
|
||||
* `comment_sort`: `hot` (default), `you` (cookies needed), `top`, `new` - choose comment sorting mode (on GameJolt's side)
|
||||
|
||||
NOTE: These options may be changed/removed in the future without concern for backward compatibility
|
||||
|
||||
<!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
|
||||
|
||||
|
||||
# PLUGINS
|
||||
|
||||
@@ -1622,7 +1677,7 @@ import yt_dlp
|
||||
|
||||
class MyLogger:
|
||||
def debug(self, msg):
|
||||
# For compatability with youtube-dl, both debug and info are passed into debug
|
||||
# For compatibility with youtube-dl, both debug and info are passed into debug
|
||||
# You can distinguish them by the prefix '[debug] '
|
||||
if msg.startswith('[debug] '):
|
||||
pass
|
||||
@@ -1675,7 +1730,7 @@ def format_selector(ctx):
|
||||
'format_id': f'{best_video["format_id"]}+{best_audio["format_id"]}',
|
||||
'ext': best_video['ext'],
|
||||
'requested_formats': [best_video, best_audio],
|
||||
# Must be + seperated list of protocols
|
||||
# Must be + separated list of protocols
|
||||
'protocol': f'{best_video["protocol"]}+{best_audio["protocol"]}'
|
||||
}
|
||||
|
||||
@@ -1711,6 +1766,8 @@ with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
**Tip**: If you are porting your code from youtube-dl to yt-dlp, one important point to look out for is that we do not guarantee the return value of `YoutubeDL.extract_info` to be json serializable, or even be a dictionary. It will be dictionary-like, but if you want to ensure it is a serializable dictionary, pass it through `YoutubeDL.sanitize_info` as shown in the example above
|
||||
|
||||
|
||||
<!-- MANPAGE: MOVE "NEW FEATURES" SECTION HERE -->
|
||||
|
||||
# DEPRECATED OPTIONS
|
||||
|
||||
These are all the deprecated options and the current alternative to achieve the same effect
|
||||
|
||||
@@ -39,12 +39,6 @@ class {name}({bases}):
|
||||
_module = '{module}'
|
||||
'''
|
||||
|
||||
make_valid_template = '''
|
||||
@classmethod
|
||||
def _make_valid_url(cls):
|
||||
return {valid_url!r}
|
||||
'''
|
||||
|
||||
|
||||
def get_base_name(base):
|
||||
if base is InfoExtractor:
|
||||
@@ -61,15 +55,14 @@ def build_lazy_ie(ie, name):
|
||||
bases=', '.join(map(get_base_name, ie.__bases__)),
|
||||
module=ie.__module__)
|
||||
valid_url = getattr(ie, '_VALID_URL', None)
|
||||
if not valid_url and hasattr(ie, '_make_valid_url'):
|
||||
valid_url = ie._make_valid_url()
|
||||
if valid_url:
|
||||
s += f' _VALID_URL = {valid_url!r}\n'
|
||||
if not ie._WORKING:
|
||||
s += ' _WORKING = False\n'
|
||||
if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
|
||||
s += f'\n{getsource(ie.suitable)}'
|
||||
if hasattr(ie, '_make_valid_url'):
|
||||
# search extractors
|
||||
s += make_valid_template.format(valid_url=ie._make_valid_url())
|
||||
return s
|
||||
|
||||
|
||||
|
||||
@@ -13,12 +13,14 @@ PREFIX = r'''%yt-dlp(1)
|
||||
|
||||
# NAME
|
||||
|
||||
youtube\-dl \- download videos from youtube.com or other video platforms
|
||||
yt\-dlp \- A youtube-dl fork with additional features and patches
|
||||
|
||||
# SYNOPSIS
|
||||
|
||||
**yt-dlp** \[OPTIONS\] URL [URL...]
|
||||
|
||||
# DESCRIPTION
|
||||
|
||||
'''
|
||||
|
||||
|
||||
@@ -33,47 +35,63 @@ def main():
|
||||
with io.open(README_FILE, encoding='utf-8') as f:
|
||||
readme = f.read()
|
||||
|
||||
readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
|
||||
readme = re.sub(r'\s+yt-dlp \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
|
||||
readme = PREFIX + readme
|
||||
|
||||
readme = filter_excluded_sections(readme)
|
||||
readme = move_sections(readme)
|
||||
readme = filter_options(readme)
|
||||
|
||||
with io.open(outfile, 'w', encoding='utf-8') as outf:
|
||||
outf.write(readme)
|
||||
outf.write(PREFIX + readme)
|
||||
|
||||
|
||||
def filter_excluded_sections(readme):
|
||||
EXCLUDED_SECTION_BEGIN_STRING = re.escape('<!-- MANPAGE: BEGIN EXCLUDED SECTION -->')
|
||||
EXCLUDED_SECTION_END_STRING = re.escape('<!-- MANPAGE: END EXCLUDED SECTION -->')
|
||||
return re.sub(
|
||||
rf'(?s){EXCLUDED_SECTION_BEGIN_STRING}.+?{EXCLUDED_SECTION_END_STRING}\n',
|
||||
'', readme)
|
||||
|
||||
|
||||
def move_sections(readme):
|
||||
MOVE_TAG_TEMPLATE = '<!-- MANPAGE: MOVE "%s" SECTION HERE -->'
|
||||
sections = re.findall(r'(?m)^%s$' % (
|
||||
re.escape(MOVE_TAG_TEMPLATE).replace(r'\%', '%') % '(.+)'), readme)
|
||||
|
||||
for section_name in sections:
|
||||
move_tag = MOVE_TAG_TEMPLATE % section_name
|
||||
if readme.count(move_tag) > 1:
|
||||
raise Exception(f'There is more than one occurrence of "{move_tag}". This is unexpected')
|
||||
|
||||
sections = re.findall(rf'(?sm)(^# {re.escape(section_name)}.+?)(?=^# )', readme)
|
||||
if len(sections) < 1:
|
||||
raise Exception(f'The section {section_name} does not exist')
|
||||
elif len(sections) > 1:
|
||||
raise Exception(f'There are multiple occurrences of section {section_name}, this is unhandled')
|
||||
|
||||
readme = readme.replace(sections[0], '', 1).replace(move_tag, sections[0], 1)
|
||||
return readme
|
||||
|
||||
|
||||
def filter_options(readme):
|
||||
ret = ''
|
||||
in_options = False
|
||||
for line in readme.split('\n'):
|
||||
if line.startswith('# '):
|
||||
if line[2:].startswith('OPTIONS'):
|
||||
in_options = True
|
||||
else:
|
||||
in_options = False
|
||||
section = re.search(r'(?sm)^# USAGE AND OPTIONS\n.+?(?=^# )', readme).group(0)
|
||||
options = '# OPTIONS\n'
|
||||
for line in section.split('\n')[1:]:
|
||||
if line.lstrip().startswith('-'):
|
||||
split = re.split(r'\s{2,}', line.lstrip())
|
||||
# Description string may start with `-` as well. If there is
|
||||
# only one piece then it's a description bit not an option.
|
||||
if len(split) > 1:
|
||||
option, description = split
|
||||
split_option = option.split(' ')
|
||||
|
||||
if in_options:
|
||||
if line.lstrip().startswith('-'):
|
||||
split = re.split(r'\s{2,}', line.lstrip())
|
||||
# Description string may start with `-` as well. If there is
|
||||
# only one piece then it's a description bit not an option.
|
||||
if len(split) > 1:
|
||||
option, description = split
|
||||
split_option = option.split(' ')
|
||||
if not split_option[-1].startswith('-'): # metavar
|
||||
option = ' '.join(split_option[:-1] + [f'*{split_option[-1]}*'])
|
||||
|
||||
if not split_option[-1].startswith('-'): # metavar
|
||||
option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
|
||||
# Pandoc's definition_lists. See http://pandoc.org/README.html
|
||||
options += f'\n{option}\n: {description}\n'
|
||||
continue
|
||||
options += line.lstrip() + '\n'
|
||||
|
||||
# Pandoc's definition_lists. See http://pandoc.org/README.html
|
||||
# for more information.
|
||||
ret += '\n%s\n: %s\n' % (option, description)
|
||||
continue
|
||||
ret += line.lstrip() + '\n'
|
||||
else:
|
||||
ret += line + '\n'
|
||||
|
||||
return ret
|
||||
return readme.replace(section, options, 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -27,13 +27,13 @@ try:
|
||||
except Exception:
|
||||
GIT_HEAD = None
|
||||
|
||||
VERSION_FILE = f'''
|
||||
VERSION_FILE = f'''\
|
||||
# Autogenerated by devscripts/update-version.py
|
||||
|
||||
__version__ = {VERSION!r}
|
||||
|
||||
RELEASE_GIT_HEAD = {GIT_HEAD!r}
|
||||
'''.lstrip()
|
||||
'''
|
||||
|
||||
with open('yt_dlp/version.py', 'wt') as f:
|
||||
f.write(VERSION_FILE)
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
orphan: true
|
||||
---
|
||||
```{include} ../Contributing.md
|
||||
```
|
||||
@@ -40,7 +40,7 @@ def main():
|
||||
'--icon=devscripts/logo.ico',
|
||||
'--upx-exclude=vcruntime140.dll',
|
||||
'--noconfirm',
|
||||
*dependancy_options(),
|
||||
*dependency_options(),
|
||||
*opts,
|
||||
'yt_dlp/__main__.py',
|
||||
]
|
||||
@@ -73,11 +73,11 @@ def version_to_list(version):
|
||||
return list(map(int, version_list)) + [0] * (4 - len(version_list))
|
||||
|
||||
|
||||
def dependancy_options():
|
||||
dependancies = [pycryptodome_module(), 'mutagen'] + collect_submodules('websockets')
|
||||
def dependency_options():
|
||||
dependencies = [pycryptodome_module(), 'mutagen'] + collect_submodules('websockets')
|
||||
excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc']
|
||||
|
||||
yield from (f'--hidden-import={module}' for module in dependancies)
|
||||
yield from (f'--hidden-import={module}' for module in dependencies)
|
||||
yield from (f'--exclude-module={module}' for module in excluded_modules)
|
||||
|
||||
|
||||
|
||||
+29
-1
@@ -21,6 +21,7 @@
|
||||
- **9now.com.au**
|
||||
- **abc.net.au**
|
||||
- **abc.net.au:iview**
|
||||
- **abc.net.au:iview:showseries**
|
||||
- **abcnews**
|
||||
- **abcnews:video**
|
||||
- **abcotvs**: ABC Owned Television Stations
|
||||
@@ -273,6 +274,7 @@
|
||||
- **DiscoveryPlus**
|
||||
- **DiscoveryPlusIndia**
|
||||
- **DiscoveryPlusIndiaShow**
|
||||
- **DiscoveryPlusItaly**
|
||||
- **DiscoveryPlusItalyShow**
|
||||
- **DiscoveryVR**
|
||||
- **Disney**
|
||||
@@ -287,6 +289,8 @@
|
||||
- **DPlay**
|
||||
- **DRBonanza**
|
||||
- **Dropbox**
|
||||
- **Dropout**
|
||||
- **DropoutSeason**
|
||||
- **DrTuber**
|
||||
- **drtv**
|
||||
- **drtv:live**
|
||||
@@ -379,6 +383,12 @@
|
||||
- **GabTV**
|
||||
- **Gaia**
|
||||
- **GameInformer**
|
||||
- **GameJolt**
|
||||
- **GameJoltCommunity**
|
||||
- **GameJoltGame**
|
||||
- **GameJoltGameSoundtrack**
|
||||
- **GameJoltSearch**
|
||||
- **GameJoltUser**
|
||||
- **GameSpot**
|
||||
- **GameStar**
|
||||
- **Gaskrank**
|
||||
@@ -399,6 +409,7 @@
|
||||
- **GloboArticle**
|
||||
- **Go**
|
||||
- **GodTube**
|
||||
- **Gofile**
|
||||
- **Golem**
|
||||
- **google:podcasts**
|
||||
- **google:podcasts:feed**
|
||||
@@ -436,6 +447,8 @@
|
||||
- **hrfernsehen**
|
||||
- **HRTi**
|
||||
- **HRTiPlaylist**
|
||||
- **HSEProduct**
|
||||
- **HSEShow**
|
||||
- **Huajiao**: 花椒直播
|
||||
- **HuffPost**: Huffington Post
|
||||
- **Hungama**
|
||||
@@ -652,6 +665,8 @@
|
||||
- **n-tv.de**
|
||||
- **N1Info:article**
|
||||
- **N1InfoAsset**
|
||||
- **Nate**
|
||||
- **NateProgram**
|
||||
- **natgeo:video**
|
||||
- **NationalGeographicTV**
|
||||
- **Naver**
|
||||
@@ -766,6 +781,8 @@
|
||||
- **OnionStudios**
|
||||
- **Ooyala**
|
||||
- **OoyalaExternal**
|
||||
- **Opencast**
|
||||
- **OpencastPlaylist**
|
||||
- **openrec**
|
||||
- **openrec:capture**
|
||||
- **OraTV**
|
||||
@@ -819,6 +836,8 @@
|
||||
- **Pinkbike**
|
||||
- **Pinterest**
|
||||
- **PinterestCollection**
|
||||
- **pixiv:sketch**
|
||||
- **pixiv:sketch:user**
|
||||
- **Pladform**
|
||||
- **PlanetMarathi**
|
||||
- **Platzi**
|
||||
@@ -941,6 +960,7 @@
|
||||
- **RTRFM**
|
||||
- **RTS**: RTS.ch
|
||||
- **rtve.es:alacarta**: RTVE a la carta
|
||||
- **rtve.es:audio**: RTVE audio
|
||||
- **rtve.es:infantil**: RTVE infantil
|
||||
- **rtve.es:live**: RTVE.es live streams
|
||||
- **rtve.es:television**
|
||||
@@ -950,11 +970,12 @@
|
||||
- **RumbleChannel**
|
||||
- **RumbleEmbed**
|
||||
- **rutube**: Rutube videos
|
||||
- **rutube:channel**: Rutube channels
|
||||
- **rutube:channel**: Rutube channel
|
||||
- **rutube:embed**: Rutube embedded videos
|
||||
- **rutube:movie**: Rutube movies
|
||||
- **rutube:person**: Rutube person videos
|
||||
- **rutube:playlist**: Rutube playlists
|
||||
- **rutube:tags**: Rutube tags
|
||||
- **RUTV**: RUTV.RU
|
||||
- **Ruutu**
|
||||
- **Ruv**
|
||||
@@ -994,6 +1015,7 @@
|
||||
- **simplecast:episode**
|
||||
- **simplecast:podcast**
|
||||
- **Sina**
|
||||
- **Skeb**
|
||||
- **sky.it**
|
||||
- **sky:news**
|
||||
- **sky:news:story**
|
||||
@@ -1013,6 +1035,7 @@
|
||||
- **SonyLIVSeries**
|
||||
- **soundcloud**
|
||||
- **soundcloud:playlist**
|
||||
- **soundcloud:related**
|
||||
- **soundcloud:search**: Soundcloud search; "scsearch:" prefix
|
||||
- **soundcloud:set**
|
||||
- **soundcloud:trackstation**
|
||||
@@ -1120,12 +1143,16 @@
|
||||
- **ThreeSpeak**
|
||||
- **ThreeSpeakUser**
|
||||
- **TikTok**
|
||||
- **tiktok:effect**
|
||||
- **tiktok:sound**
|
||||
- **tiktok:tag**
|
||||
- **tiktok:user**
|
||||
- **tinypic**: tinypic.com videos
|
||||
- **TMZ**
|
||||
- **TNAFlix**
|
||||
- **TNAFlixNetworkEmbed**
|
||||
- **toggle**
|
||||
- **toggo**
|
||||
- **Tokentube**
|
||||
- **Tokentube:channel**
|
||||
- **ToonGoggles**
|
||||
@@ -1138,6 +1165,7 @@
|
||||
- **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
|
||||
- **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
|
||||
- **TrovoVod**
|
||||
- **TrueID**
|
||||
- **TruNews**
|
||||
- **TruTV**
|
||||
- **Tube8**
|
||||
|
||||
+47
-4
@@ -194,6 +194,51 @@ def expect_dict(self, got_dict, expected_dict):
|
||||
expect_value(self, got, expected, info_field)
|
||||
|
||||
|
||||
def sanitize_got_info_dict(got_dict):
|
||||
IGNORED_FIELDS = (
|
||||
# Format keys
|
||||
'url', 'manifest_url', 'format', 'format_id', 'format_note', 'width', 'height', 'resolution',
|
||||
'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'vbr', 'fps', 'vcodec', 'container', 'filesize',
|
||||
'filesize_approx', 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'preference',
|
||||
'language', 'language_preference', 'quality', 'source_preference', 'http_headers',
|
||||
'stretched_ratio', 'no_resume', 'has_drm', 'downloader_options',
|
||||
|
||||
# RTMP formats
|
||||
'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time',
|
||||
|
||||
# Lists
|
||||
'formats', 'thumbnails', 'subtitles', 'automatic_captions', 'comments', 'entries',
|
||||
|
||||
# Auto-generated
|
||||
'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch',
|
||||
'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url',
|
||||
|
||||
# Only live_status needs to be checked
|
||||
'is_live', 'was_live',
|
||||
)
|
||||
|
||||
IGNORED_PREFIXES = ('', 'playlist', 'requested', 'webpage')
|
||||
|
||||
def sanitize(key, value):
|
||||
if isinstance(value, str) and len(value) > 100:
|
||||
return f'md5:{md5(value)}'
|
||||
elif isinstance(value, list) and len(value) > 10:
|
||||
return f'count:{len(value)}'
|
||||
return value
|
||||
|
||||
test_info_dict = {
|
||||
key: sanitize(key, value) for key, value in got_dict.items()
|
||||
if value is not None and key not in IGNORED_FIELDS and not any(
|
||||
key.startswith(f'{prefix}_') for prefix in IGNORED_PREFIXES)
|
||||
}
|
||||
|
||||
# display_id may be generated from id
|
||||
if test_info_dict.get('display_id') == test_info_dict['id']:
|
||||
test_info_dict.pop('display_id')
|
||||
|
||||
return test_info_dict
|
||||
|
||||
|
||||
def expect_info_dict(self, got_dict, expected_dict):
|
||||
expect_dict(self, got_dict, expected_dict)
|
||||
# Check for the presence of mandatory fields
|
||||
@@ -207,10 +252,8 @@ def expect_info_dict(self, got_dict, expected_dict):
|
||||
for key in ['webpage_url', 'extractor', 'extractor_key']:
|
||||
self.assertTrue(got_dict.get(key), 'Missing field: %s' % key)
|
||||
|
||||
# Are checkable fields missing from the test case definition?
|
||||
test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
|
||||
for key, value in got_dict.items()
|
||||
if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
|
||||
test_info_dict = sanitize_got_info_dict(got_dict)
|
||||
|
||||
missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
|
||||
if missing_keys:
|
||||
def _repr(v):
|
||||
|
||||
+84
-15
@@ -99,10 +99,10 @@ class TestInfoExtractor(unittest.TestCase):
|
||||
self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
|
||||
|
||||
def test_search_json_ld_realworld(self):
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/23306
|
||||
expect_dict(
|
||||
self,
|
||||
self.ie._search_json_ld(r'''<script type="application/ld+json">
|
||||
_TESTS = [
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/23306
|
||||
(
|
||||
r'''<script type="application/ld+json">
|
||||
{
|
||||
"@context": "http://schema.org/",
|
||||
"@type": "VideoObject",
|
||||
@@ -135,17 +135,86 @@ class TestInfoExtractor(unittest.TestCase):
|
||||
"name": "Kleio Valentien",
|
||||
"url": "https://www.eporner.com/pornstar/kleio-valentien/"
|
||||
}]}
|
||||
</script>''', None),
|
||||
{
|
||||
'title': '1 On 1 With Kleio',
|
||||
'description': 'Kleio Valentien',
|
||||
'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4',
|
||||
'timestamp': 1449347075,
|
||||
'duration': 743.0,
|
||||
'view_count': 1120958,
|
||||
'width': 1920,
|
||||
'height': 1080,
|
||||
})
|
||||
</script>''',
|
||||
{
|
||||
'title': '1 On 1 With Kleio',
|
||||
'description': 'Kleio Valentien',
|
||||
'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4',
|
||||
'timestamp': 1449347075,
|
||||
'duration': 743.0,
|
||||
'view_count': 1120958,
|
||||
'width': 1920,
|
||||
'height': 1080,
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
r'''<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@graph": [
|
||||
{
|
||||
"@type": "NewsArticle",
|
||||
"mainEntityOfPage": {
|
||||
"@type": "WebPage",
|
||||
"@id": "https://www.ant1news.gr/Society/article/620286/symmoria-anilikon-dikigoros-thymaton-ithelan-na-toys-apoteleiosoyn"
|
||||
},
|
||||
"headline": "Συμμορία ανηλίκων – δικηγόρος θυμάτων: ήθελαν να τους αποτελειώσουν",
|
||||
"name": "Συμμορία ανηλίκων – δικηγόρος θυμάτων: ήθελαν να τους αποτελειώσουν",
|
||||
"description": "Τα παιδιά δέχθηκαν την επίθεση επειδή αρνήθηκαν να γίνουν μέλη της συμμορίας, ανέφερε ο Γ. Ζαχαρόπουλος.",
|
||||
"image": {
|
||||
"@type": "ImageObject",
|
||||
"url": "https://ant1media.azureedge.net/imgHandler/1100/a635c968-be71-447c-bf9c-80d843ece21e.jpg",
|
||||
"width": 1100,
|
||||
"height": 756 },
|
||||
"datePublished": "2021-11-10T08:50:00+03:00",
|
||||
"dateModified": "2021-11-10T08:52:53+03:00",
|
||||
"author": {
|
||||
"@type": "Person",
|
||||
"@id": "https://www.ant1news.gr/",
|
||||
"name": "Ant1news",
|
||||
"image": "https://www.ant1news.gr/images/logo-e5d7e4b3e714c88e8d2eca96130142f6.png",
|
||||
"url": "https://www.ant1news.gr/"
|
||||
},
|
||||
"publisher": {
|
||||
"@type": "Organization",
|
||||
"@id": "https://www.ant1news.gr#publisher",
|
||||
"name": "Ant1news",
|
||||
"url": "https://www.ant1news.gr",
|
||||
"logo": {
|
||||
"@type": "ImageObject",
|
||||
"url": "https://www.ant1news.gr/images/logo-e5d7e4b3e714c88e8d2eca96130142f6.png",
|
||||
"width": 400,
|
||||
"height": 400 },
|
||||
"sameAs": [
|
||||
"https://www.facebook.com/Ant1news.gr",
|
||||
"https://twitter.com/antennanews",
|
||||
"https://www.youtube.com/channel/UC0smvAbfczoN75dP0Hw4Pzw",
|
||||
"https://www.instagram.com/ant1news/"
|
||||
]
|
||||
},
|
||||
|
||||
"keywords": "μαχαίρωμα,συμμορία ανηλίκων,ΕΙΔΗΣΕΙΣ,ΕΙΔΗΣΕΙΣ ΣΗΜΕΡΑ,ΝΕΑ,Κοινωνία - Ant1news",
|
||||
|
||||
|
||||
"articleSection": "Κοινωνία"
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>''',
|
||||
{
|
||||
'timestamp': 1636523400,
|
||||
'title': 'md5:91fe569e952e4d146485740ae927662b',
|
||||
},
|
||||
{'expected_type': 'NewsArticle'},
|
||||
),
|
||||
]
|
||||
for html, expected_dict, search_json_ld_kwargs in _TESTS:
|
||||
expect_dict(
|
||||
self,
|
||||
self.ie._search_json_ld(html, None, **search_json_ld_kwargs),
|
||||
expected_dict
|
||||
)
|
||||
|
||||
def test_download_json(self):
|
||||
uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
|
||||
|
||||
@@ -717,6 +717,7 @@ class TestYoutubeDL(unittest.TestCase):
|
||||
test('%(id)s', '.abcd', info={'id': '.abcd'})
|
||||
test('%(id)s', 'ab__cd', info={'id': 'ab__cd'})
|
||||
test('%(id)s', ('ab:cd', 'ab -cd'), info={'id': 'ab:cd'})
|
||||
test('%(id.0)s', '-', info={'id': '--'})
|
||||
|
||||
# Invalid templates
|
||||
self.assertTrue(isinstance(YoutubeDL.validate_outtmpl('%(title)'), ValueError))
|
||||
@@ -777,6 +778,10 @@ class TestYoutubeDL(unittest.TestCase):
|
||||
test('%(title5)#U', 'a\u0301e\u0301i\u0301 𝐀')
|
||||
test('%(title5)+U', 'áéí A')
|
||||
test('%(title5)+#U', 'a\u0301e\u0301i\u0301 A')
|
||||
test('%(height)D', '1K')
|
||||
test('%(height)5.2D', ' 1.08K')
|
||||
test('%(title4)#S', 'foo_bar_test')
|
||||
test('%(title4).10S', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' ')))
|
||||
if compat_os_name == 'nt':
|
||||
test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
|
||||
test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'"))
|
||||
@@ -808,6 +813,11 @@ class TestYoutubeDL(unittest.TestCase):
|
||||
test('%(width-100,height+width|def)s', 'def')
|
||||
test('%(timestamp-x>%H\\,%M\\,%S,timestamp>%H\\,%M\\,%S)s', '12,00,00')
|
||||
|
||||
# Replacement
|
||||
test('%(id&foo)s.bar', 'foo.bar')
|
||||
test('%(title&foo)s.bar', 'NA.bar')
|
||||
test('%(title&foo|baz)s.bar', 'baz.bar')
|
||||
|
||||
# Laziness
|
||||
def gen():
|
||||
yield from range(5)
|
||||
|
||||
+34
-2
@@ -8,6 +8,8 @@ from yt_dlp.cookies import (
|
||||
WindowsChromeCookieDecryptor,
|
||||
parse_safari_cookies,
|
||||
pbkdf2_sha1,
|
||||
_get_linux_desktop_environment,
|
||||
_LinuxDesktopEnvironment,
|
||||
)
|
||||
|
||||
|
||||
@@ -42,6 +44,37 @@ class MonkeyPatch:
|
||||
|
||||
|
||||
class TestCookies(unittest.TestCase):
|
||||
def test_get_desktop_environment(self):
|
||||
""" based on https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util_unittest.cc """
|
||||
test_cases = [
|
||||
({}, _LinuxDesktopEnvironment.OTHER),
|
||||
|
||||
({'DESKTOP_SESSION': 'gnome'}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'DESKTOP_SESSION': 'mate'}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'DESKTOP_SESSION': 'kde4'}, _LinuxDesktopEnvironment.KDE),
|
||||
({'DESKTOP_SESSION': 'kde'}, _LinuxDesktopEnvironment.KDE),
|
||||
({'DESKTOP_SESSION': 'xfce'}, _LinuxDesktopEnvironment.XFCE),
|
||||
|
||||
({'GNOME_DESKTOP_SESSION_ID': 1}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'KDE_FULL_SESSION': 1}, _LinuxDesktopEnvironment.KDE),
|
||||
|
||||
({'XDG_CURRENT_DESKTOP': 'X-Cinnamon'}, _LinuxDesktopEnvironment.CINNAMON),
|
||||
({'XDG_CURRENT_DESKTOP': 'GNOME'}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'XDG_CURRENT_DESKTOP': 'GNOME:GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'XDG_CURRENT_DESKTOP': 'GNOME : GNOME-Classic'}, _LinuxDesktopEnvironment.GNOME),
|
||||
|
||||
({'XDG_CURRENT_DESKTOP': 'Unity', 'DESKTOP_SESSION': 'gnome-fallback'}, _LinuxDesktopEnvironment.GNOME),
|
||||
({'XDG_CURRENT_DESKTOP': 'KDE', 'KDE_SESSION_VERSION': '5'}, _LinuxDesktopEnvironment.KDE),
|
||||
({'XDG_CURRENT_DESKTOP': 'KDE'}, _LinuxDesktopEnvironment.KDE),
|
||||
({'XDG_CURRENT_DESKTOP': 'Pantheon'}, _LinuxDesktopEnvironment.PANTHEON),
|
||||
({'XDG_CURRENT_DESKTOP': 'Unity'}, _LinuxDesktopEnvironment.UNITY),
|
||||
({'XDG_CURRENT_DESKTOP': 'Unity:Unity7'}, _LinuxDesktopEnvironment.UNITY),
|
||||
({'XDG_CURRENT_DESKTOP': 'Unity:Unity8'}, _LinuxDesktopEnvironment.UNITY),
|
||||
]
|
||||
|
||||
for env, expected_desktop_environment in test_cases:
|
||||
self.assertEqual(_get_linux_desktop_environment(env), expected_desktop_environment)
|
||||
|
||||
def test_chrome_cookie_decryptor_linux_derive_key(self):
|
||||
key = LinuxChromeCookieDecryptor.derive_key(b'abc')
|
||||
self.assertEqual(key, b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17')
|
||||
@@ -58,8 +91,7 @@ class TestCookies(unittest.TestCase):
|
||||
self.assertEqual(decryptor.decrypt(encrypted_value), value)
|
||||
|
||||
def test_chrome_cookie_decryptor_linux_v11(self):
|
||||
with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b'',
|
||||
'KEYRING_AVAILABLE': True}):
|
||||
with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}):
|
||||
encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd'
|
||||
value = 'tz=Europe.London'
|
||||
decryptor = LinuxChromeCookieDecryptor('Chrome', Logger())
|
||||
|
||||
@@ -124,11 +124,11 @@ class TestModifyChaptersPP(unittest.TestCase):
|
||||
chapters = self._chapters([70], ['c']) + [
|
||||
self._sponsor_chapter(10, 20, 'sponsor'),
|
||||
self._sponsor_chapter(30, 40, 'preview'),
|
||||
self._sponsor_chapter(50, 60, 'sponsor')]
|
||||
self._sponsor_chapter(50, 60, 'filler')]
|
||||
expected = self._chapters(
|
||||
[10, 20, 30, 40, 50, 60, 70],
|
||||
['c', '[SponsorBlock]: Sponsor', 'c', '[SponsorBlock]: Preview/Recap',
|
||||
'c', '[SponsorBlock]: Sponsor', 'c'])
|
||||
'c', '[SponsorBlock]: Filler Tangent', 'c'])
|
||||
self._remove_marked_arrange_sponsors_test_impl(chapters, expected, [])
|
||||
|
||||
def test_remove_marked_arrange_sponsors_UniqueNamesForOverlappingSponsors(self):
|
||||
|
||||
@@ -1156,9 +1156,16 @@ class TestUtil(unittest.TestCase):
|
||||
self.assertEqual(parse_count('1000'), 1000)
|
||||
self.assertEqual(parse_count('1.000'), 1000)
|
||||
self.assertEqual(parse_count('1.1k'), 1100)
|
||||
self.assertEqual(parse_count('1.1 k'), 1100)
|
||||
self.assertEqual(parse_count('1,1 k'), 1100)
|
||||
self.assertEqual(parse_count('1.1kk'), 1100000)
|
||||
self.assertEqual(parse_count('1.1kk '), 1100000)
|
||||
self.assertEqual(parse_count('1,1kk'), 1100000)
|
||||
self.assertEqual(parse_count('100 views'), 100)
|
||||
self.assertEqual(parse_count('1,100 views'), 1100)
|
||||
self.assertEqual(parse_count('1.1kk views'), 1100000)
|
||||
self.assertEqual(parse_count('10M views'), 10000000)
|
||||
self.assertEqual(parse_count('has 10M views'), 10000000)
|
||||
|
||||
def test_parse_resolution(self):
|
||||
self.assertEqual(parse_resolution(None), {})
|
||||
|
||||
@@ -82,6 +82,10 @@ _NSIG_TESTS = [
|
||||
'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
|
||||
'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
|
||||
'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
+201
-126
@@ -5,7 +5,6 @@ from __future__ import absolute_import, unicode_literals
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import copy
|
||||
import datetime
|
||||
import errno
|
||||
import fileinput
|
||||
@@ -68,8 +67,10 @@ from .utils import (
|
||||
float_or_none,
|
||||
format_bytes,
|
||||
format_field,
|
||||
format_decimal_suffix,
|
||||
formatSeconds,
|
||||
GeoRestrictedError,
|
||||
get_domain,
|
||||
HEADRequest,
|
||||
int_or_none,
|
||||
iri_to_uri,
|
||||
@@ -96,6 +97,7 @@ from .utils import (
|
||||
ReExtractInfo,
|
||||
register_socks_protocols,
|
||||
RejectedVideoReached,
|
||||
remove_terminal_sequences,
|
||||
render_table,
|
||||
replace_extension,
|
||||
SameFileError,
|
||||
@@ -142,6 +144,7 @@ from .downloader.rtmp import rtmpdump_version
|
||||
from .postprocessor import (
|
||||
get_postprocessor,
|
||||
EmbedThumbnailPP,
|
||||
FFmpegFixupDuplicateMoovPP,
|
||||
FFmpegFixupDurationPP,
|
||||
FFmpegFixupM3u8PP,
|
||||
FFmpegFixupM4aPP,
|
||||
@@ -314,10 +317,10 @@ class YoutubeDL(object):
|
||||
break_per_url: Whether break_on_reject and break_on_existing
|
||||
should act on each input URL as opposed to for the entire queue
|
||||
cookiefile: File name where cookies should be read from and dumped to
|
||||
cookiesfrombrowser: A tuple containing the name of the browser and the profile
|
||||
name/path from where cookies are loaded.
|
||||
Eg: ('chrome', ) or ('vivaldi', 'default')
|
||||
nocheckcertificate:Do not verify SSL certificates
|
||||
cookiesfrombrowser: A tuple containing the name of the browser, the profile
|
||||
name/pathfrom where cookies are loaded, and the name of the
|
||||
keyring. Eg: ('chrome', ) or ('vivaldi', 'default', 'BASICTEXT')
|
||||
nocheckcertificate: Do not verify SSL certificates
|
||||
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
|
||||
At the moment, this is only supported by YouTube.
|
||||
proxy: URL of the proxy server to use
|
||||
@@ -447,8 +450,8 @@ class YoutubeDL(object):
|
||||
The following parameters are not used by YoutubeDL itself, they are used by
|
||||
the downloader (see yt_dlp/downloader/common.py):
|
||||
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
|
||||
max_filesize, test, noresizebuffer, retries, fragment_retries, continuedl,
|
||||
noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
|
||||
max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
|
||||
continuedl, noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
|
||||
external_downloader_args, concurrent_fragment_downloads.
|
||||
|
||||
The following options are used by the post processors:
|
||||
@@ -622,7 +625,7 @@ class YoutubeDL(object):
|
||||
|
||||
# Creating format selector here allows us to catch syntax errors before the extraction
|
||||
self.format_selector = (
|
||||
None if self.params.get('format') is None
|
||||
self.params.get('format') if self.params.get('format') in (None, '-')
|
||||
else self.params['format'] if callable(self.params['format'])
|
||||
else self.build_format_selector(self.params['format']))
|
||||
|
||||
@@ -633,13 +636,6 @@ class YoutubeDL(object):
|
||||
self.print_debug_header()
|
||||
self.add_default_info_extractors()
|
||||
|
||||
for pp_def_raw in self.params.get('postprocessors', []):
|
||||
pp_def = dict(pp_def_raw)
|
||||
when = pp_def.pop('when', 'post_process')
|
||||
pp_class = get_postprocessor(pp_def.pop('key'))
|
||||
pp = pp_class(self, **compat_kwargs(pp_def))
|
||||
self.add_post_processor(pp, when=when)
|
||||
|
||||
hooks = {
|
||||
'post_hooks': self.add_post_hook,
|
||||
'progress_hooks': self.add_progress_hook,
|
||||
@@ -649,6 +645,13 @@ class YoutubeDL(object):
|
||||
for ph in self.params.get(opt, []):
|
||||
fn(ph)
|
||||
|
||||
for pp_def_raw in self.params.get('postprocessors', []):
|
||||
pp_def = dict(pp_def_raw)
|
||||
when = pp_def.pop('when', 'post_process')
|
||||
self.add_post_processor(
|
||||
get_postprocessor(pp_def.pop('key'))(self, **compat_kwargs(pp_def)),
|
||||
when=when)
|
||||
|
||||
register_socks_protocols()
|
||||
|
||||
def preload_download_archive(fn):
|
||||
@@ -735,6 +738,9 @@ class YoutubeDL(object):
|
||||
def add_postprocessor_hook(self, ph):
|
||||
"""Add the postprocessing progress hook"""
|
||||
self._postprocessor_hooks.append(ph)
|
||||
for pps in self._pps.values():
|
||||
for pp in pps:
|
||||
pp.add_progress_hook(ph)
|
||||
|
||||
def _bidi_workaround(self, message):
|
||||
if not hasattr(self, '_output_channel'):
|
||||
@@ -776,6 +782,7 @@ class YoutubeDL(object):
|
||||
def to_console_title(self, message):
|
||||
if not self.params.get('consoletitle', False):
|
||||
return
|
||||
message = remove_terminal_sequences(message)
|
||||
if compat_os_name == 'nt':
|
||||
if ctypes.windll.kernel32.GetConsoleWindow():
|
||||
# c_wchar_p() might not be necessary if `message` is
|
||||
@@ -812,14 +819,15 @@ class YoutubeDL(object):
|
||||
if self.params.get('cookiefile') is not None:
|
||||
self.cookiejar.save(ignore_discard=True, ignore_expires=True)
|
||||
|
||||
def trouble(self, message=None, tb=None):
|
||||
def trouble(self, message=None, tb=None, is_error=True):
|
||||
"""Determine action to take when a download problem appears.
|
||||
|
||||
Depending on if the downloader has been configured to ignore
|
||||
download errors or not, this method may throw an exception or
|
||||
not when errors are found, after printing the message.
|
||||
|
||||
tb, if given, is additional traceback information.
|
||||
@param tb If given, is additional traceback information
|
||||
@param is_error Whether to raise error according to ignorerrors
|
||||
"""
|
||||
if message is not None:
|
||||
self.to_stderr(message)
|
||||
@@ -835,6 +843,8 @@ class YoutubeDL(object):
|
||||
tb = ''.join(tb_data)
|
||||
if tb:
|
||||
self.to_stderr(tb)
|
||||
if not is_error:
|
||||
return
|
||||
if not self.params.get('ignoreerrors'):
|
||||
if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
|
||||
exc_info = sys.exc_info()[1].exc_info
|
||||
@@ -894,12 +904,12 @@ class YoutubeDL(object):
|
||||
else:
|
||||
self.to_stderr(f'{self._format_err("DeprecationWarning:", self.Styles.ERROR)} {message}', True)
|
||||
|
||||
def report_error(self, message, tb=None):
|
||||
def report_error(self, message, *args, **kwargs):
|
||||
'''
|
||||
Do the same as trouble, but prefixes the message with 'ERROR:', colored
|
||||
in red if stderr is a tty file.
|
||||
'''
|
||||
self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', tb)
|
||||
self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args, **kwargs)
|
||||
|
||||
def write_debug(self, message, only_once=False):
|
||||
'''Log debug message or Print message to stderr'''
|
||||
@@ -996,7 +1006,7 @@ class YoutubeDL(object):
|
||||
def validate_outtmpl(cls, outtmpl):
|
||||
''' @return None or Exception object '''
|
||||
outtmpl = re.sub(
|
||||
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljqBU]'),
|
||||
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljqBUDS]'),
|
||||
lambda mobj: f'{mobj.group(0)[:-1]}s',
|
||||
cls._outtmpl_expandpath(outtmpl))
|
||||
try:
|
||||
@@ -1012,8 +1022,12 @@ class YoutubeDL(object):
|
||||
info_dict.pop(key, None)
|
||||
return info_dict
|
||||
|
||||
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
|
||||
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
|
||||
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
|
||||
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
|
||||
@param sanitize Whether to sanitize the output as a filename.
|
||||
For backward compatibility, a function can also be passed
|
||||
"""
|
||||
|
||||
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
|
||||
|
||||
info_dict = self._copy_infodict(info_dict)
|
||||
@@ -1034,7 +1048,7 @@ class YoutubeDL(object):
|
||||
}
|
||||
|
||||
TMPL_DICT = {}
|
||||
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljqBU]'))
|
||||
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljqBUDS]'))
|
||||
MATH_FUNCTIONS = {
|
||||
'+': float.__add__,
|
||||
'-': float.__sub__,
|
||||
@@ -1042,14 +1056,15 @@ class YoutubeDL(object):
|
||||
# Field is of the form key1.key2...
|
||||
# where keys (except first) can be string, int or slice
|
||||
FIELD_RE = r'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num=r'(?:-?\d+)')
|
||||
MATH_FIELD_RE = r'''{field}|{num}'''.format(field=FIELD_RE, num=r'-?\d+(?:.\d+)?')
|
||||
MATH_FIELD_RE = r'''(?:{field}|{num})'''.format(field=FIELD_RE, num=r'-?\d+(?:.\d+)?')
|
||||
MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
|
||||
INTERNAL_FORMAT_RE = re.compile(r'''(?x)
|
||||
(?P<negate>-)?
|
||||
(?P<fields>{field})
|
||||
(?P<maths>(?:{math_op}{math_field})*)
|
||||
(?:>(?P<strf_format>.+?))?
|
||||
(?P<alternate>(?<!\\),[^|)]+)?
|
||||
(?P<alternate>(?<!\\),[^|&)]+)?
|
||||
(?:&(?P<replacement>.*?))?
|
||||
(?:\|(?P<default>.*?))?
|
||||
$'''.format(field=FIELD_RE, math_op=MATH_OPERATORS_RE, math_field=MATH_FIELD_RE))
|
||||
|
||||
@@ -1097,22 +1112,30 @@ class YoutubeDL(object):
|
||||
|
||||
na = self.params.get('outtmpl_na_placeholder', 'NA')
|
||||
|
||||
def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')):
|
||||
return sanitize_filename(str(value), restricted=restricted,
|
||||
is_id=re.search(r'(^|[_.])id(\.|$)', key))
|
||||
|
||||
sanitizer = sanitize if callable(sanitize) else filename_sanitizer
|
||||
sanitize = bool(sanitize)
|
||||
|
||||
def _dumpjson_default(obj):
|
||||
if isinstance(obj, (set, LazyList)):
|
||||
return list(obj)
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
return repr(obj)
|
||||
|
||||
def create_key(outer_mobj):
|
||||
if not outer_mobj.group('has_key'):
|
||||
return outer_mobj.group(0)
|
||||
key = outer_mobj.group('key')
|
||||
mobj = re.match(INTERNAL_FORMAT_RE, key)
|
||||
initial_field = mobj.group('fields').split('.')[-1] if mobj else ''
|
||||
value, default = None, na
|
||||
initial_field = mobj.group('fields') if mobj else ''
|
||||
value, replacement, default = None, None, na
|
||||
while mobj:
|
||||
mobj = mobj.groupdict()
|
||||
default = mobj['default'] if mobj['default'] is not None else default
|
||||
value = get_value(mobj)
|
||||
replacement = mobj['replacement']
|
||||
if value is None and mobj['alternate']:
|
||||
mobj = re.match(INTERNAL_FORMAT_RE, mobj['alternate'][1:])
|
||||
else:
|
||||
@@ -1122,13 +1145,13 @@ class YoutubeDL(object):
|
||||
if fmt == 's' and value is not None and key in field_size_compat_map.keys():
|
||||
fmt = '0{:d}d'.format(field_size_compat_map[key])
|
||||
|
||||
value = default if value is None else value
|
||||
value = default if value is None else value if replacement is None else replacement
|
||||
|
||||
flags = outer_mobj.group('conversion') or ''
|
||||
str_fmt = f'{fmt[:-1]}s'
|
||||
if fmt[-1] == 'l': # list
|
||||
delim = '\n' if '#' in flags else ', '
|
||||
value, fmt = delim.join(variadic(value)), str_fmt
|
||||
value, fmt = delim.join(variadic(value, allowed_types=(str, bytes))), str_fmt
|
||||
elif fmt[-1] == 'j': # json
|
||||
value, fmt = json.dumps(value, default=_dumpjson_default, indent=4 if '#' in flags else None), str_fmt
|
||||
elif fmt[-1] == 'q': # quoted
|
||||
@@ -1142,6 +1165,10 @@ class YoutubeDL(object):
|
||||
# "+" = compatibility equivalence, "#" = NFD
|
||||
'NF%s%s' % ('K' if '+' in flags else '', 'D' if '#' in flags else 'C'),
|
||||
value), str_fmt
|
||||
elif fmt[-1] == 'D': # decimal suffix
|
||||
value, fmt = format_decimal_suffix(value, f'%{fmt[:-1]}f%s' if fmt[:-1] else '%d%s'), 's'
|
||||
elif fmt[-1] == 'S': # filename sanitization
|
||||
value, fmt = filename_sanitizer(initial_field, value, restricted='#' in flags), str_fmt
|
||||
elif fmt[-1] == 'c':
|
||||
if value:
|
||||
value = str(value)[0]
|
||||
@@ -1158,7 +1185,7 @@ class YoutubeDL(object):
|
||||
# So we convert it to repr first
|
||||
value, fmt = repr(value), str_fmt
|
||||
if fmt[-1] in 'csr':
|
||||
value = sanitize(initial_field, value)
|
||||
value = sanitizer(initial_field, value)
|
||||
|
||||
key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format'))
|
||||
TMPL_DICT[key] = value
|
||||
@@ -1172,12 +1199,8 @@ class YoutubeDL(object):
|
||||
|
||||
def _prepare_filename(self, info_dict, tmpl_type='default'):
|
||||
try:
|
||||
sanitize = lambda k, v: sanitize_filename(
|
||||
compat_str(v),
|
||||
restricted=self.params.get('restrictfilenames'),
|
||||
is_id=(k == 'id' or k.endswith('_id')))
|
||||
outtmpl = self._outtmpl_expandpath(self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default']))
|
||||
filename = self.evaluate_outtmpl(outtmpl, info_dict, sanitize)
|
||||
filename = self.evaluate_outtmpl(outtmpl, info_dict, True)
|
||||
|
||||
force_ext = OUTTMPL_TYPES.get(tmpl_type)
|
||||
if filename and force_ext is not None:
|
||||
@@ -1325,31 +1348,33 @@ class YoutubeDL(object):
|
||||
def __handle_extraction_exceptions(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except GeoRestrictedError as e:
|
||||
msg = e.msg
|
||||
if e.countries:
|
||||
msg += '\nThis video is available in %s.' % ', '.join(
|
||||
map(ISO3166Utils.short2full, e.countries))
|
||||
msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
|
||||
self.report_error(msg)
|
||||
except ExtractorError as e: # An error we somewhat expected
|
||||
self.report_error(compat_str(e), e.format_traceback())
|
||||
except ReExtractInfo as e:
|
||||
if e.expected:
|
||||
self.to_screen(f'{e}; Re-extracting data')
|
||||
else:
|
||||
self.to_stderr('\r')
|
||||
self.report_warning(f'{e}; Re-extracting data')
|
||||
return wrapper(self, *args, **kwargs)
|
||||
except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError):
|
||||
raise
|
||||
except Exception as e:
|
||||
if self.params.get('ignoreerrors'):
|
||||
self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
|
||||
else:
|
||||
while True:
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError):
|
||||
raise
|
||||
except ReExtractInfo as e:
|
||||
if e.expected:
|
||||
self.to_screen(f'{e}; Re-extracting data')
|
||||
else:
|
||||
self.to_stderr('\r')
|
||||
self.report_warning(f'{e}; Re-extracting data')
|
||||
continue
|
||||
except GeoRestrictedError as e:
|
||||
msg = e.msg
|
||||
if e.countries:
|
||||
msg += '\nThis video is available in %s.' % ', '.join(
|
||||
map(ISO3166Utils.short2full, e.countries))
|
||||
msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
|
||||
self.report_error(msg)
|
||||
except ExtractorError as e: # An error we somewhat expected
|
||||
self.report_error(str(e), e.format_traceback())
|
||||
except Exception as e:
|
||||
if self.params.get('ignoreerrors'):
|
||||
self.report_error(str(e), tb=encode_compat_str(traceback.format_exc()))
|
||||
else:
|
||||
raise
|
||||
break
|
||||
return wrapper
|
||||
|
||||
def _wait_for_video(self, ie_result):
|
||||
@@ -1369,11 +1394,11 @@ class YoutubeDL(object):
|
||||
min_wait, max_wait = self.params.get('wait_for_video')
|
||||
diff = try_get(ie_result, lambda x: x['release_timestamp'] - time.time())
|
||||
if diff is None and ie_result.get('live_status') == 'is_upcoming':
|
||||
diff = random.randrange(min_wait or 0, max_wait) if max_wait else min_wait
|
||||
diff = random.randrange(min_wait, max_wait) if (max_wait and min_wait) else (max_wait or min_wait)
|
||||
self.report_warning('Release time of video is not known')
|
||||
elif (diff or 0) <= 0:
|
||||
self.report_warning('Video should already be available according to extracted info')
|
||||
diff = min(max(diff, min_wait or 0), max_wait or float('inf'))
|
||||
diff = min(max(diff or 0, min_wait or 0), max_wait or float('inf'))
|
||||
self.to_screen(f'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
|
||||
|
||||
wait_till = time.time() + diff
|
||||
@@ -1419,6 +1444,7 @@ class YoutubeDL(object):
|
||||
'webpage_url': url,
|
||||
'original_url': url,
|
||||
'webpage_url_basename': url_basename(url),
|
||||
'webpage_url_domain': get_domain(url),
|
||||
})
|
||||
if ie is not None:
|
||||
self.add_extra_info(ie_result, {
|
||||
@@ -1452,6 +1478,7 @@ class YoutubeDL(object):
|
||||
info_copy['id'] = ie.get_temp_id(ie_result['url'])
|
||||
self.add_default_extra_info(info_copy, ie, ie_result['url'])
|
||||
self.add_extra_info(info_copy, extra_info)
|
||||
info_copy, _ = self.pre_process(info_copy)
|
||||
self.__forced_printings(info_copy, self.prepare_filename(info_copy), incomplete=True)
|
||||
if self.params.get('force_write_download_archive', False):
|
||||
self.record_download_archive(info_copy)
|
||||
@@ -1470,7 +1497,7 @@ class YoutubeDL(object):
|
||||
self.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls))
|
||||
ie_result['additional_entries'] = [
|
||||
self.extract_info(
|
||||
url, download, extra_info,
|
||||
url, download, extra_info=extra_info,
|
||||
force_generic_extractor=self.params.get('force_generic_extractor'))
|
||||
for url in additional_urls
|
||||
]
|
||||
@@ -1542,6 +1569,7 @@ class YoutubeDL(object):
|
||||
'extractor': ie_result['extractor'],
|
||||
'webpage_url': ie_result['webpage_url'],
|
||||
'webpage_url_basename': url_basename(ie_result['webpage_url']),
|
||||
'webpage_url_domain': get_domain(ie_result['webpage_url']),
|
||||
'extractor_key': ie_result['extractor_key'],
|
||||
})
|
||||
return r
|
||||
@@ -1703,6 +1731,7 @@ class YoutubeDL(object):
|
||||
'extractor': ie_result['extractor'],
|
||||
'webpage_url': ie_result['webpage_url'],
|
||||
'webpage_url_basename': url_basename(ie_result['webpage_url']),
|
||||
'webpage_url_domain': get_domain(ie_result['webpage_url']),
|
||||
'extractor_key': ie_result['extractor_key'],
|
||||
}
|
||||
|
||||
@@ -2059,8 +2088,7 @@ class YoutubeDL(object):
|
||||
selector_1, selector_2 = map(_build_selector_function, selector.selector)
|
||||
|
||||
def selector_function(ctx):
|
||||
for pair in itertools.product(
|
||||
selector_1(copy.deepcopy(ctx)), selector_2(copy.deepcopy(ctx))):
|
||||
for pair in itertools.product(selector_1(ctx), selector_2(ctx)):
|
||||
yield _merge(pair)
|
||||
|
||||
elif selector.type == SINGLE: # atom
|
||||
@@ -2130,7 +2158,7 @@ class YoutubeDL(object):
|
||||
filters = [self._build_format_filter(f) for f in selector.filters]
|
||||
|
||||
def final_selector(ctx):
|
||||
ctx_copy = copy.deepcopy(ctx)
|
||||
ctx_copy = dict(ctx)
|
||||
for _filter in filters:
|
||||
ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
|
||||
return selector_function(ctx_copy)
|
||||
@@ -2342,6 +2370,10 @@ class YoutubeDL(object):
|
||||
if not self.params.get('allow_unplayable_formats'):
|
||||
formats = [f for f in formats if not f.get('has_drm')]
|
||||
|
||||
if info_dict.get('is_live'):
|
||||
get_from_start = bool(self.params.get('live_from_start'))
|
||||
formats = [f for f in formats if bool(f.get('is_from_start')) == get_from_start]
|
||||
|
||||
if not formats:
|
||||
self.raise_no_formats(info_dict)
|
||||
|
||||
@@ -2433,20 +2465,18 @@ class YoutubeDL(object):
|
||||
# The pre-processors may have modified the formats
|
||||
formats = info_dict.get('formats', [info_dict])
|
||||
|
||||
list_only = self.params.get('simulate') is None and (
|
||||
self.params.get('list_thumbnails') or self.params.get('listformats') or self.params.get('listsubtitles'))
|
||||
interactive_format_selection = not list_only and self.format_selector == '-'
|
||||
if self.params.get('list_thumbnails'):
|
||||
self.list_thumbnails(info_dict)
|
||||
if self.params.get('listformats'):
|
||||
if not info_dict.get('formats') and not info_dict.get('url'):
|
||||
self.to_screen('%s has no formats' % info_dict['id'])
|
||||
else:
|
||||
self.list_formats(info_dict)
|
||||
if self.params.get('listsubtitles'):
|
||||
if 'automatic_captions' in info_dict:
|
||||
self.list_subtitles(
|
||||
info_dict['id'], automatic_captions, 'automatic captions')
|
||||
self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
|
||||
list_only = self.params.get('simulate') is None and (
|
||||
self.params.get('list_thumbnails') or self.params.get('listformats') or self.params.get('listsubtitles'))
|
||||
if self.params.get('listformats') or interactive_format_selection:
|
||||
self.list_formats(info_dict)
|
||||
if list_only:
|
||||
# Without this printing, -F --print-json will not work
|
||||
self.__forced_printings(info_dict, self.prepare_filename(info_dict), incomplete=True)
|
||||
@@ -2458,33 +2488,48 @@ class YoutubeDL(object):
|
||||
self.write_debug('Default format spec: %s' % req_format)
|
||||
format_selector = self.build_format_selector(req_format)
|
||||
|
||||
# While in format selection we may need to have an access to the original
|
||||
# format set in order to calculate some metrics or do some processing.
|
||||
# For now we need to be able to guess whether original formats provided
|
||||
# by extractor are incomplete or not (i.e. whether extractor provides only
|
||||
# video-only or audio-only formats) for proper formats selection for
|
||||
# extractors with such incomplete formats (see
|
||||
# https://github.com/ytdl-org/youtube-dl/pull/5556).
|
||||
# Since formats may be filtered during format selection and may not match
|
||||
# the original formats the results may be incorrect. Thus original formats
|
||||
# or pre-calculated metrics should be passed to format selection routines
|
||||
# as well.
|
||||
# We will pass a context object containing all necessary additional data
|
||||
# instead of just formats.
|
||||
# This fixes incorrect format selection issue (see
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/10083).
|
||||
incomplete_formats = (
|
||||
# All formats are video-only or
|
||||
all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats)
|
||||
# all formats are audio-only
|
||||
or all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
|
||||
while True:
|
||||
if interactive_format_selection:
|
||||
req_format = input(
|
||||
self._format_screen('\nEnter format selector: ', self.Styles.EMPHASIS))
|
||||
try:
|
||||
format_selector = self.build_format_selector(req_format)
|
||||
except SyntaxError as err:
|
||||
self.report_error(err, tb=False, is_error=False)
|
||||
continue
|
||||
|
||||
ctx = {
|
||||
'formats': formats,
|
||||
'incomplete_formats': incomplete_formats,
|
||||
}
|
||||
# While in format selection we may need to have an access to the original
|
||||
# format set in order to calculate some metrics or do some processing.
|
||||
# For now we need to be able to guess whether original formats provided
|
||||
# by extractor are incomplete or not (i.e. whether extractor provides only
|
||||
# video-only or audio-only formats) for proper formats selection for
|
||||
# extractors with such incomplete formats (see
|
||||
# https://github.com/ytdl-org/youtube-dl/pull/5556).
|
||||
# Since formats may be filtered during format selection and may not match
|
||||
# the original formats the results may be incorrect. Thus original formats
|
||||
# or pre-calculated metrics should be passed to format selection routines
|
||||
# as well.
|
||||
# We will pass a context object containing all necessary additional data
|
||||
# instead of just formats.
|
||||
# This fixes incorrect format selection issue (see
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/10083).
|
||||
incomplete_formats = (
|
||||
# All formats are video-only or
|
||||
all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats)
|
||||
# all formats are audio-only
|
||||
or all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
|
||||
|
||||
ctx = {
|
||||
'formats': formats,
|
||||
'incomplete_formats': incomplete_formats,
|
||||
}
|
||||
|
||||
formats_to_download = list(format_selector(ctx))
|
||||
if interactive_format_selection and not formats_to_download:
|
||||
self.report_error('Requested format is not available', tb=False, is_error=False)
|
||||
continue
|
||||
break
|
||||
|
||||
formats_to_download = list(format_selector(ctx))
|
||||
if not formats_to_download:
|
||||
if not self.params.get('ignore_no_formats_error'):
|
||||
raise ExtractorError('Requested format is not available', expected=True,
|
||||
@@ -2648,7 +2693,9 @@ class YoutubeDL(object):
|
||||
urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']])
|
||||
self.write_debug('Invoking downloader on "%s"' % urls)
|
||||
|
||||
new_info = copy.deepcopy(self._copy_infodict(info))
|
||||
# Note: Ideally info should be a deep-copied so that hooks cannot modify it.
|
||||
# But it may contain objects that are not deep-copyable
|
||||
new_info = self._copy_infodict(info)
|
||||
if new_info.get('http_headers') is None:
|
||||
new_info['http_headers'] = self._calc_headers(new_info)
|
||||
return fd.download(name, new_info, subtitle)
|
||||
@@ -2663,6 +2710,9 @@ class YoutubeDL(object):
|
||||
if self._num_downloads >= int(max_downloads):
|
||||
raise MaxDownloadsReached()
|
||||
|
||||
if info_dict.get('is_live') and not self.params.get('live_from_start'):
|
||||
info_dict['title'] += ' ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
|
||||
|
||||
# TODO: backward compatibility, to be removed
|
||||
info_dict['fulltitle'] = info_dict['title']
|
||||
|
||||
@@ -2715,7 +2765,7 @@ class YoutubeDL(object):
|
||||
_infojson_written = self._write_info_json('video', info_dict, infofn)
|
||||
if _infojson_written:
|
||||
info_dict['infojson_filename'] = infofn
|
||||
# For backward compatability, even though it was a private field
|
||||
# For backward compatibility, even though it was a private field
|
||||
info_dict['__infojson_filename'] = infofn
|
||||
elif _infojson_written is None:
|
||||
return
|
||||
@@ -2874,15 +2924,22 @@ class YoutubeDL(object):
|
||||
dl_filename = existing_file(full_filename, temp_filename)
|
||||
info_dict['__real_download'] = False
|
||||
|
||||
downloaded = []
|
||||
merger = FFmpegMergerPP(self)
|
||||
|
||||
fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
|
||||
if dl_filename is not None:
|
||||
self.report_file_already_downloaded(dl_filename)
|
||||
elif get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-'):
|
||||
elif fd:
|
||||
for f in requested_formats if fd != FFmpegFD else []:
|
||||
f['filepath'] = fname = prepend_extension(
|
||||
correct_ext(temp_filename, info_dict['ext']),
|
||||
'f%s' % f['format_id'], info_dict['ext'])
|
||||
downloaded.append(fname)
|
||||
info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
|
||||
success, real_download = self.dl(temp_filename, info_dict)
|
||||
info_dict['__real_download'] = real_download
|
||||
else:
|
||||
downloaded = []
|
||||
merger = FFmpegMergerPP(self)
|
||||
if self.params.get('allow_unplayable_formats'):
|
||||
self.report_warning(
|
||||
'You have requested merging of multiple formats '
|
||||
@@ -2894,7 +2951,7 @@ class YoutubeDL(object):
|
||||
'The formats won\'t be merged.')
|
||||
|
||||
if temp_filename == '-':
|
||||
reason = ('using a downloader other than ffmpeg' if FFmpegFD.can_merge_formats(info_dict)
|
||||
reason = ('using a downloader other than ffmpeg' if FFmpegFD.can_merge_formats(info_dict, self.params)
|
||||
else 'but the formats are incompatible for simultaneous download' if merger.available
|
||||
else 'but ffmpeg is not installed')
|
||||
self.report_warning(
|
||||
@@ -2916,14 +2973,15 @@ class YoutubeDL(object):
|
||||
partial_success, real_download = self.dl(fname, new_info)
|
||||
info_dict['__real_download'] = info_dict['__real_download'] or real_download
|
||||
success = success and partial_success
|
||||
if merger.available and not self.params.get('allow_unplayable_formats'):
|
||||
info_dict['__postprocessors'].append(merger)
|
||||
info_dict['__files_to_merge'] = downloaded
|
||||
# Even if there were no downloads, it is being merged only now
|
||||
info_dict['__real_download'] = True
|
||||
else:
|
||||
for file in downloaded:
|
||||
files_to_move[file] = None
|
||||
|
||||
if downloaded and merger.available and not self.params.get('allow_unplayable_formats'):
|
||||
info_dict['__postprocessors'].append(merger)
|
||||
info_dict['__files_to_merge'] = downloaded
|
||||
# Even if there were no downloads, it is being merged only now
|
||||
info_dict['__real_download'] = True
|
||||
else:
|
||||
for file in downloaded:
|
||||
files_to_move[file] = None
|
||||
else:
|
||||
# Just a single file
|
||||
dl_filename = existing_file(full_filename, temp_filename)
|
||||
@@ -2990,9 +3048,14 @@ class YoutubeDL(object):
|
||||
|
||||
downloader = get_suitable_downloader(info_dict, self.params) if 'protocol' in info_dict else None
|
||||
downloader = downloader.__name__ if downloader else None
|
||||
ffmpeg_fixup(info_dict.get('requested_formats') is None and downloader == 'HlsFD',
|
||||
'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
|
||||
FFmpegFixupM3u8PP)
|
||||
|
||||
if info_dict.get('requested_formats') is None: # Not necessary if doing merger
|
||||
ffmpeg_fixup(downloader == 'HlsFD',
|
||||
'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
|
||||
FFmpegFixupM3u8PP)
|
||||
ffmpeg_fixup(info_dict.get('is_live') and downloader == 'DashSegmentsFD',
|
||||
'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP)
|
||||
|
||||
ffmpeg_fixup(downloader == 'WebSocketFragmentFD', 'Malformed timestamps detected', FFmpegFixupTimestampPP)
|
||||
ffmpeg_fixup(downloader == 'WebSocketFragmentFD', 'Malformed duration detected', FFmpegFixupDurationPP)
|
||||
|
||||
@@ -3089,10 +3152,17 @@ class YoutubeDL(object):
|
||||
k.startswith('_') or k in remove_keys or v in empty_values)
|
||||
else:
|
||||
reject = lambda k, v: k in remove_keys
|
||||
filter_fn = lambda obj: (
|
||||
list(map(filter_fn, obj)) if isinstance(obj, (LazyList, list, tuple, set))
|
||||
else obj if not isinstance(obj, dict)
|
||||
else dict((k, filter_fn(v)) for k, v in obj.items() if not reject(k, v)))
|
||||
|
||||
def filter_fn(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)}
|
||||
elif isinstance(obj, (list, tuple, set, LazyList)):
|
||||
return list(map(filter_fn, obj))
|
||||
elif obj is None or isinstance(obj, (str, int, float, bool)):
|
||||
return obj
|
||||
else:
|
||||
return repr(obj)
|
||||
|
||||
return filter_fn(info_dict)
|
||||
|
||||
@staticmethod
|
||||
@@ -3290,6 +3360,11 @@ class YoutubeDL(object):
|
||||
return headers
|
||||
|
||||
def list_formats(self, info_dict):
|
||||
if not info_dict.get('formats') and not info_dict.get('url'):
|
||||
self.to_screen('%s has no formats' % info_dict['id'])
|
||||
return
|
||||
self.to_screen('[info] Available formats for %s:' % info_dict['id'])
|
||||
|
||||
formats = info_dict.get('formats', [info_dict])
|
||||
new_format = self.params.get('listformats_table', True) is not False
|
||||
if new_format:
|
||||
@@ -3304,7 +3379,7 @@ class YoutubeDL(object):
|
||||
delim,
|
||||
format_field(f, 'filesize', ' \t%s', func=format_bytes) + format_field(f, 'filesize_approx', '~\t%s', func=format_bytes),
|
||||
format_field(f, 'tbr', '\t%dk'),
|
||||
shorten_protocol_name(f.get('protocol', '').replace('native', 'n')),
|
||||
shorten_protocol_name(f.get('protocol', '')),
|
||||
delim,
|
||||
format_field(f, 'vcodec', default='unknown').replace(
|
||||
'none',
|
||||
@@ -3340,8 +3415,6 @@ class YoutubeDL(object):
|
||||
if f.get('preference') is None or f['preference'] >= -1000]
|
||||
header_line = ['format code', 'extension', 'resolution', 'note']
|
||||
|
||||
self.to_screen(
|
||||
'[info] Available formats for %s:' % info_dict['id'])
|
||||
self.to_stdout(render_table(
|
||||
header_line, table,
|
||||
extra_gap=(0 if new_format else 1),
|
||||
@@ -3391,7 +3464,8 @@ class YoutubeDL(object):
|
||||
def get_encoding(stream):
|
||||
ret = getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__)
|
||||
if not supports_terminal_sequences(stream):
|
||||
ret += ' (No ANSI)'
|
||||
from .compat import WINDOWS_VT_MODE
|
||||
ret += ' (No VT)' if WINDOWS_VT_MODE is False else ' (No ANSI)'
|
||||
return ret
|
||||
|
||||
encoding_str = 'Encodings: locale %s, fs %s, out %s, err %s, pref %s' % (
|
||||
@@ -3468,11 +3542,11 @@ class YoutubeDL(object):
|
||||
|
||||
from .downloader.websocket import has_websockets
|
||||
from .postprocessor.embedthumbnail import has_mutagen
|
||||
from .cookies import SQLITE_AVAILABLE, KEYRING_AVAILABLE
|
||||
from .cookies import SQLITE_AVAILABLE, SECRETSTORAGE_AVAILABLE
|
||||
|
||||
lib_str = join_nonempty(
|
||||
compat_pycrypto_AES and compat_pycrypto_AES.__name__.split('.')[0],
|
||||
KEYRING_AVAILABLE and 'keyring',
|
||||
SECRETSTORAGE_AVAILABLE and 'secretstorage',
|
||||
has_mutagen and 'mutagen',
|
||||
SQLITE_AVAILABLE and 'sqlite',
|
||||
has_websockets and 'websockets',
|
||||
@@ -3669,7 +3743,7 @@ class YoutubeDL(object):
|
||||
self.write_debug(f'Skipping writing {label} thumbnail')
|
||||
return ret
|
||||
|
||||
for t in thumbnails[::-1]:
|
||||
for idx, t in list(enumerate(thumbnails))[::-1]:
|
||||
thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
|
||||
thumb_display_id = f'{label} thumbnail {t["id"]}'
|
||||
thumb_filename = replace_extension(filename, thumb_ext, info_dict.get('ext'))
|
||||
@@ -3690,6 +3764,7 @@ class YoutubeDL(object):
|
||||
ret.append((thumb_filename, thumb_filename_final))
|
||||
t['filepath'] = thumb_filename
|
||||
except network_exceptions as err:
|
||||
thumbnails.pop(idx)
|
||||
self.report_warning(f'Unable to download {thumb_display_id}: {err}')
|
||||
if ret and not write_all:
|
||||
break
|
||||
|
||||
+36
-17
@@ -18,10 +18,11 @@ from .options import (
|
||||
)
|
||||
from .compat import (
|
||||
compat_getpass,
|
||||
compat_os_name,
|
||||
compat_shlex_quote,
|
||||
workaround_optparse_bug9161,
|
||||
)
|
||||
from .cookies import SUPPORTED_BROWSERS
|
||||
from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
|
||||
from .utils import (
|
||||
DateRange,
|
||||
decodeOption,
|
||||
@@ -95,7 +96,8 @@ def _real_main(argv=None):
|
||||
if opts.batchfile is not None:
|
||||
try:
|
||||
if opts.batchfile == '-':
|
||||
write_string('Reading URLs from stdin:\n')
|
||||
write_string('Reading URLs from stdin - EOF (%s) to end:\n' % (
|
||||
'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'))
|
||||
batchfd = sys.stdin
|
||||
else:
|
||||
batchfd = io.open(
|
||||
@@ -136,6 +138,11 @@ def _real_main(argv=None):
|
||||
sys.exit(0)
|
||||
|
||||
# Conflicting, missing and erroneous options
|
||||
if opts.format == 'best':
|
||||
warnings.append('.\n '.join((
|
||||
'"-f best" selects the best pre-merged format which is often not the best option',
|
||||
'To let yt-dlp download and merge the best available formats, simply do not pass any format selection',
|
||||
'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning')))
|
||||
if opts.usenetrc and (opts.username is not None or opts.password is not None):
|
||||
parser.error('using .netrc conflicts with giving username/password')
|
||||
if opts.password is not None and opts.username is None:
|
||||
@@ -197,12 +204,11 @@ def _real_main(argv=None):
|
||||
if opts.concurrent_fragment_downloads <= 0:
|
||||
parser.error('Concurrent fragments must be positive')
|
||||
if opts.wait_for_video is not None:
|
||||
mobj = re.match(r'(?P<min>\d+)(?:-(?P<max>\d+))?$', opts.wait_for_video)
|
||||
if not mobj:
|
||||
parser.error('Invalid time range to wait')
|
||||
min_wait, max_wait = map(int_or_none, mobj.group('min', 'max'))
|
||||
if max_wait is not None and max_wait < min_wait:
|
||||
min_wait, max_wait, *_ = map(parse_duration, opts.wait_for_video.split('-', 1) + [None])
|
||||
if min_wait is None or (max_wait is None and '-' in opts.wait_for_video):
|
||||
parser.error('Invalid time range to wait')
|
||||
elif max_wait is not None and max_wait < min_wait:
|
||||
parser.error('Minimum time range to wait must not be longer than the maximum')
|
||||
opts.wait_for_video = (min_wait, max_wait)
|
||||
|
||||
def parse_retries(retries, name=''):
|
||||
@@ -216,6 +222,8 @@ def _real_main(argv=None):
|
||||
return parsed_retries
|
||||
if opts.retries is not None:
|
||||
opts.retries = parse_retries(opts.retries)
|
||||
if opts.file_access_retries is not None:
|
||||
opts.file_access_retries = parse_retries(opts.file_access_retries, 'file access ')
|
||||
if opts.fragment_retries is not None:
|
||||
opts.fragment_retries = parse_retries(opts.fragment_retries, 'fragment ')
|
||||
if opts.extractor_retries is not None:
|
||||
@@ -258,10 +266,20 @@ def _real_main(argv=None):
|
||||
if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
|
||||
parser.error('invalid thumbnail format specified')
|
||||
if opts.cookiesfrombrowser is not None:
|
||||
opts.cookiesfrombrowser = [
|
||||
part.strip() or None for part in opts.cookiesfrombrowser.split(':', 1)]
|
||||
if opts.cookiesfrombrowser[0].lower() not in SUPPORTED_BROWSERS:
|
||||
parser.error('unsupported browser specified for cookies')
|
||||
mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
|
||||
if mobj is None:
|
||||
parser.error(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
|
||||
browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
|
||||
browser_name = browser_name.lower()
|
||||
if browser_name not in SUPPORTED_BROWSERS:
|
||||
parser.error(f'unsupported browser specified for cookies: "{browser_name}". '
|
||||
f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
|
||||
if keyring is not None:
|
||||
keyring = keyring.upper()
|
||||
if keyring not in SUPPORTED_KEYRINGS:
|
||||
parser.error(f'unsupported keyring specified for cookies: "{keyring}". '
|
||||
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
|
||||
opts.cookiesfrombrowser = (browser_name, profile, keyring)
|
||||
geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
|
||||
if geo_bypass_code is not None:
|
||||
try:
|
||||
@@ -514,7 +532,7 @@ def _real_main(argv=None):
|
||||
if len(dur) == 2 and all(t is not None for t in dur):
|
||||
remove_ranges.append(tuple(dur))
|
||||
continue
|
||||
parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form ?start-end')
|
||||
parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form *start-end')
|
||||
try:
|
||||
remove_chapters_patterns.append(re.compile(regex))
|
||||
except re.error as err:
|
||||
@@ -559,13 +577,12 @@ def _real_main(argv=None):
|
||||
'_from_cli': True,
|
||||
})
|
||||
if opts.embedthumbnail:
|
||||
already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
|
||||
postprocessors.append({
|
||||
'key': 'EmbedThumbnail',
|
||||
# already_have_thumbnail = True prevents the file from being deleted after embedding
|
||||
'already_have_thumbnail': already_have_thumbnail
|
||||
'already_have_thumbnail': opts.writethumbnail
|
||||
})
|
||||
if not already_have_thumbnail:
|
||||
if not opts.writethumbnail:
|
||||
opts.writethumbnail = True
|
||||
opts.outtmpl['pl_thumbnail'] = ''
|
||||
if opts.split_chapters:
|
||||
@@ -668,6 +685,7 @@ def _real_main(argv=None):
|
||||
'throttledratelimit': opts.throttledratelimit,
|
||||
'overwrites': opts.overwrites,
|
||||
'retries': opts.retries,
|
||||
'file_access_retries': opts.file_access_retries,
|
||||
'fragment_retries': opts.fragment_retries,
|
||||
'extractor_retries': opts.extractor_retries,
|
||||
'skip_unavailable_fragments': opts.skip_unavailable_fragments,
|
||||
@@ -695,8 +713,8 @@ def _real_main(argv=None):
|
||||
'allow_playlist_files': opts.allow_playlist_files,
|
||||
'clean_infojson': opts.clean_infojson,
|
||||
'getcomments': opts.getcomments,
|
||||
'writethumbnail': opts.writethumbnail,
|
||||
'write_all_thumbnails': opts.write_all_thumbnails,
|
||||
'writethumbnail': opts.writethumbnail is True,
|
||||
'write_all_thumbnails': opts.writethumbnail == 'all',
|
||||
'writelink': opts.writelink,
|
||||
'writeurllink': opts.writeurllink,
|
||||
'writewebloclink': opts.writewebloclink,
|
||||
@@ -747,6 +765,7 @@ def _real_main(argv=None):
|
||||
'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
|
||||
'encoding': opts.encoding,
|
||||
'extract_flat': opts.extract_flat,
|
||||
'live_from_start': opts.live_from_start,
|
||||
'wait_for_video': opts.wait_for_video,
|
||||
'mark_watched': opts.mark_watched,
|
||||
'merge_output_format': opts.merge_output_format,
|
||||
|
||||
+10
-1
@@ -160,12 +160,20 @@ except ImportError:
|
||||
compat_pycrypto_AES = None
|
||||
|
||||
|
||||
WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
|
||||
|
||||
|
||||
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
|
||||
if compat_os_name != 'nt':
|
||||
return
|
||||
global WINDOWS_VT_MODE
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
subprocess.Popen('', shell=True, startupinfo=startupinfo)
|
||||
try:
|
||||
subprocess.Popen('', shell=True, startupinfo=startupinfo)
|
||||
WINDOWS_VT_MODE = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# Deprecated
|
||||
@@ -226,6 +234,7 @@ compat_xml_parse_error = etree.ParseError
|
||||
# Set public objects
|
||||
|
||||
__all__ = [
|
||||
'WINDOWS_VT_MODE',
|
||||
'compat_HTMLParseError',
|
||||
'compat_HTMLParser',
|
||||
'compat_HTTPError',
|
||||
|
||||
+264
-56
@@ -1,3 +1,4 @@
|
||||
import contextlib
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
@@ -7,6 +8,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum, auto
|
||||
from hashlib import pbkdf2_hmac
|
||||
|
||||
from .aes import aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes
|
||||
@@ -15,7 +17,6 @@ from .compat import (
|
||||
compat_cookiejar_Cookie,
|
||||
)
|
||||
from .utils import (
|
||||
bug_reports_message,
|
||||
expand_path,
|
||||
Popen,
|
||||
YoutubeDLCookieJar,
|
||||
@@ -31,19 +32,16 @@ except ImportError:
|
||||
|
||||
|
||||
try:
|
||||
import keyring
|
||||
KEYRING_AVAILABLE = True
|
||||
KEYRING_UNAVAILABLE_REASON = f'due to unknown reasons{bug_reports_message()}'
|
||||
import secretstorage
|
||||
SECRETSTORAGE_AVAILABLE = True
|
||||
except ImportError:
|
||||
KEYRING_AVAILABLE = False
|
||||
KEYRING_UNAVAILABLE_REASON = (
|
||||
'as the `keyring` module is not installed. '
|
||||
'Please install by running `python3 -m pip install keyring`. '
|
||||
'Depending on your platform, additional packages may be required '
|
||||
'to access the keyring; see https://pypi.org/project/keyring')
|
||||
SECRETSTORAGE_AVAILABLE = False
|
||||
SECRETSTORAGE_UNAVAILABLE_REASON = (
|
||||
'as the `secretstorage` module is not installed. '
|
||||
'Please install by running `python3 -m pip install secretstorage`.')
|
||||
except Exception as _err:
|
||||
KEYRING_AVAILABLE = False
|
||||
KEYRING_UNAVAILABLE_REASON = 'as the `keyring` module could not be initialized: %s' % _err
|
||||
SECRETSTORAGE_AVAILABLE = False
|
||||
SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}'
|
||||
|
||||
|
||||
CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
|
||||
@@ -74,8 +72,8 @@ class YDLLogger:
|
||||
def load_cookies(cookie_file, browser_specification, ydl):
|
||||
cookie_jars = []
|
||||
if browser_specification is not None:
|
||||
browser_name, profile = _parse_browser_specification(*browser_specification)
|
||||
cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl)))
|
||||
browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
|
||||
cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
|
||||
|
||||
if cookie_file is not None:
|
||||
cookie_file = expand_path(cookie_file)
|
||||
@@ -87,13 +85,13 @@ def load_cookies(cookie_file, browser_specification, ydl):
|
||||
return _merge_cookie_jars(cookie_jars)
|
||||
|
||||
|
||||
def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()):
|
||||
def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
|
||||
if browser_name == 'firefox':
|
||||
return _extract_firefox_cookies(profile, logger)
|
||||
elif browser_name == 'safari':
|
||||
return _extract_safari_cookies(profile, logger)
|
||||
elif browser_name in CHROMIUM_BASED_BROWSERS:
|
||||
return _extract_chrome_cookies(browser_name, profile, logger)
|
||||
return _extract_chrome_cookies(browser_name, profile, keyring, logger)
|
||||
else:
|
||||
raise ValueError('unknown browser: {}'.format(browser_name))
|
||||
|
||||
@@ -207,7 +205,7 @@ def _get_chromium_based_browser_settings(browser_name):
|
||||
}
|
||||
|
||||
|
||||
def _extract_chrome_cookies(browser_name, profile, logger):
|
||||
def _extract_chrome_cookies(browser_name, profile, keyring, logger):
|
||||
logger.info('Extracting cookies from {}'.format(browser_name))
|
||||
|
||||
if not SQLITE_AVAILABLE:
|
||||
@@ -234,7 +232,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
|
||||
raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
|
||||
logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
|
||||
|
||||
decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger)
|
||||
decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
|
||||
cursor = None
|
||||
@@ -247,6 +245,7 @@ def _extract_chrome_cookies(browser_name, profile, logger):
|
||||
'expires_utc, {} FROM cookies'.format(secure_column))
|
||||
jar = YoutubeDLCookieJar()
|
||||
failed_cookies = 0
|
||||
unencrypted_cookies = 0
|
||||
for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
|
||||
host_key = host_key.decode('utf-8')
|
||||
name = name.decode('utf-8')
|
||||
@@ -258,6 +257,8 @@ def _extract_chrome_cookies(browser_name, profile, logger):
|
||||
if value is None:
|
||||
failed_cookies += 1
|
||||
continue
|
||||
else:
|
||||
unencrypted_cookies += 1
|
||||
|
||||
cookie = compat_cookiejar_Cookie(
|
||||
version=0, name=name, value=value, port=None, port_specified=False,
|
||||
@@ -270,6 +271,9 @@ def _extract_chrome_cookies(browser_name, profile, logger):
|
||||
else:
|
||||
failed_message = ''
|
||||
logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
|
||||
counts = decryptor.cookie_counts.copy()
|
||||
counts['unencrypted'] = unencrypted_cookies
|
||||
logger.debug('cookie version breakdown: {}'.format(counts))
|
||||
return jar
|
||||
finally:
|
||||
if cursor is not None:
|
||||
@@ -305,10 +309,14 @@ class ChromeCookieDecryptor:
|
||||
def decrypt(self, encrypted_value):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
|
||||
|
||||
def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
|
||||
if sys.platform in ('linux', 'linux2'):
|
||||
return LinuxChromeCookieDecryptor(browser_keyring_name, logger)
|
||||
return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
|
||||
elif sys.platform == 'darwin':
|
||||
return MacChromeCookieDecryptor(browser_keyring_name, logger)
|
||||
elif sys.platform == 'win32':
|
||||
@@ -319,13 +327,12 @@ def get_cookie_decryptor(browser_root, browser_keyring_name, logger):
|
||||
|
||||
|
||||
class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
def __init__(self, browser_keyring_name, logger):
|
||||
def __init__(self, browser_keyring_name, logger, *, keyring=None):
|
||||
self._logger = logger
|
||||
self._v10_key = self.derive_key(b'peanuts')
|
||||
if KEYRING_AVAILABLE:
|
||||
self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name))
|
||||
else:
|
||||
self._v11_key = None
|
||||
password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
|
||||
self._v11_key = None if password is None else self.derive_key(password)
|
||||
self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
|
||||
|
||||
@staticmethod
|
||||
def derive_key(password):
|
||||
@@ -333,20 +340,27 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
|
||||
return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
|
||||
|
||||
elif version == b'v11':
|
||||
self._cookie_counts['v11'] += 1
|
||||
if self._v11_key is None:
|
||||
self._logger.warning(f'cannot decrypt cookie {KEYRING_UNAVAILABLE_REASON}', only_once=True)
|
||||
self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
|
||||
return None
|
||||
return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
return None
|
||||
|
||||
|
||||
@@ -355,6 +369,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
self._logger = logger
|
||||
password = _get_mac_keyring_password(browser_keyring_name, logger)
|
||||
self._v10_key = None if password is None else self.derive_key(password)
|
||||
self._cookie_counts = {'v10': 0, 'other': 0}
|
||||
|
||||
@staticmethod
|
||||
def derive_key(password):
|
||||
@@ -362,11 +377,16 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
|
||||
return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
if self._v10_key is None:
|
||||
self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
|
||||
return None
|
||||
@@ -374,6 +394,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
# other prefixes are considered 'old data' which were stored as plaintext
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
|
||||
return encrypted_value
|
||||
@@ -383,12 +404,18 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
def __init__(self, browser_root, logger):
|
||||
self._logger = logger
|
||||
self._v10_key = _get_windows_v10_key(browser_root, logger)
|
||||
self._cookie_counts = {'v10': 0, 'other': 0}
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
if self._v10_key is None:
|
||||
self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
|
||||
return None
|
||||
@@ -408,6 +435,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
# any other prefix means the data is DPAPI encrypted
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
|
||||
return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
|
||||
@@ -577,42 +605,221 @@ def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
|
||||
return jar
|
||||
|
||||
|
||||
def _get_linux_keyring_password(browser_keyring_name):
|
||||
password = keyring.get_password('{} Keys'.format(browser_keyring_name),
|
||||
'{} Safe Storage'.format(browser_keyring_name))
|
||||
if password is None:
|
||||
# this sometimes occurs in KDE because chrome does not check hasEntry and instead
|
||||
# just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry
|
||||
# to verify this:
|
||||
# dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
|
||||
# while starting chrome.
|
||||
# this may be a bug as the intended behaviour is to generate a random password and store
|
||||
# it, but that doesn't matter here.
|
||||
password = ''
|
||||
return password.encode('utf-8')
|
||||
class _LinuxDesktopEnvironment(Enum):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
|
||||
DesktopEnvironment
|
||||
"""
|
||||
OTHER = auto()
|
||||
CINNAMON = auto()
|
||||
GNOME = auto()
|
||||
KDE = auto()
|
||||
PANTHEON = auto()
|
||||
UNITY = auto()
|
||||
XFCE = auto()
|
||||
|
||||
|
||||
class _LinuxKeyring(Enum):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
|
||||
SelectedLinuxBackend
|
||||
"""
|
||||
KWALLET = auto()
|
||||
GNOMEKEYRING = auto()
|
||||
BASICTEXT = auto()
|
||||
|
||||
|
||||
SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
|
||||
|
||||
|
||||
def _get_linux_desktop_environment(env):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
|
||||
GetDesktopEnvironment
|
||||
"""
|
||||
xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
|
||||
desktop_session = env.get('DESKTOP_SESSION', None)
|
||||
if xdg_current_desktop is not None:
|
||||
xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
|
||||
|
||||
if xdg_current_desktop == 'Unity':
|
||||
if desktop_session is not None and 'gnome-fallback' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
else:
|
||||
return _LinuxDesktopEnvironment.UNITY
|
||||
elif xdg_current_desktop == 'GNOME':
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif xdg_current_desktop == 'X-Cinnamon':
|
||||
return _LinuxDesktopEnvironment.CINNAMON
|
||||
elif xdg_current_desktop == 'KDE':
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
elif xdg_current_desktop == 'Pantheon':
|
||||
return _LinuxDesktopEnvironment.PANTHEON
|
||||
elif xdg_current_desktop == 'XFCE':
|
||||
return _LinuxDesktopEnvironment.XFCE
|
||||
elif desktop_session is not None:
|
||||
if desktop_session in ('mate', 'gnome'):
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif 'kde' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
elif 'xfce' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.XFCE
|
||||
else:
|
||||
if 'GNOME_DESKTOP_SESSION_ID' in env:
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif 'KDE_FULL_SESSION' in env:
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
else:
|
||||
return _LinuxDesktopEnvironment.OTHER
|
||||
|
||||
|
||||
def _choose_linux_keyring(logger):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
|
||||
SelectBackend
|
||||
"""
|
||||
desktop_environment = _get_linux_desktop_environment(os.environ)
|
||||
logger.debug('detected desktop environment: {}'.format(desktop_environment.name))
|
||||
if desktop_environment == _LinuxDesktopEnvironment.KDE:
|
||||
linux_keyring = _LinuxKeyring.KWALLET
|
||||
elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
|
||||
linux_keyring = _LinuxKeyring.BASICTEXT
|
||||
else:
|
||||
linux_keyring = _LinuxKeyring.GNOMEKEYRING
|
||||
return linux_keyring
|
||||
|
||||
|
||||
def _get_kwallet_network_wallet(logger):
|
||||
""" The name of the wallet used to store network passwords.
|
||||
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
|
||||
KWalletDBus::NetworkWallet
|
||||
which does a dbus call to the following function:
|
||||
https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
|
||||
Wallet::NetworkWallet
|
||||
"""
|
||||
default_wallet = 'kdewallet'
|
||||
try:
|
||||
proc = Popen([
|
||||
'dbus-send', '--session', '--print-reply=literal',
|
||||
'--dest=org.kde.kwalletd5',
|
||||
'/modules/kwalletd5',
|
||||
'org.kde.KWallet.networkWallet'
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode != 0:
|
||||
logger.warning('failed to read NetworkWallet')
|
||||
return default_wallet
|
||||
else:
|
||||
network_wallet = stdout.decode('utf-8').strip()
|
||||
logger.debug('NetworkWallet = "{}"'.format(network_wallet))
|
||||
return network_wallet
|
||||
except BaseException as e:
|
||||
logger.warning('exception while obtaining NetworkWallet: {}'.format(e))
|
||||
return default_wallet
|
||||
|
||||
|
||||
def _get_kwallet_password(browser_keyring_name, logger):
|
||||
logger.debug('using kwallet-query to obtain password from kwallet')
|
||||
|
||||
if shutil.which('kwallet-query') is None:
|
||||
logger.error('kwallet-query command not found. KWallet and kwallet-query '
|
||||
'must be installed to read from KWallet. kwallet-query should be'
|
||||
'included in the kwallet package for your distribution')
|
||||
return b''
|
||||
|
||||
network_wallet = _get_kwallet_network_wallet(logger)
|
||||
|
||||
try:
|
||||
proc = Popen([
|
||||
'kwallet-query',
|
||||
'--read-password', '{} Safe Storage'.format(browser_keyring_name),
|
||||
'--folder', '{} Keys'.format(browser_keyring_name),
|
||||
network_wallet
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode != 0:
|
||||
logger.error('kwallet-query failed with return code {}. Please consult '
|
||||
'the kwallet-query man page for details'.format(proc.returncode))
|
||||
return b''
|
||||
else:
|
||||
if stdout.lower().startswith(b'failed to read'):
|
||||
logger.debug('failed to read password from kwallet. Using empty string instead')
|
||||
# this sometimes occurs in KDE because chrome does not check hasEntry and instead
|
||||
# just tries to read the value (which kwallet returns "") whereas kwallet-query
|
||||
# checks hasEntry. To verify this:
|
||||
# dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
|
||||
# while starting chrome.
|
||||
# this may be a bug as the intended behaviour is to generate a random password and store
|
||||
# it, but that doesn't matter here.
|
||||
return b''
|
||||
else:
|
||||
logger.debug('password found')
|
||||
if stdout[-1:] == b'\n':
|
||||
stdout = stdout[:-1]
|
||||
return stdout
|
||||
except BaseException as e:
|
||||
logger.warning(f'exception running kwallet-query: {type(e).__name__}({e})')
|
||||
return b''
|
||||
|
||||
|
||||
def _get_gnome_keyring_password(browser_keyring_name, logger):
|
||||
if not SECRETSTORAGE_AVAILABLE:
|
||||
logger.error('secretstorage not available {}'.format(SECRETSTORAGE_UNAVAILABLE_REASON))
|
||||
return b''
|
||||
# the Gnome keyring does not seem to organise keys in the same way as KWallet,
|
||||
# using `dbus-monitor` during startup, it can be observed that chromium lists all keys
|
||||
# and presumably searches for its key in the list. It appears that we must do the same.
|
||||
# https://github.com/jaraco/keyring/issues/556
|
||||
with contextlib.closing(secretstorage.dbus_init()) as con:
|
||||
col = secretstorage.get_default_collection(con)
|
||||
for item in col.get_all_items():
|
||||
if item.get_label() == '{} Safe Storage'.format(browser_keyring_name):
|
||||
return item.get_secret()
|
||||
else:
|
||||
logger.error('failed to read from keyring')
|
||||
return b''
|
||||
|
||||
|
||||
def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
|
||||
# note: chrome/chromium can be run with the following flags to determine which keyring backend
|
||||
# it has chosen to use
|
||||
# chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
|
||||
# Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
|
||||
# will not be sufficient in all cases.
|
||||
|
||||
keyring = _LinuxKeyring[keyring] or _choose_linux_keyring(logger)
|
||||
logger.debug(f'Chosen keyring: {keyring.name}')
|
||||
|
||||
if keyring == _LinuxKeyring.KWALLET:
|
||||
return _get_kwallet_password(browser_keyring_name, logger)
|
||||
elif keyring == _LinuxKeyring.GNOMEKEYRING:
|
||||
return _get_gnome_keyring_password(browser_keyring_name, logger)
|
||||
elif keyring == _LinuxKeyring.BASICTEXT:
|
||||
# when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
|
||||
return None
|
||||
assert False, f'Unknown keyring {keyring}'
|
||||
|
||||
|
||||
def _get_mac_keyring_password(browser_keyring_name, logger):
|
||||
if KEYRING_AVAILABLE:
|
||||
logger.debug('using keyring to obtain password')
|
||||
password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name)
|
||||
return password.encode('utf-8')
|
||||
else:
|
||||
logger.debug('using find-generic-password to obtain password')
|
||||
logger.debug('using find-generic-password to obtain password from OSX keychain')
|
||||
try:
|
||||
proc = Popen(
|
||||
['security', 'find-generic-password',
|
||||
'-w', # write password to stdout
|
||||
'-a', browser_keyring_name, # match 'account'
|
||||
'-s', '{} Safe Storage'.format(browser_keyring_name)], # match 'service'
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if stdout[-1:] == b'\n':
|
||||
stdout = stdout[:-1]
|
||||
return stdout
|
||||
except BaseException as e:
|
||||
logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
|
||||
return None
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if stdout[-1:] == b'\n':
|
||||
stdout = stdout[:-1]
|
||||
return stdout
|
||||
except BaseException as e:
|
||||
logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
|
||||
return None
|
||||
|
||||
|
||||
def _get_windows_v10_key(browser_root, logger):
|
||||
@@ -736,10 +943,11 @@ def _is_path(value):
|
||||
return os.path.sep in value
|
||||
|
||||
|
||||
def _parse_browser_specification(browser_name, profile=None):
|
||||
browser_name = browser_name.lower()
|
||||
def _parse_browser_specification(browser_name, profile=None, keyring=None):
|
||||
if browser_name not in SUPPORTED_BROWSERS:
|
||||
raise ValueError(f'unsupported browser: "{browser_name}"')
|
||||
if keyring not in (None, *SUPPORTED_KEYRINGS):
|
||||
raise ValueError(f'unsupported keyring: "{keyring}"')
|
||||
if profile is not None and _is_path(profile):
|
||||
profile = os.path.expanduser(profile)
|
||||
return browser_name, profile
|
||||
return browser_name, profile, keyring
|
||||
|
||||
@@ -12,10 +12,15 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
|
||||
info_copy = info_dict.copy()
|
||||
info_copy['to_stdout'] = to_stdout
|
||||
|
||||
downloaders = [_get_suitable_downloader(info_copy, proto, params, default)
|
||||
for proto in (protocol or info_copy['protocol']).split('+')]
|
||||
protocols = (protocol or info_copy['protocol']).split('+')
|
||||
downloaders = [_get_suitable_downloader(info_copy, proto, params, default) for proto in protocols]
|
||||
|
||||
if set(downloaders) == {FFmpegFD} and FFmpegFD.can_merge_formats(info_copy, params):
|
||||
return FFmpegFD
|
||||
elif (set(downloaders) == {DashSegmentsFD}
|
||||
and not (to_stdout and len(protocols) > 1)
|
||||
and set(protocols) == {'http_dash_segments_generator'}):
|
||||
return DashSegmentsFD
|
||||
elif len(downloaders) == 1:
|
||||
return downloaders[0]
|
||||
return None
|
||||
@@ -49,6 +54,7 @@ PROTOCOL_MAP = {
|
||||
'rtsp': RtspFD,
|
||||
'f4m': F4mFD,
|
||||
'http_dash_segments': DashSegmentsFD,
|
||||
'http_dash_segments_generator': DashSegmentsFD,
|
||||
'ism': IsmFD,
|
||||
'mhtml': MhtmlFD,
|
||||
'niconico_dmc': NiconicoDmcFD,
|
||||
@@ -63,6 +69,7 @@ def shorten_protocol_name(proto, simplify=False):
|
||||
'm3u8_native': 'm3u8_n',
|
||||
'rtmp_ffmpeg': 'rtmp_f',
|
||||
'http_dash_segments': 'dash',
|
||||
'http_dash_segments_generator': 'dash_g',
|
||||
'niconico_dmc': 'dmc',
|
||||
'websocket_frag': 'WSfrag',
|
||||
}
|
||||
@@ -71,6 +78,7 @@ def shorten_protocol_name(proto, simplify=False):
|
||||
'https': 'http',
|
||||
'ftps': 'ftp',
|
||||
'm3u8_native': 'm3u8',
|
||||
'http_dash_segments_generator': 'dash',
|
||||
'rtmp_ffmpeg': 'rtmp',
|
||||
'm3u8_frag_urls': 'm3u8',
|
||||
'dash_frag_urls': 'dash',
|
||||
|
||||
@@ -4,12 +4,14 @@ import os
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import errno
|
||||
|
||||
from ..utils import (
|
||||
decodeArgument,
|
||||
encodeFilename,
|
||||
error_to_compat_str,
|
||||
format_bytes,
|
||||
sanitize_open,
|
||||
shell_quote,
|
||||
timeconvert,
|
||||
timetuple_from_msec,
|
||||
@@ -39,6 +41,7 @@ class FileDownloader(object):
|
||||
ratelimit: Download speed limit, in bytes/sec.
|
||||
throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
|
||||
retries: Number of times to retry for HTTP error 5xx
|
||||
file_access_retries: Number of times to retry on file access error
|
||||
buffersize: Size of download buffer in bytes.
|
||||
noresizebuffer: Do not automatically resize the download buffer.
|
||||
continuedl: Try to continue downloads if possible.
|
||||
@@ -207,6 +210,21 @@ class FileDownloader(object):
|
||||
def ytdl_filename(self, filename):
|
||||
return filename + '.ytdl'
|
||||
|
||||
def sanitize_open(self, filename, open_mode):
|
||||
file_access_retries = self.params.get('file_access_retries', 10)
|
||||
retry = 0
|
||||
while True:
|
||||
try:
|
||||
return sanitize_open(filename, open_mode)
|
||||
except (IOError, OSError) as err:
|
||||
retry = retry + 1
|
||||
if retry > file_access_retries or err.errno not in (errno.EACCES,):
|
||||
raise
|
||||
self.to_screen(
|
||||
'[download] Got file access error. Retrying (attempt %d of %s) ...'
|
||||
% (retry, self.format_retries(file_access_retries)))
|
||||
time.sleep(0.01)
|
||||
|
||||
def try_rename(self, old_filename, new_filename):
|
||||
if old_filename == new_filename:
|
||||
return
|
||||
@@ -397,6 +415,7 @@ class FileDownloader(object):
|
||||
'status': 'finished',
|
||||
'total_bytes': os.path.getsize(encodeFilename(filename)),
|
||||
}, info_dict)
|
||||
self._finish_multiline_status()
|
||||
return True, False
|
||||
|
||||
if subtitle is False:
|
||||
|
||||
+43
-25
@@ -1,4 +1,5 @@
|
||||
from __future__ import unicode_literals
|
||||
import time
|
||||
|
||||
from ..downloader import get_suitable_downloader
|
||||
from .fragment import FragmentFD
|
||||
@@ -15,27 +16,53 @@ class DashSegmentsFD(FragmentFD):
|
||||
FD_NAME = 'dashsegments'
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
if info_dict.get('is_live'):
|
||||
if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}:
|
||||
self.report_error('Live DASH videos are not supported')
|
||||
|
||||
fragment_base_url = info_dict.get('fragment_base_url')
|
||||
fragments = info_dict['fragments'][:1] if self.params.get(
|
||||
'test', False) else info_dict['fragments']
|
||||
|
||||
real_start = time.time()
|
||||
real_downloader = get_suitable_downloader(
|
||||
info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': len(fragments),
|
||||
}
|
||||
requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
|
||||
args = []
|
||||
for fmt in requested_formats or [info_dict]:
|
||||
try:
|
||||
fragment_count = 1 if self.params.get('test') else len(fmt['fragments'])
|
||||
except TypeError:
|
||||
fragment_count = None
|
||||
ctx = {
|
||||
'filename': fmt.get('filepath') or filename,
|
||||
'live': 'is_from_start' if fmt.get('is_from_start') else fmt.get('is_live'),
|
||||
'total_frags': fragment_count,
|
||||
}
|
||||
|
||||
if real_downloader:
|
||||
self._prepare_external_frag_download(ctx)
|
||||
else:
|
||||
self._prepare_and_start_frag_download(ctx, info_dict)
|
||||
if real_downloader:
|
||||
self._prepare_external_frag_download(ctx)
|
||||
else:
|
||||
self._prepare_and_start_frag_download(ctx, fmt)
|
||||
ctx['start'] = real_start
|
||||
|
||||
fragments_to_download = self._get_fragments(fmt, ctx)
|
||||
|
||||
if real_downloader:
|
||||
self.to_screen(
|
||||
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
|
||||
info_dict['fragments'] = list(fragments_to_download)
|
||||
fd = real_downloader(self.ydl, self.params)
|
||||
return fd.real_download(filename, info_dict)
|
||||
|
||||
args.append([ctx, fragments_to_download, fmt])
|
||||
|
||||
return self.download_and_append_fragments_multiple(*args)
|
||||
|
||||
def _resolve_fragments(self, fragments, ctx):
|
||||
fragments = fragments(ctx) if callable(fragments) else fragments
|
||||
return [next(iter(fragments))] if self.params.get('test') else fragments
|
||||
|
||||
def _get_fragments(self, fmt, ctx):
|
||||
fragment_base_url = fmt.get('fragment_base_url')
|
||||
fragments = self._resolve_fragments(fmt['fragments'], ctx)
|
||||
|
||||
fragments_to_download = []
|
||||
frag_index = 0
|
||||
for i, fragment in enumerate(fragments):
|
||||
frag_index += 1
|
||||
@@ -46,17 +73,8 @@ class DashSegmentsFD(FragmentFD):
|
||||
assert fragment_base_url
|
||||
fragment_url = urljoin(fragment_base_url, fragment['path'])
|
||||
|
||||
fragments_to_download.append({
|
||||
yield {
|
||||
'frag_index': frag_index,
|
||||
'index': i,
|
||||
'url': fragment_url,
|
||||
})
|
||||
|
||||
if real_downloader:
|
||||
self.to_screen(
|
||||
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
|
||||
info_dict['fragments'] = fragments_to_download
|
||||
fd = real_downloader(self.ydl, self.params)
|
||||
return fd.real_download(filename, info_dict)
|
||||
|
||||
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ from ..utils import (
|
||||
handle_youtubedl_headers,
|
||||
check_executable,
|
||||
Popen,
|
||||
sanitize_open,
|
||||
)
|
||||
|
||||
|
||||
@@ -144,11 +143,11 @@ class ExternalFD(FragmentFD):
|
||||
return -1
|
||||
|
||||
decrypt_fragment = self.decrypter(info_dict)
|
||||
dest, _ = sanitize_open(tmpfilename, 'wb')
|
||||
dest, _ = self.sanitize_open(tmpfilename, 'wb')
|
||||
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||
fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index)
|
||||
try:
|
||||
src, _ = sanitize_open(fragment_filename, 'rb')
|
||||
src, _ = self.sanitize_open(fragment_filename, 'rb')
|
||||
except IOError as err:
|
||||
if skip_unavailable_fragments and frag_index > 1:
|
||||
self.report_skip_fragment(frag_index, err)
|
||||
@@ -266,6 +265,7 @@ class Aria2cFD(ExternalFD):
|
||||
cmd += self._option('--all-proxy', 'proxy')
|
||||
cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
|
||||
cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
|
||||
cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
|
||||
cmd += self._configuration_args()
|
||||
|
||||
# aria2c strips out spaces from the beginning/end of filenames and paths.
|
||||
@@ -290,7 +290,7 @@ class Aria2cFD(ExternalFD):
|
||||
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||
fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
|
||||
url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename))
|
||||
stream, _ = sanitize_open(url_list_file, 'wb')
|
||||
stream, _ = self.sanitize_open(url_list_file, 'wb')
|
||||
stream.write('\n'.join(url_list).encode('utf-8'))
|
||||
stream.close()
|
||||
cmd += ['-i', url_list_file]
|
||||
|
||||
@@ -366,7 +366,7 @@ class F4mFD(FragmentFD):
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': total_frags,
|
||||
'live': live,
|
||||
'live': bool(live),
|
||||
}
|
||||
|
||||
self._prepare_frag_download(ctx)
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
from math import ceil
|
||||
|
||||
try:
|
||||
import concurrent.futures
|
||||
@@ -15,6 +16,7 @@ from .common import FileDownloader
|
||||
from .http import HttpFD
|
||||
from ..aes import aes_cbc_decrypt_bytes
|
||||
from ..compat import (
|
||||
compat_os_name,
|
||||
compat_urllib_error,
|
||||
compat_struct_pack,
|
||||
)
|
||||
@@ -22,7 +24,6 @@ from ..utils import (
|
||||
DownloadError,
|
||||
error_to_compat_str,
|
||||
encodeFilename,
|
||||
sanitize_open,
|
||||
sanitized_Request,
|
||||
)
|
||||
|
||||
@@ -90,11 +91,11 @@ class FragmentFD(FileDownloader):
|
||||
self._start_frag_download(ctx, info_dict)
|
||||
|
||||
def __do_ytdl_file(self, ctx):
|
||||
return not ctx['live'] and not ctx['tmpfilename'] == '-' and not self.params.get('_no_ytdl_file')
|
||||
return ctx['live'] is not True and ctx['tmpfilename'] != '-' and not self.params.get('_no_ytdl_file')
|
||||
|
||||
def _read_ytdl_file(self, ctx):
|
||||
assert 'ytdl_corrupt' not in ctx
|
||||
stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
|
||||
stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
|
||||
try:
|
||||
ytdl_data = json.loads(stream.read())
|
||||
ctx['fragment_index'] = ytdl_data['downloader']['current_fragment']['index']
|
||||
@@ -106,7 +107,7 @@ class FragmentFD(FileDownloader):
|
||||
stream.close()
|
||||
|
||||
def _write_ytdl_file(self, ctx):
|
||||
frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
|
||||
frag_index_stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
|
||||
try:
|
||||
downloader = {
|
||||
'current_fragment': {
|
||||
@@ -138,7 +139,7 @@ class FragmentFD(FileDownloader):
|
||||
return True, self._read_fragment(ctx)
|
||||
|
||||
def _read_fragment(self, ctx):
|
||||
down, frag_sanitized = sanitize_open(ctx['fragment_filename_sanitized'], 'rb')
|
||||
down, frag_sanitized = self.sanitize_open(ctx['fragment_filename_sanitized'], 'rb')
|
||||
ctx['fragment_filename_sanitized'] = frag_sanitized
|
||||
frag_content = down.read()
|
||||
down.close()
|
||||
@@ -214,7 +215,7 @@ class FragmentFD(FileDownloader):
|
||||
self._write_ytdl_file(ctx)
|
||||
assert ctx['fragment_index'] == 0
|
||||
|
||||
dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
|
||||
dest_stream, tmpfilename = self.sanitize_open(tmpfilename, open_mode)
|
||||
|
||||
ctx.update({
|
||||
'dl': dl,
|
||||
@@ -375,17 +376,20 @@ class FragmentFD(FileDownloader):
|
||||
@params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
|
||||
all args must be either tuple or list
|
||||
'''
|
||||
interrupt_trigger = [True]
|
||||
max_progress = len(args)
|
||||
if max_progress == 1:
|
||||
return self.download_and_append_fragments(*args[0], pack_func=pack_func, finish_func=finish_func)
|
||||
max_workers = self.params.get('concurrent_fragment_downloads', max_progress)
|
||||
max_workers = self.params.get('concurrent_fragment_downloads', 1)
|
||||
if max_progress > 1:
|
||||
self._prepare_multiline_status(max_progress)
|
||||
|
||||
def thread_func(idx, ctx, fragments, info_dict, tpe):
|
||||
ctx['max_progress'] = max_progress
|
||||
ctx['progress_idx'] = idx
|
||||
return self.download_and_append_fragments(ctx, fragments, info_dict, pack_func=pack_func, finish_func=finish_func, tpe=tpe)
|
||||
return self.download_and_append_fragments(
|
||||
ctx, fragments, info_dict, pack_func=pack_func, finish_func=finish_func,
|
||||
tpe=tpe, interrupt_trigger=interrupt_trigger)
|
||||
|
||||
class FTPE(concurrent.futures.ThreadPoolExecutor):
|
||||
# has to stop this or it's going to wait on the worker thread itself
|
||||
@@ -393,8 +397,11 @@ class FragmentFD(FileDownloader):
|
||||
pass
|
||||
|
||||
spins = []
|
||||
if compat_os_name == 'nt':
|
||||
self.report_warning('Ctrl+C does not work on Windows when used with parallel threads. '
|
||||
'This is a known issue and patches are welcome')
|
||||
for idx, (ctx, fragments, info_dict) in enumerate(args):
|
||||
tpe = FTPE(ceil(max_workers / max_progress))
|
||||
tpe = FTPE(math.ceil(max_workers / max_progress))
|
||||
job = tpe.submit(thread_func, idx, ctx, fragments, info_dict, tpe)
|
||||
spins.append((tpe, job))
|
||||
|
||||
@@ -402,18 +409,33 @@ class FragmentFD(FileDownloader):
|
||||
for tpe, job in spins:
|
||||
try:
|
||||
result = result and job.result()
|
||||
except KeyboardInterrupt:
|
||||
interrupt_trigger[0] = False
|
||||
finally:
|
||||
tpe.shutdown(wait=True)
|
||||
if not interrupt_trigger[0]:
|
||||
raise KeyboardInterrupt()
|
||||
return result
|
||||
|
||||
def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None):
|
||||
def download_and_append_fragments(
|
||||
self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None,
|
||||
tpe=None, interrupt_trigger=None):
|
||||
if not interrupt_trigger:
|
||||
interrupt_trigger = (True, )
|
||||
|
||||
fragment_retries = self.params.get('fragment_retries', 0)
|
||||
is_fatal = (lambda idx: idx == 0) if self.params.get('skip_unavailable_fragments', True) else (lambda _: True)
|
||||
is_fatal = (
|
||||
((lambda _: False) if info_dict.get('is_live') else (lambda idx: idx == 0))
|
||||
if self.params.get('skip_unavailable_fragments', True) else (lambda _: True))
|
||||
|
||||
if not pack_func:
|
||||
pack_func = lambda frag_content, _: frag_content
|
||||
|
||||
def download_fragment(fragment, ctx):
|
||||
frag_index = ctx['fragment_index'] = fragment['frag_index']
|
||||
ctx['last_error'] = None
|
||||
if not interrupt_trigger[0]:
|
||||
return False, frag_index
|
||||
headers = info_dict.get('http_headers', {}).copy()
|
||||
byte_range = fragment.get('byte_range')
|
||||
if byte_range:
|
||||
@@ -428,12 +450,13 @@ class FragmentFD(FileDownloader):
|
||||
if not success:
|
||||
return False, frag_index
|
||||
break
|
||||
except compat_urllib_error.HTTPError as err:
|
||||
except (compat_urllib_error.HTTPError, http.client.IncompleteRead) as err:
|
||||
# Unavailable (possibly temporary) fragments may be served.
|
||||
# First we try to retry then either skip or abort.
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/10165,
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/10448).
|
||||
count += 1
|
||||
ctx['last_error'] = err
|
||||
if count <= fragment_retries:
|
||||
self.report_retry_fragment(err, frag_index, count, fragment_retries)
|
||||
except DownloadError:
|
||||
@@ -466,7 +489,8 @@ class FragmentFD(FileDownloader):
|
||||
|
||||
decrypt_fragment = self.decrypter(info_dict)
|
||||
|
||||
max_workers = self.params.get('concurrent_fragment_downloads', 1)
|
||||
max_workers = math.ceil(
|
||||
self.params.get('concurrent_fragment_downloads', 1) / ctx.get('max_progress', 1))
|
||||
if can_threaded_download and max_workers > 1:
|
||||
|
||||
def _download_fragment(fragment):
|
||||
@@ -477,6 +501,8 @@ class FragmentFD(FileDownloader):
|
||||
self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome')
|
||||
with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
|
||||
for fragment, frag_content, frag_index, frag_filename in pool.map(_download_fragment, fragments):
|
||||
if not interrupt_trigger[0]:
|
||||
break
|
||||
ctx['fragment_filename_sanitized'] = frag_filename
|
||||
ctx['fragment_index'] = frag_index
|
||||
result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
|
||||
@@ -484,6 +510,8 @@ class FragmentFD(FileDownloader):
|
||||
return False
|
||||
else:
|
||||
for fragment in fragments:
|
||||
if not interrupt_trigger[0]:
|
||||
break
|
||||
frag_content, frag_index = download_fragment(fragment, ctx)
|
||||
result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
|
||||
if not result:
|
||||
|
||||
@@ -16,7 +16,6 @@ from ..utils import (
|
||||
ContentTooShortError,
|
||||
encodeFilename,
|
||||
int_or_none,
|
||||
sanitize_open,
|
||||
sanitized_Request,
|
||||
ThrottledDownload,
|
||||
write_xattr,
|
||||
@@ -263,7 +262,7 @@ class HttpFD(FileDownloader):
|
||||
# Open destination file just in time
|
||||
if ctx.stream is None:
|
||||
try:
|
||||
ctx.stream, ctx.tmpfilename = sanitize_open(
|
||||
ctx.stream, ctx.tmpfilename = self.sanitize_open(
|
||||
ctx.tmpfilename, ctx.open_mode)
|
||||
assert ctx.stream is not None
|
||||
ctx.filename = self.undo_temp_name(ctx.tmpfilename)
|
||||
|
||||
+64
-2
@@ -8,6 +8,7 @@ import time
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
dict_get,
|
||||
ExtractorError,
|
||||
js_to_json,
|
||||
int_or_none,
|
||||
@@ -233,8 +234,6 @@ class ABCIViewIE(InfoExtractor):
|
||||
}]
|
||||
|
||||
is_live = video_params.get('livestream') == '1'
|
||||
if is_live:
|
||||
title = self._live_title(title)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
@@ -255,3 +254,66 @@ class ABCIViewIE(InfoExtractor):
|
||||
'subtitles': subtitles,
|
||||
'is_live': is_live,
|
||||
}
|
||||
|
||||
|
||||
class ABCIViewShowSeriesIE(InfoExtractor):
|
||||
IE_NAME = 'abc.net.au:iview:showseries'
|
||||
_VALID_URL = r'https?://iview\.abc\.net\.au/show/(?P<id>[^/]+)(?:/series/\d+)?$'
|
||||
_GEO_COUNTRIES = ['AU']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
|
||||
'info_dict': {
|
||||
'id': '124870-1',
|
||||
'title': 'Series 1',
|
||||
'description': 'md5:93119346c24a7c322d446d8eece430ff',
|
||||
'series': 'Upper Middle Bogan',
|
||||
'season': 'Series 1',
|
||||
'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$'
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
|
||||
'info_dict': {
|
||||
'id': 'CO1108V001S00',
|
||||
'ext': 'mp4',
|
||||
'title': 'Series 1 Ep 1 I\'m A Swan',
|
||||
'description': 'md5:7b676758c1de11a30b79b4d301e8da93',
|
||||
'series': 'Upper Middle Bogan',
|
||||
'uploader_id': 'abc1',
|
||||
'upload_date': '20210630',
|
||||
'timestamp': 1625036400,
|
||||
},
|
||||
'params': {
|
||||
'noplaylist': True,
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
show_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, show_id)
|
||||
webpage_data = self._search_regex(
|
||||
r'window\.__INITIAL_STATE__\s*=\s*[\'"](.+?)[\'"]\s*;',
|
||||
webpage, 'initial state')
|
||||
video_data = self._parse_json(
|
||||
unescapeHTML(webpage_data).encode('utf-8').decode('unicode_escape'), show_id)
|
||||
video_data = video_data['route']['pageData']['_embedded']
|
||||
|
||||
if self.get_param('noplaylist') and 'highlightVideo' in video_data:
|
||||
self.to_screen('Downloading just the highlight video because of --no-playlist')
|
||||
return self.url_result(video_data['highlightVideo']['shareUrl'], ie=ABCIViewIE.ie_key())
|
||||
|
||||
self.to_screen(f'Downloading playlist {show_id} - add --no-playlist to just download the highlight video')
|
||||
series = video_data['selectedSeries']
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'entries': [self.url_result(episode['shareUrl'])
|
||||
for episode in series['_embedded']['videoEpisodes']],
|
||||
'id': series.get('id'),
|
||||
'title': dict_get(series, ('title', 'displaySubtitle')),
|
||||
'description': series.get('description'),
|
||||
'series': dict_get(series, ('showTitle', 'displayTitle')),
|
||||
'season': dict_get(series, ('title', 'displaySubtitle')),
|
||||
'thumbnail': series.get('thumbnail'),
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ class AdobeConnectIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'is_live': is_live,
|
||||
}
|
||||
|
||||
+360
-107
@@ -3,33 +3,36 @@ from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .youtube import YoutubeIE
|
||||
from .youtube import YoutubeIE, YoutubeBaseInfoExtractor
|
||||
from ..compat import (
|
||||
compat_urllib_parse_unquote,
|
||||
compat_urllib_parse_unquote_plus,
|
||||
compat_HTTPError
|
||||
)
|
||||
from ..utils import (
|
||||
bug_reports_message,
|
||||
clean_html,
|
||||
determine_ext,
|
||||
dict_get,
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
get_element_by_id,
|
||||
HEADRequest,
|
||||
int_or_none,
|
||||
KNOWN_EXTENSIONS,
|
||||
merge_dicts,
|
||||
mimetype2ext,
|
||||
orderedSet,
|
||||
parse_duration,
|
||||
parse_qs,
|
||||
RegexNotFoundError,
|
||||
str_to_int,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
unified_strdate,
|
||||
unified_timestamp,
|
||||
urlhandle_detect_ext,
|
||||
url_or_none
|
||||
)
|
||||
|
||||
|
||||
@@ -262,12 +265,12 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
_VALID_URL = r"""(?x)^
|
||||
(?:https?://)?web\.archive\.org/
|
||||
(?:web/)?
|
||||
(?:[0-9A-Za-z_*]+/)? # /web and the version index is optional
|
||||
(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)? # /web and the version index is optional
|
||||
|
||||
(?:https?(?::|%3[Aa])//)?
|
||||
(?:
|
||||
(?:\w+\.)?youtube\.com/watch(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD]) # Youtube URL
|
||||
|(wayback-fakeurl\.archive\.org/yt/) # Or the internal fake url
|
||||
(?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD]) # Youtube URL
|
||||
|(?:wayback-fakeurl\.archive\.org/yt/) # Or the internal fake url
|
||||
)
|
||||
(?P<id>[0-9A-Za-z_-]{11})(?:%26|\#|&|$)
|
||||
"""
|
||||
@@ -278,141 +281,391 @@ class YoutubeWebArchiveIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': 'aYAGB11YrSs',
|
||||
'ext': 'webm',
|
||||
'title': 'Team Fortress 2 - Sandviches!'
|
||||
'title': 'Team Fortress 2 - Sandviches!',
|
||||
'description': 'md5:4984c0f9a07f349fc5d8e82ab7af4eaf',
|
||||
'upload_date': '20110926',
|
||||
'uploader': 'Zeurel',
|
||||
'channel_id': 'UCukCyHaD-bK3in_pKpfH9Eg',
|
||||
'duration': 32,
|
||||
'uploader_id': 'Zeurel',
|
||||
'uploader_url': 'http://www.youtube.com/user/Zeurel'
|
||||
}
|
||||
},
|
||||
{
|
||||
}, {
|
||||
# Internal link
|
||||
'url': 'https://web.archive.org/web/2oe/http://wayback-fakeurl.archive.org/yt/97t7Xj_iBv0',
|
||||
'info_dict': {
|
||||
'id': '97t7Xj_iBv0',
|
||||
'ext': 'mp4',
|
||||
'title': 'How Flexible Machines Could Save The World'
|
||||
'title': 'Why Machines That Bend Are Better',
|
||||
'description': 'md5:00404df2c632d16a674ff8df1ecfbb6c',
|
||||
'upload_date': '20190312',
|
||||
'uploader': 'Veritasium',
|
||||
'channel_id': 'UCHnyfMqiRRG1u-2MsSQLbXA',
|
||||
'duration': 771,
|
||||
'uploader_id': '1veritasium',
|
||||
'uploader_url': 'http://www.youtube.com/user/1veritasium'
|
||||
}
|
||||
},
|
||||
{
|
||||
# Video from 2012, webm format itag 45.
|
||||
}, {
|
||||
# Video from 2012, webm format itag 45. Newest capture is deleted video, with an invalid description.
|
||||
# Should use the date in the link. Title ends with '- Youtube'. Capture has description in eow-description
|
||||
'url': 'https://web.archive.org/web/20120712231619/http://www.youtube.com/watch?v=AkhihxRKcrs&gl=US&hl=en',
|
||||
'info_dict': {
|
||||
'id': 'AkhihxRKcrs',
|
||||
'ext': 'webm',
|
||||
'title': 'Limited Run: Mondo\'s Modern Classic 1 of 3 (SDCC 2012)'
|
||||
'title': 'Limited Run: Mondo\'s Modern Classic 1 of 3 (SDCC 2012)',
|
||||
'upload_date': '20120712',
|
||||
'duration': 398,
|
||||
'description': 'md5:ff4de6a7980cb65d951c2f6966a4f2f3',
|
||||
'uploader_id': 'machinima',
|
||||
'uploader_url': 'http://www.youtube.com/user/machinima'
|
||||
}
|
||||
},
|
||||
{
|
||||
# Old flash-only video. Webpage title starts with "YouTube - ".
|
||||
}, {
|
||||
# FLV video. Video file URL does not provide itag information
|
||||
'url': 'https://web.archive.org/web/20081211103536/http://www.youtube.com/watch?v=jNQXAC9IVRw',
|
||||
'info_dict': {
|
||||
'id': 'jNQXAC9IVRw',
|
||||
'ext': 'unknown_video',
|
||||
'title': 'Me at the zoo'
|
||||
'ext': 'flv',
|
||||
'title': 'Me at the zoo',
|
||||
'upload_date': '20050423',
|
||||
'channel_id': 'UC4QobU6STFB0P71PMvOGN5A',
|
||||
'duration': 19,
|
||||
'description': 'md5:10436b12e07ac43ff8df65287a56efb4',
|
||||
'uploader_id': 'jawed',
|
||||
'uploader_url': 'http://www.youtube.com/user/jawed'
|
||||
}
|
||||
},
|
||||
{
|
||||
# Flash video with .flv extension (itag 34). Title has prefix "YouTube -"
|
||||
# Title has some weird unicode characters too.
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/20110712231407/http://www.youtube.com/watch?v=lTx3G6h2xyA',
|
||||
'info_dict': {
|
||||
'id': 'lTx3G6h2xyA',
|
||||
'ext': 'flv',
|
||||
'title': 'Madeon - Pop Culture (live mashup)'
|
||||
'title': 'Madeon - Pop Culture (live mashup)',
|
||||
'upload_date': '20110711',
|
||||
'uploader': 'Madeon',
|
||||
'channel_id': 'UCqMDNf3Pn5L7pcNkuSEeO3w',
|
||||
'duration': 204,
|
||||
'description': 'md5:f7535343b6eda34a314eff8b85444680',
|
||||
'uploader_id': 'itsmadeon',
|
||||
'uploader_url': 'http://www.youtube.com/user/itsmadeon'
|
||||
}
|
||||
},
|
||||
{ # Some versions of Youtube have have "YouTube" as page title in html (and later rewritten by js).
|
||||
}, {
|
||||
# First capture is of dead video, second is the oldest from CDX response.
|
||||
'url': 'https://web.archive.org/https://www.youtube.com/watch?v=1JYutPM8O6E',
|
||||
'info_dict': {
|
||||
'id': '1JYutPM8O6E',
|
||||
'ext': 'mp4',
|
||||
'title': 'Fake Teen Doctor Strikes AGAIN! - Weekly Weird News',
|
||||
'upload_date': '20160218',
|
||||
'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
|
||||
'duration': 1236,
|
||||
'description': 'md5:21032bae736421e89c2edf36d1936947',
|
||||
'uploader_id': 'MachinimaETC',
|
||||
'uploader_url': 'http://www.youtube.com/user/MachinimaETC'
|
||||
}
|
||||
}, {
|
||||
# First capture of dead video, capture date in link links to dead capture.
|
||||
'url': 'https://web.archive.org/web/20180803221945/https://www.youtube.com/watch?v=6FPhZJGvf4E',
|
||||
'info_dict': {
|
||||
'id': '6FPhZJGvf4E',
|
||||
'ext': 'mp4',
|
||||
'title': 'WTF: Video Games Still Launch BROKEN?! - T.U.G.S.',
|
||||
'upload_date': '20160219',
|
||||
'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
|
||||
'duration': 798,
|
||||
'description': 'md5:a1dbf12d9a3bd7cb4c5e33b27d77ffe7',
|
||||
'uploader_id': 'MachinimaETC',
|
||||
'uploader_url': 'http://www.youtube.com/user/MachinimaETC'
|
||||
},
|
||||
'expected_warnings': [
|
||||
r'unable to download capture webpage \(it may not be archived\)'
|
||||
]
|
||||
}, { # Very old YouTube page, has - YouTube in title.
|
||||
'url': 'http://web.archive.org/web/20070302011044/http://youtube.com/watch?v=-06-KB9XTzg',
|
||||
'info_dict': {
|
||||
'id': '-06-KB9XTzg',
|
||||
'ext': 'flv',
|
||||
'title': 'New Coin Hack!! 100% Safe!!'
|
||||
}
|
||||
}, {
|
||||
'url': 'web.archive.org/https://www.youtube.com/watch?v=dWW7qP423y8',
|
||||
'info_dict': {
|
||||
'id': 'dWW7qP423y8',
|
||||
'ext': 'mp4',
|
||||
'title': 'It\'s Bootleg AirPods Time.',
|
||||
'upload_date': '20211021',
|
||||
'channel_id': 'UC7Jwj9fkrf1adN4fMmTkpug',
|
||||
'channel_url': 'http://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpug',
|
||||
'duration': 810,
|
||||
'description': 'md5:7b567f898d8237b256f36c1a07d6d7bc',
|
||||
'uploader': 'DankPods',
|
||||
'uploader_id': 'UC7Jwj9fkrf1adN4fMmTkpug',
|
||||
'uploader_url': 'http://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpug'
|
||||
}
|
||||
}, {
|
||||
# player response contains '};' See: https://github.com/ytdl-org/youtube-dl/issues/27093
|
||||
'url': 'https://web.archive.org/web/20200827003909if_/http://www.youtube.com/watch?v=6Dh-RL__uN4',
|
||||
'info_dict': {
|
||||
'id': '6Dh-RL__uN4',
|
||||
'ext': 'mp4',
|
||||
'title': 'bitch lasagna',
|
||||
'upload_date': '20181005',
|
||||
'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
|
||||
'channel_url': 'http://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
|
||||
'duration': 135,
|
||||
'description': 'md5:2dbe4051feeff2dab5f41f82bb6d11d0',
|
||||
'uploader': 'PewDiePie',
|
||||
'uploader_id': 'PewDiePie',
|
||||
'uploader_url': 'http://www.youtube.com/user/PewDiePie'
|
||||
}
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFw',
|
||||
'info_dict': {
|
||||
'id': 'kH-G_aIBlFw',
|
||||
'ext': 'mp4',
|
||||
'title': 'kH-G_aIBlFw'
|
||||
},
|
||||
'expected_warnings': [
|
||||
'unable to extract title',
|
||||
]
|
||||
},
|
||||
{
|
||||
# First capture is a 302 redirect intermediary page.
|
||||
'url': 'https://web.archive.org/web/20050214000000/http://www.youtube.com/watch?v=0altSZ96U4M',
|
||||
'info_dict': {
|
||||
'id': '0altSZ96U4M',
|
||||
'ext': 'mp4',
|
||||
'title': '0altSZ96U4M'
|
||||
},
|
||||
'expected_warnings': [
|
||||
'unable to extract title',
|
||||
]
|
||||
},
|
||||
{
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/20050214000000_if/http://www.youtube.com/watch?v=0altSZ96U4M',
|
||||
'only_matching': True
|
||||
}, {
|
||||
# Video not archived, only capture is unavailable video page
|
||||
'url': 'https://web.archive.org/web/20210530071008/https://www.youtube.com/watch?v=lHJTf93HL1s&spfreload=10',
|
||||
'only_matching': True,
|
||||
},
|
||||
{ # Encoded url
|
||||
'only_matching': True
|
||||
}, { # Encoded url
|
||||
'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fgl%3DUS%26v%3DAkhihxRKcrs%26hl%3Den',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fv%3DAkhihxRKcrs%26gl%3DUS%26hl%3Den',
|
||||
'only_matching': True,
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://web.archive.org/web/20060527081937/http://www.youtube.com:80/watch.php?v=ELTFsLT73fA&search=soccer',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://web.archive.org/http://www.youtube.com:80/watch?v=-05VVye-ffg',
|
||||
'only_matching': True
|
||||
}
|
||||
]
|
||||
_YT_INITIAL_DATA_RE = r'(?:(?:(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;)|%s)' % YoutubeBaseInfoExtractor._YT_INITIAL_DATA_RE
|
||||
_YT_INITIAL_PLAYER_RESPONSE_RE = r'(?:(?:(?:window\s*\[\s*["\']ytInitialPlayerResponse["\']\s*\]|ytInitialPlayerResponse)\s*=[(\s]*({.+?})[)\s]*;)|%s)' % YoutubeBaseInfoExtractor._YT_INITIAL_PLAYER_RESPONSE_RE
|
||||
_YT_INITIAL_BOUNDARY_RE = r'(?:(?:var\s+meta|</script|\n)|%s)' % YoutubeBaseInfoExtractor._YT_INITIAL_BOUNDARY_RE
|
||||
|
||||
_YT_DEFAULT_THUMB_SERVERS = ['i.ytimg.com'] # thumbnails most likely archived on these servers
|
||||
_YT_ALL_THUMB_SERVERS = orderedSet(
|
||||
_YT_DEFAULT_THUMB_SERVERS + ['img.youtube.com', *[f'{c}{n or ""}.ytimg.com' for c in ('i', 's') for n in (*range(0, 5), 9)]])
|
||||
|
||||
_WAYBACK_BASE_URL = 'https://web.archive.org/web/%sif_/'
|
||||
_OLDEST_CAPTURE_DATE = 20050214000000
|
||||
_NEWEST_CAPTURE_DATE = 20500101000000
|
||||
|
||||
def _call_cdx_api(self, item_id, url, filters: list = None, collapse: list = None, query: dict = None, note='Downloading CDX API JSON'):
|
||||
# CDX docs: https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md
|
||||
query = {
|
||||
'url': url,
|
||||
'output': 'json',
|
||||
'fl': 'original,mimetype,length,timestamp',
|
||||
'limit': 500,
|
||||
'filter': ['statuscode:200'] + (filters or []),
|
||||
'collapse': collapse or [],
|
||||
**(query or {})
|
||||
}
|
||||
res = self._download_json('https://web.archive.org/cdx/search/cdx', item_id, note, query=query)
|
||||
if isinstance(res, list) and len(res) >= 2:
|
||||
# format response to make it easier to use
|
||||
return list(dict(zip(res[0], v)) for v in res[1:])
|
||||
elif not isinstance(res, list) or len(res) != 0:
|
||||
self.report_warning('Error while parsing CDX API response' + bug_reports_message())
|
||||
|
||||
def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
|
||||
return self._parse_json(self._search_regex(
|
||||
(r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
|
||||
regex), webpage, name, default='{}'), video_id, fatal=False)
|
||||
|
||||
def _extract_webpage_title(self, webpage):
|
||||
page_title = self._html_search_regex(
|
||||
r'<title>([^<]*)</title>', webpage, 'title', default='')
|
||||
# YouTube video pages appear to always have either 'YouTube -' as prefix or '- YouTube' as suffix.
|
||||
return self._html_search_regex(
|
||||
r'(?:YouTube\s*-\s*(.*)$)|(?:(.*)\s*-\s*YouTube$)',
|
||||
page_title, 'title', default='')
|
||||
|
||||
def _extract_metadata(self, video_id, webpage):
|
||||
|
||||
search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None)) if webpage else (lambda x: None))
|
||||
player_response = self._extract_yt_initial_variable(
|
||||
webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE, video_id, 'initial player response') or {}
|
||||
initial_data = self._extract_yt_initial_variable(
|
||||
webpage, self._YT_INITIAL_DATA_RE, video_id, 'initial player response') or {}
|
||||
|
||||
initial_data_video = traverse_obj(
|
||||
initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'videoPrimaryInfoRenderer'),
|
||||
expected_type=dict, get_all=False, default={})
|
||||
|
||||
video_details = traverse_obj(
|
||||
player_response, 'videoDetails', expected_type=dict, get_all=False, default={})
|
||||
|
||||
microformats = traverse_obj(
|
||||
player_response, ('microformat', 'playerMicroformatRenderer'), expected_type=dict, get_all=False, default={})
|
||||
|
||||
video_title = (
|
||||
video_details.get('title')
|
||||
or YoutubeBaseInfoExtractor._get_text(microformats, 'title')
|
||||
or YoutubeBaseInfoExtractor._get_text(initial_data_video, 'title')
|
||||
or self._extract_webpage_title(webpage)
|
||||
or search_meta(['og:title', 'twitter:title', 'title']))
|
||||
|
||||
channel_id = str_or_none(
|
||||
video_details.get('channelId')
|
||||
or microformats.get('externalChannelId')
|
||||
or search_meta('channelId')
|
||||
or self._search_regex(
|
||||
r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1', # @b45a9e6
|
||||
webpage, 'channel id', default=None, group='id'))
|
||||
channel_url = f'http://www.youtube.com/channel/{channel_id}' if channel_id else None
|
||||
|
||||
duration = int_or_none(
|
||||
video_details.get('lengthSeconds')
|
||||
or microformats.get('lengthSeconds')
|
||||
or parse_duration(search_meta('duration')))
|
||||
description = (
|
||||
video_details.get('shortDescription')
|
||||
or YoutubeBaseInfoExtractor._get_text(microformats, 'description')
|
||||
or clean_html(get_element_by_id('eow-description', webpage)) # @9e6dd23
|
||||
or search_meta(['description', 'og:description', 'twitter:description']))
|
||||
|
||||
uploader = video_details.get('author')
|
||||
|
||||
# Uploader ID and URL
|
||||
uploader_mobj = re.search(
|
||||
r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">', # @fd05024
|
||||
webpage)
|
||||
if uploader_mobj is not None:
|
||||
uploader_id, uploader_url = uploader_mobj.group('uploader_id'), uploader_mobj.group('uploader_url')
|
||||
else:
|
||||
# @a6211d2
|
||||
uploader_url = url_or_none(microformats.get('ownerProfileUrl'))
|
||||
uploader_id = self._search_regex(
|
||||
r'(?:user|channel)/([^/]+)', uploader_url or '', 'uploader id', default=None)
|
||||
|
||||
upload_date = unified_strdate(
|
||||
dict_get(microformats, ('uploadDate', 'publishDate'))
|
||||
or search_meta(['uploadDate', 'datePublished'])
|
||||
or self._search_regex(
|
||||
[r'(?s)id="eow-date.*?>(.*?)</span>',
|
||||
r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'], # @7998520
|
||||
webpage, 'upload date', default=None))
|
||||
|
||||
return {
|
||||
'title': video_title,
|
||||
'description': description,
|
||||
'upload_date': upload_date,
|
||||
'uploader': uploader,
|
||||
'channel_id': channel_id,
|
||||
'channel_url': channel_url,
|
||||
'duration': duration,
|
||||
'uploader_url': uploader_url,
|
||||
'uploader_id': uploader_id,
|
||||
}
|
||||
|
||||
def _extract_thumbnails(self, video_id):
|
||||
try_all = 'thumbnails' in self._configuration_arg('check_all')
|
||||
thumbnail_base_urls = ['http://{server}/vi{webp}/{video_id}'.format(
|
||||
webp='_webp' if ext == 'webp' else '', video_id=video_id, server=server)
|
||||
for server in (self._YT_ALL_THUMB_SERVERS if try_all else self._YT_DEFAULT_THUMB_SERVERS) for ext in (('jpg', 'webp') if try_all else ('jpg',))]
|
||||
|
||||
thumbnails = []
|
||||
for url in thumbnail_base_urls:
|
||||
response = self._call_cdx_api(
|
||||
video_id, url, filters=['mimetype:image/(?:webp|jpeg)'],
|
||||
collapse=['urlkey'], query={'matchType': 'prefix'})
|
||||
if not response:
|
||||
continue
|
||||
thumbnails.extend(
|
||||
{
|
||||
'url': (self._WAYBACK_BASE_URL % (int_or_none(thumbnail_dict.get('timestamp')) or self._OLDEST_CAPTURE_DATE)) + thumbnail_dict.get('original'),
|
||||
'filesize': int_or_none(thumbnail_dict.get('length')),
|
||||
'preference': int_or_none(thumbnail_dict.get('length'))
|
||||
} for thumbnail_dict in response)
|
||||
if not try_all:
|
||||
break
|
||||
|
||||
self._remove_duplicate_formats(thumbnails)
|
||||
return thumbnails
|
||||
|
||||
def _get_capture_dates(self, video_id, url_date):
|
||||
capture_dates = []
|
||||
# Note: CDX API will not find watch pages with extra params in the url.
|
||||
response = self._call_cdx_api(
|
||||
video_id, f'https://www.youtube.com/watch?v={video_id}',
|
||||
filters=['mimetype:text/html'], collapse=['timestamp:6', 'digest'], query={'matchType': 'prefix'}) or []
|
||||
all_captures = sorted([int_or_none(r['timestamp']) for r in response if int_or_none(r['timestamp']) is not None])
|
||||
|
||||
# Prefer the new polymer UI captures as we support extracting more metadata from them
|
||||
# WBM captures seem to all switch to this layout ~July 2020
|
||||
modern_captures = list(filter(lambda x: x >= 20200701000000, all_captures))
|
||||
if modern_captures:
|
||||
capture_dates.append(modern_captures[0])
|
||||
capture_dates.append(url_date)
|
||||
if all_captures:
|
||||
capture_dates.append(all_captures[0])
|
||||
|
||||
if 'captures' in self._configuration_arg('check_all'):
|
||||
capture_dates.extend(modern_captures + all_captures)
|
||||
|
||||
# Fallbacks if any of the above fail
|
||||
capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE])
|
||||
return orderedSet(capture_dates)
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
title = video_id # if we are not able get a title
|
||||
|
||||
def _extract_title(webpage):
|
||||
page_title = self._html_search_regex(
|
||||
r'<title>([^<]*)</title>', webpage, 'title', fatal=False) or ''
|
||||
# YouTube video pages appear to always have either 'YouTube -' as suffix or '- YouTube' as prefix.
|
||||
try:
|
||||
page_title = self._html_search_regex(
|
||||
r'(?:YouTube\s*-\s*(.*)$)|(?:(.*)\s*-\s*YouTube$)',
|
||||
page_title, 'title', default='')
|
||||
except RegexNotFoundError:
|
||||
page_title = None
|
||||
url_date, video_id = self._match_valid_url(url).groups()
|
||||
|
||||
if not page_title:
|
||||
self.report_warning('unable to extract title', video_id=video_id)
|
||||
return
|
||||
return page_title
|
||||
|
||||
# If the video is no longer available, the oldest capture may be one before it was removed.
|
||||
# Setting the capture date in url to early date seems to redirect to earliest capture.
|
||||
webpage = self._download_webpage(
|
||||
'https://web.archive.org/web/20050214000000/http://www.youtube.com/watch?v=%s' % video_id,
|
||||
video_id=video_id, fatal=False, errnote='unable to download video webpage (probably not archived).')
|
||||
if webpage:
|
||||
title = _extract_title(webpage) or title
|
||||
|
||||
# Use link translator mentioned in https://github.com/ytdl-org/youtube-dl/issues/13655
|
||||
internal_fake_url = 'https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/%s' % video_id
|
||||
urlh = None
|
||||
try:
|
||||
video_file_webpage = self._request_webpage(
|
||||
HEADRequest(internal_fake_url), video_id,
|
||||
note='Fetching video file url', expected_status=True)
|
||||
urlh = self._request_webpage(
|
||||
HEADRequest('https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/%s' % video_id),
|
||||
video_id, note='Fetching archived video file url', expected_status=True)
|
||||
except ExtractorError as e:
|
||||
# HTTP Error 404 is expected if the video is not saved.
|
||||
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
|
||||
raise ExtractorError(
|
||||
'HTTP Error %s. Most likely the video is not archived or issue with web.archive.org.' % e.cause.code,
|
||||
self.raise_no_formats(
|
||||
'The requested video is not archived, indexed, or there is an issue with web.archive.org',
|
||||
expected=True)
|
||||
raise
|
||||
video_file_url = compat_urllib_parse_unquote(video_file_webpage.url)
|
||||
video_file_url_qs = parse_qs(video_file_url)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Attempt to recover any ext & format info from playback url
|
||||
format = {'url': video_file_url}
|
||||
itag = try_get(video_file_url_qs, lambda x: x['itag'][0])
|
||||
if itag and itag in YoutubeIE._formats: # Naughty access but it works
|
||||
format.update(YoutubeIE._formats[itag])
|
||||
format.update({'format_id': itag})
|
||||
else:
|
||||
mime = try_get(video_file_url_qs, lambda x: x['mime'][0])
|
||||
ext = mimetype2ext(mime) or determine_ext(video_file_url)
|
||||
format.update({'ext': ext})
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': [format],
|
||||
'duration': str_to_int(try_get(video_file_url_qs, lambda x: x['dur'][0]))
|
||||
}
|
||||
capture_dates = self._get_capture_dates(video_id, int_or_none(url_date))
|
||||
self.write_debug('Captures to try: ' + ', '.join(str(i) for i in capture_dates if i is not None))
|
||||
info = {'id': video_id}
|
||||
for capture in capture_dates:
|
||||
if not capture:
|
||||
continue
|
||||
webpage = self._download_webpage(
|
||||
(self._WAYBACK_BASE_URL + 'http://www.youtube.com/watch?v=%s') % (capture, video_id),
|
||||
video_id=video_id, fatal=False, errnote='unable to download capture webpage (it may not be archived)',
|
||||
note='Downloading capture webpage')
|
||||
current_info = self._extract_metadata(video_id, webpage or '')
|
||||
# Try avoid getting deleted video metadata
|
||||
if current_info.get('title'):
|
||||
info = merge_dicts(info, current_info)
|
||||
if 'captures' not in self._configuration_arg('check_all'):
|
||||
break
|
||||
|
||||
info['thumbnails'] = self._extract_thumbnails(video_id)
|
||||
|
||||
if urlh:
|
||||
url = compat_urllib_parse_unquote(urlh.url)
|
||||
video_file_url_qs = parse_qs(url)
|
||||
# Attempt to recover any ext & format info from playback url & response headers
|
||||
format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
|
||||
itag = try_get(video_file_url_qs, lambda x: x['itag'][0])
|
||||
if itag and itag in YoutubeIE._formats:
|
||||
format.update(YoutubeIE._formats[itag])
|
||||
format.update({'format_id': itag})
|
||||
else:
|
||||
mime = try_get(video_file_url_qs, lambda x: x['mime'][0])
|
||||
ext = (mimetype2ext(mime)
|
||||
or urlhandle_detect_ext(urlh)
|
||||
or mimetype2ext(urlh.headers.get('x-archive-guessed-content-type')))
|
||||
format.update({'ext': ext})
|
||||
info['formats'] = [format]
|
||||
if not info.get('duration'):
|
||||
info['duration'] = str_to_int(try_get(video_file_url_qs, lambda x: x['dur'][0]))
|
||||
|
||||
if not info.get('title'):
|
||||
info['title'] = video_id
|
||||
return info
|
||||
|
||||
@@ -158,7 +158,7 @@ class ArcPublishingIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': uuid,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'thumbnail': try_get(video, lambda x: x['promo_image']['url']),
|
||||
'description': try_get(video, lambda x: x['subheadlines']['basic']),
|
||||
'formats': formats,
|
||||
|
||||
@@ -280,7 +280,7 @@ class ARDMediathekIE(ARDMediathekBaseIE):
|
||||
|
||||
info.update({
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if info.get('is_live') else title,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
})
|
||||
|
||||
@@ -14,7 +14,7 @@ from ..utils import (
|
||||
|
||||
|
||||
class AudiomackIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?audiomack\.com/(?:song/|(?=.+/song/))(?P<id>[\w/-]+)'
|
||||
IE_NAME = 'audiomack'
|
||||
_TESTS = [
|
||||
# hosted on audiomack
|
||||
@@ -39,15 +39,16 @@ class AudiomackIE(InfoExtractor):
|
||||
'title': 'Black Mamba Freestyle [Prod. By Danny Wolf]',
|
||||
'uploader': 'ILOVEMAKONNEN',
|
||||
'upload_date': '20160414',
|
||||
}
|
||||
},
|
||||
'skip': 'Song has been removed from the site',
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
# URLs end with [uploader name]/[uploader title]
|
||||
# URLs end with [uploader name]/song/[uploader title]
|
||||
# this title is whatever the user types in, and is rarely
|
||||
# the proper song title. Real metadata is in the api response
|
||||
album_url_tag = self._match_id(url)
|
||||
album_url_tag = self._match_id(url).replace('/song/', '/')
|
||||
|
||||
# Request the extended version of the api for extra fields like artist and title
|
||||
api_response = self._download_json(
|
||||
@@ -73,13 +74,13 @@ class AudiomackIE(InfoExtractor):
|
||||
|
||||
|
||||
class AudiomackAlbumIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?audiomack\.com/(?:album/|(?=.+/album/))(?P<id>[\w/-]+)'
|
||||
IE_NAME = 'audiomack:album'
|
||||
_TESTS = [
|
||||
# Standard album playlist
|
||||
{
|
||||
'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
|
||||
'playlist_count': 15,
|
||||
'playlist_count': 11,
|
||||
'info_dict':
|
||||
{
|
||||
'id': '812251',
|
||||
@@ -95,24 +96,27 @@ class AudiomackAlbumIE(InfoExtractor):
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
|
||||
'id': '837577',
|
||||
'title': 'PPP (Pistol P Project) - 8. Real (prod by SYK SENSE )',
|
||||
'id': '837576',
|
||||
'ext': 'mp3',
|
||||
'uploader': 'Lil Herb a.k.a. G Herbo',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'title': 'PPP (Pistol P Project) - 10. 4 Minutes Of Hell Part 4 (prod by DY OF 808 MAFIA)',
|
||||
'id': '837580',
|
||||
'ext': 'mp3',
|
||||
'uploader': 'Lil Herb a.k.a. G Herbo',
|
||||
}
|
||||
}],
|
||||
'params': {
|
||||
'playliststart': 9,
|
||||
'playlistend': 9,
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
# URLs end with [uploader name]/[uploader title]
|
||||
# URLs end with [uploader name]/album/[uploader title]
|
||||
# this title is whatever the user types in, and is rarely
|
||||
# the proper song title. Real metadata is in the api response
|
||||
album_url_tag = self._match_id(url)
|
||||
album_url_tag = self._match_id(url).replace('/album/', '/')
|
||||
result = {'_type': 'playlist', 'entries': []}
|
||||
# There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
|
||||
# Therefore we don't know how many songs the album has and must infi-loop until failure
|
||||
@@ -134,7 +138,7 @@ class AudiomackAlbumIE(InfoExtractor):
|
||||
# Pull out the album metadata and add to result (if it exists)
|
||||
for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
|
||||
if apikey in api_response and resultkey not in result:
|
||||
result[resultkey] = api_response[apikey]
|
||||
result[resultkey] = compat_str(api_response[apikey])
|
||||
song_id = url_basename(api_response['url']).rpartition('.')[0]
|
||||
result['entries'].append({
|
||||
'id': compat_str(api_response.get('id', song_id)),
|
||||
|
||||
@@ -41,7 +41,7 @@ class AWAANBaseIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': video_data.get('description_en') or video_data.get('description_ar'),
|
||||
'thumbnail': 'http://admin.mangomolo.com/analytics/%s' % img if img else None,
|
||||
'duration': int_or_none(video_data.get('duration')),
|
||||
|
||||
@@ -19,14 +19,15 @@ from ..utils import (
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
parse_count,
|
||||
smuggle_url,
|
||||
srt_subtitles_timecode,
|
||||
str_or_none,
|
||||
str_to_int,
|
||||
strip_jsonp,
|
||||
unified_timestamp,
|
||||
unsmuggle_url,
|
||||
urlencode_postdata,
|
||||
url_or_none,
|
||||
OnDemandPagedList
|
||||
)
|
||||
|
||||
@@ -722,10 +723,10 @@ class BiliBiliPlayerIE(InfoExtractor):
|
||||
|
||||
|
||||
class BiliIntlBaseIE(InfoExtractor):
|
||||
_API_URL = 'https://api.bili{}/intl/gateway{}'
|
||||
_API_URL = 'https://api.bilibili.tv/intl/gateway'
|
||||
|
||||
def _call_api(self, type, endpoint, id):
|
||||
return self._download_json(self._API_URL.format(type, endpoint), id)['data']
|
||||
def _call_api(self, endpoint, *args, **kwargs):
|
||||
return self._download_json(self._API_URL + endpoint, *args, **kwargs)['data']
|
||||
|
||||
def json2srt(self, json):
|
||||
data = '\n\n'.join(
|
||||
@@ -733,29 +734,40 @@ class BiliIntlBaseIE(InfoExtractor):
|
||||
for i, line in enumerate(json['body']))
|
||||
return data
|
||||
|
||||
def _get_subtitles(self, type, ep_id):
|
||||
sub_json = self._call_api(type, f'/m/subtitle?ep_id={ep_id}&platform=web', ep_id)
|
||||
def _get_subtitles(self, ep_id):
|
||||
sub_json = self._call_api(f'/web/v2/subtitle?episode_id={ep_id}&platform=web', ep_id)
|
||||
subtitles = {}
|
||||
for sub in sub_json.get('subtitles', []):
|
||||
for sub in sub_json.get('subtitles') or []:
|
||||
sub_url = sub.get('url')
|
||||
if not sub_url:
|
||||
continue
|
||||
sub_data = self._download_json(sub_url, ep_id, fatal=False)
|
||||
sub_data = self._download_json(
|
||||
sub_url, ep_id, errnote='Unable to download subtitles', fatal=False,
|
||||
note='Downloading subtitles%s' % f' for {sub["lang"]}' if sub.get('lang') else '')
|
||||
if not sub_data:
|
||||
continue
|
||||
subtitles.setdefault(sub.get('key', 'en'), []).append({
|
||||
subtitles.setdefault(sub.get('lang_key', 'en'), []).append({
|
||||
'ext': 'srt',
|
||||
'data': self.json2srt(sub_data)
|
||||
})
|
||||
return subtitles
|
||||
|
||||
def _get_formats(self, type, ep_id):
|
||||
video_json = self._call_api(type, f'/web/playurl?ep_id={ep_id}&platform=web', ep_id)
|
||||
if not video_json:
|
||||
self.raise_login_required(method='cookies')
|
||||
def _get_formats(self, ep_id):
|
||||
video_json = self._call_api(f'/web/playurl?ep_id={ep_id}&platform=web', ep_id,
|
||||
note='Downloading video formats', errnote='Unable to download video formats')
|
||||
if video_json.get('code'):
|
||||
if video_json['code'] in (10004004, 10004005, 10023006):
|
||||
self.raise_login_required(method='cookies')
|
||||
elif video_json['code'] == 10004001:
|
||||
self.raise_geo_restricted()
|
||||
elif video_json.get('message') and str(video_json['code']) != video_json['message']:
|
||||
raise ExtractorError(
|
||||
f'Unable to download video formats: {self.IE_NAME} said: {video_json["message"]}', expected=True)
|
||||
else:
|
||||
raise ExtractorError('Unable to download video formats')
|
||||
video_json = video_json['playurl']
|
||||
formats = []
|
||||
for vid in video_json.get('video', []):
|
||||
for vid in video_json.get('video') or []:
|
||||
video_res = vid.get('video_resource') or {}
|
||||
video_info = vid.get('stream_info') or {}
|
||||
if not video_res.get('url'):
|
||||
@@ -771,7 +783,7 @@ class BiliIntlBaseIE(InfoExtractor):
|
||||
'vcodec': video_res.get('codecs'),
|
||||
'filesize': video_res.get('size'),
|
||||
})
|
||||
for aud in video_json.get('audio_resource', []):
|
||||
for aud in video_json.get('audio_resource') or []:
|
||||
if not aud.get('url'):
|
||||
continue
|
||||
formats.append({
|
||||
@@ -786,85 +798,93 @@ class BiliIntlBaseIE(InfoExtractor):
|
||||
self._sort_formats(formats)
|
||||
return formats
|
||||
|
||||
def _extract_ep_info(self, type, episode_data, ep_id):
|
||||
def _extract_ep_info(self, episode_data, ep_id):
|
||||
return {
|
||||
'id': ep_id,
|
||||
'title': episode_data.get('long_title') or episode_data['title'],
|
||||
'title': episode_data.get('title_display') or episode_data['title'],
|
||||
'thumbnail': episode_data.get('cover'),
|
||||
'episode_number': str_to_int(episode_data.get('title')),
|
||||
'formats': self._get_formats(type, ep_id),
|
||||
'subtitles': self._get_subtitles(type, ep_id),
|
||||
'episode_number': int_or_none(self._search_regex(
|
||||
r'^E(\d+)(?:$| - )', episode_data.get('title_display'), 'episode number', default=None)),
|
||||
'formats': self._get_formats(ep_id),
|
||||
'subtitles': self._get_subtitles(ep_id),
|
||||
'extractor_key': BiliIntlIE.ie_key(),
|
||||
}
|
||||
|
||||
|
||||
class BiliIntlIE(BiliIntlBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?bili(?P<type>bili\.tv|intl.com)/(?:[a-z]{2}/)?play/(?P<season_id>\d+)/(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-z]{2}/)?play/(?P<season_id>\d+)/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.tv/en/play/34613/341736',
|
||||
'info_dict': {
|
||||
'id': '341736',
|
||||
'ext': 'mp4',
|
||||
'title': 'The First Night',
|
||||
'thumbnail': 'https://i0.hdslb.com/bfs/intl/management/91e30e5521235d9b163339a26a0b030ebda54310.png',
|
||||
'title': 'E2 - The First Night',
|
||||
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
|
||||
'episode_number': 2,
|
||||
},
|
||||
'params': {
|
||||
'format': 'bv',
|
||||
},
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.bilibili.tv/en/play/1033760/11005006',
|
||||
'info_dict': {
|
||||
'id': '11005006',
|
||||
'ext': 'mp4',
|
||||
'title': 'E3 - Who?',
|
||||
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
|
||||
'episode_number': 3,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.biliintl.com/en/play/34613/341736',
|
||||
'info_dict': {
|
||||
'id': '341736',
|
||||
'ext': 'mp4',
|
||||
'title': 'The First Night',
|
||||
'thumbnail': 'https://i0.hdslb.com/bfs/intl/management/91e30e5521235d9b163339a26a0b030ebda54310.png',
|
||||
'episode_number': 2,
|
||||
},
|
||||
'params': {
|
||||
'format': 'bv',
|
||||
},
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
type, season_id, id = self._match_valid_url(url).groups()
|
||||
data_json = self._call_api(type, f'/web/view/ogv_collection?season_id={season_id}', id)
|
||||
episode_data = next(
|
||||
episode for episode in data_json.get('episodes', [])
|
||||
if str(episode.get('ep_id')) == id)
|
||||
return self._extract_ep_info(type, episode_data, id)
|
||||
season_id, video_id = self._match_valid_url(url).groups()
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
# Bstation layout
|
||||
initial_data = self._parse_json(self._search_regex(
|
||||
r'window\.__INITIAL_DATA__\s*=\s*({.+?});', webpage,
|
||||
'preload state', default='{}'), video_id, fatal=False) or {}
|
||||
episode_data = traverse_obj(initial_data, ('OgvVideo', 'epDetail'), expected_type=dict)
|
||||
|
||||
if not episode_data:
|
||||
# Non-Bstation layout, read through episode list
|
||||
season_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={season_id}&platform=web', video_id)
|
||||
episode_data = next(
|
||||
episode for episode in traverse_obj(season_json, ('sections', ..., 'episodes', ...), expected_type=dict)
|
||||
if str(episode.get('episode_id')) == video_id)
|
||||
return self._extract_ep_info(episode_data, video_id)
|
||||
|
||||
|
||||
class BiliIntlSeriesIE(BiliIntlBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?bili(?P<type>bili\.tv|intl.com)/(?:[a-z]{2}/)?play/(?P<id>\d+)$'
|
||||
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-z]{2}/)?play/(?P<id>\d+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bilibili.tv/en/play/34613',
|
||||
'playlist_mincount': 15,
|
||||
'info_dict': {
|
||||
'id': '34613',
|
||||
'title': 'Fly Me to the Moon',
|
||||
'description': 'md5:a861ee1c4dc0acfad85f557cc42ac627',
|
||||
'categories': ['Romance', 'Comedy', 'Slice of life'],
|
||||
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
|
||||
'view_count': int,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
'format': 'bv',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.biliintl.com/en/play/34613',
|
||||
'playlist_mincount': 15,
|
||||
'info_dict': {
|
||||
'id': '34613',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
'format': 'bv',
|
||||
},
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _entries(self, id, type):
|
||||
data_json = self._call_api(type, f'/web/view/ogv_collection?season_id={id}', id)
|
||||
for episode in data_json.get('episodes', []):
|
||||
episode_id = str(episode.get('ep_id'))
|
||||
yield self._extract_ep_info(type, episode, episode_id)
|
||||
def _entries(self, series_id):
|
||||
series_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={series_id}&platform=web', series_id)
|
||||
for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict, default=[]):
|
||||
episode_id = str(episode.get('episode_id'))
|
||||
yield self._extract_ep_info(episode, episode_id)
|
||||
|
||||
def _real_extract(self, url):
|
||||
type, id = self._match_valid_url(url).groups()
|
||||
return self.playlist_result(self._entries(id, type), playlist_id=id)
|
||||
series_id = self._match_id(url)
|
||||
series_info = self._call_api(f'/web/v2/ogv/play/season_info?season_id={series_id}&platform=web', series_id).get('season') or {}
|
||||
return self.playlist_result(
|
||||
self._entries(series_id), series_id, series_info.get('title'), series_info.get('description'),
|
||||
categories=traverse_obj(series_info, ('styles', ..., 'title'), expected_type=str_or_none),
|
||||
thumbnail=url_or_none(series_info.get('horizontal_cover')), view_count=parse_count(series_info.get('view')))
|
||||
|
||||
@@ -51,7 +51,7 @@ class BitwaveStreamIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': username,
|
||||
'title': self._live_title(channel['data']['title']),
|
||||
'title': channel['data']['title'],
|
||||
'uploader': username,
|
||||
'uploader_id': username,
|
||||
'formats': formats,
|
||||
|
||||
@@ -49,7 +49,7 @@ class BongaCamsIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': channel_id,
|
||||
'title': self._live_title(uploader or uploader_id),
|
||||
'title': uploader or uploader_id,
|
||||
'uploader': uploader,
|
||||
'uploader_id': uploader_id,
|
||||
'like_count': like_count,
|
||||
|
||||
@@ -16,6 +16,7 @@ from ..compat import (
|
||||
)
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
dict_get,
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
find_xpath_attr,
|
||||
@@ -471,32 +472,22 @@ class BrightcoveNewIE(AdobePassIE):
|
||||
def _parse_brightcove_metadata(self, json_data, video_id, headers={}):
|
||||
title = json_data['name'].strip()
|
||||
|
||||
num_drm_sources = 0
|
||||
formats, subtitles = [], {}
|
||||
sources = json_data.get('sources') or []
|
||||
for source in sources:
|
||||
container = source.get('container')
|
||||
ext = mimetype2ext(source.get('type'))
|
||||
src = source.get('src')
|
||||
skip_unplayable = not self.get_param('allow_unplayable_formats')
|
||||
# https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
|
||||
if skip_unplayable and (container == 'WVM' or source.get('key_systems')):
|
||||
num_drm_sources += 1
|
||||
continue
|
||||
elif ext == 'ism' and skip_unplayable:
|
||||
continue
|
||||
elif ext == 'm3u8' or container == 'M2TS':
|
||||
if ext == 'm3u8' or container == 'M2TS':
|
||||
if not src:
|
||||
continue
|
||||
f, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
|
||||
formats.extend(f)
|
||||
subtitles = self._merge_subtitles(subtitles, subs)
|
||||
elif ext == 'mpd':
|
||||
if not src:
|
||||
continue
|
||||
f, subs = self._extract_mpd_formats_and_subtitles(src, video_id, 'dash', fatal=False)
|
||||
formats.extend(f)
|
||||
fmts, subs = self._extract_mpd_formats_and_subtitles(src, video_id, 'dash', fatal=False)
|
||||
subtitles = self._merge_subtitles(subtitles, subs)
|
||||
else:
|
||||
streaming_src = source.get('streaming_src')
|
||||
@@ -543,7 +534,13 @@ class BrightcoveNewIE(AdobePassIE):
|
||||
'play_path': stream_name,
|
||||
'format_id': build_format_id('rtmp'),
|
||||
})
|
||||
formats.append(f)
|
||||
fmts = [f]
|
||||
|
||||
# https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
|
||||
if container == 'WVM' or source.get('key_systems') or ext == 'ism':
|
||||
for f in fmts:
|
||||
f['has_drm'] = True
|
||||
formats.extend(fmts)
|
||||
|
||||
if not formats:
|
||||
errors = json_data.get('errors')
|
||||
@@ -551,9 +548,6 @@ class BrightcoveNewIE(AdobePassIE):
|
||||
error = errors[0]
|
||||
self.raise_no_formats(
|
||||
error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
|
||||
elif (not self.get_param('allow_unplayable_formats')
|
||||
and sources and num_drm_sources == len(sources)):
|
||||
self.report_drm(video_id)
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
@@ -577,11 +571,19 @@ class BrightcoveNewIE(AdobePassIE):
|
||||
if duration is not None and duration <= 0:
|
||||
is_live = True
|
||||
|
||||
common_res = [(160, 90), (320, 180), (480, 720), (640, 360), (768, 432), (1024, 576), (1280, 720), (1366, 768), (1920, 1080)]
|
||||
thumb_base_url = dict_get(json_data, ('poster', 'thumbnail'))
|
||||
thumbnails = [{
|
||||
'url': re.sub(r'\d+x\d+', f'{w}x{h}', thumb_base_url),
|
||||
'width': w,
|
||||
'height': h,
|
||||
} for w, h in common_res] if thumb_base_url else None
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': clean_html(json_data.get('description')),
|
||||
'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
|
||||
'thumbnails': thumbnails,
|
||||
'duration': duration,
|
||||
'timestamp': parse_iso8601(json_data.get('published_at')),
|
||||
'uploader_id': json_data.get('account_id'),
|
||||
|
||||
@@ -25,7 +25,7 @@ class CAM4IE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': channel_id,
|
||||
'title': self._live_title(channel_id),
|
||||
'title': channel_id,
|
||||
'is_live': True,
|
||||
'age_limit': 18,
|
||||
'formats': formats,
|
||||
|
||||
@@ -91,7 +91,7 @@ class CamModelsIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': user_id,
|
||||
'title': self._live_title(user_id),
|
||||
'title': user_id,
|
||||
'is_live': True,
|
||||
'formats': formats,
|
||||
'age_limit': 18
|
||||
|
||||
+37
-2
@@ -11,11 +11,13 @@ from ..compat import (
|
||||
compat_str,
|
||||
)
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
js_to_json,
|
||||
smuggle_url,
|
||||
try_get,
|
||||
orderedSet,
|
||||
smuggle_url,
|
||||
strip_or_none,
|
||||
try_get,
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
@@ -313,6 +315,38 @@ class CBCGemIE(InfoExtractor):
|
||||
return
|
||||
self._claims_token = self._downloader.cache.load(self._NETRC_MACHINE, 'claims_token')
|
||||
|
||||
def _find_secret_formats(self, formats, video_id):
|
||||
""" Find a valid video url and convert it to the secret variant """
|
||||
base_format = next((f for f in formats if f.get('vcodec') != 'none'), None)
|
||||
if not base_format:
|
||||
return
|
||||
|
||||
base_url = re.sub(r'(Manifest\(.*?),filter=[\w-]+(.*?\))', r'\1\2', base_format['url'])
|
||||
url = re.sub(r'(Manifest\(.*?),format=[\w-]+(.*?\))', r'\1\2', base_url)
|
||||
|
||||
secret_xml = self._download_xml(url, video_id, note='Downloading secret XML', fatal=False)
|
||||
if not secret_xml:
|
||||
return
|
||||
|
||||
for child in secret_xml:
|
||||
if child.attrib.get('Type') != 'video':
|
||||
continue
|
||||
for video_quality in child:
|
||||
bitrate = int_or_none(video_quality.attrib.get('Bitrate'))
|
||||
if not bitrate or 'Index' not in video_quality.attrib:
|
||||
continue
|
||||
height = int_or_none(video_quality.attrib.get('MaxHeight'))
|
||||
|
||||
yield {
|
||||
**base_format,
|
||||
'format_id': join_nonempty('sec', height),
|
||||
# Note: \g<1> is necessary instead of \1 since bitrate is a number
|
||||
'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\g<1>{bitrate}\2', base_url),
|
||||
'width': int_or_none(video_quality.attrib.get('MaxWidth')),
|
||||
'tbr': bitrate / 1000.0,
|
||||
'height': height,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_info = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/assets/' + video_id, video_id)
|
||||
@@ -335,6 +369,7 @@ class CBCGemIE(InfoExtractor):
|
||||
|
||||
formats = self._extract_m3u8_formats(m3u8_url, video_id, m3u8_id='hls')
|
||||
self._remove_duplicate_formats(formats)
|
||||
formats.extend(self._find_secret_formats(formats, video_id))
|
||||
|
||||
for format in formats:
|
||||
if format.get('vcodec') == 'none':
|
||||
|
||||
@@ -12,8 +12,7 @@ from ..utils import (
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
sanitized_Request,
|
||||
unescapeHTML,
|
||||
update_url_query,
|
||||
traverse_obj,
|
||||
urlencode_postdata,
|
||||
USER_AGENTS,
|
||||
)
|
||||
@@ -99,11 +98,13 @@ class CeskaTelevizeIE(InfoExtractor):
|
||||
playlist_description = playlist_description.replace('\xa0', ' ')
|
||||
|
||||
if parsed_url.path.startswith('/porady/'):
|
||||
refer_url = update_url_query(unescapeHTML(self._search_regex(
|
||||
(r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
|
||||
r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
|
||||
webpage, 'iframe player url', group='url')), query={'autoStart': 'true'})
|
||||
webpage = self._download_webpage(refer_url, playlist_id)
|
||||
next_data = self._search_nextjs_data(webpage, playlist_id)
|
||||
idec = traverse_obj(next_data, ('props', 'pageProps', 'data', ('show', 'mediaMeta'), 'idec'), get_all=False)
|
||||
if not idec:
|
||||
raise ExtractorError('Failed to find IDEC id')
|
||||
iframe_hash = self._download_webpage('https://www.ceskatelevize.cz/v-api/iframe-hash/', playlist_id)
|
||||
webpage = self._download_webpage('https://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php', playlist_id,
|
||||
query={'hash': iframe_hash, 'origin': 'iVysilani', 'autoStart': 'true', 'IDEC': idec})
|
||||
|
||||
NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
|
||||
if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
|
||||
@@ -211,8 +212,6 @@ class CeskaTelevizeIE(InfoExtractor):
|
||||
|
||||
if playlist_len == 1:
|
||||
final_title = playlist_title or title
|
||||
if is_live:
|
||||
final_title = self._live_title(final_title)
|
||||
else:
|
||||
final_title = '%s (%s)' % (playlist_title, title)
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ class ChaturbateIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(video_id),
|
||||
'title': video_id,
|
||||
'thumbnail': 'https://roomimg.stream.highwebmedia.com/ri/%s.jpg' % video_id,
|
||||
'age_limit': self._rta_search(webpage),
|
||||
'is_live': True,
|
||||
|
||||
+58
-35
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
|
||||
|
||||
import base64
|
||||
import collections
|
||||
import datetime
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
@@ -164,9 +163,8 @@ class InfoExtractor(object):
|
||||
* filesize_approx An estimate for the number of bytes
|
||||
* player_url SWF Player URL (used for rtmpdump).
|
||||
* protocol The protocol that will be used for the actual
|
||||
download, lower-case.
|
||||
"http", "https", "rtsp", "rtmp", "rtmp_ffmpeg", "rtmpe",
|
||||
"m3u8", "m3u8_native" or "http_dash_segments".
|
||||
download, lower-case. One of "http", "https" or
|
||||
one of the protocols defined in downloader.PROTOCOL_MAP
|
||||
* fragment_base_url
|
||||
Base URL for fragments. Each fragment's path
|
||||
value (if present) will be relative to
|
||||
@@ -182,6 +180,8 @@ class InfoExtractor(object):
|
||||
fragment_base_url
|
||||
* "duration" (optional, int or float)
|
||||
* "filesize" (optional, int)
|
||||
* is_from_start Is a live format that can be downloaded
|
||||
from the start. Boolean
|
||||
* preference Order number of this format. If this field is
|
||||
present and not None, the formats get sorted
|
||||
by this field, regardless of all other values.
|
||||
@@ -466,6 +466,8 @@ class InfoExtractor(object):
|
||||
# we have cached the regexp for *this* class, whereas getattr would also
|
||||
# match the superclass
|
||||
if '_VALID_URL_RE' not in cls.__dict__:
|
||||
if '_VALID_URL' not in cls.__dict__:
|
||||
cls._VALID_URL = cls._make_valid_url()
|
||||
cls._VALID_URL_RE = re.compile(cls._VALID_URL)
|
||||
return cls._VALID_URL_RE.match(url)
|
||||
|
||||
@@ -614,7 +616,7 @@ class InfoExtractor(object):
|
||||
kwargs = {
|
||||
'video_id': e.video_id or self.get_temp_id(url),
|
||||
'ie': self.IE_NAME,
|
||||
'tb': e.traceback,
|
||||
'tb': e.traceback or sys.exc_info()[2],
|
||||
'expected': e.expected,
|
||||
'cause': e.cause
|
||||
}
|
||||
@@ -1450,8 +1452,13 @@ class InfoExtractor(object):
|
||||
})
|
||||
extract_interaction_statistic(e)
|
||||
|
||||
for e in json_ld:
|
||||
if '@context' in e:
|
||||
def traverse_json_ld(json_ld, at_top_level=True):
|
||||
for e in json_ld:
|
||||
if at_top_level and '@context' not in e:
|
||||
continue
|
||||
if at_top_level and set(e.keys()) == {'@context', '@graph'}:
|
||||
traverse_json_ld(variadic(e['@graph'], allowed_types=(dict,)), at_top_level=False)
|
||||
break
|
||||
item_type = e.get('@type')
|
||||
if expected_type is not None and expected_type != item_type:
|
||||
continue
|
||||
@@ -1487,7 +1494,7 @@ class InfoExtractor(object):
|
||||
info.update({
|
||||
'timestamp': parse_iso8601(e.get('datePublished')),
|
||||
'title': unescapeHTML(e.get('headline')),
|
||||
'description': unescapeHTML(e.get('articleBody')),
|
||||
'description': unescapeHTML(e.get('articleBody') or e.get('description')),
|
||||
})
|
||||
elif item_type == 'VideoObject':
|
||||
extract_video_object(e)
|
||||
@@ -1502,6 +1509,8 @@ class InfoExtractor(object):
|
||||
continue
|
||||
else:
|
||||
break
|
||||
traverse_json_ld(json_ld)
|
||||
|
||||
return dict((k, v) for k, v in info.items() if v is not None)
|
||||
|
||||
def _search_nextjs_data(self, webpage, video_id, **kw):
|
||||
@@ -1511,6 +1520,24 @@ class InfoExtractor(object):
|
||||
webpage, 'next.js data', **kw),
|
||||
video_id, **kw)
|
||||
|
||||
def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__'):
|
||||
''' Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function. '''
|
||||
# not all website do this, but it can be changed
|
||||
# https://stackoverflow.com/questions/67463109/how-to-change-or-hide-nuxt-and-nuxt-keyword-in-page-source
|
||||
rectx = re.escape(context_name)
|
||||
js, arg_keys, arg_vals = self._search_regex(
|
||||
(r'<script>window\.%s=\(function\((?P<arg_keys>.*?)\)\{return\s(?P<js>\{.*?\})\}\((?P<arg_vals>.+?)\)\);?</script>' % rectx,
|
||||
r'%s\(.*?\(function\((?P<arg_keys>.*?)\)\{return\s(?P<js>\{.*?\})\}\((?P<arg_vals>.*?)\)' % rectx),
|
||||
webpage, context_name, group=['js', 'arg_keys', 'arg_vals'])
|
||||
|
||||
args = dict(zip(arg_keys.split(','), arg_vals.split(',')))
|
||||
|
||||
for key, val in args.items():
|
||||
if val in ('undefined', 'void 0'):
|
||||
args[key] = 'null'
|
||||
|
||||
return self._parse_json(js_to_json(js, args), video_id)['data'][0]
|
||||
|
||||
@staticmethod
|
||||
def _hidden_inputs(html):
|
||||
html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
|
||||
@@ -1547,7 +1574,7 @@ class InfoExtractor(object):
|
||||
'vcodec': {'type': 'ordered', 'regex': True,
|
||||
'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
|
||||
'acodec': {'type': 'ordered', 'regex': True,
|
||||
'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
|
||||
'order': ['[af]lac', 'wav|aiff', 'opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e-?a?c-?3', 'ac-?3', 'dts', '', None, 'none']},
|
||||
'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
|
||||
'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
|
||||
'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
|
||||
@@ -1586,6 +1613,11 @@ class InfoExtractor(object):
|
||||
'res': {'type': 'multiple', 'field': ('height', 'width'),
|
||||
'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
|
||||
|
||||
# For compatibility with youtube-dl
|
||||
'format_id': {'type': 'alias', 'field': 'id'},
|
||||
'preference': {'type': 'alias', 'field': 'ie_pref'},
|
||||
'language_preference': {'type': 'alias', 'field': 'lang'},
|
||||
|
||||
# Deprecated
|
||||
'dimension': {'type': 'alias', 'field': 'res'},
|
||||
'resolution': {'type': 'alias', 'field': 'res'},
|
||||
@@ -1595,7 +1627,6 @@ class InfoExtractor(object):
|
||||
'video_bitrate': {'type': 'alias', 'field': 'vbr'},
|
||||
'audio_bitrate': {'type': 'alias', 'field': 'abr'},
|
||||
'framerate': {'type': 'alias', 'field': 'fps'},
|
||||
'language_preference': {'type': 'alias', 'field': 'lang'},
|
||||
'protocol': {'type': 'alias', 'field': 'proto'},
|
||||
'source_preference': {'type': 'alias', 'field': 'source'},
|
||||
'filesize_approx': {'type': 'alias', 'field': 'fs_approx'},
|
||||
@@ -1610,9 +1641,7 @@ class InfoExtractor(object):
|
||||
'audio': {'type': 'alias', 'field': 'hasaud'},
|
||||
'has_audio': {'type': 'alias', 'field': 'hasaud'},
|
||||
'extractor': {'type': 'alias', 'field': 'ie_pref'},
|
||||
'preference': {'type': 'alias', 'field': 'ie_pref'},
|
||||
'extractor_preference': {'type': 'alias', 'field': 'ie_pref'},
|
||||
'format_id': {'type': 'alias', 'field': 'id'},
|
||||
}
|
||||
|
||||
def __init__(self, ie, field_preference):
|
||||
@@ -1712,9 +1741,10 @@ class InfoExtractor(object):
|
||||
continue
|
||||
if self._get_field_setting(field, 'type') == 'alias':
|
||||
alias, field = field, self._get_field_setting(field, 'field')
|
||||
self.ydl.deprecation_warning(
|
||||
f'Format sorting alias {alias} is deprecated '
|
||||
f'and may be removed in a future version. Please use {field} instead')
|
||||
if alias not in ('format_id', 'preference', 'language_preference'):
|
||||
self.ydl.deprecation_warning(
|
||||
f'Format sorting alias {alias} is deprecated '
|
||||
f'and may be removed in a future version. Please use {field} instead')
|
||||
reverse = match.group('reverse') is not None
|
||||
closest = match.group('separator') == '~'
|
||||
limit_text = match.group('limit')
|
||||
@@ -2302,7 +2332,7 @@ class InfoExtractor(object):
|
||||
|
||||
if smil is False:
|
||||
assert not fatal
|
||||
return []
|
||||
return [], {}
|
||||
|
||||
namespace = self._parse_smil_namespace(smil)
|
||||
|
||||
@@ -3433,10 +3463,8 @@ class InfoExtractor(object):
|
||||
return formats
|
||||
|
||||
def _live_title(self, name):
|
||||
""" Generate the title for a live video """
|
||||
now = datetime.datetime.now()
|
||||
now_str = now.strftime('%Y-%m-%d %H:%M')
|
||||
return name + ' ' + now_str
|
||||
self._downloader.deprecation_warning('yt_dlp.InfoExtractor._live_title is deprecated and does not work as expected')
|
||||
return name
|
||||
|
||||
def _int(self, v, name, fatal=False, **kwargs):
|
||||
res = int_or_none(v, **kwargs)
|
||||
@@ -3546,14 +3574,18 @@ class InfoExtractor(object):
|
||||
|
||||
def extractor():
|
||||
comments = []
|
||||
interrupted = True
|
||||
try:
|
||||
while True:
|
||||
comments.append(next(generator))
|
||||
except KeyboardInterrupt:
|
||||
interrupted = True
|
||||
self.to_screen('Interrupted by user')
|
||||
except StopIteration:
|
||||
interrupted = False
|
||||
except KeyboardInterrupt:
|
||||
self.to_screen('Interrupted by user')
|
||||
except Exception as e:
|
||||
if self.get_param('ignoreerrors') is not True:
|
||||
raise
|
||||
self._downloader.report_error(e)
|
||||
comment_count = len(comments)
|
||||
self.to_screen(f'Extracted {comment_count} comments')
|
||||
return {
|
||||
@@ -3631,7 +3663,7 @@ class InfoExtractor(object):
|
||||
else 'public' if all_known
|
||||
else None)
|
||||
|
||||
def _configuration_arg(self, key, default=NO_DEFAULT, casesense=False):
|
||||
def _configuration_arg(self, key, default=NO_DEFAULT, *, ie_key=None, casesense=False):
|
||||
'''
|
||||
@returns A list of values for the extractor argument given by "key"
|
||||
or "default" if no such key is present
|
||||
@@ -3639,7 +3671,7 @@ class InfoExtractor(object):
|
||||
@param casesense When false, the values are converted to lower case
|
||||
'''
|
||||
val = traverse_obj(
|
||||
self._downloader.params, ('extractor_args', self.ie_key().lower(), key))
|
||||
self._downloader.params, ('extractor_args', (ie_key or self.ie_key()).lower(), key))
|
||||
if val is None:
|
||||
return [] if default is NO_DEFAULT else default
|
||||
return list(val) if casesense else [x.lower() for x in val]
|
||||
@@ -3658,17 +3690,8 @@ class SearchInfoExtractor(InfoExtractor):
|
||||
def _make_valid_url(cls):
|
||||
return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
|
||||
|
||||
@classmethod
|
||||
def suitable(cls, url):
|
||||
return re.match(cls._make_valid_url(), url) is not None
|
||||
|
||||
def _real_extract(self, query):
|
||||
mobj = re.match(self._make_valid_url(), query)
|
||||
if mobj is None:
|
||||
raise ExtractorError('Invalid search query "%s"' % query)
|
||||
|
||||
prefix = mobj.group('prefix')
|
||||
query = mobj.group('query')
|
||||
prefix, query = self._match_valid_url(query).group('prefix', 'query')
|
||||
if prefix == '':
|
||||
return self._get_n_results(query, 1)
|
||||
elif prefix == 'all':
|
||||
|
||||
+21
-19
@@ -23,32 +23,35 @@ from ..utils import (
|
||||
class CrackleIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?(?:sony)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
# geo restricted to CA
|
||||
'url': 'https://www.crackle.com/andromeda/2502343',
|
||||
# Crackle is available in the United States and territories
|
||||
'url': 'https://www.crackle.com/thanksgiving/2510064',
|
||||
'info_dict': {
|
||||
'id': '2502343',
|
||||
'id': '2510064',
|
||||
'ext': 'mp4',
|
||||
'title': 'Under The Night',
|
||||
'description': 'md5:d2b8ca816579ae8a7bf28bfff8cefc8a',
|
||||
'duration': 2583,
|
||||
'title': 'Touch Football',
|
||||
'description': 'md5:cfbb513cf5de41e8b56d7ab756cff4df',
|
||||
'duration': 1398,
|
||||
'view_count': int,
|
||||
'average_rating': 0,
|
||||
'age_limit': 14,
|
||||
'genre': 'Action, Sci-Fi',
|
||||
'creator': 'Allan Kroeker',
|
||||
'artist': 'Keith Hamilton Cobb, Kevin Sorbo, Lisa Ryder, Lexa Doig, Robert Hewitt Wolfe',
|
||||
'release_year': 2000,
|
||||
'series': 'Andromeda',
|
||||
'episode': 'Under The Night',
|
||||
'age_limit': 17,
|
||||
'genre': 'Comedy',
|
||||
'creator': 'Daniel Powell',
|
||||
'artist': 'Chris Elliott, Amy Sedaris',
|
||||
'release_year': 2016,
|
||||
'series': 'Thanksgiving',
|
||||
'episode': 'Touch Football',
|
||||
'season_number': 1,
|
||||
'episode_number': 1,
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
}
|
||||
},
|
||||
'expected_warnings': [
|
||||
'Trying with a list of known countries'
|
||||
],
|
||||
}, {
|
||||
'url': 'https://www.sonycrackle.com/andromeda/2502343',
|
||||
'url': 'https://www.sonycrackle.com/thanksgiving/2510064',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@@ -129,7 +132,6 @@ class CrackleIE(InfoExtractor):
|
||||
break
|
||||
|
||||
ignore_no_formats = self.get_param('ignore_no_formats_error')
|
||||
allow_unplayable_formats = self.get_param('allow_unplayable_formats')
|
||||
|
||||
if not media or (not media.get('MediaURLs') and not ignore_no_formats):
|
||||
raise ExtractorError(
|
||||
@@ -143,9 +145,9 @@ class CrackleIE(InfoExtractor):
|
||||
for e in media.get('MediaURLs') or []:
|
||||
if e.get('UseDRM'):
|
||||
has_drm = True
|
||||
if not allow_unplayable_formats:
|
||||
continue
|
||||
format_url = url_or_none(e.get('Path'))
|
||||
format_url = url_or_none(e.get('DRMPath'))
|
||||
else:
|
||||
format_url = url_or_none(e.get('Path'))
|
||||
if not format_url:
|
||||
continue
|
||||
ext = determine_ext(format_url)
|
||||
|
||||
@@ -305,7 +305,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': clean_html(media.get('description')),
|
||||
'thumbnails': thumbnails,
|
||||
'duration': int_or_none(metadata.get('duration')) or None,
|
||||
|
||||
@@ -84,7 +84,7 @@ class DLiveStreamIE(InfoExtractor):
|
||||
self._sort_formats(formats)
|
||||
return {
|
||||
'id': display_name,
|
||||
'title': self._live_title(title),
|
||||
'title': title,
|
||||
'uploader': display_name,
|
||||
'uploader_id': username,
|
||||
'formats': formats,
|
||||
|
||||
@@ -105,7 +105,7 @@ class DouyuTVIE(InfoExtractor):
|
||||
'aid': 'pcclient'
|
||||
})['data']['live_url']
|
||||
|
||||
title = self._live_title(unescapeHTML(room['room_name']))
|
||||
title = unescapeHTML(room['room_name'])
|
||||
description = room.get('show_details')
|
||||
thumbnail = room.get('room_src')
|
||||
uploader = room.get('nickname')
|
||||
|
||||
@@ -348,7 +348,7 @@ class HGTVDeIE(DPlayBaseIE):
|
||||
|
||||
|
||||
class DiscoveryPlusIE(DPlayBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/(?:\w{2}/)?video' + DPlayBaseIE._PATH_REGEX
|
||||
_VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/(?!it/)(?:\w{2}/)?video' + DPlayBaseIE._PATH_REGEX
|
||||
_TESTS = [{
|
||||
'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
|
||||
'info_dict': {
|
||||
@@ -564,10 +564,10 @@ class DiscoveryPlusShowBaseIE(DPlayBaseIE):
|
||||
total_pages = try_get(season_json, lambda x: x['meta']['totalPages'], int) or 1
|
||||
episodes_json = season_json['data']
|
||||
for episode in episodes_json:
|
||||
video_id = episode['attributes']['path']
|
||||
video_path = episode['attributes']['path']
|
||||
yield self.url_result(
|
||||
'%svideos/%s' % (self._DOMAIN, video_id),
|
||||
ie=self._VIDEO_IE.ie_key(), video_id=video_id)
|
||||
'%svideos/%s' % (self._DOMAIN, video_path),
|
||||
ie=self._VIDEO_IE.ie_key(), video_id=episode.get('id') or video_path)
|
||||
page_num += 1
|
||||
|
||||
def _real_extract(self, url):
|
||||
@@ -575,6 +575,18 @@ class DiscoveryPlusShowBaseIE(DPlayBaseIE):
|
||||
return self.playlist_result(self._entries(show_name), playlist_id=show_name)
|
||||
|
||||
|
||||
class DiscoveryPlusItalyIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/it/video' + DPlayBaseIE._PATH_REGEX
|
||||
_TESTS = [{
|
||||
'url': 'https://www.discoveryplus.com/it/video/i-signori-della-neve/stagione-2-episodio-1-i-preparativi',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
return self.url_result(f'https://discoveryplus.it/video/{video_id}', DPlayIE.ie_key(), video_id)
|
||||
|
||||
|
||||
class DiscoveryPlusItalyShowIE(DiscoveryPlusShowBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?discoveryplus\.it/programmi/(?P<show_name>[^/]+)/?(?:[?#]|$)'
|
||||
_TESTS = [{
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# coding: utf-8
|
||||
from .common import InfoExtractor
|
||||
from .vimeo import VHXEmbedIE
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
ExtractorError,
|
||||
get_element_by_class,
|
||||
get_element_by_id,
|
||||
get_elements_by_class,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
unified_strdate,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
class DropoutIE(InfoExtractor):
|
||||
_LOGIN_URL = 'https://www.dropout.tv/login'
|
||||
_NETRC_MACHINE = 'dropout'
|
||||
|
||||
_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?:[^/]+/)*videos/(?P<id>[^/]+)/?$'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'https://www.dropout.tv/game-changer/season:2/videos/yes-or-no',
|
||||
'note': 'Episode in a series',
|
||||
'md5': '5e000fdfd8d8fa46ff40456f1c2af04a',
|
||||
'info_dict': {
|
||||
'id': '738153',
|
||||
'display_id': 'yes-or-no',
|
||||
'ext': 'mp4',
|
||||
'title': 'Yes or No',
|
||||
'description': 'Ally, Brennan, and Zac are asked a simple question, but is there a correct answer?',
|
||||
'release_date': '20200508',
|
||||
'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/351e3f24-c4a3-459a-8b79-dc80f1e5b7fd.jpg',
|
||||
'series': 'Game Changer',
|
||||
'season_number': 2,
|
||||
'season': 'Season 2',
|
||||
'episode_number': 6,
|
||||
'episode': 'Yes or No',
|
||||
'duration': 1180,
|
||||
'uploader_id': 'user80538407',
|
||||
'uploader_url': 'https://vimeo.com/user80538407',
|
||||
'uploader': 'OTT Videos'
|
||||
},
|
||||
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
|
||||
},
|
||||
{
|
||||
'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1/videos/episode-1',
|
||||
'note': 'Episode in a series (missing release_date)',
|
||||
'md5': '712caf7c191f1c47c8f1879520c2fa5c',
|
||||
'info_dict': {
|
||||
'id': '320562',
|
||||
'display_id': 'episode-1',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Beginning Begins',
|
||||
'description': 'The cast introduces their PCs, including a neurotic elf, a goblin PI, and a corn-worshipping cleric.',
|
||||
'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/4421ed0d-f630-4c88-9004-5251b2b8adfa.jpg',
|
||||
'series': 'Dimension 20: Fantasy High',
|
||||
'season_number': 1,
|
||||
'season': 'Season 1',
|
||||
'episode_number': 1,
|
||||
'episode': 'The Beginning Begins',
|
||||
'duration': 6838,
|
||||
'uploader_id': 'user80538407',
|
||||
'uploader_url': 'https://vimeo.com/user80538407',
|
||||
'uploader': 'OTT Videos'
|
||||
},
|
||||
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
|
||||
},
|
||||
{
|
||||
'url': 'https://www.dropout.tv/videos/misfits-magic-holiday-special',
|
||||
'note': 'Episode not in a series',
|
||||
'md5': 'c30fa18999c5880d156339f13c953a26',
|
||||
'info_dict': {
|
||||
'id': '1915774',
|
||||
'display_id': 'misfits-magic-holiday-special',
|
||||
'ext': 'mp4',
|
||||
'title': 'Misfits & Magic Holiday Special',
|
||||
'description': 'The magical misfits spend Christmas break at Gowpenny, with an unwelcome visitor.',
|
||||
'release_date': '20211215',
|
||||
'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/d91ea8a6-b250-42ed-907e-b30fb1c65176-8e24b8e5.jpg',
|
||||
'duration': 11698,
|
||||
'uploader_id': 'user80538407',
|
||||
'uploader_url': 'https://vimeo.com/user80538407',
|
||||
'uploader': 'OTT Videos'
|
||||
},
|
||||
'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
|
||||
}
|
||||
]
|
||||
|
||||
def _get_authenticity_token(self, display_id):
|
||||
signin_page = self._download_webpage(
|
||||
self._LOGIN_URL, display_id, note='Getting authenticity token')
|
||||
return self._html_search_regex(
|
||||
r'name=["\']authenticity_token["\'] value=["\'](.+?)["\']',
|
||||
signin_page, 'authenticity_token')
|
||||
|
||||
def _login(self, display_id):
|
||||
username, password = self._get_login_info()
|
||||
if not (username and password):
|
||||
self.raise_login_required(method='password')
|
||||
|
||||
response = self._download_webpage(
|
||||
self._LOGIN_URL, display_id, note='Logging in', data=urlencode_postdata({
|
||||
'email': username,
|
||||
'password': password,
|
||||
'authenticity_token': self._get_authenticity_token(display_id),
|
||||
'utf8': True
|
||||
}))
|
||||
|
||||
user_has_subscription = self._search_regex(
|
||||
r'user_has_subscription:\s*["\'](.+?)["\']', response, 'subscription status', default='none')
|
||||
if user_has_subscription.lower() == 'true':
|
||||
return response
|
||||
elif user_has_subscription.lower() == 'false':
|
||||
raise ExtractorError('Account is not subscribed')
|
||||
else:
|
||||
raise ExtractorError('Incorrect username/password')
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
try:
|
||||
self._login(display_id)
|
||||
webpage = self._download_webpage(url, display_id, note='Downloading video webpage')
|
||||
finally:
|
||||
self._download_webpage('https://www.dropout.tv/logout', display_id, note='Logging out')
|
||||
|
||||
embed_url = self._search_regex(r'embed_url:\s*["\'](.+?)["\']', webpage, 'embed url')
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
watch_info = get_element_by_id('watch-info', webpage) or ''
|
||||
|
||||
title = clean_html(get_element_by_class('video-title', watch_info))
|
||||
season_episode = get_element_by_class(
|
||||
'site-font-secondary-color', get_element_by_class('text', watch_info))
|
||||
episode_number = int_or_none(self._search_regex(
|
||||
r'Episode (\d+)', season_episode or '', 'episode', default=None))
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': VHXEmbedIE.ie_key(),
|
||||
'url': embed_url,
|
||||
'id': self._search_regex(r'embed.vhx.tv/videos/(.+?)\?', embed_url, 'id'),
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': self._html_search_meta('description', webpage, fatal=False),
|
||||
'thumbnail': thumbnail.split('?')[0] if thumbnail else None, # Ignore crop/downscale
|
||||
'series': clean_html(get_element_by_class('series-title', watch_info)),
|
||||
'episode_number': episode_number,
|
||||
'episode': title if episode_number else None,
|
||||
'season_number': int_or_none(self._search_regex(
|
||||
r'Season (\d+),', season_episode or '', 'season', default=None)),
|
||||
'release_date': unified_strdate(self._search_regex(
|
||||
r'data-meta-field-name=["\']release_dates["\'] data-meta-field-value=["\'](.+?)["\']',
|
||||
watch_info, 'release date', default=None)),
|
||||
}
|
||||
|
||||
|
||||
class DropoutSeasonIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:[0-9]+/?$)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1',
|
||||
'note': 'Multi-season series with the season in the url',
|
||||
'playlist_count': 17,
|
||||
'info_dict': {
|
||||
'id': 'dimension-20-fantasy-high-season-1',
|
||||
'title': 'Dimension 20 Fantasy High - Season 1'
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'https://www.dropout.tv/dimension-20-fantasy-high',
|
||||
'note': 'Multi-season series with the season not in the url',
|
||||
'playlist_count': 17,
|
||||
'info_dict': {
|
||||
'id': 'dimension-20-fantasy-high-season-1',
|
||||
'title': 'Dimension 20 Fantasy High - Season 1'
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'https://www.dropout.tv/dimension-20-shriek-week',
|
||||
'note': 'Single-season series',
|
||||
'playlist_count': 4,
|
||||
'info_dict': {
|
||||
'id': 'dimension-20-shriek-week-season-1',
|
||||
'title': 'Dimension 20 Shriek Week - Season 1'
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
season_id = self._match_id(url)
|
||||
season_title = season_id.replace('-', ' ').title()
|
||||
webpage = self._download_webpage(url, season_id)
|
||||
|
||||
entries = [
|
||||
self.url_result(
|
||||
url=self._search_regex(r'<a href=["\'](.+?)["\'] class=["\']browse-item-link["\']',
|
||||
item, 'item_url'),
|
||||
ie=DropoutIE.ie_key()
|
||||
) for item in get_elements_by_class('js-collection-item', webpage)
|
||||
]
|
||||
|
||||
seasons = (get_element_by_class('select-dropdown-wrapper', webpage) or '').strip().replace('\n', '')
|
||||
current_season = self._search_regex(r'<option[^>]+selected>([^<]+)</option>',
|
||||
seasons, 'current_season', default='').strip()
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': join_nonempty(season_id, current_season.lower().replace(' ', '-')),
|
||||
'title': join_nonempty(season_title, current_season, delim=' - '),
|
||||
'entries': entries
|
||||
}
|
||||
@@ -321,7 +321,7 @@ class DRTVLiveIE(InfoExtractor):
|
||||
channel_data = self._download_json(
|
||||
'https://www.dr.dk/mu-online/api/1.0/channel/' + channel_id,
|
||||
channel_id)
|
||||
title = self._live_title(channel_data['Title'])
|
||||
title = channel_data['Title']
|
||||
|
||||
formats = []
|
||||
for streaming_server in channel_data.get('StreamingServers', []):
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
|
||||
from .abc import (
|
||||
ABCIE,
|
||||
ABCIViewIE,
|
||||
ABCIViewShowSeriesIE,
|
||||
)
|
||||
from .abcnews import (
|
||||
AbcNewsIE,
|
||||
@@ -356,6 +357,7 @@ from .dplay import (
|
||||
AnimalPlanetIE,
|
||||
DiscoveryPlusIndiaIE,
|
||||
DiscoveryNetworksDeIE,
|
||||
DiscoveryPlusItalyIE,
|
||||
DiscoveryPlusItalyShowIE,
|
||||
DiscoveryPlusIndiaShowIE,
|
||||
)
|
||||
@@ -384,6 +386,10 @@ from .disney import DisneyIE
|
||||
from .dispeak import DigitallySpeakingIE
|
||||
from .doodstream import DoodStreamIE
|
||||
from .dropbox import DropboxIE
|
||||
from .dropout import (
|
||||
DropoutSeasonIE,
|
||||
DropoutIE
|
||||
)
|
||||
from .dw import (
|
||||
DWIE,
|
||||
DWArticleIE,
|
||||
@@ -434,6 +440,7 @@ from .eyedotv import EyedoTVIE
|
||||
from .facebook import (
|
||||
FacebookIE,
|
||||
FacebookPluginsVideoIE,
|
||||
FacebookRedirectURLIE,
|
||||
)
|
||||
from .fancode import (
|
||||
FancodeVodIE,
|
||||
@@ -505,6 +512,14 @@ from .gab import (
|
||||
)
|
||||
from .gaia import GaiaIE
|
||||
from .gameinformer import GameInformerIE
|
||||
from .gamejolt import (
|
||||
GameJoltIE,
|
||||
GameJoltUserIE,
|
||||
GameJoltGameIE,
|
||||
GameJoltGameSoundtrackIE,
|
||||
GameJoltCommunityIE,
|
||||
GameJoltSearchIE,
|
||||
)
|
||||
from .gamespot import GameSpotIE
|
||||
from .gamestar import GameStarIE
|
||||
from .gaskrank import GaskrankIE
|
||||
@@ -523,6 +538,7 @@ from .globo import (
|
||||
)
|
||||
from .go import GoIE
|
||||
from .godtube import GodTubeIE
|
||||
from .gofile import GofileIE
|
||||
from .golem import GolemIE
|
||||
from .googledrive import GoogleDriveIE
|
||||
from .googlepodcasts import (
|
||||
@@ -562,6 +578,10 @@ from .hrti import (
|
||||
HRTiIE,
|
||||
HRTiPlaylistIE,
|
||||
)
|
||||
from .hse import (
|
||||
HSEShowIE,
|
||||
HSEProductIE,
|
||||
)
|
||||
from .huajiao import HuajiaoIE
|
||||
from .huffpost import HuffPostIE
|
||||
from .hungama import (
|
||||
@@ -856,6 +876,10 @@ from .n1 import (
|
||||
N1InfoAssetIE,
|
||||
N1InfoIIE,
|
||||
)
|
||||
from .nate import (
|
||||
NateIE,
|
||||
NateProgramIE,
|
||||
)
|
||||
from .nationalgeographic import (
|
||||
NationalGeographicVideoIE,
|
||||
NationalGeographicTVIE,
|
||||
@@ -1025,6 +1049,10 @@ from .ooyala import (
|
||||
OoyalaIE,
|
||||
OoyalaExternalIE,
|
||||
)
|
||||
from .opencast import (
|
||||
OpencastIE,
|
||||
OpencastPlaylistIE,
|
||||
)
|
||||
from .openrec import (
|
||||
OpenRecIE,
|
||||
OpenRecCaptureIE,
|
||||
@@ -1098,6 +1126,10 @@ from .pinterest import (
|
||||
PinterestIE,
|
||||
PinterestCollectionIE,
|
||||
)
|
||||
from .pixivsketch import (
|
||||
PixivSketchIE,
|
||||
PixivSketchUserIE,
|
||||
)
|
||||
from .pladform import PladformIE
|
||||
from .planetmarathi import PlanetMarathiIE
|
||||
from .platzi import (
|
||||
@@ -1249,7 +1281,13 @@ from .rtl2 import (
|
||||
from .rtp import RTPIE
|
||||
from .rtrfm import RTRFMIE
|
||||
from .rts import RTSIE
|
||||
from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
|
||||
from .rtve import (
|
||||
RTVEALaCartaIE,
|
||||
RTVEAudioIE,
|
||||
RTVELiveIE,
|
||||
RTVEInfantilIE,
|
||||
RTVETelevisionIE,
|
||||
)
|
||||
from .rtvnh import RTVNHIE
|
||||
from .rtvs import RTVSIE
|
||||
from .ruhd import RUHDIE
|
||||
@@ -1264,6 +1302,7 @@ from .rutube import (
|
||||
RutubeMovieIE,
|
||||
RutubePersonIE,
|
||||
RutubePlaylistIE,
|
||||
RutubeTagsIE,
|
||||
)
|
||||
from .rutv import RUTVIE
|
||||
from .ruutu import RuutuIE
|
||||
@@ -1315,6 +1354,7 @@ from .simplecast import (
|
||||
)
|
||||
from .sina import SinaIE
|
||||
from .sixplay import SixPlayIE
|
||||
from .skeb import SkebIE
|
||||
from .skyit import (
|
||||
SkyItPlayerIE,
|
||||
SkyItVideoIE,
|
||||
@@ -1350,6 +1390,7 @@ from .soundcloud import (
|
||||
SoundcloudEmbedIE,
|
||||
SoundcloudIE,
|
||||
SoundcloudSetIE,
|
||||
SoundcloudRelatedIE,
|
||||
SoundcloudUserIE,
|
||||
SoundcloudTrackStationIE,
|
||||
SoundcloudPlaylistIE,
|
||||
@@ -1497,6 +1538,9 @@ from .threeqsdn import ThreeQSDNIE
|
||||
from .tiktok import (
|
||||
TikTokIE,
|
||||
TikTokUserIE,
|
||||
TikTokSoundIE,
|
||||
TikTokEffectIE,
|
||||
TikTokTagIE,
|
||||
DouyinIE,
|
||||
)
|
||||
from .tinypic import TinyPicIE
|
||||
@@ -1511,6 +1555,9 @@ from .toggle import (
|
||||
ToggleIE,
|
||||
MeWatchIE,
|
||||
)
|
||||
from .toggo import (
|
||||
ToggoIE,
|
||||
)
|
||||
from .tokentube import (
|
||||
TokentubeIE,
|
||||
TokentubeChannelIE
|
||||
@@ -1527,6 +1574,7 @@ from .trovo import (
|
||||
TrovoChannelVodIE,
|
||||
TrovoChannelClipIE,
|
||||
)
|
||||
from .trueid import TrueIDIE
|
||||
from .trunews import TruNewsIE
|
||||
from .trutv import TruTVIE
|
||||
from .tube8 import Tube8IE
|
||||
|
||||
@@ -23,9 +23,11 @@ from ..utils import (
|
||||
merge_dicts,
|
||||
network_exceptions,
|
||||
parse_count,
|
||||
parse_qs,
|
||||
qualities,
|
||||
sanitized_Request,
|
||||
try_get,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
urljoin,
|
||||
)
|
||||
@@ -746,3 +748,42 @@ class FacebookPluginsVideoIE(InfoExtractor):
|
||||
return self.url_result(
|
||||
compat_urllib_parse_unquote(self._match_id(url)),
|
||||
FacebookIE.ie_key())
|
||||
|
||||
|
||||
class FacebookRedirectURLIE(InfoExtractor):
|
||||
IE_DESC = False # Do not list
|
||||
_VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/flx/warn[/?]'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.facebook.com/flx/warn/?h=TAQHsoToz&u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&s=1',
|
||||
'info_dict': {
|
||||
'id': 'pO8h3EaFRdo',
|
||||
'ext': 'mp4',
|
||||
'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
|
||||
'description': 'md5:2d713ccbb45b686a1888397b2c77ca6b',
|
||||
'channel_id': 'UCGBpxWJr9FNOcFYA5GkKrMg',
|
||||
'playable_in_embed': True,
|
||||
'categories': ['Music'],
|
||||
'channel': 'Boiler Room',
|
||||
'uploader_id': 'brtvofficial',
|
||||
'uploader': 'Boiler Room',
|
||||
'tags': 'count:11',
|
||||
'duration': 3332,
|
||||
'live_status': 'not_live',
|
||||
'thumbnail': 'https://i.ytimg.com/vi/pO8h3EaFRdo/maxresdefault.jpg',
|
||||
'channel_url': 'https://www.youtube.com/channel/UCGBpxWJr9FNOcFYA5GkKrMg',
|
||||
'availability': 'public',
|
||||
'uploader_url': 'http://www.youtube.com/user/brtvofficial',
|
||||
'upload_date': '20150917',
|
||||
'age_limit': 0,
|
||||
'view_count': int,
|
||||
'like_count': int,
|
||||
},
|
||||
'add_ie': ['Youtube'],
|
||||
'params': {'skip_download': 'Youtube'},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
redirect_url = url_or_none(parse_qs(url).get('u', [None])[-1])
|
||||
if not redirect_url:
|
||||
raise ExtractorError('Invalid facebook redirect URL', expected=True)
|
||||
return self.url_result(redirect_url)
|
||||
|
||||
@@ -41,7 +41,7 @@ class FancodeVodIE(InfoExtractor):
|
||||
_ACCESS_TOKEN = None
|
||||
_NETRC_MACHINE = 'fancode'
|
||||
|
||||
_LOGIN_HINT = 'Use "--user refresh --password <refresh_token>" to login using a refresh token'
|
||||
_LOGIN_HINT = 'Use "--username refresh --password <refresh_token>" to login using a refresh token'
|
||||
|
||||
headers = {
|
||||
'content-type': 'application/json',
|
||||
|
||||
@@ -170,7 +170,7 @@ class FilmOnChannelIE(InfoExtractor):
|
||||
return {
|
||||
'id': channel_id,
|
||||
'display_id': channel_data.get('alias'),
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': channel_data.get('description'),
|
||||
'thumbnails': thumbnails,
|
||||
'formats': formats,
|
||||
|
||||
@@ -203,7 +203,7 @@ class FranceTVIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'thumbnail': image,
|
||||
'duration': duration,
|
||||
'timestamp': timestamp,
|
||||
|
||||
@@ -59,9 +59,6 @@ class FreshLiveIE(InfoExtractor):
|
||||
stream_url, video_id, 'mp4',
|
||||
'm3u8_native', m3u8_id='hls')
|
||||
|
||||
if is_live:
|
||||
title = self._live_title(title)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
|
||||
@@ -5,19 +5,32 @@ from .common import InfoExtractor
|
||||
|
||||
|
||||
class FujiTVFODPlus7IE(InfoExtractor):
|
||||
_VALID_URL = r'https?://i\.fod\.fujitv\.co\.jp/plus7/web/[0-9a-z]{4}/(?P<id>[0-9a-z]+)'
|
||||
_VALID_URL = r'https?://fod\.fujitv\.co\.jp/title/[0-9a-z]{4}/(?P<id>[0-9a-z]+)'
|
||||
_BASE_URL = 'http://i.fod.fujitv.co.jp/'
|
||||
_BITRATE_MAP = {
|
||||
300: (320, 180),
|
||||
800: (640, 360),
|
||||
1200: (1280, 720),
|
||||
2000: (1280, 720),
|
||||
4000: (1920, 1080),
|
||||
}
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://fod.fujitv.co.jp/title/5d40/5d40810075',
|
||||
'info_dict': {
|
||||
'id': '5d40810075',
|
||||
'title': '5d40810075',
|
||||
'ext': 'mp4',
|
||||
'format_id': '4000',
|
||||
'thumbnail': 'http://i.fod.fujitv.co.jp/pc/image/wbtn/wbtn_5d40810075.jpg'
|
||||
},
|
||||
'skip': 'Expires after a week'
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
formats = self._extract_m3u8_formats(
|
||||
self._BASE_URL + 'abr/pc_html5/%s.m3u8' % video_id, video_id, 'mp4')
|
||||
self._BASE_URL + 'abr/tv_android/%s.m3u8' % video_id, video_id, 'mp4')
|
||||
for f in formats:
|
||||
wh = self._BITRATE_MAP.get(f.get('tbr'))
|
||||
if wh:
|
||||
|
||||
@@ -0,0 +1,540 @@
|
||||
# coding: utf-8
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_urllib_parse_unquote
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
try_get
|
||||
)
|
||||
|
||||
|
||||
class GameJoltBaseIE(InfoExtractor):
|
||||
_API_BASE = 'https://gamejolt.com/site-api/'
|
||||
|
||||
def _call_api(self, endpoint, *args, **kwargs):
|
||||
kwargs.setdefault('headers', {}).update({'Accept': 'image/webp,*/*'})
|
||||
return self._download_json(self._API_BASE + endpoint, *args, **kwargs)['payload']
|
||||
|
||||
def _parse_content_as_text(self, content):
|
||||
outer_contents, joined_contents = content.get('content') or [], []
|
||||
for outer_content in outer_contents:
|
||||
if outer_content.get('type') != 'paragraph':
|
||||
joined_contents.append(self._parse_content_as_text(outer_content))
|
||||
continue
|
||||
inner_contents, inner_content_text = outer_content.get('content') or [], ''
|
||||
for inner_content in inner_contents:
|
||||
if inner_content.get('text'):
|
||||
inner_content_text += inner_content['text']
|
||||
elif inner_content.get('type') == 'hardBreak':
|
||||
inner_content_text += '\n'
|
||||
joined_contents.append(inner_content_text)
|
||||
|
||||
return '\n'.join(joined_contents)
|
||||
|
||||
def _get_comments(self, post_num_id, post_hash_id):
|
||||
sort_by, scroll_id = self._configuration_arg('comment_sort', ['hot'], ie_key=GameJoltIE.ie_key())[0], -1
|
||||
is_scrolled = sort_by in ('new', 'you')
|
||||
for page in itertools.count(1):
|
||||
comments_data = self._call_api(
|
||||
'comments/Fireside_Post/%s/%s?%s=%d' % (
|
||||
post_num_id, sort_by,
|
||||
'scroll_id' if is_scrolled else 'page', scroll_id if is_scrolled else page),
|
||||
post_hash_id, note='Downloading comments list page %d' % page)
|
||||
if not comments_data.get('comments'):
|
||||
break
|
||||
for comment in traverse_obj(comments_data, (('comments', 'childComments'), ...), expected_type=dict, default=[]):
|
||||
yield {
|
||||
'id': comment['id'],
|
||||
'text': self._parse_content_as_text(
|
||||
self._parse_json(comment['comment_content'], post_hash_id)),
|
||||
'timestamp': int_or_none(comment.get('posted_on'), scale=1000),
|
||||
'like_count': comment.get('votes'),
|
||||
'author': traverse_obj(comment, ('user', ('display_name', 'name')), expected_type=str_or_none, get_all=False),
|
||||
'author_id': traverse_obj(comment, ('user', 'username'), expected_type=str_or_none),
|
||||
'author_thumbnail': traverse_obj(comment, ('user', 'image_avatar'), expected_type=str_or_none),
|
||||
'parent': comment.get('parent_id') or None,
|
||||
}
|
||||
scroll_id = int_or_none(comments_data['comments'][-1].get('posted_on'))
|
||||
|
||||
def _parse_post(self, post_data):
|
||||
post_id = post_data['hash']
|
||||
lead_content = self._parse_json(post_data.get('lead_content') or '{}', post_id, fatal=False) or {}
|
||||
description, full_description = post_data.get('leadStr') or self._parse_content_as_text(
|
||||
self._parse_json(post_data.get('lead_content'), post_id)), None
|
||||
if post_data.get('has_article'):
|
||||
article_content = self._parse_json(
|
||||
post_data.get('article_content')
|
||||
or self._call_api(f'web/posts/article/{post_data.get("id", post_id)}', post_id,
|
||||
note='Downloading article metadata', errnote='Unable to download article metadata', fatal=False).get('article'),
|
||||
post_id, fatal=False)
|
||||
full_description = self._parse_content_as_text(article_content)
|
||||
|
||||
user_data = post_data.get('user') or {}
|
||||
info_dict = {
|
||||
'extractor_key': GameJoltIE.ie_key(),
|
||||
'extractor': 'GameJolt',
|
||||
'webpage_url': str_or_none(post_data.get('url')) or f'https://gamejolt.com/p/{post_id}',
|
||||
'id': post_id,
|
||||
'title': description,
|
||||
'description': full_description or description,
|
||||
'display_id': post_data.get('slug'),
|
||||
'uploader': user_data.get('display_name') or user_data.get('name'),
|
||||
'uploader_id': user_data.get('username'),
|
||||
'uploader_url': 'https://gamejolt.com' + user_data['url'] if user_data.get('url') else None,
|
||||
'categories': [try_get(category, lambda x: '%s - %s' % (x['community']['name'], x['channel'].get('display_title') or x['channel']['title']))
|
||||
for category in post_data.get('communities' or [])],
|
||||
'tags': traverse_obj(
|
||||
lead_content, ('content', ..., 'content', ..., 'marks', ..., 'attrs', 'tag'), expected_type=str_or_none),
|
||||
'like_count': int_or_none(post_data.get('like_count')),
|
||||
'comment_count': int_or_none(post_data.get('comment_count'), default=0),
|
||||
'timestamp': int_or_none(post_data.get('added_on'), scale=1000),
|
||||
'release_timestamp': int_or_none(post_data.get('published_on'), scale=1000),
|
||||
'__post_extractor': self.extract_comments(post_data.get('id'), post_id)
|
||||
}
|
||||
|
||||
# TODO: Handle multiple videos/embeds?
|
||||
video_data = traverse_obj(post_data, ('videos', ...), expected_type=dict, get_all=False) or {}
|
||||
formats, subtitles, thumbnails = [], {}, []
|
||||
for media in video_data.get('media') or []:
|
||||
media_url, mimetype, ext, media_id = media['img_url'], media.get('filetype', ''), determine_ext(media['img_url']), media.get('type')
|
||||
if mimetype == 'application/vnd.apple.mpegurl' or ext == 'm3u8':
|
||||
hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(media_url, post_id, 'mp4', m3u8_id=media_id)
|
||||
formats.extend(hls_formats)
|
||||
subtitles.update(hls_subs)
|
||||
elif mimetype == 'application/dash+xml' or ext == 'mpd':
|
||||
dash_formats, dash_subs = self._extract_mpd_formats_and_subtitles(media_url, post_id, mpd_id=media_id)
|
||||
formats.extend(dash_formats)
|
||||
subtitles.update(dash_subs)
|
||||
elif 'image' in mimetype:
|
||||
thumbnails.append({
|
||||
'id': media_id,
|
||||
'url': media_url,
|
||||
'width': media.get('width'),
|
||||
'height': media.get('height'),
|
||||
'filesize': media.get('filesize'),
|
||||
})
|
||||
else:
|
||||
formats.append({
|
||||
'format_id': media_id,
|
||||
'url': media_url,
|
||||
'width': media.get('width'),
|
||||
'height': media.get('height'),
|
||||
'filesize': media.get('filesize'),
|
||||
'acodec': 'none' if 'video-card' in media_url else None,
|
||||
})
|
||||
|
||||
if formats:
|
||||
return {
|
||||
**info_dict,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnails': thumbnails,
|
||||
'view_count': int_or_none(video_data.get('view_count')),
|
||||
}
|
||||
|
||||
gif_entries = []
|
||||
for media in post_data.get('media', []):
|
||||
if determine_ext(media['img_url']) != 'gif' or 'gif' not in media.get('filetype', ''):
|
||||
continue
|
||||
gif_entries.append({
|
||||
'id': media['hash'],
|
||||
'title': media['filename'].split('.')[0],
|
||||
'formats': [{
|
||||
'format_id': url_key,
|
||||
'url': media[url_key],
|
||||
'width': media.get('width') if url_key == 'img_url' else None,
|
||||
'height': media.get('height') if url_key == 'img_url' else None,
|
||||
'filesize': media.get('filesize') if url_key == 'img_url' else None,
|
||||
'acodec': 'none',
|
||||
} for url_key in ('img_url', 'mediaserver_url', 'mediaserver_url_mp4', 'mediaserver_url_webm') if media.get(url_key)]
|
||||
})
|
||||
if gif_entries:
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
**info_dict,
|
||||
'entries': gif_entries,
|
||||
}
|
||||
|
||||
embed_url = traverse_obj(post_data, ('embeds', ..., 'url'), expected_type=str_or_none, get_all=False)
|
||||
if embed_url:
|
||||
return self.url_result(embed_url)
|
||||
return info_dict
|
||||
|
||||
|
||||
class GameJoltIE(GameJoltBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/p/(?:[\w-]*-)?(?P<id>\w{8})'
|
||||
_TESTS = [{
|
||||
# No audio
|
||||
'url': 'https://gamejolt.com/p/introducing-ramses-jackson-some-fnf-himbo-i-ve-been-animating-fo-c6achnzu',
|
||||
'md5': 'cd5f733258f6678b0ce500dd88166d86',
|
||||
'info_dict': {
|
||||
'id': 'c6achnzu',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'introducing-ramses-jackson-some-fnf-himbo-i-ve-been-animating-fo-c6achnzu',
|
||||
'title': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin',
|
||||
'description': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin',
|
||||
'uploader': 'Jakeneutron',
|
||||
'uploader_id': 'Jakeneutron',
|
||||
'uploader_url': 'https://gamejolt.com/@Jakeneutron',
|
||||
'categories': ['Friday Night Funkin\' - Videos'],
|
||||
'tags': ['fnfmod', 'fridaynightfunkin'],
|
||||
'timestamp': 1633499590,
|
||||
'upload_date': '20211006',
|
||||
'release_timestamp': 1633499655,
|
||||
'release_date': '20211006',
|
||||
'thumbnail': 're:^https?://.+wgch9mhq.png$',
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
# YouTube embed
|
||||
'url': 'https://gamejolt.com/p/hey-hey-if-there-s-anyone-who-s-looking-to-get-into-learning-a-n6g4jzpq',
|
||||
'md5': '79a931ff500a5c783ef6c3bda3272e32',
|
||||
'info_dict': {
|
||||
'id': 'XsNA_mzC0q4',
|
||||
'title': 'Adobe Animate CC 2021 Tutorial || Part 1 - The Basics',
|
||||
'description': 'md5:9d1ab9e2625b3fe1f42b2a44c67fdd13',
|
||||
'uploader': 'Jakeneutron',
|
||||
'uploader_id': 'Jakeneutron',
|
||||
'uploader_url': 'http://www.youtube.com/user/Jakeneutron',
|
||||
'ext': 'mp4',
|
||||
'duration': 1749,
|
||||
'tags': ['Adobe Animate CC', 'Tutorial', 'Animation', 'The Basics', 'For Beginners'],
|
||||
'like_count': int,
|
||||
'playable_in_embed': True,
|
||||
'categories': ['Education'],
|
||||
'availability': 'public',
|
||||
'thumbnail': 'https://i.ytimg.com/vi_webp/XsNA_mzC0q4/maxresdefault.webp',
|
||||
'age_limit': 0,
|
||||
'live_status': 'not_live',
|
||||
'channel_url': 'https://www.youtube.com/channel/UC6_L7fnczNalFZyBthUE9oA',
|
||||
'channel': 'Jakeneutron',
|
||||
'channel_id': 'UC6_L7fnczNalFZyBthUE9oA',
|
||||
'upload_date': '20211015',
|
||||
'view_count': int,
|
||||
'chapters': 'count:18',
|
||||
}
|
||||
}, {
|
||||
# Article
|
||||
'url': 'https://gamejolt.com/p/i-fuckin-broke-chaos-d56h3eue',
|
||||
'md5': '786c1ccf98fde02c03a2768acb4258d0',
|
||||
'info_dict': {
|
||||
'id': 'd56h3eue',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'i-fuckin-broke-chaos-d56h3eue',
|
||||
'title': 'I fuckin broke Chaos.',
|
||||
'description': 'I moved my tab durning the cutscene so now it\'s stuck like this.',
|
||||
'uploader': 'Jeff____________',
|
||||
'uploader_id': 'The_Nyesh_Man',
|
||||
'uploader_url': 'https://gamejolt.com/@The_Nyesh_Man',
|
||||
'categories': ['Friday Night Funkin\' - Videos'],
|
||||
'timestamp': 1639800264,
|
||||
'upload_date': '20211218',
|
||||
'release_timestamp': 1639800330,
|
||||
'release_date': '20211218',
|
||||
'thumbnail': 're:^https?://.+euksy8bd.png$',
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
'view_count': int,
|
||||
}
|
||||
}, {
|
||||
# Single GIF
|
||||
'url': 'https://gamejolt.com/p/hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
|
||||
'info_dict': {
|
||||
'id': 'vs4gdrd8',
|
||||
'display_id': 'hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
|
||||
'title': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
|
||||
'description': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
|
||||
'uploader': 'Quesoguy',
|
||||
'uploader_id': 'CheeseguyDev',
|
||||
'uploader_url': 'https://gamejolt.com/@CheeseguyDev',
|
||||
'categories': ['Game Dev - General', 'Arts n\' Crafts - Creations', 'Pixel Art - showcase',
|
||||
'Friday Night Funkin\' - Mods', 'Newgrounds - Friday Night Funkin (13+)'],
|
||||
'timestamp': 1639517122,
|
||||
'release_timestamp': 1639519966,
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'dszyjnwi',
|
||||
'ext': 'webm',
|
||||
'title': 'gif-presentacion-mejorado-dszyjnwi',
|
||||
'n_entries': 1,
|
||||
}
|
||||
}]
|
||||
}, {
|
||||
# Multiple GIFs
|
||||
'url': 'https://gamejolt.com/p/gif-yhsqkumq',
|
||||
'playlist_count': 35,
|
||||
'info_dict': {
|
||||
'id': 'yhsqkumq',
|
||||
'display_id': 'gif-yhsqkumq',
|
||||
'title': 'GIF',
|
||||
'description': 'GIF',
|
||||
'uploader': 'DaniilTvman',
|
||||
'uploader_id': 'DaniilTvman',
|
||||
'uploader_url': 'https://gamejolt.com/@DaniilTvman',
|
||||
'categories': ['Five Nights At The AGK Studio Comunity - NEWS game'],
|
||||
'timestamp': 1638721559,
|
||||
'release_timestamp': 1638722276,
|
||||
'like_count': int,
|
||||
'comment_count': int,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
post_id = self._match_id(url)
|
||||
post_data = self._call_api(
|
||||
f'web/posts/view/{post_id}', post_id)['post']
|
||||
return self._parse_post(post_data)
|
||||
|
||||
|
||||
class GameJoltPostListBaseIE(GameJoltBaseIE):
|
||||
def _entries(self, endpoint, list_id, note='Downloading post list', errnote='Unable to download post list', initial_items=[]):
|
||||
page_num, scroll_id = 1, None
|
||||
items = initial_items or self._call_api(endpoint, list_id, note=note, errnote=errnote)['items']
|
||||
while items:
|
||||
for item in items:
|
||||
yield self._parse_post(item['action_resource_model'])
|
||||
scroll_id = items[-1]['scroll_id']
|
||||
page_num += 1
|
||||
items = self._call_api(
|
||||
endpoint, list_id, note=f'{note} page {page_num}', errnote=errnote, data=json.dumps({
|
||||
'scrollDirection': 'from',
|
||||
'scrollId': scroll_id,
|
||||
}).encode('utf-8')).get('items')
|
||||
|
||||
|
||||
class GameJoltUserIE(GameJoltPostListBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/@(?P<id>[\w-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://gamejolt.com/@BlazikenSuperStar',
|
||||
'playlist_mincount': 1,
|
||||
'info_dict': {
|
||||
'id': '6116784',
|
||||
'title': 'S. Blaze',
|
||||
'description': 'md5:5ba7fbbb549e8ea2545aafbfe22eb03a',
|
||||
},
|
||||
'params': {
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
user_id = self._match_id(url)
|
||||
user_data = self._call_api(
|
||||
f'web/profile/@{user_id}', user_id, note='Downloading user info', errnote='Unable to download user info')['user']
|
||||
bio = self._parse_content_as_text(
|
||||
self._parse_json(user_data.get('bio_content', '{}'), user_id, fatal=False) or {})
|
||||
return self.playlist_result(
|
||||
self._entries(f'web/posts/fetch/user/@{user_id}?tab=active', user_id, 'Downloading user posts', 'Unable to download user posts'),
|
||||
str_or_none(user_data.get('id')), user_data.get('display_name') or user_data.get('name'), bio)
|
||||
|
||||
|
||||
class GameJoltGameIE(GameJoltPostListBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/games/[\w-]+/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://gamejolt.com/games/Friday4Fun/655124',
|
||||
'playlist_mincount': 2,
|
||||
'info_dict': {
|
||||
'id': '655124',
|
||||
'title': 'Friday Night Funkin\': Friday 4 Fun',
|
||||
'description': 'md5:576a7dd87912a2dcf33c50d2bd3966d3'
|
||||
},
|
||||
'params': {
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
game_id = self._match_id(url)
|
||||
game_data = self._call_api(
|
||||
f'web/discover/games/{game_id}', game_id, note='Downloading game info', errnote='Unable to download game info')['game']
|
||||
description = self._parse_content_as_text(
|
||||
self._parse_json(game_data.get('description_content', '{}'), game_id, fatal=False) or {})
|
||||
return self.playlist_result(
|
||||
self._entries(f'web/posts/fetch/game/{game_id}', game_id, 'Downloading game posts', 'Unable to download game posts'),
|
||||
game_id, game_data.get('title'), description)
|
||||
|
||||
|
||||
class GameJoltGameSoundtrackIE(GameJoltBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/get/soundtrack(?:\?|\#!?)(?:.*?[&;])??game=(?P<id>(?:\d+)+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://gamejolt.com/get/soundtrack?foo=bar&game=657899',
|
||||
'info_dict': {
|
||||
'id': '657899',
|
||||
'title': 'Friday Night Funkin\': Vs Oswald',
|
||||
'n_entries': None,
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '184434',
|
||||
'ext': 'mp3',
|
||||
'title': 'Gettin\' Lucky (Menu Music)',
|
||||
'url': r're:^https://.+vs-oswald-menu-music\.mp3$',
|
||||
'release_timestamp': 1635190816,
|
||||
'release_date': '20211025',
|
||||
'n_entries': 3,
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '184435',
|
||||
'ext': 'mp3',
|
||||
'title': 'Rabbit\'s Luck (Extended Version)',
|
||||
'url': r're:^https://.+rabbit-s-luck--full-version-\.mp3$',
|
||||
'release_timestamp': 1635190841,
|
||||
'release_date': '20211025',
|
||||
'n_entries': 3,
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '185228',
|
||||
'ext': 'mp3',
|
||||
'title': 'Last Straw',
|
||||
'url': r're:^https://.+last-straw\.mp3$',
|
||||
'release_timestamp': 1635881104,
|
||||
'release_date': '20211102',
|
||||
'n_entries': 3,
|
||||
}
|
||||
}]
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
game_id = self._match_id(url)
|
||||
game_overview = self._call_api(
|
||||
f'web/discover/games/overview/{game_id}', game_id, note='Downloading soundtrack info', errnote='Unable to download soundtrack info')
|
||||
return self.playlist_result([{
|
||||
'id': str_or_none(song.get('id')),
|
||||
'title': str_or_none(song.get('title')),
|
||||
'url': str_or_none(song.get('url')),
|
||||
'release_timestamp': int_or_none(song.get('posted_on'), scale=1000),
|
||||
} for song in game_overview.get('songs') or []], game_id, traverse_obj(
|
||||
game_overview, ('microdata', 'name'), (('twitter', 'fb'), 'title'), expected_type=str_or_none, get_all=False))
|
||||
|
||||
|
||||
class GameJoltCommunityIE(GameJoltPostListBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/c/(?P<id>(?P<community>[\w-]+)(?:/(?P<channel>[\w-]+))?)(?:(?:\?|\#!?)(?:.*?[&;])??sort=(?P<sort>\w+))?'
|
||||
_TESTS = [{
|
||||
'url': 'https://gamejolt.com/c/fnf/videos',
|
||||
'playlist_mincount': 50,
|
||||
'info_dict': {
|
||||
'id': 'fnf/videos',
|
||||
'title': 'Friday Night Funkin\' - Videos',
|
||||
'description': 'md5:6d8c06f27460f7d35c1554757ffe53c8'
|
||||
},
|
||||
'params': {
|
||||
'playlistend': 50,
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
|
||||
}, {
|
||||
'url': 'https://gamejolt.com/c/youtubers',
|
||||
'playlist_mincount': 50,
|
||||
'info_dict': {
|
||||
'id': 'youtubers/featured',
|
||||
'title': 'Youtubers - featured',
|
||||
'description': 'md5:53e5582c93dcc467ab597bfca4db17d4'
|
||||
},
|
||||
'params': {
|
||||
'playlistend': 50,
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id, community_id, channel_id, sort_by = self._match_valid_url(url).group('id', 'community', 'channel', 'sort')
|
||||
channel_id, sort_by = channel_id or 'featured', sort_by or 'new'
|
||||
|
||||
community_data = self._call_api(
|
||||
f'web/communities/view/{community_id}', display_id,
|
||||
note='Downloading community info', errnote='Unable to download community info')['community']
|
||||
channel_data = traverse_obj(self._call_api(
|
||||
f'web/communities/view-channel/{community_id}/{channel_id}', display_id,
|
||||
note='Downloading channel info', errnote='Unable to download channel info', fatal=False), 'channel') or {}
|
||||
|
||||
title = f'{community_data.get("name") or community_id} - {channel_data.get("display_title") or channel_id}'
|
||||
description = self._parse_content_as_text(
|
||||
self._parse_json(community_data.get('description_content') or '{}', display_id, fatal=False) or {})
|
||||
return self.playlist_result(
|
||||
self._entries(
|
||||
f'web/posts/fetch/community/{community_id}?channels[]={sort_by}&channels[]={channel_id}',
|
||||
display_id, 'Downloading community posts', 'Unable to download community posts'),
|
||||
f'{community_id}/{channel_id}', title, description)
|
||||
|
||||
|
||||
class GameJoltSearchIE(GameJoltPostListBaseIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?gamejolt\.com/search(?:/(?P<filter>communities|users|games))?(?:\?|\#!?)(?:.*?[&;])??q=(?P<id>(?:[^&#]+)+)'
|
||||
_URL_FORMATS = {
|
||||
'users': 'https://gamejolt.com/@{username}',
|
||||
'communities': 'https://gamejolt.com/c/{path}',
|
||||
'games': 'https://gamejolt.com/games/{slug}/{id}',
|
||||
}
|
||||
_TESTS = [{
|
||||
'url': 'https://gamejolt.com/search?foo=bar&q=%23fnf',
|
||||
'playlist_mincount': 50,
|
||||
'info_dict': {
|
||||
'id': '#fnf',
|
||||
'title': '#fnf',
|
||||
},
|
||||
'params': {
|
||||
'playlistend': 50,
|
||||
'ignore_no_formats_error': True,
|
||||
},
|
||||
'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
|
||||
}, {
|
||||
'url': 'https://gamejolt.com/search/communities?q=cookie%20run',
|
||||
'playlist_mincount': 10,
|
||||
'info_dict': {
|
||||
'id': 'cookie run',
|
||||
'title': 'cookie run',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://gamejolt.com/search/users?q=mlp',
|
||||
'playlist_mincount': 278,
|
||||
'info_dict': {
|
||||
'id': 'mlp',
|
||||
'title': 'mlp',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://gamejolt.com/search/games?q=roblox',
|
||||
'playlist_mincount': 688,
|
||||
'info_dict': {
|
||||
'id': 'roblox',
|
||||
'title': 'roblox',
|
||||
},
|
||||
}]
|
||||
|
||||
def _search_entries(self, query, filter_mode, display_query):
|
||||
initial_search_data = self._call_api(
|
||||
f'web/search/{filter_mode}?q={query}', display_query,
|
||||
note=f'Downloading {filter_mode} list', errnote=f'Unable to download {filter_mode} list')
|
||||
entries_num = traverse_obj(initial_search_data, 'count', f'{filter_mode}Count')
|
||||
if not entries_num:
|
||||
return
|
||||
for page in range(1, math.ceil(entries_num / initial_search_data['perPage']) + 1):
|
||||
search_results = self._call_api(
|
||||
f'web/search/{filter_mode}?q={query}&page={page}', display_query,
|
||||
note=f'Downloading {filter_mode} list page {page}', errnote=f'Unable to download {filter_mode} list')
|
||||
for result in search_results[filter_mode]:
|
||||
yield self.url_result(self._URL_FORMATS[filter_mode].format(**result))
|
||||
|
||||
def _real_extract(self, url):
|
||||
filter_mode, query = self._match_valid_url(url).group('filter', 'id')
|
||||
display_query = compat_urllib_parse_unquote(query)
|
||||
return self.playlist_result(
|
||||
self._search_entries(query, filter_mode, display_query) if filter_mode else self._entries(
|
||||
f'web/posts/fetch/search/{query}', display_query, initial_items=self._call_api(
|
||||
f'web/search?q={query}', display_query,
|
||||
note='Downloading initial post list', errnote='Unable to download initial post list')['posts']),
|
||||
display_query, display_query)
|
||||
@@ -2344,6 +2344,18 @@ class GenericIE(InfoExtractor):
|
||||
'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
|
||||
}
|
||||
},
|
||||
{
|
||||
# KVS Player (for sites that serve kt_player.js via non-https urls)
|
||||
'url': 'http://www.camhub.world/embed/389508',
|
||||
'md5': 'fbe89af4cfb59c8fd9f34a202bb03e32',
|
||||
'info_dict': {
|
||||
'id': '389508',
|
||||
'display_id': 'syren-de-mer-onlyfans-05-07-2020have-a-happy-safe-holiday5f014e68a220979bdb8cd-source',
|
||||
'ext': 'mp4',
|
||||
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
|
||||
'thumbnail': 'http://www.camhub.world/contents/videos_screenshots/389000/389508/preview.mp4.jpg',
|
||||
}
|
||||
},
|
||||
{
|
||||
# Reddit-hosted video that will redirect and be processed by RedditIE
|
||||
# Redirects to https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/
|
||||
@@ -3653,6 +3665,10 @@ class GenericIE(InfoExtractor):
|
||||
json_ld = self._search_json_ld(webpage, video_id, default={})
|
||||
if json_ld.get('url'):
|
||||
self.report_detected('JSON LD')
|
||||
if determine_ext(json_ld.get('url')) == 'm3u8':
|
||||
json_ld['formats'], json_ld['subtitles'] = self._extract_m3u8_formats_and_subtitles(
|
||||
json_ld['url'], video_id, 'mp4')
|
||||
json_ld.pop('url')
|
||||
return merge_dicts(json_ld, info_dict)
|
||||
|
||||
def check_video(vurl):
|
||||
@@ -3685,7 +3701,7 @@ class GenericIE(InfoExtractor):
|
||||
self.report_detected('JW Player embed')
|
||||
if not found:
|
||||
# Look for generic KVS player
|
||||
found = re.search(r'<script [^>]*?src="https://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
|
||||
found = re.search(r'<script [^>]*?src="https?://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
|
||||
if found:
|
||||
self.report_detected('KWS Player')
|
||||
if found.group('maj_ver') not in ['4', '5']:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# coding: utf-8
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
try_get
|
||||
)
|
||||
|
||||
|
||||
class GofileIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?gofile\.io/d/(?P<id>[^/]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://gofile.io/d/AMZyDw',
|
||||
'info_dict': {
|
||||
'id': 'AMZyDw',
|
||||
},
|
||||
'playlist_mincount': 2,
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'de571ac1-5edc-42e2-8ec2-bdac83ad4a31',
|
||||
'filesize': 928116,
|
||||
'ext': 'mp4',
|
||||
'title': 'nuuh'
|
||||
}
|
||||
}]
|
||||
}, { # URL to test mixed file types
|
||||
'url': 'https://gofile.io/d/avt34h',
|
||||
'info_dict': {
|
||||
'id': 'avt34h',
|
||||
},
|
||||
'playlist_mincount': 1,
|
||||
}, { # URL to test no video/audio error
|
||||
'url': 'https://gofile.io/d/aB03lZ',
|
||||
'info_dict': {
|
||||
'id': 'aB03lZ',
|
||||
},
|
||||
'playlist_count': 0,
|
||||
'skip': 'No video/audio found at provided URL.',
|
||||
}]
|
||||
_TOKEN = None
|
||||
|
||||
def _real_initialize(self):
|
||||
token = self._get_cookies('https://gofile.io/').get('accountToken')
|
||||
if token:
|
||||
self._TOKEN = token.value
|
||||
return
|
||||
|
||||
account_data = self._download_json(
|
||||
'https://api.gofile.io/createAccount', None, note='Getting a new guest account')
|
||||
self._TOKEN = account_data['data']['token']
|
||||
self._set_cookie('gofile.io', 'accountToken', self._TOKEN)
|
||||
|
||||
def _entries(self, file_id):
|
||||
files = self._download_json(
|
||||
f'https://api.gofile.io/getContent?contentId={file_id}&token={self._TOKEN}&websiteToken=websiteToken&cache=true',
|
||||
'Gofile', note='Getting filelist')
|
||||
|
||||
status = files['status']
|
||||
if status != 'ok':
|
||||
raise ExtractorError(f'{self.IE_NAME} said: status {status}', expected=True)
|
||||
|
||||
found_files = False
|
||||
for file in (try_get(files, lambda x: x['data']['contents'], dict) or {}).values():
|
||||
file_type, file_format = file.get('mimetype').split('/', 1)
|
||||
if file_type not in ('video', 'audio') and file_format != 'vnd.mts':
|
||||
continue
|
||||
|
||||
found_files = True
|
||||
file_url = file.get('directLink')
|
||||
if file_url:
|
||||
yield {
|
||||
'id': file['id'],
|
||||
'title': file['name'].rsplit('.', 1)[0],
|
||||
'url': file_url,
|
||||
'filesize': file.get('size'),
|
||||
'release_timestamp': file.get('createTime')
|
||||
}
|
||||
|
||||
if not found_files:
|
||||
raise ExtractorError('No video/audio found at provided URL.', expected=True)
|
||||
|
||||
def _real_extract(self, url):
|
||||
file_id = self._match_id(url)
|
||||
return self.playlist_result(self._entries(file_id), playlist_id=file_id)
|
||||
@@ -6,7 +6,7 @@ from ..utils import unified_strdate
|
||||
|
||||
|
||||
class GronkhIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?gronkh\.tv/stream/(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?gronkh\.tv/(?:watch/)?stream/(?P<id>\d+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://gronkh.tv/stream/536',
|
||||
@@ -19,6 +19,9 @@ class GronkhIE(InfoExtractor):
|
||||
'upload_date': '20211001'
|
||||
},
|
||||
'params': {'skip_download': True}
|
||||
}, {
|
||||
'url': 'https://gronkh.tv/watch/stream/546',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
||||
@@ -209,6 +209,6 @@ class HitboxLiveIE(HitboxIE):
|
||||
'https://www.smashcast.tv/api/media/live', video_id)
|
||||
metadata['formats'] = formats
|
||||
metadata['is_live'] = True
|
||||
metadata['title'] = self._live_title(metadata.get('title'))
|
||||
metadata['title'] = metadata.get('title')
|
||||
|
||||
return metadata
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# coding: utf-8
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
traverse_obj,
|
||||
unified_timestamp,
|
||||
)
|
||||
|
||||
|
||||
class HSEShowBaseInfoExtractor(InfoExtractor):
|
||||
_GEO_COUNTRIES = ['DE']
|
||||
|
||||
def _extract_redux_data(self, url, video_id):
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
redux = self._html_search_regex(
|
||||
r'window\.__REDUX_DATA__\s*=\s*({.*});?', webpage, 'redux data')
|
||||
return self._parse_json(redux.replace('\n', ''), video_id)
|
||||
|
||||
def _extract_formats_and_subtitles(self, sources, video_id):
|
||||
if not sources:
|
||||
raise ExtractorError('No video found', expected=True, video_id=video_id)
|
||||
formats, subtitles = [], {}
|
||||
for src in sources:
|
||||
if src['mimetype'] != 'application/x-mpegURL':
|
||||
continue
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, ext='mp4')
|
||||
formats.extend(fmts)
|
||||
subtitles = self._merge_subtitles(subtitles, subs)
|
||||
self._sort_formats(formats)
|
||||
return formats, subtitles
|
||||
|
||||
|
||||
class HSEShowIE(HSEShowBaseInfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?hse\.de/dpl/c/tv-shows/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.hse.de/dpl/c/tv-shows/505350',
|
||||
'info_dict': {
|
||||
'id': '505350',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pfeffinger Mode & Accessoires',
|
||||
'timestamp': 1638810000,
|
||||
'upload_date': '20211206',
|
||||
'channel': 'HSE24',
|
||||
'uploader': 'Arina Pirayesh'
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
json_data = self._extract_redux_data(url, video_id)
|
||||
formats, subtitles = self._extract_formats_and_subtitles(
|
||||
traverse_obj(json_data, ('tvShowPage', 'tvShowVideo', 'sources')), video_id)
|
||||
|
||||
show = traverse_obj(json_data, ('tvShowPage', 'tvShow')) or {}
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': show.get('title') or video_id,
|
||||
'formats': formats,
|
||||
'timestamp': unified_timestamp(f'{show.get("date")} {show.get("hour")}:00'),
|
||||
'thumbnail': traverse_obj(json_data, ('tvShowVideo', 'poster')),
|
||||
'channel': self._search_regex(
|
||||
r'tvShow \| ([A-Z0-9]+)_', show.get('actionFieldText') or '', video_id, fatal=False),
|
||||
'uploader': show.get('presenter'),
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
|
||||
class HSEProductIE(HSEShowBaseInfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?hse\.de/dpl/p/product/(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.hse.de/dpl/p/product/408630',
|
||||
'info_dict': {
|
||||
'id': '408630',
|
||||
'ext': 'mp4',
|
||||
'title': 'Hose im Ponte-Mix',
|
||||
'uploader': 'Judith Williams'
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
json_data = self._extract_redux_data(url, video_id)
|
||||
video = traverse_obj(json_data, ('productContent', 'productContent', 'videos', 0)) or {}
|
||||
formats, subtitles = self._extract_formats_and_subtitles(video.get('sources'), video_id)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': traverse_obj(json_data, ('productDetail', 'product', 'name', 'short')) or video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnail': video.get('poster'),
|
||||
'uploader': traverse_obj(json_data, ('productDetail', 'product', 'brand', 'brandName')),
|
||||
}
|
||||
@@ -88,7 +88,7 @@ class ImgGamingBaseIE(InfoExtractor):
|
||||
video_data = self._download_json(dve_api_url, media_id)
|
||||
is_live = media_type == 'live'
|
||||
if is_live:
|
||||
title = self._live_title(self._call_api('event/', media_id)['title'])
|
||||
title = self._call_api('event/', media_id)['title']
|
||||
else:
|
||||
title = video_data['name']
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from ..utils import (
|
||||
int_or_none,
|
||||
lowercase_escape,
|
||||
std_headers,
|
||||
str_to_int,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
@@ -170,7 +171,7 @@ class InstagramIOSIE(InfoExtractor):
|
||||
|
||||
|
||||
class InstagramIE(InstagramBaseIE):
|
||||
_VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
|
||||
_VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com(?:/[^/]+)?/(?:p|tv|reel)/(?P<id>[^/?#&]+))'
|
||||
_TESTS = [{
|
||||
'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
|
||||
'md5': '0d2da106a9d2631273e192b372806516',
|
||||
@@ -266,6 +267,9 @@ class InstagramIE(InstagramBaseIE):
|
||||
}, {
|
||||
'url': 'https://www.instagram.com/reel/CDUMkliABpa/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.instagram.com/marvelskies.fc/reel/CWqAgUZgCku/',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
@@ -290,7 +294,10 @@ class InstagramIE(InstagramBaseIE):
|
||||
video_id, url = self._match_valid_url(url).group('id', 'url')
|
||||
webpage, urlh = self._download_webpage_handle(url, video_id)
|
||||
if 'www.instagram.com/accounts/login' in urlh.geturl():
|
||||
self.raise_login_required('You need to log in to access this content')
|
||||
self.report_warning('Main webpage is locked behind the login page. '
|
||||
'Retrying with embed webpage (Note that some metadata might be missing)')
|
||||
webpage = self._download_webpage(
|
||||
'https://www.instagram.com/p/%s/embed/' % video_id, video_id, note='Downloading embed webpage')
|
||||
|
||||
shared_data = self._parse_json(
|
||||
self._search_regex(
|
||||
@@ -311,7 +318,10 @@ class InstagramIE(InstagramBaseIE):
|
||||
r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*({.+?})\s*\)\s*;',
|
||||
webpage, 'additional data', default='{}'),
|
||||
video_id, fatal=False)
|
||||
media = traverse_obj(additional_data, ('graphql', 'shortcode_media'), expected_type=dict) or {}
|
||||
media = traverse_obj(additional_data, ('graphql', 'shortcode_media'), 'shortcode_media', expected_type=dict) or {}
|
||||
|
||||
if not media and 'www.instagram.com/accounts/login' in urlh.geturl():
|
||||
self.raise_login_required('You need to log in to access this content')
|
||||
|
||||
uploader_id = traverse_obj(media, ('owner', 'username')) or self._search_regex(
|
||||
r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', webpage, 'uploader id', fatal=False)
|
||||
@@ -345,13 +355,14 @@ class InstagramIE(InstagramBaseIE):
|
||||
formats.extend(self._parse_mpd_formats(self._parse_xml(dash, video_id), mpd_id='dash'))
|
||||
self._sort_formats(formats)
|
||||
|
||||
comment_data = traverse_obj(media, ('edge_media_to_parent_comment', 'edges'))
|
||||
comments = [{
|
||||
'author': traverse_obj(comment_dict, ('node', 'owner', 'username')),
|
||||
'author_id': traverse_obj(comment_dict, ('node', 'owner', 'id')),
|
||||
'id': traverse_obj(comment_dict, ('node', 'id')),
|
||||
'text': traverse_obj(comment_dict, ('node', 'text')),
|
||||
'timestamp': traverse_obj(comment_dict, ('node', 'created_at'), expected_type=int_or_none),
|
||||
} for comment_dict in traverse_obj(media, ('edge_media_to_parent_comment', 'edges'))]
|
||||
} for comment_dict in comment_data] if comment_data else None
|
||||
|
||||
display_resources = (
|
||||
media.get('display_resources')
|
||||
@@ -372,7 +383,8 @@ class InstagramIE(InstagramBaseIE):
|
||||
'timestamp': traverse_obj(media, 'taken_at_timestamp', 'date', expected_type=int_or_none),
|
||||
'uploader_id': uploader_id,
|
||||
'uploader': traverse_obj(media, ('owner', 'full_name')),
|
||||
'like_count': self._get_count(media, 'likes', 'preview_like'),
|
||||
'like_count': self._get_count(media, 'likes', 'preview_like') or str_to_int(self._search_regex(
|
||||
r'data-log-event="likeCountClick"[^>]*>[^\d]*([\d,\.]+)', webpage, 'like count', fatal=False)),
|
||||
'comment_count': self._get_count(media, 'comments', 'preview_comment', 'to_comment', 'to_parent_comment'),
|
||||
'comments': comments,
|
||||
'thumbnails': thumbnails,
|
||||
|
||||
@@ -75,7 +75,7 @@ class IvideonIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': server_id,
|
||||
'title': self._live_title(camera_name or server_id),
|
||||
'title': camera_name or server_id,
|
||||
'description': description,
|
||||
'is_live': True,
|
||||
'formats': formats,
|
||||
|
||||
@@ -59,7 +59,7 @@ class JamendoIE(InfoExtractor):
|
||||
})[0]
|
||||
|
||||
def _real_extract(self, url):
|
||||
track_id, display_id = self._VALID_URL_RE.match(url).groups()
|
||||
track_id, display_id = self._match_valid_url(url).groups()
|
||||
# webpage = self._download_webpage(
|
||||
# 'https://www.jamendo.com/track/' + track_id, track_id)
|
||||
# models = self._parse_json(self._html_search_regex(
|
||||
|
||||
@@ -112,7 +112,7 @@ class Laola1TvEmbedIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'upload_date': unified_strdate(_v('time_date')),
|
||||
'uploader': _v('meta_organisation'),
|
||||
'categories': categories,
|
||||
@@ -161,7 +161,7 @@ class Laola1TvBaseIE(Laola1TvEmbedIE):
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': video_data.get('description'),
|
||||
'thumbnail': video_data.get('image'),
|
||||
'categories': categories,
|
||||
|
||||
+25
-15
@@ -184,28 +184,38 @@ class LBRYIE(LBRYBaseIE):
|
||||
display_id = compat_urllib_parse_unquote(display_id)
|
||||
uri = 'lbry://' + display_id
|
||||
result = self._resolve_url(uri, display_id, 'stream')
|
||||
result_value = result['value']
|
||||
if result_value.get('stream_type') not in self._SUPPORTED_STREAM_TYPES:
|
||||
if result['value'].get('stream_type') in self._SUPPORTED_STREAM_TYPES:
|
||||
claim_id, is_live, headers = result['claim_id'], False, None
|
||||
streaming_url = self._call_api_proxy(
|
||||
'get', claim_id, {'uri': uri}, 'streaming url')['streaming_url']
|
||||
final_url = self._request_webpage(
|
||||
streaming_url, display_id, note='Downloading streaming redirect url info').geturl()
|
||||
elif result.get('value_type') == 'stream':
|
||||
claim_id, is_live = result['signing_channel']['claim_id'], True
|
||||
headers = {'referer': 'https://player.odysee.live/'}
|
||||
live_data = self._download_json(
|
||||
f'https://api.live.odysee.com/v1/odysee/live/{claim_id}', claim_id,
|
||||
note='Downloading livestream JSON metadata')['data']
|
||||
if not live_data['live']:
|
||||
raise ExtractorError('This stream is not live', expected=True)
|
||||
streaming_url = final_url = live_data['url']
|
||||
else:
|
||||
raise ExtractorError('Unsupported URL', expected=True)
|
||||
claim_id = result['claim_id']
|
||||
title = result_value['title']
|
||||
streaming_url = self._call_api_proxy(
|
||||
'get', claim_id, {'uri': uri}, 'streaming url')['streaming_url']
|
||||
|
||||
info = self._parse_stream(result, url)
|
||||
urlh = self._request_webpage(
|
||||
streaming_url, display_id, note='Downloading streaming redirect url info')
|
||||
if determine_ext(urlh.geturl()) == 'm3u8':
|
||||
if determine_ext(final_url) == 'm3u8':
|
||||
info['formats'] = self._extract_m3u8_formats(
|
||||
urlh.geturl(), display_id, 'mp4', entry_protocol='m3u8_native',
|
||||
m3u8_id='hls')
|
||||
final_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', live=is_live, headers=headers)
|
||||
self._sort_formats(info['formats'])
|
||||
else:
|
||||
info['url'] = streaming_url
|
||||
info.update({
|
||||
return {
|
||||
**info,
|
||||
'id': claim_id,
|
||||
'title': title,
|
||||
})
|
||||
return info
|
||||
'title': result['value']['title'],
|
||||
'is_live': is_live,
|
||||
'http_headers': headers,
|
||||
}
|
||||
|
||||
|
||||
class LBRYChannelIE(LBRYBaseIE):
|
||||
|
||||
@@ -116,7 +116,7 @@ class LineLiveBaseIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': broadcast_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'thumbnails': thumbnails,
|
||||
'timestamp': int_or_none(item.get('createdAt')),
|
||||
'channel': channel.get('name'),
|
||||
|
||||
@@ -176,7 +176,7 @@ class LivestreamIE(InfoExtractor):
|
||||
return {
|
||||
'id': broadcast_id,
|
||||
'formats': formats,
|
||||
'title': self._live_title(stream_info['stream_title']) if is_live else stream_info['stream_title'],
|
||||
'title': stream_info['stream_title'],
|
||||
'thumbnail': stream_info.get('thumbnail_url'),
|
||||
'is_live': is_live,
|
||||
}
|
||||
@@ -344,7 +344,7 @@ class LivestreamOriginalIE(InfoExtractor):
|
||||
is_live = video_data.get('isLive')
|
||||
info.update({
|
||||
'id': content_id,
|
||||
'title': self._live_title(info['title']) if is_live else info['title'],
|
||||
'title': info['title'],
|
||||
'formats': self._extract_video_formats(video_data, content_id),
|
||||
'is_live': is_live,
|
||||
})
|
||||
|
||||
@@ -33,7 +33,7 @@ class MangomoloBaseIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': page_id,
|
||||
'title': self._live_title(page_id) if self._IS_LIVE else page_id,
|
||||
'title': page_id,
|
||||
'uploader_id': hidden_inputs.get('userid'),
|
||||
'duration': int_or_none(hidden_inputs.get('duration')),
|
||||
'is_live': self._IS_LIVE,
|
||||
|
||||
@@ -49,7 +49,7 @@ class MatchTVIE(InfoExtractor):
|
||||
self._sort_formats(formats)
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title('Матч ТВ - Прямой эфир'),
|
||||
'title': 'Матч ТВ - Прямой эфир',
|
||||
'is_live': True,
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ class MuenchenTVIE(InfoExtractor):
|
||||
display_id = 'live'
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
title = self._live_title(self._og_search_title(webpage))
|
||||
title = self._og_search_title(webpage)
|
||||
|
||||
data_js = self._search_regex(
|
||||
r'(?s)\nplaylist:\s*(\[.*?}\]),',
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import itertools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class NateIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://tv\.nate\.com/clip/(?P<id>[0-9]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://tv.nate.com/clip/1848976',
|
||||
'info_dict': {
|
||||
'id': '1848976',
|
||||
'ext': 'mp4',
|
||||
'title': '[결승 오프닝 타이틀] 2018 LCK 서머 스플릿 결승전 kt Rolster VS Griffin',
|
||||
'description': 'md5:e1b79a7dcf0d8d586443f11366f50e6f',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
'upload_date': '20180908',
|
||||
'age_limit': 15,
|
||||
'duration': 73,
|
||||
'uploader': '2018 LCK 서머 스플릿(롤챔스)',
|
||||
'channel': '2018 LCK 서머 스플릿(롤챔스)',
|
||||
'channel_id': '3606',
|
||||
'uploader_id': '3606',
|
||||
'tags': 'count:59',
|
||||
},
|
||||
'params': {'skip_download': True}
|
||||
}, {
|
||||
'url': 'https://tv.nate.com/clip/4300566',
|
||||
'info_dict': {
|
||||
'id': '4300566',
|
||||
'ext': 'mp4',
|
||||
'title': '[심쿵엔딩] 이준호x이세영, 서로를 기억하며 끌어안는 두 사람!💕, MBC 211204 방송',
|
||||
'description': 'md5:be1653502d9c13ce344ddf7828e089fa',
|
||||
'thumbnail': r're:^https?://.*\.jpg',
|
||||
'upload_date': '20211204',
|
||||
'age_limit': 15,
|
||||
'duration': 201,
|
||||
'uploader': '옷소매 붉은 끝동',
|
||||
'channel': '옷소매 붉은 끝동',
|
||||
'channel_id': '27987',
|
||||
'uploader_id': '27987',
|
||||
'tags': 'count:20',
|
||||
},
|
||||
'params': {'skip_download': True}
|
||||
}]
|
||||
|
||||
_QUALITY = {
|
||||
'36': 2160,
|
||||
'35': 1080,
|
||||
'34': 720,
|
||||
'33': 480,
|
||||
'32': 360,
|
||||
'31': 270,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
id = self._match_id(url)
|
||||
video_data = self._download_json(f'https://tv.nate.com/api/v1/clip/{id}', id)
|
||||
formats = [{
|
||||
'format_id': f_url[-2:],
|
||||
'url': f_url,
|
||||
'height': self._QUALITY.get(f_url[-2:]),
|
||||
'quality': int_or_none(f_url[-2:]),
|
||||
} for f_url in video_data.get('smcUriList') or []]
|
||||
self._sort_formats(formats)
|
||||
return {
|
||||
'id': id,
|
||||
'title': video_data.get('clipTitle'),
|
||||
'description': video_data.get('synopsis'),
|
||||
'thumbnail': video_data.get('contentImg'),
|
||||
'upload_date': unified_strdate(traverse_obj(video_data, 'broadDate', 'regDate')),
|
||||
'age_limit': video_data.get('targetAge'),
|
||||
'duration': video_data.get('playTime'),
|
||||
'formats': formats,
|
||||
'uploader': video_data.get('programTitle'),
|
||||
'channel': video_data.get('programTitle'),
|
||||
'channel_id': str_or_none(video_data.get('programSeq')),
|
||||
'uploader_id': str_or_none(video_data.get('programSeq')),
|
||||
'tags': video_data['hashTag'].split(',') if video_data.get('hashTag') else None,
|
||||
}
|
||||
|
||||
|
||||
class NateProgramIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://tv\.nate\.com/program/clips/(?P<id>[0-9]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://tv.nate.com/program/clips/27987',
|
||||
'playlist_mincount': 191,
|
||||
'info_dict': {
|
||||
'id': '27987',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://tv.nate.com/program/clips/3606',
|
||||
'playlist_mincount': 15,
|
||||
'info_dict': {
|
||||
'id': '3606',
|
||||
},
|
||||
}]
|
||||
|
||||
def _entries(self, id):
|
||||
for page_num in itertools.count(1):
|
||||
program_data = self._download_json(f'https://tv.nate.com/api/v1/program/{id}/clip/ranking?size=20&page={page_num}',
|
||||
id, note=f'Downloading page {page_num}')
|
||||
for clip in program_data.get('content') or []:
|
||||
clip_id = clip.get('clipSeq')
|
||||
if clip_id:
|
||||
yield self.url_result(
|
||||
'https://tv.nate.com/clip/%s' % clip_id,
|
||||
ie=NateIE.ie_key(), video_id=clip_id)
|
||||
if program_data.get('last'):
|
||||
break
|
||||
|
||||
def _real_extract(self, url):
|
||||
id = self._match_id(url)
|
||||
return self.playlist_result(self._entries(id), playlist_id=id)
|
||||
@@ -305,7 +305,7 @@ class NBCSportsStreamIE(AdobePassIE):
|
||||
self._sort_formats(formats)
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'description': live_source.get('description'),
|
||||
'formats': formats,
|
||||
'is_live': is_live,
|
||||
@@ -545,8 +545,6 @@ class NBCOlympicsStreamIE(AdobePassIE):
|
||||
|
||||
title = event_config['eventTitle']
|
||||
is_live = {'live': True, 'replay': False}.get(event_config.get('eventStatus'))
|
||||
if is_live:
|
||||
title = self._live_title(title)
|
||||
|
||||
source_url = self._download_json(
|
||||
f'https://api-leap.nbcsports.com/feeds/assets/{pid}?application=NBCOlympics&platform=desktop&format=nbc-player&env=staging',
|
||||
|
||||
@@ -245,8 +245,6 @@ class NDREmbedBaseIE(InfoExtractor):
|
||||
|
||||
live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
|
||||
title = config['title']
|
||||
if live:
|
||||
title = self._live_title(title)
|
||||
uploader = ppjson.get('config', {}).get('branding')
|
||||
upload_date = ppjson.get('config', {}).get('publicationDate')
|
||||
duration = int_or_none(config.get('duration'))
|
||||
|
||||
@@ -89,7 +89,7 @@ class NFLBaseIE(InfoExtractor):
|
||||
'ext': determine_ext(image_url, 'jpg'),
|
||||
}]
|
||||
info.update({
|
||||
'title': self._live_title(title) if is_live else title,
|
||||
'title': title,
|
||||
'is_live': is_live,
|
||||
'description': clean_html(item.get('description')),
|
||||
'thumbnails': thumbnails,
|
||||
|
||||
@@ -662,11 +662,32 @@ class NiconicoPlaylistIE(InfoExtractor):
|
||||
}
|
||||
|
||||
|
||||
NicovideoSearchIE_NAME = 'nicovideo:search'
|
||||
class NicovideoSearchBaseIE(InfoExtractor):
|
||||
def _entries(self, url, item_id, query=None, note='Downloading page %(page)s'):
|
||||
query = query or {}
|
||||
pages = [query['page']] if 'page' in query else itertools.count(1)
|
||||
for page_num in pages:
|
||||
query['page'] = str(page_num)
|
||||
webpage = self._download_webpage(url, item_id, query=query, note=note % {'page': page_num})
|
||||
results = re.findall(r'(?<=data-video-id=)["\']?(?P<videoid>.*?)(?=["\'])', webpage)
|
||||
for item in results:
|
||||
yield self.url_result(f'http://www.nicovideo.jp/watch/{item}', 'Niconico', item)
|
||||
if not results:
|
||||
break
|
||||
|
||||
def _search_results(self, query):
|
||||
return self._entries(
|
||||
self._proto_relative_url(f'//www.nicovideo.jp/search/{query}'), query)
|
||||
|
||||
|
||||
class NicovideoSearchURLIE(InfoExtractor):
|
||||
IE_NAME = f'{NicovideoSearchIE_NAME}_url'
|
||||
class NicovideoSearchIE(NicovideoSearchBaseIE, SearchInfoExtractor):
|
||||
IE_DESC = 'Nico video search'
|
||||
IE_NAME = 'nicovideo:search'
|
||||
_SEARCH_KEY = 'nicosearch'
|
||||
|
||||
|
||||
class NicovideoSearchURLIE(NicovideoSearchBaseIE):
|
||||
IE_NAME = f'{NicovideoSearchIE.IE_NAME}_url'
|
||||
IE_DESC = 'Nico video search URLs'
|
||||
_VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/search/(?P<id>[^?#&]+)?'
|
||||
_TESTS = [{
|
||||
@@ -685,37 +706,14 @@ class NicovideoSearchURLIE(InfoExtractor):
|
||||
'playlist_count': 31,
|
||||
}]
|
||||
|
||||
def _entries(self, url, item_id, query=None, note='Downloading page %(page)s'):
|
||||
query = query or {}
|
||||
pages = [query['page']] if 'page' in query else itertools.count(1)
|
||||
for page_num in pages:
|
||||
query['page'] = str(page_num)
|
||||
webpage = self._download_webpage(url, item_id, query=query, note=note % {'page': page_num})
|
||||
results = re.findall(r'(?<=data-video-id=)["\']?(?P<videoid>.*?)(?=["\'])', webpage)
|
||||
for item in results:
|
||||
yield self.url_result(f'http://www.nicovideo.jp/watch/{item}', 'Niconico', item)
|
||||
if not results:
|
||||
break
|
||||
|
||||
def _real_extract(self, url):
|
||||
query = self._match_id(url)
|
||||
return self.playlist_result(self._entries(url, query), query, query)
|
||||
|
||||
|
||||
class NicovideoSearchIE(SearchInfoExtractor, NicovideoSearchURLIE):
|
||||
IE_DESC = 'Nico video search'
|
||||
IE_NAME = NicovideoSearchIE_NAME
|
||||
_SEARCH_KEY = 'nicosearch'
|
||||
_TESTS = []
|
||||
|
||||
def _search_results(self, query):
|
||||
return self._entries(
|
||||
self._proto_relative_url(f'//www.nicovideo.jp/search/{query}'), query)
|
||||
|
||||
|
||||
class NicovideoSearchDateIE(NicovideoSearchIE):
|
||||
class NicovideoSearchDateIE(NicovideoSearchBaseIE, SearchInfoExtractor):
|
||||
IE_DESC = 'Nico video search, newest first'
|
||||
IE_NAME = f'{NicovideoSearchIE_NAME}:date'
|
||||
IE_NAME = f'{NicovideoSearchIE.IE_NAME}:date'
|
||||
_SEARCH_KEY = 'nicosearchdate'
|
||||
_TESTS = [{
|
||||
'url': 'nicosearchdateall:a',
|
||||
@@ -756,7 +754,7 @@ class NicovideoSearchDateIE(NicovideoSearchIE):
|
||||
if page_num:
|
||||
query['page'] = str(page_num)
|
||||
|
||||
yield from NicovideoSearchURLIE._entries(self, url, item_id, query=query, note=note)
|
||||
yield from super()._entries(url, item_id, query=query, note=note)
|
||||
|
||||
|
||||
class NiconicoUserIE(InfoExtractor):
|
||||
|
||||
@@ -77,13 +77,8 @@ class NJPWWorldIE(InfoExtractor):
|
||||
for kind, vid in re.findall(r'if\s+\(\s*imageQualityType\s*==\s*\'([^\']+)\'\s*\)\s*{\s*video_id\s*=\s*"(\d+)"', webpage):
|
||||
player_path = '/intent?id=%s&type=url' % vid
|
||||
player_url = compat_urlparse.urljoin(url, player_path)
|
||||
formats.append({
|
||||
'url': player_url,
|
||||
'format_id': kind,
|
||||
'ext': 'mp4',
|
||||
'protocol': 'm3u8',
|
||||
'quality': 2 if kind == 'high' else 1,
|
||||
})
|
||||
formats += self._extract_m3u8_formats(
|
||||
player_url, video_id, 'mp4', 'm3u8_native', m3u8_id=kind, fatal=False, quality=int(kind == 'high'))
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ class NovaEmbedIE(InfoExtractor):
|
||||
|
||||
player = self._parse_json(
|
||||
self._search_regex(
|
||||
(r'(?:replacePlaceholders.*?:\s*)?replacePlaceholders\s*\(\s*(?P<json>{.*})\s*\)(?:\s*\))?\s*,',
|
||||
(r'(?:(?:replacePlaceholders|processAdTagModifier).*?:\s*)?(?:replacePlaceholders|processAdTagModifier)\s*\(\s*(?P<json>{.*?})\s*\)(?:\s*\))?\s*,',
|
||||
r'Player\.init\s*\([^,]+,(?P<cndn>\s*\w+\s*\?)?\s*(?P<json>{(?(cndn).+?|.+)})\s*(?(cndn):|,\s*{.+?}\s*\)\s*;)'),
|
||||
webpage, 'player', default='{}', group='json'), video_id, fatal=False)
|
||||
if player:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user