artem/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2026-08-23 20:38:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'yt-dlp:master' into generic

Browse Source
This commit is contained in:
Damiano Amatruda
2023-02-07 14:38:43 +01:00
committed by GitHub
parent 91d6f0cfb6 7aefd19afe
commit a111b3ab58
142 changed files with 7705 additions and 1782 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yt_dlp/YoutubeDL.py
+64 -34
View File
@@ -32,7 +32,8 @@ from .extractor import gen_extractor_classes, get_info_extractor
from .extractor.common import UnsupportedURLIE
from .extractor.openload import PhantomJSwrapper
from .minicurses import format_text
from .postprocessor import _PLUGIN_CLASSES as plugin_postprocessors
from .plugins import directories as plugin_directories
from .postprocessor import _PLUGIN_CLASSES as plugin_pps
from .postprocessor import (
EmbedThumbnailPP,
FFmpegFixupDuplicateMoovPP,
@@ -317,6 +318,7 @@ class YoutubeDL:
If not provided and the key is encrypted, yt-dlp will ask interactively
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
(Only supported by some extractors)
enable_file_urls: Enable file:// URLs. This is disabled by default for security reasons.
http_headers: A dictionary of custom headers to be used for all requests
proxy: URL of the proxy server to use
geo_verification_proxy: URL of the proxy to use for IP address verification
@@ -584,7 +586,6 @@ class YoutubeDL:
self._playlist_urls = set()
self.cache = Cache(self)
windows_enable_vt_mode()
stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout
self._out_files = Namespace(
out=stdout,
@@ -593,6 +594,12 @@ class YoutubeDL:
console=None if compat_os_name == 'nt' else next(
filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None)
)
try:
windows_enable_vt_mode()
except Exception as e:
self.write_debug(f'Failed to enable VT mode: {e}')
self._allow_colors = Namespace(**{
type_: not self.params.get('no_color') and supports_terminal_sequences(stream)
for type_, stream in self._out_files.items_ if type_ != 'console'
@@ -1068,7 +1075,7 @@ class YoutubeDL:
# correspondingly that is not what we want since we need to keep
# '%%' intact for template dict substitution step. Working around
# with boundary-alike separator hack.
sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
sep = ''.join(random.choices(ascii_letters, k=32))
outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$')
# outtmpl should be expand_path'ed before template dict substitution
@@ -1626,8 +1633,8 @@ class YoutubeDL:
if result_type in ('url', 'url_transparent'):
ie_result['url'] = sanitize_url(
ie_result['url'], scheme='http' if self.params.get('prefer_insecure') else 'https')
if ie_result.get('original_url'):
extra_info.setdefault('original_url', ie_result['original_url'])
if ie_result.get('original_url') and not extra_info.get('original_url'):
extra_info = {'original_url': ie_result['original_url'], **extra_info}
extract_flat = self.params.get('extract_flat', False)
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
@@ -1770,7 +1777,7 @@ class YoutubeDL:
return {
**info,
'playlist_index': 0,
'__last_playlist_index': max(ie_result['requested_entries'] or (0, 0)),
'__last_playlist_index': max(ie_result.get('requested_entries') or (0, 0)),
'extractor': ie_result['extractor'],
'extractor_key': ie_result['extractor_key'],
}
@@ -1862,11 +1869,10 @@ class YoutubeDL:
self.to_screen('[download] Downloading item %s of %s' % (
self._format_screen(i + 1, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
extra.update({
entry_result = self.__process_iterable_entry(entry, download, collections.ChainMap({
'playlist_index': playlist_index,
'playlist_autonumber': i + 1,
})
entry_result = self.__process_iterable_entry(entry, download, extra)
}, extra))
if not entry_result:
failures += 1
if failures >= max_failures:
@@ -2405,11 +2411,7 @@ class YoutubeDL:
def _fill_common_fields(self, info_dict, final=True):
# TODO: move sanitization here
if final:
title = info_dict.get('title', NO_DEFAULT)
if title is NO_DEFAULT:
raise ExtractorError('Missing "title" field in extractor result',
video_id=info_dict['id'], ie=info_dict['extractor'])
info_dict['fulltitle'] = title
title = info_dict['fulltitle'] = info_dict.get('title')
if not title:
if title == '':
self.write_debug('Extractor gave empty title. Creating a generic title')
@@ -2977,6 +2979,16 @@ class YoutubeDL:
# Does nothing under normal operation - for backward compatibility of process_info
self.post_extract(info_dict)
def replace_info_dict(new_info):
nonlocal info_dict
if new_info == info_dict:
return
info_dict.clear()
info_dict.update(new_info)
new_info, _ = self.pre_process(info_dict, 'video')
replace_info_dict(new_info)
self._num_downloads += 1
# info_dict['_filename'] needs to be set for backward compatibility
@@ -3090,13 +3102,6 @@ class YoutubeDL:
for link_type, should_write in write_links.items()):
return
def replace_info_dict(new_info):
nonlocal info_dict
if new_info == info_dict:
return
info_dict.clear()
info_dict.update(new_info)
new_info, files_to_move = self.pre_process(info_dict, 'before_dl', files_to_move)
replace_info_dict(new_info)
@@ -3123,7 +3128,7 @@ class YoutubeDL:
fd, success = None, True
if info_dict.get('protocol') or info_dict.get('url'):
fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
if fd is not FFmpegFD and (
if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
info_dict.get('section_start') or info_dict.get('section_end')):
msg = ('This format cannot be partially downloaded' if FFmpegFD.available()
else 'You have requested downloading the video partially, but ffmpeg is not installed')
@@ -3388,6 +3393,7 @@ class YoutubeDL:
reject = lambda k, v: v is None or k.startswith('__') or k in {
'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber',
'_format_sort_fields',
}
else:
reject = lambda k, v: False
@@ -3457,7 +3463,8 @@ class YoutubeDL:
return infodict
def run_all_pps(self, key, info, *, additional_pps=None):
self._forceprint(key, info)
if key != 'video':
self._forceprint(key, info)
for pp in (additional_pps or []) + self._pps[key]:
info = self.run_pp(pp, info)
return info
@@ -3726,7 +3733,10 @@ class YoutubeDL:
# These imports can be slow. So import them only as needed
from .extractor.extractors import _LAZY_LOADER
from .extractor.extractors import _PLUGIN_CLASSES as plugin_extractors
from .extractor.extractors import (
_PLUGIN_CLASSES as plugin_ies,
_PLUGIN_OVERRIDES as plugin_ie_overrides
)
def get_encoding(stream):
ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__))
@@ -3771,10 +3781,6 @@ class YoutubeDL:
write_debug('Lazy loading extractors is forcibly disabled')
else:
write_debug('Lazy loading extractors is disabled')
if plugin_extractors or plugin_postprocessors:
write_debug('Plugins: %s' % [
'%s%s' % (klass.__name__, '' if klass.__name__ == name else f' as {name}')
for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())])
if self.params['compat_opts']:
write_debug('Compatibility options: %s' % ', '.join(self.params['compat_opts']))
@@ -3808,6 +3814,21 @@ class YoutubeDL:
proxy_map.update(handler.proxies)
write_debug(f'Proxy map: {proxy_map}')
for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items():
display_list = ['%s%s' % (
klass.__name__, '' if klass.__name__ == name else f' as {name}')
for name, klass in plugins.items()]
if plugin_type == 'Extractor':
display_list.extend(f'{plugins[-1].IE_NAME.partition("+")[2]} ({parent.__name__})'
for parent, plugins in plugin_ie_overrides.items())
if not display_list:
continue
write_debug(f'{plugin_type} Plugins: {", ".join(sorted(display_list))}')
plugin_dirs = plugin_directories()
if plugin_dirs:
write_debug(f'Plugin directories: {plugin_dirs}')
# Not implemented
if False and self.params.get('call_home'):
ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode()
@@ -3857,9 +3878,12 @@ class YoutubeDL:
# https://github.com/ytdl-org/youtube-dl/issues/8227)
file_handler = urllib.request.FileHandler()
def file_open(*args, **kwargs):
raise urllib.error.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
file_handler.file_open = file_open
if not self.params.get('enable_file_urls'):
def file_open(*args, **kwargs):
raise urllib.error.URLError(
'file:// URLs are explicitly disabled in yt-dlp for security reasons. '
'Use --enable-file-urls to enable at your own risk.')
file_handler.file_open = file_open
opener = urllib.request.build_opener(
proxy_handler, https_handler, cookie_processor, ydlh, redirect_handler, data_handler, file_handler)
@@ -3921,7 +3945,7 @@ class YoutubeDL:
elif not self.params.get('overwrites', True) and os.path.exists(descfn):
self.to_screen(f'[info] {label.title()} description is already present')
elif ie_result.get('description') is None:
self.report_warning(f'There\'s no {label} description to write')
self.to_screen(f'[info] There\'s no {label} description to write')
return False
else:
try:
@@ -3937,15 +3961,18 @@ class YoutubeDL:
''' Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error'''
ret = []
subtitles = info_dict.get('requested_subtitles')
if not subtitles or not (self.params.get('writesubtitles') or self.params.get('writeautomaticsub')):
if not (self.params.get('writesubtitles') or self.params.get('writeautomaticsub')):
# subtitles download errors are already managed as troubles in relevant IE
# that way it will silently go on when used with unsupporting IE
return ret
elif not subtitles:
self.to_screen('[info] There\'s no subtitles for the requested languages')
return ret
sub_filename_base = self.prepare_filename(info_dict, 'subtitle')
if not sub_filename_base:
self.to_screen('[info] Skipping writing video subtitles')
return ret
for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext']
sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
@@ -3992,6 +4019,9 @@ class YoutubeDL:
thumbnails, ret = [], []
if write_all or self.params.get('writethumbnail', False):
thumbnails = info_dict.get('thumbnails') or []
if not thumbnails:
self.to_screen(f'[info] There\'s no {label} thumbnails to download')
return ret
multiple = write_all and len(thumbnails) > 1
if thumb_filename_base is None:
yt_dlp/__init__.py
+15 -13
View File
@@ -91,12 +91,11 @@ def get_urls(urls, batchfile, verbose):
def print_extractor_information(opts, urls):
# Importing GenericIE is currently slow since it imports other extractors
# TODO: Move this back to module level after generalization of embed detection
from .extractor.generic import GenericIE
out = ''
if opts.list_extractors:
# Importing GenericIE is currently slow since it imports YoutubeIE
from .extractor.generic import GenericIE
urls = dict.fromkeys(urls, False)
for ie in list_extractor_classes(opts.age_limit):
out += ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie.working() else '') + '\n'
@@ -333,7 +332,7 @@ def validate_options(opts):
mobj = range_ != '-' and re.fullmatch(r'([^-]+)?\s*-\s*([^-]+)?', range_)
dur = mobj and (parse_timestamp(mobj.group(1) or '0'), parse_timestamp(mobj.group(2) or 'inf'))
if None in (dur or [None]):
raise ValueError(f'invalid {name} time range "{regex}". Must be of the form *start-end')
raise ValueError(f'invalid {name} time range "{regex}". Must be of the form "*start-end"')
ranges.append(dur)
continue
try:
@@ -351,7 +350,7 @@ def validate_options(opts):
mobj = re.fullmatch(r'''(?x)
(?P<name>[^+:]+)
(?:\s*\+\s*(?P<keyring>[^:]+))?
(?:\s*:\s*(?P<profile>.+?))?
(?:\s*:\s*(?!:)(?P<profile>.+?))?
(?:\s*::\s*(?P<container>.+))?
''', opts.cookiesfrombrowser)
if mobj is None:
@@ -387,10 +386,12 @@ def validate_options(opts):
raise ValueError(f'{cmd} is invalid; {err}')
yield action
parse_metadata = opts.parse_metadata or []
if opts.metafromtitle is not None:
parse_metadata.append('title:%s' % opts.metafromtitle)
opts.parse_metadata = list(itertools.chain(*map(metadataparser_actions, parse_metadata)))
opts.parse_metadata.setdefault('pre_process', []).append('title:%s' % opts.metafromtitle)
opts.parse_metadata = {
k: list(itertools.chain(*map(metadataparser_actions, v)))
for k, v in opts.parse_metadata.items()
}
# Other options
if opts.playlist_items is not None:
@@ -562,11 +563,11 @@ def validate_options(opts):
def get_postprocessors(opts):
yield from opts.add_postprocessors
if opts.parse_metadata:
for when, actions in opts.parse_metadata.items():
yield {
'key': 'MetadataParser',
'actions': opts.parse_metadata,
'when': 'pre_process'
'actions': actions,
'when': when
}
sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove
if sponsorblock_query:
@@ -702,7 +703,7 @@ def parse_options(argv=None):
postprocessors = list(get_postprocessors(opts))
print_only = bool(opts.forceprint) and all(k not in opts.forceprint for k in POSTPROCESS_WHEN[2:])
print_only = bool(opts.forceprint) and all(k not in opts.forceprint for k in POSTPROCESS_WHEN[3:])
any_getting = any(getattr(opts, k) for k in (
'dumpjson', 'dump_single_json', 'getdescription', 'getduration', 'getfilename',
'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl'
@@ -854,6 +855,7 @@ def parse_options(argv=None):
'legacyserverconnect': opts.legacy_server_connect,
'nocheckcertificate': opts.no_check_certificate,
'prefer_insecure': opts.prefer_insecure,
'enable_file_urls': opts.enable_file_urls,
'http_headers': opts.headers,
'proxy': opts.proxy,
'socket_timeout': opts.socket_timeout,
yt_dlp/cache.py
+4 -5
View File
@@ -5,6 +5,7 @@ import os
import re
import shutil
import traceback
import urllib.parse
from .utils import expand_path, traverse_obj, version_tuple, write_json_file
from .version import __version__
@@ -22,11 +23,9 @@ class Cache:
return expand_path(res)
def _get_cache_fn(self, section, key, dtype):
assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
'invalid section %r' % section
assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
return os.path.join(
self._get_root_dir(), section, f'{key}.{dtype}')
assert re.match(r'^[\w.-]+$', section), f'invalid section {section!r}'
key = urllib.parse.quote(key, safe='').replace('%', ',') # encode non-ascii characters
return os.path.join(self._get_root_dir(), section, f'{key}.{dtype}')
@property
def enabled(self):
yt_dlp/downloader/common.py
+4 -1
View File
@@ -20,6 +20,7 @@ from ..utils import (
RetryManager,
classproperty,
decodeArgument,
deprecation_warning,
encodeFilename,
format_bytes,
join_nonempty,
@@ -180,7 +181,9 @@ class FileDownloader:
@staticmethod
def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into an integer."""
parse_bytes(bytestr)
deprecation_warning('yt_dlp.FileDownloader.parse_bytes is deprecated and '
'may be removed in the future. Use yt_dlp.utils.parse_bytes instead')
return parse_bytes(bytestr)
def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit."""
yt_dlp/downloader/dash.py
+11 -3
View File
@@ -1,8 +1,9 @@
import time
import urllib.parse
from . import get_suitable_downloader
from .fragment import FragmentFD
from ..utils import urljoin
from ..utils import update_url_query, urljoin
class DashSegmentsFD(FragmentFD):
@@ -40,7 +41,12 @@ class DashSegmentsFD(FragmentFD):
self._prepare_and_start_frag_download(ctx, fmt)
ctx['start'] = real_start
fragments_to_download = self._get_fragments(fmt, ctx)
extra_query = None
extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
if extra_param_to_segment_url:
extra_query = urllib.parse.parse_qs(extra_param_to_segment_url)
fragments_to_download = self._get_fragments(fmt, ctx, extra_query)
if real_downloader:
self.to_screen(
@@ -57,7 +63,7 @@ class DashSegmentsFD(FragmentFD):
fragments = fragments(ctx) if callable(fragments) else fragments
return [next(iter(fragments))] if self.params.get('test') else fragments
def _get_fragments(self, fmt, ctx):
def _get_fragments(self, fmt, ctx, extra_query):
fragment_base_url = fmt.get('fragment_base_url')
fragments = self._resolve_fragments(fmt['fragments'], ctx)
@@ -70,6 +76,8 @@ class DashSegmentsFD(FragmentFD):
if not fragment_url:
assert fragment_base_url
fragment_url = urljoin(fragment_base_url, fragment['path'])
if extra_query:
fragment_url = update_url_query(fragment_url, extra_query)
yield {
'frag_index': frag_index,
yt_dlp/downloader/external.py
+119 -22
View File
@@ -1,9 +1,11 @@
import enum
import json
import os.path
import re
import subprocess
import sys
import time
import uuid
from .fragment import FragmentFD
from ..compat import functools
@@ -20,8 +22,10 @@ from ..utils import (
determine_ext,
encodeArgument,
encodeFilename,
find_available_port,
handle_youtubedl_headers,
remove_end,
sanitized_Request,
traverse_obj,
)
@@ -60,7 +64,6 @@ class ExternalFD(FragmentFD):
}
if filename != '-':
fsize = os.path.getsize(encodeFilename(tmpfilename))
self.to_screen(f'\r[{self.get_basename()}] Downloaded {fsize} bytes')
self.try_rename(tmpfilename, filename)
status.update({
'downloaded_bytes': fsize,
@@ -129,8 +132,7 @@ class ExternalFD(FragmentFD):
self._debug_cmd(cmd)
if 'fragments' not in info_dict:
_, stderr, returncode = Popen.run(
cmd, text=True, stderr=subprocess.PIPE if self._CAPTURE_STDERR else None)
_, stderr, returncode = self._call_process(cmd, info_dict)
if returncode and stderr:
self.to_stderr(stderr)
return returncode
@@ -140,7 +142,7 @@ class ExternalFD(FragmentFD):
retry_manager = RetryManager(self.params.get('fragment_retries'), self.report_retry,
frag_index=None, fatal=not skip_unavailable_fragments)
for retry in retry_manager:
_, stderr, returncode = Popen.run(cmd, text=True, stderr=subprocess.PIPE)
_, stderr, returncode = self._call_process(cmd, info_dict)
if not returncode:
break
# TODO: Decide whether to retry based on error code
@@ -172,6 +174,9 @@ class ExternalFD(FragmentFD):
self.try_remove(encodeFilename('%s.frag.urls' % tmpfilename))
return 0
def _call_process(self, cmd, info_dict):
return Popen.run(cmd, text=True, stderr=subprocess.PIPE)
class CurlFD(ExternalFD):
AVAILABLE_OPT = '-V'
@@ -256,6 +261,15 @@ class Aria2cFD(ExternalFD):
def _aria2c_filename(fn):
return fn if os.path.isabs(fn) else f'.{os.path.sep}{fn}'
def _call_downloader(self, tmpfilename, info_dict):
# FIXME: Disabled due to https://github.com/yt-dlp/yt-dlp/issues/5931
if False and 'no-external-downloader-progress' not in self.params.get('compat_opts', []):
info_dict['__rpc'] = {
'port': find_available_port() or 19190,
'secret': str(uuid.uuid4()),
}
return super()._call_downloader(tmpfilename, info_dict)
def _make_cmd(self, tmpfilename, info_dict):
cmd = [self.exe, '-c',
'--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
@@ -276,6 +290,12 @@ class Aria2cFD(ExternalFD):
cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
cmd += self._configuration_args()
if '__rpc' in info_dict:
cmd += [
'--enable-rpc',
f'--rpc-listen-port={info_dict["__rpc"]["port"]}',
f'--rpc-secret={info_dict["__rpc"]["secret"]}']
# aria2c strips out spaces from the beginning/end of filenames and paths.
# We work around this issue by adding a "./" to the beginning of the
# filename and relative path, and adding a "/" at the end of the path.
@@ -304,6 +324,88 @@ class Aria2cFD(ExternalFD):
cmd += ['--', info_dict['url']]
return cmd
def aria2c_rpc(self, rpc_port, rpc_secret, method, params=()):
# Does not actually need to be UUID, just unique
sanitycheck = str(uuid.uuid4())
d = json.dumps({
'jsonrpc': '2.0',
'id': sanitycheck,
'method': method,
'params': [f'token:{rpc_secret}', *params],
}).encode('utf-8')
request = sanitized_Request(
f'http://localhost:{rpc_port}/jsonrpc',
data=d, headers={
'Content-Type': 'application/json',
'Content-Length': f'{len(d)}',
'Ytdl-request-proxy': '__noproxy__',
})
with self.ydl.urlopen(request) as r:
resp = json.load(r)
assert resp.get('id') == sanitycheck, 'Something went wrong with RPC server'
return resp['result']
def _call_process(self, cmd, info_dict):
if '__rpc' not in info_dict:
return super()._call_process(cmd, info_dict)
send_rpc = functools.partial(self.aria2c_rpc, info_dict['__rpc']['port'], info_dict['__rpc']['secret'])
started = time.time()
fragmented = 'fragments' in info_dict
frag_count = len(info_dict['fragments']) if fragmented else 1
status = {
'filename': info_dict.get('_filename'),
'status': 'downloading',
'elapsed': 0,
'downloaded_bytes': 0,
'fragment_count': frag_count if fragmented else None,
'fragment_index': 0 if fragmented else None,
}
self._hook_progress(status, info_dict)
def get_stat(key, *obj, average=False):
val = tuple(filter(None, map(float, traverse_obj(obj, (..., ..., key))))) or [0]
return sum(val) / (len(val) if average else 1)
with Popen(cmd, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) as p:
# Add a small sleep so that RPC client can receive response,
# or the connection stalls infinitely
time.sleep(0.2)
retval = p.poll()
while retval is None:
# We don't use tellStatus as we won't know the GID without reading stdout
# Ref: https://aria2.github.io/manual/en/html/aria2c.html#aria2.tellActive
active = send_rpc('aria2.tellActive')
completed = send_rpc('aria2.tellStopped', [0, frag_count])
downloaded = get_stat('totalLength', completed) + get_stat('completedLength', active)
speed = get_stat('downloadSpeed', active)
total = frag_count * get_stat('totalLength', active, completed, average=True)
if total < downloaded:
total = None
status.update({
'downloaded_bytes': int(downloaded),
'speed': speed,
'total_bytes': None if fragmented else total,
'total_bytes_estimate': total,
'eta': (total - downloaded) / (speed or 1),
'fragment_index': min(frag_count, len(completed) + 1) if fragmented else None,
'elapsed': time.time() - started
})
self._hook_progress(status, info_dict)
if not active and len(completed) >= frag_count:
send_rpc('aria2.shutdown')
retval = p.wait()
break
time.sleep(0.1)
retval = p.poll()
return '', p.stderr.read(), retval
class HttpieFD(ExternalFD):
AVAILABLE_OPT = '--version'
@@ -342,7 +444,6 @@ class FFmpegFD(ExternalFD):
and cls.can_download(info_dict))
def _call_downloader(self, tmpfilename, info_dict):
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
ffpp = FFmpegPostProcessor(downloader=self)
if not ffpp.available:
self.report_error('m3u8 download detected but ffmpeg could not be found. Please install')
@@ -372,16 +473,6 @@ class FFmpegFD(ExternalFD):
# http://trac.ffmpeg.org/ticket/6125#comment:10
args += ['-seekable', '1' if seekable else '0']
http_headers = None
if info_dict.get('http_headers'):
youtubedl_headers = handle_youtubedl_headers(info_dict['http_headers'])
http_headers = [
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
'-headers',
''.join(f'{key}: {val}\r\n' for key, val in youtubedl_headers.items())
]
env = None
proxy = self.params.get('proxy')
if proxy:
@@ -434,21 +525,26 @@ class FFmpegFD(ExternalFD):
start_time, end_time = info_dict.get('section_start') or 0, info_dict.get('section_end')
for i, url in enumerate(urls):
if http_headers is not None and re.match(r'^https?://', url):
args += http_headers
selected_formats = info_dict.get('requested_formats') or [info_dict]
for i, fmt in enumerate(selected_formats):
if fmt.get('http_headers') and re.match(r'^https?://', fmt['url']):
headers_dict = handle_youtubedl_headers(fmt['http_headers'])
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
args.extend(['-headers', ''.join(f'{key}: {val}\r\n' for key, val in headers_dict.items())])
if start_time:
args += ['-ss', str(start_time)]
if end_time:
args += ['-t', str(end_time - start_time)]
args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', url]
args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', fmt['url']]
if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'):
args += ['-c', 'copy']
if info_dict.get('requested_formats') or protocol == 'http_dash_segments':
for (i, fmt) in enumerate(info_dict.get('requested_formats') or [info_dict]):
for i, fmt in enumerate(selected_formats):
stream_number = fmt.get('manifest_stream_number', 0)
args.extend(['-map', f'{i}:{stream_number}'])
@@ -488,8 +584,9 @@ class FFmpegFD(ExternalFD):
args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
self._debug_cmd(args)
piped = any(fmt['url'] in ('-', 'pipe:') for fmt in selected_formats)
with Popen(args, stdin=subprocess.PIPE, env=env) as proc:
if url in ('-', 'pipe:'):
if piped:
self.on_process_started(proc, proc.stdin)
try:
retval = proc.wait()
@@ -499,7 +596,7 @@ class FFmpegFD(ExternalFD):
# produces a file that is playable (this is mostly useful for live
# streams). Note that Windows is not affected and produces playable
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and url not in ('-', 'pipe:'):
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and not piped:
proc.communicate_or_kill(b'q')
else:
proc.kill(timeout=None)
yt_dlp/extractor/_extractors.py
+65 -10
View File
@@ -21,7 +21,8 @@ from .youtube import ( # Youtube is moved to the top to improve performance
YoutubeYtBeIE,
YoutubeYtUserIE,
YoutubeWatchLaterIE,
YoutubeShortsAudioPivotIE
YoutubeShortsAudioPivotIE,
YoutubeConsentRedirectIE,
)
from .abc import (
@@ -78,6 +79,8 @@ from .agora import (
WyborczaVideoIE,
)
from .airmozilla import AirMozillaIE
from .airtv import AirTVIE
from .aitube import AitubeKZVideoIE
from .aljazeera import AlJazeeraIE
from .alphaporno import AlphaPornoIE
from .amara import AmaraIE
@@ -86,7 +89,15 @@ from .alura import (
AluraCourseIE
)
from .amcnetworks import AMCNetworksIE
from .amazon import AmazonStoreIE
from .amazon import (
AmazonStoreIE,
AmazonReviewsIE,
)
from .amazonminitv import (
AmazonMiniTVIE,
AmazonMiniTVSeasonIE,
AmazonMiniTVSeriesIE,
)
from .americastestkitchen import (
AmericasTestKitchenIE,
AmericasTestKitchenSeasonIE,
@@ -178,6 +189,10 @@ from .bbc import (
from .beeg import BeegIE
from .behindkink import BehindKinkIE
from .bellmedia import BellMediaIE
from .beatbump import (
BeatBumpVideoIE,
BeatBumpPlaylistIE,
)
from .beatport import BeatportIE
from .berufetv import BerufeTVIE
from .bet import BetIE
@@ -461,6 +476,8 @@ from .drtuber import DrTuberIE
from .drtv import (
DRTVIE,
DRTVLiveIE,
DRTVSeasonIE,
DRTVSeriesIE,
)
from .dtube import DTubeIE
from .dvtv import DVTVIE
@@ -531,7 +548,7 @@ from .espn import (
ESPNCricInfoIE,
)
from .esri import EsriVideoIE
from .europa import EuropaIE
from .europa import EuropaIE, EuroParlWebstreamIE
from .europeantour import EuropeanTourIE
from .eurosport import EurosportIE
from .euscreen import EUScreenIE
@@ -820,6 +837,8 @@ from .joj import JojIE
from .jwplatform import JWPlatformIE
from .kakao import KakaoIE
from .kaltura import KalturaIE
from .kanal2 import Kanal2IE
from .kankanews import KankaNewsIE
from .karaoketv import KaraoketvIE
from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
@@ -829,6 +848,10 @@ from .khanacademy import (
KhanAcademyIE,
KhanAcademyUnitIE,
)
from .kick import (
KickIE,
KickVODIE,
)
from .kicker import KickerIE
from .kickstarter import KickStarterIE
from .kinja import KinjaEmbedIE
@@ -976,6 +999,10 @@ from .mediasite import (
MediasiteCatalogIE,
MediasiteNamedCatalogIE,
)
from .mediastream import (
MediaStreamIE,
WinSportsVideoIE,
)
from .mediaworksnz import MediaWorksNZVODIE
from .medici import MediciIE
from .megaphone import MegaphoneIE
@@ -1144,6 +1171,7 @@ from .neteasemusic import (
from .netverse import (
NetverseIE,
NetversePlaylistIE,
NetverseSearchIE,
)
from .newgrounds import (
NewgroundsIE,
@@ -1205,6 +1233,7 @@ from .nintendo import NintendoIE
from .nitter import NitterIE
from .njpwworld import NJPWWorldIE
from .nobelprize import NobelPrizeIE
from .noice import NoicePodcastIE
from .nonktube import NonkTubeIE
from .noodlemagazine import NoodleMagazineIE
from .noovo import NoovoIE
@@ -1270,6 +1299,7 @@ from .on24 import On24IE
from .ondemandkorea import OnDemandKoreaIE
from .onefootball import OneFootballIE
from .onenewsnz import OneNewsNZIE
from .oneplace import OnePlacePodcastIE
from .onet import (
OnetIE,
OnetChannelIE,
@@ -1392,6 +1422,8 @@ from .pokergo import (
from .polsatgo import PolsatGoIE
from .polskieradio import (
PolskieRadioIE,
PolskieRadioLegacyIE,
PolskieRadioAuditionIE,
PolskieRadioCategoryIE,
PolskieRadioPlayerIE,
PolskieRadioPodcastIE,
@@ -1524,7 +1556,10 @@ from .rokfin import (
)
from .roosterteeth import RoosterTeethIE, RoosterTeethSeriesIE
from .rottentomatoes import RottenTomatoesIE
from .rozhlas import RozhlasIE
from .rozhlas import (
RozhlasIE,
RozhlasVltavaIE,
)
from .rte import RteIE, RteRadioIE
from .rtlnl import (
RtlNlIE,
@@ -1561,6 +1596,7 @@ from .ruhd import RUHDIE
from .rule34video import Rule34VideoIE
from .rumble import (
RumbleEmbedIE,
RumbleIE,
RumbleChannelIE,
)
from .rutube import (
@@ -1603,6 +1639,7 @@ from .savefrom import SaveFromIE
from .sbs import SBSIE
from .screen9 import Screen9IE
from .screencast import ScreencastIE
from .screencastify import ScreencastifyIE
from .screencastomatic import ScreencastOMaticIE
from .scrippsnetworks import (
ScrippsNetworksWatchIE,
@@ -1632,6 +1669,7 @@ from .shared import (
VivoIE,
)
from .sharevideos import ShareVideosEmbedIE
from .sibnet import SibnetEmbedIE
from .shemaroome import ShemarooMeIE
from .showroomlive import ShowRoomLiveIE
from .simplecast import (
@@ -1679,6 +1717,7 @@ from .soundcloud import (
SoundcloudSetIE,
SoundcloudRelatedIE,
SoundcloudUserIE,
SoundcloudUserPermalinkIE,
SoundcloudTrackStationIE,
SoundcloudPlaylistIE,
SoundcloudSearchIE,
@@ -1840,6 +1879,11 @@ from .theweatherchannel import TheWeatherChannelIE
from .thisamericanlife import ThisAmericanLifeIE
from .thisav import ThisAVIE
from .thisoldhouse import ThisOldHouseIE
from .thisvid import (
ThisVidIE,
ThisVidMemberIE,
ThisVidPlaylistIE,
)
from .threespeak import (
ThreeSpeakIE,
ThreeSpeakUserIE,
@@ -1852,6 +1896,7 @@ from .tiktok import (
TikTokEffectIE,
TikTokTagIE,
TikTokVMIE,
TikTokLiveIE,
DouyinIE,
)
from .tinypic import TinyPicIE
@@ -1889,6 +1934,7 @@ from .trovo import (
TrovoChannelVodIE,
TrovoChannelClipIE,
)
from .trtcocuk import TrtCocukVideoIE
from .trueid import TrueIDIE
from .trunews import TruNewsIE
from .truth import TruthIE
@@ -2002,6 +2048,10 @@ from .twitter import (
TwitterSpacesIE,
TwitterShortenerIE,
)
from .txxx import (
TxxxIE,
PornTopIE,
)
from .udemy import (
UdemyIE,
UdemyCourseIE
@@ -2072,6 +2122,13 @@ from .videocampus_sachsen import (
)
from .videodetective import VideoDetectiveIE
from .videofyme import VideofyMeIE
from .videoken import (
VideoKenIE,
VideoKenPlayerIE,
VideoKenPlaylistIE,
VideoKenCategoryIE,
VideoKenTopicIE,
)
from .videomore import (
VideomoreIE,
VideomoreVideoIE,
@@ -2096,6 +2153,7 @@ from .vimeo import (
VimeoGroupsIE,
VimeoLikesIE,
VimeoOndemandIE,
VimeoProIE,
VimeoReviewIE,
VimeoUserIE,
VimeoWatchLaterIE,
@@ -2138,6 +2196,7 @@ from .voicy import (
VoicyIE,
VoicyChannelIE,
)
from .volejtv import VolejTVIE
from .voot import (
VootIE,
VootSeriesIE,
@@ -2183,6 +2242,7 @@ from .wdr import (
WDRElefantIE,
WDRMobileIE,
)
from .webcamerapl import WebcameraplIE
from .webcaster import (
WebcasterIE,
WebcasterFeedIE,
@@ -2219,6 +2279,7 @@ from .wsj import (
WSJArticleIE,
)
from .wwe import WWEIE
from .xanimu import XanimuIE
from .xbef import XBefIE
from .xboxclips import XboxClipsIE
from .xfileshare import XFileShareIE
@@ -2227,12 +2288,6 @@ from .xhamster import (
XHamsterEmbedIE,
XHamsterUserIE,
)
from .xiami import (
XiamiSongIE,
XiamiAlbumIE,
XiamiArtistIE,
XiamiCollectionIE
)
from .ximalaya import (
XimalayaIE,
XimalayaAlbumIE
yt_dlp/extractor/adn.py
+1 -1
View File
@@ -168,7 +168,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
}, data=b'')['token']
links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
self._K = ''.join(random.choices('0123456789abcdef', k=16))
message = bytes_to_intlist(json.dumps({
'k': self._K,
't': token,
yt_dlp/extractor/airtv.py
+96
View File
@@ -0,0 +1,96 @@
from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import (
determine_ext,
int_or_none,
mimetype2ext,
parse_iso8601,
traverse_obj
)
class AirTVIE(InfoExtractor):
_VALID_URL = r'https?://www\.air\.tv/watch\?v=(?P<id>\w+)'
_TESTS = [{
# without youtube_id
'url': 'https://www.air.tv/watch?v=W87jcWleSn2hXZN47zJZsQ',
'info_dict': {
'id': 'W87jcWleSn2hXZN47zJZsQ',
'ext': 'mp4',
'release_date': '20221003',
'release_timestamp': 1664792603,
'channel_id': 'vgfManQlRQKgoFQ8i8peFQ',
'title': 'md5:c12d49ed367c3dadaa67659aff43494c',
'upload_date': '20221003',
'duration': 151,
'view_count': int,
'thumbnail': 'https://cdn-sp-gcs.air.tv/videos/W/8/W87jcWleSn2hXZN47zJZsQ/b13fc56464f47d9d62a36d110b9b5a72-4096x2160_9.jpg',
'timestamp': 1664792603,
}
}, {
# with youtube_id
'url': 'https://www.air.tv/watch?v=sv57EC8tRXG6h8dNXFUU1Q',
'info_dict': {
'id': '2ZTqmpee-bQ',
'ext': 'mp4',
'comment_count': int,
'tags': 'count:11',
'channel_follower_count': int,
'like_count': int,
'uploader': 'Newsflare',
'thumbnail': 'https://i.ytimg.com/vi_webp/2ZTqmpee-bQ/maxresdefault.webp',
'availability': 'public',
'title': 'Geese Chase Alligator Across Golf Course',
'uploader_id': 'NewsflareBreaking',
'channel_url': 'https://www.youtube.com/channel/UCzSSoloGEz10HALUAbYhngQ',
'description': 'md5:99b21d9cea59330149efbd9706e208f5',
'age_limit': 0,
'channel_id': 'UCzSSoloGEz10HALUAbYhngQ',
'uploader_url': 'http://www.youtube.com/user/NewsflareBreaking',
'view_count': int,
'categories': ['News & Politics'],
'live_status': 'not_live',
'playable_in_embed': True,
'channel': 'Newsflare',
'duration': 37,
'upload_date': '20180511',
}
}]
def _get_formats_and_subtitle(self, json_data, video_id):
formats, subtitles = [], {}
for source in traverse_obj(json_data, 'sources', 'sources_desktop', ...):
ext = determine_ext(source.get('src'), mimetype2ext(source.get('type')))
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(source.get('src'), video_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
formats.append({'url': source.get('src'), 'ext': ext})
return formats, subtitles
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
nextjs_json = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['initialState']['videos'][display_id]
if nextjs_json.get('youtube_id'):
return self.url_result(
f'https://www.youtube.com/watch?v={nextjs_json.get("youtube_id")}', YoutubeIE)
formats, subtitles = self._get_formats_and_subtitle(nextjs_json, display_id)
return {
'id': display_id,
'title': nextjs_json.get('title') or self._html_search_meta('og:title', webpage),
'formats': formats,
'subtitles': subtitles,
'description': nextjs_json.get('description') or None,
'duration': int_or_none(nextjs_json.get('duration')),
'thumbnails': [
{'url': thumbnail}
for thumbnail in traverse_obj(nextjs_json, ('default_thumbnails', ...))],
'channel_id': traverse_obj(nextjs_json, 'channel', 'channel_slug'),
'timestamp': parse_iso8601(nextjs_json.get('created')),
'release_timestamp': parse_iso8601(nextjs_json.get('published')),
'view_count': int_or_none(nextjs_json.get('views')),
}
yt_dlp/extractor/aitube.py
+60
View File
@@ -0,0 +1,60 @@
from .common import InfoExtractor
from ..utils import int_or_none, merge_dicts
class AitubeKZVideoIE(InfoExtractor):
_VALID_URL = r'https?://aitube\.kz/(?:video|embed/)\?(?:[^\?]+)?id=(?P<id>[\w-]+)'
_TESTS = [{
# id paramater as first parameter
'url': 'https://aitube.kz/video?id=9291d29b-c038-49a1-ad42-3da2051d353c&playlistId=d55b1f5f-ef2a-4f23-b646-2a86275b86b7&season=1',
'info_dict': {
'id': '9291d29b-c038-49a1-ad42-3da2051d353c',
'ext': 'mp4',
'duration': 2174.0,
'channel_id': '94962f73-013b-432c-8853-1bd78ca860fe',
'like_count': int,
'channel': 'ASTANA TV',
'comment_count': int,
'view_count': int,
'description': 'Смотреть любимые сериалы и видео, поделиться видео и сериалами с друзьями и близкими',
'thumbnail': 'https://cdn.static02.aitube.kz/kz.aitudala.aitube.staticaccess/files/ddf2a2ff-bee3-409b-b5f2-2a8202bba75b',
'upload_date': '20221102',
'timestamp': 1667370519,
'title': 'Ангел хранитель 1 серия',
'channel_follower_count': int,
}
}, {
# embed url
'url': 'https://aitube.kz/embed/?id=9291d29b-c038-49a1-ad42-3da2051d353c',
'only_matching': True,
}, {
# id parameter is not as first paramater
'url': 'https://aitube.kz/video?season=1&id=9291d29b-c038-49a1-ad42-3da2051d353c&playlistId=d55b1f5f-ef2a-4f23-b646-2a86275b86b7',
'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
nextjs_data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['videoInfo']
json_ld_data = self._search_json_ld(webpage, video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
f'https://api-http.aitube.kz/kz.aitudala.aitube.staticaccess/video/{video_id}/video', video_id)
return merge_dicts({
'id': video_id,
'title': nextjs_data.get('title') or self._html_search_meta(['name', 'og:title'], webpage),
'description': nextjs_data.get('description'),
'formats': formats,
'subtitles': subtitles,
'view_count': (nextjs_data.get('viewCount')
or int_or_none(self._html_search_meta('ya:ovs:views_total', webpage))),
'like_count': nextjs_data.get('likeCount'),
'channel': nextjs_data.get('channelTitle'),
'channel_id': nextjs_data.get('channelId'),
'thumbnail': nextjs_data.get('coverUrl'),
'comment_count': nextjs_data.get('commentCount'),
'channel_follower_count': int_or_none(nextjs_data.get('channelSubscriberCount')),
}, json_ld_data)
yt_dlp/extractor/amazon.py
+109 -7
View File
@@ -1,5 +1,17 @@
import re
from .common import InfoExtractor
from ..utils import ExtractorError, int_or_none
from ..utils import (
ExtractorError,
clean_html,
float_or_none,
get_element_by_attribute,
get_element_by_class,
int_or_none,
js_to_json,
traverse_obj,
url_or_none,
)
class AmazonStoreIE(InfoExtractor):
@@ -9,7 +21,7 @@ class AmazonStoreIE(InfoExtractor):
'url': 'https://www.amazon.co.uk/dp/B098XNCHLD/',
'info_dict': {
'id': 'B098XNCHLD',
'title': 'md5:dae240564cbb2642170c02f7f0d7e472',
'title': str,
},
'playlist_mincount': 1,
'playlist': [{
@@ -20,28 +32,32 @@ class AmazonStoreIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 34,
},
}]
}],
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.in/Sony-WH-1000XM4-Cancelling-Headphones-Bluetooth/dp/B0863TXGM3',
'info_dict': {
'id': 'B0863TXGM3',
'title': 'md5:d1d3352428f8f015706c84b31e132169',
'title': str,
},
'playlist_mincount': 4,
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.com/dp/B0845NXCXF/',
'info_dict': {
'id': 'B0845NXCXF',
'title': 'md5:f3fa12779bf62ddb6a6ec86a360a858e',
'title': str,
},
'playlist-mincount': 1,
'expected_warnings': ['Unable to extract data'],
}, {
'url': 'https://www.amazon.es/Samsung-Smartphone-s-AMOLED-Quad-c%C3%A1mara-espa%C3%B1ola/dp/B08WX337PQ',
'info_dict': {
'id': 'B08WX337PQ',
'title': 'md5:f3fa12779bf62ddb6a6ec86a360a858e',
'title': str,
},
'playlist_mincount': 1,
'expected_warnings': ['Unable to extract data'],
}]
def _real_extract(self, url):
@@ -52,7 +68,7 @@ class AmazonStoreIE(InfoExtractor):
try:
data_json = self._search_json(
r'var\s?obj\s?=\s?jQuery\.parseJSON\(\'', webpage, 'data', id,
transform_source=lambda x: x.replace(R'\\u', R'\u'))
transform_source=js_to_json)
except ExtractorError as e:
retry.error = e
@@ -66,3 +82,89 @@ class AmazonStoreIE(InfoExtractor):
'width': int_or_none(video.get('videoWidth')),
} for video in (data_json.get('videos') or []) if video.get('isVideo') and video.get('url')]
return self.playlist_result(entries, playlist_id=id, playlist_title=data_json.get('title'))
class AmazonReviewsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?amazon\.(?:[a-z]{2,3})(?:\.[a-z]{2})?/gp/customer-reviews/(?P<id>[^/&#$?]+)'
_TESTS = [{
'url': 'https://www.amazon.com/gp/customer-reviews/R10VE9VUSY19L3/ref=cm_cr_arp_d_rvw_ttl',
'info_dict': {
'id': 'R10VE9VUSY19L3',
'ext': 'mp4',
'title': 'Get squad #Suspicious',
'description': 'md5:7012695052f440a1e064e402d87e0afb',
'uploader': 'Kimberly Cronkright',
'average_rating': 1.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}, {
'url': 'https://www.amazon.com/gp/customer-reviews/R10VE9VUSY19L3/ref=cm_cr_arp_d_rvw_ttl?language=es_US',
'info_dict': {
'id': 'R10VE9VUSY19L3',
'ext': 'mp4',
'title': 'Get squad #Suspicious',
'description': 'md5:7012695052f440a1e064e402d87e0afb',
'uploader': 'Kimberly Cronkright',
'average_rating': 1.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}, {
'url': 'https://www.amazon.in/gp/customer-reviews/RV1CO8JN5VGXV/',
'info_dict': {
'id': 'RV1CO8JN5VGXV',
'ext': 'mp4',
'title': 'Not sure about its durability',
'description': 'md5:1a252c106357f0a3109ebf37d2e87494',
'uploader': 'Shoaib Gulzar',
'average_rating': 2.0,
'thumbnail': r're:^https?://.*\.jpg$',
},
'expected_warnings': ['Review body was not found in webpage'],
}]
def _real_extract(self, url):
video_id = self._match_id(url)
for retry in self.RetryManager():
webpage = self._download_webpage(url, video_id)
review_body = get_element_by_attribute('data-hook', 'review-body', webpage)
if not review_body:
retry.error = ExtractorError('Review body was not found in webpage', expected=True)
formats, subtitles = [], {}
manifest_url = self._search_regex(
r'data-video-url="([^"]+)"', review_body, 'm3u8 url', default=None)
if url_or_none(manifest_url):
fmts, subtitles = self._extract_m3u8_formats_and_subtitles(
manifest_url, video_id, 'mp4', fatal=False)
formats.extend(fmts)
video_url = self._search_regex(
r'<input[^>]+\bvalue="([^"]+)"[^>]+\bclass="video-url"', review_body, 'mp4 url', default=None)
if url_or_none(video_url):
formats.append({
'url': video_url,
'ext': 'mp4',
'format_id': 'http-mp4',
})
if not formats:
self.raise_no_formats('No video found for this customer review', expected=True)
return {
'id': video_id,
'title': (clean_html(get_element_by_attribute('data-hook', 'review-title', webpage))
or self._html_extract_title(webpage)),
'description': clean_html(traverse_obj(re.findall(
r'<span(?:\s+class="cr-original-review-content")?>(.+?)</span>', review_body), -1)),
'uploader': clean_html(get_element_by_class('a-profile-name', webpage)),
'average_rating': float_or_none(clean_html(get_element_by_attribute(
'data-hook', 'review-star-rating', webpage) or '').partition(' ')[0]),
'thumbnail': self._search_regex(
r'data-thumbnail-url="([^"]+)"', review_body, 'thumbnail', default=None),
'formats': formats,
'subtitles': subtitles,
}
yt_dlp/extractor/amazonminitv.py
+290
View File
@@ -0,0 +1,290 @@
import json
from .common import InfoExtractor
from ..utils import ExtractorError, int_or_none, traverse_obj, try_get
class AmazonMiniTVBaseIE(InfoExtractor):
def _real_initialize(self):
self._download_webpage(
'https://www.amazon.in/minitv', None,
note='Fetching guest session cookies')
AmazonMiniTVBaseIE.session_id = self._get_cookies('https://www.amazon.in')['session-id'].value
def _call_api(self, asin, data=None, note=None):
device = {'clientId': 'ATVIN', 'deviceLocale': 'en_GB'}
if data:
data['variables'].update({
'contentType': 'VOD',
'sessionIdToken': self.session_id,
**device,
})
resp = self._download_json(
f'https://www.amazon.in/minitv/api/web/{"graphql" if data else "prs"}',
asin, note=note, headers={'Content-Type': 'application/json'},
data=json.dumps(data).encode() if data else None,
query=None if data else {
'deviceType': 'A1WMMUXPCUJL4N',
'contentId': asin,
**device,
})
if resp.get('errors'):
raise ExtractorError(f'MiniTV said: {resp["errors"][0]["message"]}')
elif not data:
return resp
return resp['data'][data['operationName']]
class AmazonMiniTVIE(AmazonMiniTVBaseIE):
_VALID_URL = r'(?:https?://(?:www\.)?amazon\.in/minitv/tp/|amazonminitv:(?:amzn1\.dv\.gti\.)?)(?P<id>[a-f0-9-]+)'
_TESTS = [{
'url': 'https://www.amazon.in/minitv/tp/75fe3a75-b8fe-4499-8100-5c9424344840?referrer=https%3A%2F%2Fwww.amazon.in%2Fminitv',
'info_dict': {
'id': 'amzn1.dv.gti.75fe3a75-b8fe-4499-8100-5c9424344840',
'ext': 'mp4',
'title': 'May I Kiss You?',
'language': 'Hindi',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'md5:a549bfc747973e04feb707833474e59d',
'release_timestamp': 1644710400,
'release_date': '20220213',
'duration': 846,
'chapters': 'count:2',
'series': 'Couple Goals',
'series_id': 'amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
'season': 'Season 3',
'season_number': 3,
'season_id': 'amzn1.dv.gti.20331016-d9b9-4968-b991-c89fa4927a36',
'episode': 'May I Kiss You?',
'episode_number': 2,
'episode_id': 'amzn1.dv.gti.75fe3a75-b8fe-4499-8100-5c9424344840',
},
}, {
'url': 'https://www.amazon.in/minitv/tp/280d2564-584f-452f-9c98-7baf906e01ab?referrer=https%3A%2F%2Fwww.amazon.in%2Fminitv',
'info_dict': {
'id': 'amzn1.dv.gti.280d2564-584f-452f-9c98-7baf906e01ab',
'ext': 'mp4',
'title': 'Jahaan',
'language': 'Hindi',
'thumbnail': r're:^https?://.*\.jpg',
'description': 'md5:05eb765a77bf703f322f120ec6867339',
'release_timestamp': 1647475200,
'release_date': '20220317',
'duration': 783,
'chapters': [],
},
}, {
'url': 'https://www.amazon.in/minitv/tp/280d2564-584f-452f-9c98-7baf906e01ab',
'only_matching': True,
}, {
'url': 'amazonminitv:amzn1.dv.gti.280d2564-584f-452f-9c98-7baf906e01ab',
'only_matching': True,
}, {
'url': 'amazonminitv:280d2564-584f-452f-9c98-7baf906e01ab',
'only_matching': True,
}]
_GRAPHQL_QUERY_CONTENT = '''
query content($sessionIdToken: String!, $deviceLocale: String, $contentId: ID!, $contentType: ContentType!, $clientId: String) {
content(
applicationContextInput: {deviceLocale: $deviceLocale, sessionIdToken: $sessionIdToken, clientId: $clientId}
contentId: $contentId
contentType: $contentType
) {
contentId
name
... on Episode {
contentId
vodType
name
images
description {
synopsis
contentLengthInSeconds
}
publicReleaseDateUTC
audioTracks
seasonId
seriesId
seriesName
seasonNumber
episodeNumber
timecode {
endCreditsTime
}
}
... on MovieContent {
contentId
vodType
name
description {
synopsis
contentLengthInSeconds
}
images
publicReleaseDateUTC
audioTracks
}
}
}'''
def _real_extract(self, url):
asin = f'amzn1.dv.gti.{self._match_id(url)}'
prs = self._call_api(asin, note='Downloading playback info')
formats, subtitles = [], {}
for type_, asset in prs['playbackAssets'].items():
if not traverse_obj(asset, 'manifestUrl'):
continue
if type_ == 'hls':
m3u8_fmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
asset['manifestUrl'], asin, ext='mp4', entry_protocol='m3u8_native',
m3u8_id=type_, fatal=False)
formats.extend(m3u8_fmts)
subtitles = self._merge_subtitles(subtitles, m3u8_subs)
elif type_ == 'dash':
mpd_fmts, mpd_subs = self._extract_mpd_formats_and_subtitles(
asset['manifestUrl'], asin, mpd_id=type_, fatal=False)
formats.extend(mpd_fmts)
subtitles = self._merge_subtitles(subtitles, mpd_subs)
else:
self.report_warning(f'Unknown asset type: {type_}')
title_info = self._call_api(
asin, note='Downloading title info', data={
'operationName': 'content',
'variables': {'contentId': asin},
'query': self._GRAPHQL_QUERY_CONTENT,
})
credits_time = try_get(title_info, lambda x: x['timecode']['endCreditsTime'] / 1000)
is_episode = title_info.get('vodType') == 'EPISODE'
return {
'id': asin,
'title': title_info.get('name'),
'formats': formats,
'subtitles': subtitles,
'language': traverse_obj(title_info, ('audioTracks', 0)),
'thumbnails': [{
'id': type_,
'url': url,
} for type_, url in (title_info.get('images') or {}).items()],
'description': traverse_obj(title_info, ('description', 'synopsis')),
'release_timestamp': int_or_none(try_get(title_info, lambda x: x['publicReleaseDateUTC'] / 1000)),
'duration': traverse_obj(title_info, ('description', 'contentLengthInSeconds')),
'chapters': [{
'start_time': credits_time,
'title': 'End Credits',
}] if credits_time else [],
'series': title_info.get('seriesName'),
'series_id': title_info.get('seriesId'),
'season_number': title_info.get('seasonNumber'),
'season_id': title_info.get('seasonId'),
'episode': title_info.get('name') if is_episode else None,
'episode_number': title_info.get('episodeNumber'),
'episode_id': asin if is_episode else None,
}
class AmazonMiniTVSeasonIE(AmazonMiniTVBaseIE):
IE_NAME = 'amazonminitv:season'
_VALID_URL = r'amazonminitv:season:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
IE_DESC = 'Amazon MiniTV Series, "minitv:season:" prefix'
_TESTS = [{
'url': 'amazonminitv:season:amzn1.dv.gti.0aa996eb-6a1b-4886-a342-387fbd2f1db0',
'playlist_mincount': 6,
'info_dict': {
'id': 'amzn1.dv.gti.0aa996eb-6a1b-4886-a342-387fbd2f1db0',
},
}, {
'url': 'amazonminitv:season:0aa996eb-6a1b-4886-a342-387fbd2f1db0',
'only_matching': True,
}]
_GRAPHQL_QUERY = '''
query getEpisodes($sessionIdToken: String!, $clientId: String, $episodeOrSeasonId: ID!, $deviceLocale: String) {
getEpisodes(
applicationContextInput: {sessionIdToken: $sessionIdToken, deviceLocale: $deviceLocale, clientId: $clientId}
episodeOrSeasonId: $episodeOrSeasonId
) {
episodes {
... on Episode {
contentId
name
images
seriesName
seasonId
seriesId
seasonNumber
episodeNumber
description {
synopsis
contentLengthInSeconds
}
publicReleaseDateUTC
}
}
}
}
'''
def _entries(self, asin):
season_info = self._call_api(
asin, note='Downloading season info', data={
'operationName': 'getEpisodes',
'variables': {'episodeOrSeasonId': asin},
'query': self._GRAPHQL_QUERY,
})
for episode in season_info['episodes']:
yield self.url_result(
f'amazonminitv:{episode["contentId"]}', AmazonMiniTVIE, episode['contentId'])
def _real_extract(self, url):
asin = f'amzn1.dv.gti.{self._match_id(url)}'
return self.playlist_result(self._entries(asin), asin)
class AmazonMiniTVSeriesIE(AmazonMiniTVBaseIE):
IE_NAME = 'amazonminitv:series'
_VALID_URL = r'amazonminitv:series:(?:amzn1\.dv\.gti\.)?(?P<id>[a-f0-9-]+)'
_TESTS = [{
'url': 'amazonminitv:series:amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
'playlist_mincount': 3,
'info_dict': {
'id': 'amzn1.dv.gti.56521d46-b040-4fd5-872e-3e70476a04b0',
},
}, {
'url': 'amazonminitv:series:56521d46-b040-4fd5-872e-3e70476a04b0',
'only_matching': True,
}]
_GRAPHQL_QUERY = '''
query getSeasons($sessionIdToken: String!, $deviceLocale: String, $episodeOrSeasonOrSeriesId: ID!, $clientId: String) {
getSeasons(
applicationContextInput: {deviceLocale: $deviceLocale, sessionIdToken: $sessionIdToken, clientId: $clientId}
episodeOrSeasonOrSeriesId: $episodeOrSeasonOrSeriesId
) {
seasons {
seasonId
}
}
}
'''
def _entries(self, asin):
season_info = self._call_api(
asin, note='Downloading series info', data={
'operationName': 'getSeasons',
'variables': {'episodeOrSeasonOrSeriesId': asin},
'query': self._GRAPHQL_QUERY,
})
for season in season_info['seasons']:
yield self.url_result(f'amazonminitv:season:{season["seasonId"]}', AmazonMiniTVSeasonIE, season['seasonId'])
def _real_extract(self, url):
asin = f'amzn1.dv.gti.{self._match_id(url)}'
return self.playlist_result(self._entries(asin), asin)
yt_dlp/extractor/ard.py
+11 -8
View File
@@ -46,6 +46,9 @@ class ARDMediathekBaseIE(InfoExtractor):
subtitles['de'] = [{
'ext': 'ttml',
'url': subtitle_url,
}, {
'ext': 'vtt',
'url': subtitle_url.replace('/ebutt/', '/webvtt/') + '.vtt',
}]
return {
@@ -286,16 +289,16 @@ class ARDMediathekIE(ARDMediathekBaseIE):
class ARDIE(InfoExtractor):
_VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
_TESTS = [{
# available till 7.01.2022
'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
# available till 7.12.2023
'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
'md5': 'a438f671e87a7eba04000336a119ccc4',
'info_dict': {
'id': 'maischberger-die-woche-video100',
'display_id': 'maischberger-die-woche-video100',
'id': 'maischberger-video-424',
'display_id': 'maischberger-video-424',
'ext': 'mp4',
'duration': 3687.0,
'title': 'maischberger. die woche vom 7. Januar 2021',
'upload_date': '20210107',
'duration': 4452.0,
'title': 'maischberger am 07.12.2022',
'upload_date': '20221207',
'thumbnail': r're:^https?://.*\.jpg$',
},
}, {
yt_dlp/extractor/arte.py
+20 -4
View File
@@ -65,6 +65,21 @@ class ArteTVIE(ArteTVBaseIE):
}, {
'url': 'https://api.arte.tv/api/player/v2/config/de/LIVE',
'only_matching': True,
}, {
'url': 'https://www.arte.tv/de/videos/110203-006-A/zaz/',
'info_dict': {
'id': '110203-006-A',
'chapters': 'count:16',
'description': 'md5:cf592f1df52fe52007e3f8eac813c084',
'alt_title': 'Zaz',
'title': 'Baloise Session 2022',
'timestamp': 1668445200,
'duration': 4054,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/ubQjmVCGyRx3hmBuZEK9QZ/940x530',
'upload_date': '20221114',
'ext': 'mp4',
},
'expected_warnings': ['geo restricted']
}]
_GEO_BYPASS = True
@@ -180,10 +195,6 @@ class ArteTVIE(ArteTVBaseIE):
else:
self.report_warning(f'Skipping stream with unknown protocol {stream["protocol"]}')
# TODO: chapters from stream['segments']?
# The JS also looks for chapters in config['data']['attributes']['chapters'],
# but I am yet to find a video having those
formats.extend(secondary_formats)
self._remove_duplicate_formats(formats)
@@ -205,6 +216,11 @@ class ArteTVIE(ArteTVBaseIE):
{'url': image['url'], 'id': image.get('caption')}
for image in metadata.get('images') or [] if url_or_none(image.get('url'))
],
# TODO: chapters may also be in stream['segments']?
'chapters': traverse_obj(config, ('data', 'attributes', 'chapters', 'elements', ..., {
'start_time': 'startTime',
'title': 'title',
})) or None,
}
yt_dlp/extractor/bandcamp.py
+45 -3
View File
@@ -29,11 +29,18 @@ class BandcampIE(InfoExtractor):
'info_dict': {
'id': '1812978515',
'ext': 'mp3',
'title': "youtube-dl \"'/\\ä↭ - youtube-dl \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭",
'title': 'youtube-dl "\'/\\ä↭ - youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
'duration': 9.8485,
'uploader': 'youtube-dl "\'/\\ä↭',
'uploader': 'youtube-dl "\'/\\ä↭',
'upload_date': '20121129',
'timestamp': 1354224127,
'track': 'youtube-dl "\'/\\ä↭ - youtube-dl test song "\'/\\ä↭',
'album_artist': 'youtube-dl "\'/\\ä↭',
'track_id': '1812978515',
'artist': 'youtube-dl "\'/\\ä↭',
'uploader_url': 'https://youtube-dl.bandcamp.com',
'uploader_id': 'youtube-dl',
'thumbnail': 'https://f4.bcbits.com/img/a3216802731_5.jpg',
},
'_skip': 'There is a limit of 200 free downloads / month for the test song'
}, {
@@ -41,7 +48,8 @@ class BandcampIE(InfoExtractor):
'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
'info_dict': {
'id': '2650410135',
'ext': 'aiff',
'ext': 'm4a',
'acodec': r're:[fa]lac',
'title': 'Ben Prunty - Lanius (Battle)',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Ben Prunty',
@@ -54,7 +62,10 @@ class BandcampIE(InfoExtractor):
'track_number': 1,
'track_id': '2650410135',
'artist': 'Ben Prunty',
'album_artist': 'Ben Prunty',
'album': 'FTL: Advanced Edition Soundtrack',
'uploader_url': 'https://benprunty.bandcamp.com',
'uploader_id': 'benprunty',
},
}, {
# no free download, mp3 128
@@ -75,7 +86,34 @@ class BandcampIE(InfoExtractor):
'track_number': 5,
'track_id': '2584466013',
'artist': 'Mastodon',
'album_artist': 'Mastodon',
'album': 'Call of the Mastodon',
'uploader_url': 'https://relapsealumni.bandcamp.com',
'uploader_id': 'relapsealumni',
},
}, {
# track from compilation album (artist/album_artist difference)
'url': 'https://diskotopia.bandcamp.com/track/safehouse',
'md5': '19c5337bca1428afa54129f86a2f6a69',
'info_dict': {
'id': '1978174799',
'ext': 'mp3',
'title': 'submerse - submerse - Safehouse',
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'submerse',
'timestamp': 1480779297,
'upload_date': '20161203',
'release_timestamp': 1481068800,
'release_date': '20161207',
'duration': 154.066,
'track': 'submerse - Safehouse',
'track_number': 3,
'track_id': '1978174799',
'artist': 'submerse',
'album_artist': 'Diskotopia',
'album': 'DSK F/W 2016-2017 Free Compilation',
'uploader_url': 'https://diskotopia.bandcamp.com',
'uploader_id': 'diskotopia',
},
}]
@@ -121,6 +159,9 @@ class BandcampIE(InfoExtractor):
embed = self._extract_data_attr(webpage, title, 'embed', False)
current = tralbum.get('current') or {}
artist = embed.get('artist') or current.get('artist') or tralbum.get('artist')
album_artist = self._html_search_regex(
r'<h3 class="albumTitle">[\S\s]*?by\s*<span>\s*<a href="[^>]+">\s*([^>]+?)\s*</a>',
webpage, 'album artist', fatal=False)
timestamp = unified_timestamp(
current.get('publish_date') or tralbum.get('album_publish_date'))
@@ -205,6 +246,7 @@ class BandcampIE(InfoExtractor):
'track_id': track_id,
'artist': artist,
'album': embed.get('album_title'),
'album_artist': album_artist,
'formats': formats,
}
yt_dlp/extractor/beatbump.py
+101
View File
@@ -0,0 +1,101 @@
from .common import InfoExtractor
from .youtube import YoutubeIE, YoutubeTabIE
class BeatBumpVideoIE(InfoExtractor):
_VALID_URL = r'https://beatbump\.ml/listen\?id=(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs',
'md5': '5ff3fff41d3935b9810a9731e485fe66',
'info_dict': {
'id': 'MgNrAu2pzNs',
'ext': 'mp4',
'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
'artist': 'Stephen',
'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
'upload_date': '20190312',
'categories': ['Music'],
'playable_in_embed': True,
'duration': 169,
'like_count': int,
'alt_title': 'Voyeur Girl',
'view_count': int,
'track': 'Voyeur Girl',
'uploader': 'Stephen - Topic',
'title': 'Voyeur Girl',
'channel_follower_count': int,
'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
'age_limit': 0,
'availability': 'public',
'live_status': 'not_live',
'album': 'it\'s too much love to know my dear',
'channel': 'Stephen',
'comment_count': int,
'description': 'md5:7ae382a65843d6df2685993e90a8628f',
'tags': 'count:11',
'creator': 'Stephen',
'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
}
}]
def _real_extract(self, url):
id_ = self._match_id(url)
return self.url_result(f'https://music.youtube.com/watch?v={id_}', YoutubeIE, id_)
class BeatBumpPlaylistIE(InfoExtractor):
_VALID_URL = r'https://beatbump\.ml/(?:release\?id=|artist/|playlist/)(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE',
'playlist_count': 50,
'info_dict': {
'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
'availability': 'unlisted',
'view_count': int,
'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
'description': '',
'tags': [],
'modified_date': '20221223',
}
}, {
'url': 'https://beatbump.ml/artist/UC_aEa8K-EOJ3D6gOs7HcyNg',
'playlist_mincount': 1,
'params': {'flatplaylist': True},
'info_dict': {
'id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'uploader_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'channel_follower_count': int,
'title': 'NoCopyrightSounds - Videos',
'uploader': 'NoCopyrightSounds',
'description': 'md5:cd4fd53d81d363d05eee6c1b478b491a',
'channel': 'NoCopyrightSounds',
'tags': 'count:12',
'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
},
}, {
'url': 'https://beatbump.ml/playlist/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
'playlist_mincount': 1,
'params': {'flatplaylist': True},
'info_dict': {
'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
'view_count': int,
'channel_url': 'https://www.youtube.com/@NoCopyrightSounds',
'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
'title': 'NCS : All Releases 💿',
'uploader': 'NoCopyrightSounds',
'availability': 'public',
'channel': 'NoCopyrightSounds',
'tags': [],
'modified_date': '20221225',
'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
}
}]
def _real_extract(self, url):
id_ = self._match_id(url)
return self.url_result(f'https://music.youtube.com/browse/{id_}', YoutubeTabIE, id_)
yt_dlp/extractor/bilibili.py
+84 -22
View File
@@ -16,13 +16,16 @@ from ..utils import (
format_field,
int_or_none,
make_archive_id,
merge_dicts,
mimetype2ext,
parse_count,
parse_qs,
qualities,
smuggle_url,
srt_subtitles_timecode,
str_or_none,
traverse_obj,
unsmuggle_url,
url_or_none,
urlencode_postdata,
)
@@ -303,7 +306,8 @@ class BiliBiliIE(BilibiliBaseIE):
getter=lambda entry: f'https://www.bilibili.com/video/{video_id}?p={entry["page"]}')
if is_anthology:
title += f' p{part_id:02d} {traverse_obj(page_list_json, ((part_id or 1) - 1, "part")) or ""}'
part_id = part_id or 1
title += f' p{part_id:02d} {traverse_obj(page_list_json, (part_id - 1, "part")) or ""}'
aid = video_data.get('aid')
old_video_id = format_field(aid, None, f'%s_part{part_id or 1}')
@@ -880,16 +884,12 @@ class BiliIntlBaseIE(InfoExtractor):
return formats
def _extract_video_info(self, video_data, *, ep_id=None, aid=None):
def _parse_video_metadata(self, video_data):
return {
'id': ep_id or aid,
'title': video_data.get('title_display') or video_data.get('title'),
'thumbnail': video_data.get('cover'),
'episode_number': int_or_none(self._search_regex(
r'^E(\d+)(?:$| - )', video_data.get('title_display') or '', 'episode number', default=None)),
'formats': self._get_formats(ep_id=ep_id, aid=aid),
'subtitles': self._get_subtitles(ep_id=ep_id, aid=aid),
'extractor_key': BiliIntlIE.ie_key(),
}
def _perform_login(self, username, password):
@@ -935,6 +935,10 @@ class BiliIntlIE(BiliIntlBaseIE):
'title': 'E2 - The First Night',
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'episode_number': 2,
'upload_date': '20201009',
'episode': 'Episode 2',
'timestamp': 1602259500,
'description': 'md5:297b5a17155eb645e14a14b385ab547e',
}
}, {
# Non-Bstation page
@@ -945,6 +949,10 @@ class BiliIntlIE(BiliIntlBaseIE):
'title': 'E3 - Who?',
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'episode_number': 3,
'description': 'md5:e1a775e71a35c43f141484715470ad09',
'episode': 'Episode 3',
'upload_date': '20211219',
'timestamp': 1639928700,
}
}, {
# Subtitle with empty content
@@ -957,6 +965,17 @@ class BiliIntlIE(BiliIntlBaseIE):
'episode_number': 140,
},
'skip': 'According to the copyright owner\'s request, you may only watch the video after you log in.'
}, {
'url': 'https://www.bilibili.tv/en/video/2041863208',
'info_dict': {
'id': '2041863208',
'ext': 'mp4',
'timestamp': 1670874843,
'description': 'Scheduled for April 2023.\nStudio: ufotable',
'thumbnail': r're:https?://pic[-\.]bstarstatic.+/ugc/.+\.jpg$',
'upload_date': '20221212',
'title': 'Kimetsu no Yaiba Season 3 Official Trailer - Bstation',
}
}, {
'url': 'https://www.biliintl.com/en/play/34613/341736',
'only_matching': True,
@@ -974,42 +993,78 @@ class BiliIntlIE(BiliIntlBaseIE):
'only_matching': True,
}]
def _real_extract(self, url):
season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
video_id = ep_id or aid
def _make_url(video_id, series_id=None):
if series_id:
return f'https://www.bilibili.tv/en/play/{series_id}/{video_id}'
return f'https://www.bilibili.tv/en/video/{video_id}'
def _extract_video_metadata(self, url, video_id, season_id):
url, smuggled_data = unsmuggle_url(url, {})
if smuggled_data.get('title'):
return smuggled_data
webpage = self._download_webpage(url, video_id)
# Bstation layout
initial_data = (
self._search_json(r'window\.__INITIAL_(?:DATA|STATE)__\s*=', webpage, 'preload state', video_id, default={})
or self._search_nuxt_data(webpage, video_id, '__initialState', fatal=False, traverse=None))
video_data = traverse_obj(
initial_data, ('OgvVideo', 'epDetail'), ('UgcVideo', 'videoData'), ('ugc', 'archive'), expected_type=dict)
initial_data, ('OgvVideo', 'epDetail'), ('UgcVideo', 'videoData'), ('ugc', 'archive'), expected_type=dict) or {}
if season_id and not video_data:
# Non-Bstation layout, read through episode list
season_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={season_id}&platform=web', video_id)
video_data = traverse_obj(season_json,
('sections', ..., 'episodes', lambda _, v: str(v['episode_id']) == ep_id),
expected_type=dict, get_all=False)
return self._extract_video_info(video_data or {}, ep_id=ep_id, aid=aid)
video_data = traverse_obj(season_json, (
'sections', ..., 'episodes', lambda _, v: str(v['episode_id']) == video_id
), expected_type=dict, get_all=False)
# XXX: webpage metadata may not accurate, it just used to not crash when video_data not found
return merge_dicts(
self._parse_video_metadata(video_data), self._search_json_ld(webpage, video_id), {
'title': self._html_search_meta('og:title', webpage),
'description': self._html_search_meta('og:description', webpage)
})
def _real_extract(self, url):
season_id, ep_id, aid = self._match_valid_url(url).group('season_id', 'ep_id', 'aid')
video_id = ep_id or aid
return {
'id': video_id,
**self._extract_video_metadata(url, video_id, season_id),
'formats': self._get_formats(ep_id=ep_id, aid=aid),
'subtitles': self.extract_subtitles(ep_id=ep_id, aid=aid),
}
class BiliIntlSeriesIE(BiliIntlBaseIE):
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-zA-Z]{2}/)?play/(?P<id>\d+)/?(?:[?#]|$)'
IE_NAME = 'biliIntl:series'
_VALID_URL = r'https?://(?:www\.)?bili(?:bili\.tv|intl\.com)/(?:[a-zA-Z]{2}/)?(?:play|media)/(?P<id>\d+)/?(?:[?#]|$)'
_TESTS = [{
'url': 'https://www.bilibili.tv/en/play/34613',
'playlist_mincount': 15,
'info_dict': {
'id': '34613',
'title': 'Fly Me to the Moon',
'description': 'md5:a861ee1c4dc0acfad85f557cc42ac627',
'categories': ['Romance', 'Comedy', 'Slice of life'],
'title': 'TONIKAWA: Over the Moon For You',
'description': 'md5:297b5a17155eb645e14a14b385ab547e',
'categories': ['Slice of life', 'Comedy', 'Romance'],
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.png$',
'view_count': int,
},
'params': {
'skip_download': True,
},
}, {
'url': 'https://www.bilibili.tv/en/media/1048837',
'info_dict': {
'id': '1048837',
'title': 'SPY×FAMILY',
'description': 'md5:b4434eb1a9a97ad2bccb779514b89f17',
'categories': ['Adventure', 'Action', 'Comedy'],
'thumbnail': r're:^https://pic\.bstarstatic\.com/ogv/.+\.jpg$',
'view_count': int,
},
'playlist_mincount': 25,
}, {
'url': 'https://www.biliintl.com/en/play/34613',
'only_matching': True,
@@ -1020,9 +1075,12 @@ class BiliIntlSeriesIE(BiliIntlBaseIE):
def _entries(self, series_id):
series_json = self._call_api(f'/web/v2/ogv/play/episodes?season_id={series_id}&platform=web', series_id)
for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict, default=[]):
episode_id = str(episode.get('episode_id'))
yield self._extract_video_info(episode, ep_id=episode_id)
for episode in traverse_obj(series_json, ('sections', ..., 'episodes', ...), expected_type=dict):
episode_id = str(episode['episode_id'])
yield self.url_result(smuggle_url(
BiliIntlIE._make_url(episode_id, series_id),
self._parse_video_metadata(episode)
), BiliIntlIE, episode_id)
def _real_extract(self, url):
series_id = self._match_id(url)
@@ -1034,7 +1092,7 @@ class BiliIntlSeriesIE(BiliIntlBaseIE):
class BiliLiveIE(InfoExtractor):
_VALID_URL = r'https?://live.bilibili.com/(?P<id>\d+)'
_VALID_URL = r'https?://live.bilibili.com/(?:blanc/)?(?P<id>\d+)'
_TESTS = [{
'url': 'https://live.bilibili.com/196',
@@ -1050,6 +1108,9 @@ class BiliLiveIE(InfoExtractor):
}, {
'url': 'https://live.bilibili.com/196?broadcast_type=0&is_room_feed=1?spm_id_from=333.999.space_home.strengthen_live_card.click',
'only_matching': True
}, {
'url': 'https://live.bilibili.com/blanc/196',
'only_matching': True
}]
_FORMATS = {
@@ -1111,6 +1172,7 @@ class BiliLiveIE(InfoExtractor):
'thumbnail': room_data.get('user_cover'),
'timestamp': stream_data.get('live_time'),
'formats': formats,
'is_live': True,
'http_headers': {
'Referer': url,
},
yt_dlp/extractor/cda.py
+40 -7
View File
@@ -4,6 +4,7 @@ import datetime
import hashlib
import hmac
import json
import random
import re
from .common import InfoExtractor
@@ -27,11 +28,10 @@ class CDAIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
_NETRC_MACHINE = 'cdapl'
_BASE_URL = 'http://www.cda.pl/'
_BASE_URL = 'https://www.cda.pl'
_BASE_API_URL = 'https://api.cda.pl'
_API_HEADERS = {
'Accept': 'application/vnd.cda.public+json',
'User-Agent': 'pl.cda 1.0 (version 1.2.88 build 15306; Android 9; Xiaomi Redmi 3S)',
}
# hardcoded in the app
_LOGIN_REQUEST_AUTH = 'Basic YzU3YzBlZDUtYTIzOC00MWQwLWI2NjQtNmZmMWMxY2Y2YzVlOklBTm95QlhRRVR6U09MV1hnV3MwMW0xT2VyNWJNZzV4clRNTXhpNGZJUGVGZ0lWUlo5UGVYTDhtUGZaR1U1U3Q'
@@ -101,6 +101,38 @@ class CDAIE(InfoExtractor):
}, **kwargs)
def _perform_login(self, username, password):
app_version = random.choice((
'1.2.88 build 15306',
'1.2.174 build 18469',
))
android_version = random.randrange(8, 14)
phone_model = random.choice((
# x-kom.pl top selling Android smartphones, as of 2022-12-26
# https://www.x-kom.pl/g-4/c/1590-smartfony-i-telefony.html?f201-system-operacyjny=61322-android
'ASUS ZenFone 8',
'Motorola edge 20 5G',
'Motorola edge 30 neo 5G',
'Motorola moto g22',
'OnePlus Nord 2T 5G',
'Samsung Galaxy A32 SM‑A325F',
'Samsung Galaxy M13',
'Samsung Galaxy S20 FE 5G',
'Xiaomi 11T',
'Xiaomi POCO M4 Pro',
'Xiaomi Redmi 10',
'Xiaomi Redmi 10C',
'Xiaomi Redmi 9C NFC',
'Xiaomi Redmi Note 10 Pro',
'Xiaomi Redmi Note 11 Pro',
'Xiaomi Redmi Note 11',
'Xiaomi Redmi Note 11S 5G',
'Xiaomi Redmi Note 11S',
'realme 10',
'realme 9 Pro+',
'vivo Y33s',
))
self._API_HEADERS['User-Agent'] = f'pl.cda 1.0 (version {app_version}; Android {android_version}; {phone_model})'
cached_bearer = self.cache.load(self._BEARER_CACHE, username) or {}
if cached_bearer.get('valid_until', 0) > datetime.datetime.now().timestamp() + 5:
self._API_HEADERS['Authorization'] = f'Bearer {cached_bearer["token"]}'
@@ -138,9 +170,6 @@ class CDAIE(InfoExtractor):
meta = self._download_json(
f'{self._BASE_API_URL}/video/{video_id}', video_id, headers=self._API_HEADERS)['video']
if meta.get('premium') and not meta.get('premium_free'):
self.report_drm(video_id)
uploader = traverse_obj(meta, 'author', 'login')
formats = [{
@@ -151,6 +180,10 @@ class CDAIE(InfoExtractor):
'filesize': quality.get('length'),
} for quality in meta['qualities'] if quality.get('file')]
if meta.get('premium') and not meta.get('premium_free') and not formats:
raise ExtractorError(
'Video requires CDA Premium - subscription needed', expected=True)
return {
'id': video_id,
'title': meta.get('title'),
@@ -167,10 +200,10 @@ class CDAIE(InfoExtractor):
def _web_extract(self, video_id, url):
self._set_cookie('cda.pl', 'cda.player', 'html5')
webpage = self._download_webpage(
self._BASE_URL + '/video/' + video_id, video_id)
f'{self._BASE_URL}/video/{video_id}/vfilm', video_id)
if 'Ten film jest dostępny dla użytkowników premium' in webpage:
raise ExtractorError('This video is only available for premium users.', expected=True)
self.raise_login_required('This video is only available for premium users')
if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
self.raise_geo_restricted()
yt_dlp/extractor/ciscowebex.py
+25 -5
View File
@@ -1,5 +1,6 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
try_get,
unified_timestamp,
@@ -38,11 +39,30 @@ class CiscoWebexIE(InfoExtractor):
siteurl = mobj.group('siteurl_1') or mobj.group('siteurl_2')
video_id = mobj.group('id')
stream = self._download_json(
password = self.get_param('videopassword')
headers = {'Accept': 'application/json'}
if password:
headers['accessPwd'] = password
stream, urlh = self._download_json_handle(
'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
video_id, fatal=False, query={'siteurl': siteurl})
if not stream:
self.raise_login_required(method='cookies')
video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
if urlh.status == 403:
if stream['code'] == 53004:
self.raise_login_required()
if stream['code'] == 53005:
if password:
raise ExtractorError('Wrong password', expected=True)
raise ExtractorError(
'This video is protected by a password, use the --video-password option', expected=True)
raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
if urlh.status == 429:
self.raise_login_required(
f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
method='cookies')
video_id = stream.get('recordUUID') or video_id
@@ -78,7 +98,7 @@ class CiscoWebexIE(InfoExtractor):
'title': stream['recordName'],
'description': stream.get('description'),
'uploader': stream.get('ownerDisplayName'),
'uploader_id': stream.get('ownerUserName') or stream.get('ownerId'), # mail or id
'uploader_id': stream.get('ownerUserName') or stream.get('ownerId'),
'timestamp': unified_timestamp(stream.get('createTime')),
'duration': int_or_none(stream.get('duration'), 1000),
'webpage_url': 'https://%s.webex.com/recordingservice/sites/%s/recording/playback/%s' % (subdomain, siteurl, video_id),
yt_dlp/extractor/common.py
+96 -41
View File
@@ -32,6 +32,7 @@ from ..utils import (
FormatSorter,
GeoRestrictedError,
GeoUtils,
HEADRequest,
LenientJSONDecoder,
RegexNotFoundError,
RetryManager,
@@ -71,6 +72,7 @@ from ..utils import (
str_to_int,
strip_or_none,
traverse_obj,
truncate_string,
try_call,
try_get,
unescapeHTML,
@@ -79,6 +81,7 @@ from ..utils import (
update_Request,
update_url_query,
url_basename,
urlhandle_detect_ext,
url_or_none,
urljoin,
variadic,
@@ -674,7 +677,8 @@ class InfoExtractor:
for _ in range(2):
try:
self.initialize()
self.write_debug('Extracting URL: %s' % url)
self.to_screen('Extracting URL: %s' % (
url if self.get_param('verbose') else truncate_string(url, 100, 20)))
ie_result = self._real_extract(url)
if ie_result is None:
return None
@@ -692,16 +696,10 @@ class InfoExtractor:
except UnsupportedError:
raise
except ExtractorError as e:
kwargs = {
'video_id': e.video_id or self.get_temp_id(url),
'ie': self.IE_NAME,
'tb': e.traceback or sys.exc_info()[2],
'expected': e.expected,
'cause': e.cause
}
if hasattr(e, 'countries'):
kwargs['countries'] = e.countries
raise type(e)(e.orig_msg, **kwargs)
e.video_id = e.video_id or self.get_temp_id(url),
e.ie = e.ie or self.IE_NAME,
e.traceback = e.traceback or sys.exc_info()[2]
raise
except http.client.IncompleteRead as e:
raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
except (KeyError, StopIteration) as e:
@@ -1266,10 +1264,9 @@ class InfoExtractor:
Like _search_regex, but strips HTML tags and unescapes entities.
"""
res = self._search_regex(pattern, string, name, default, fatal, flags, group)
if res:
return clean_html(res).strip()
else:
return res
if isinstance(res, tuple):
return tuple(map(clean_html, res))
return clean_html(res)
def _get_netrc_login_info(self, netrc_machine=None):
username = None
@@ -1400,10 +1397,16 @@ class InfoExtractor:
# And then there are the jokers who advertise that they use RTA, but actually don't.
AGE_LIMIT_MARKERS = [
r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
r'>[^<]*you acknowledge you are at least (\d+) years old',
r'>\s*(?:18\s+U(?:\.S\.C\.|SC)\s+)?(?:§+\s*)?2257\b',
]
if any(re.search(marker, html) for marker in AGE_LIMIT_MARKERS):
return 18
return 0
age_limit = 0
for marker in AGE_LIMIT_MARKERS:
mobj = re.search(marker, html)
if mobj:
age_limit = max(age_limit, int(traverse_obj(mobj, 1, default=18)))
return age_limit
def _media_rating_search(self, html):
# See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
@@ -1763,6 +1766,9 @@ class InfoExtractor:
def _extract_f4m_formats(self, manifest_url, video_id, preference=None, quality=None, f4m_id=None,
transform_source=lambda s: fix_xml_ampersands(s).strip(),
fatal=True, m3u8_id=None, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
manifest_url, video_id, 'Downloading f4m manifest',
'Unable to download f4m manifest',
@@ -1912,6 +1918,17 @@ class InfoExtractor:
errnote=None, fatal=True, live=False, data=None, headers={},
query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
if not m3u8_url:
if errnote is not False:
errnote = errnote or 'Failed to obtain m3u8 URL'
if fatal:
raise ExtractorError(errnote, video_id=video_id)
self.report_warning(f'{errnote}{bug_reports_message()}')
return [], {}
res = self._download_webpage_handle(
m3u8_url, video_id,
note='Downloading m3u8 information' if note is None else note,
@@ -2163,13 +2180,23 @@ class InfoExtractor:
return self._parse_m3u8_vod_duration(m3u8_vod or '', video_id)
def _parse_m3u8_vod_duration(self, m3u8_vod, video_id):
if '#EXT-X-PLAYLIST-TYPE:VOD' not in m3u8_vod:
if '#EXT-X-ENDLIST' not in m3u8_vod:
return None
return int(sum(
float(line[len('#EXTINF:'):].split(',')[0])
for line in m3u8_vod.splitlines() if line.startswith('#EXTINF:'))) or None
def _extract_mpd_vod_duration(
self, mpd_url, video_id, note=None, errnote=None, data=None, headers={}, query={}):
mpd_doc = self._download_xml(
mpd_url, video_id,
note='Downloading MPD VOD manifest' if note is None else note,
errnote='Failed to download VOD manifest' if errnote is None else errnote,
fatal=False, data=data, headers=headers, query=query) or {}
return int_or_none(parse_duration(mpd_doc.get('mediaPresentationDuration')))
@staticmethod
def _xpath_ns(path, namespace=None):
if not namespace:
@@ -2183,6 +2210,9 @@ class InfoExtractor:
return '/'.join(out)
def _extract_smil_formats_and_subtitles(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
if res is False:
assert not fatal
@@ -2293,7 +2323,8 @@ class InfoExtractor:
height = int_or_none(medium.get('height'))
proto = medium.get('proto')
ext = medium.get('ext')
src_ext = determine_ext(src)
src_ext = determine_ext(src, default_ext=None) or ext or urlhandle_detect_ext(
self._request_webpage(HEADRequest(src), video_id, note='Requesting extension info', fatal=False))
streamer = medium.get('streamer') or base
if proto == 'rtmp' or streamer.startswith('rtmp'):
@@ -2458,6 +2489,10 @@ class InfoExtractor:
def _extract_mpd_formats_and_subtitles(
self, mpd_url, video_id, mpd_id=None, note=None, errnote=None,
fatal=True, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
mpd_url, video_id,
note='Downloading MPD manifest' if note is None else note,
@@ -2827,6 +2862,9 @@ class InfoExtractor:
return fmts
def _extract_ism_formats_and_subtitles(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
if self.get_param('ignore_no_formats_error'):
fatal = False
res = self._download_xml_handle(
ism_url, video_id,
note='Downloading ISM manifest' if note is None else note,
@@ -3196,7 +3234,7 @@ class InfoExtractor:
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
mobj = re.search(
r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
webpage)
if mobj:
try:
@@ -3217,19 +3255,20 @@ class InfoExtractor:
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
# JWPlayer backward compatibility: flattened playlists
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
if 'playlist' not in jwplayer_data:
jwplayer_data = {'playlist': [jwplayer_data]}
entries = []
if not isinstance(jwplayer_data, dict):
return entries
# JWPlayer backward compatibility: single playlist item
playlist_items = jwplayer_data.get('playlist')
# JWPlayer backward compatibility: single playlist item/flattened playlists
# https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
if not isinstance(jwplayer_data['playlist'], list):
jwplayer_data['playlist'] = [jwplayer_data['playlist']]
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
if not isinstance(playlist_items, list):
playlist_items = (playlist_items or jwplayer_data, )
for video_data in jwplayer_data['playlist']:
for video_data in playlist_items:
if not isinstance(video_data, dict):
continue
# JWPlayer backward compatibility: flattened sources
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
if 'sources' not in video_data:
@@ -3267,6 +3306,13 @@ class InfoExtractor:
'timestamp': int_or_none(video_data.get('pubdate')),
'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
'subtitles': subtitles,
'alt_title': clean_html(video_data.get('subtitle')), # attributes used e.g. by Tele5 ...
'genre': clean_html(video_data.get('genre')),
'channel': clean_html(dict_get(video_data, ('category', 'channel'))),
'season_number': int_or_none(video_data.get('season')),
'episode_number': int_or_none(video_data.get('episode')),
'release_year': int_or_none(video_data.get('releasedate')),
'age_limit': int_or_none(video_data.get('age_restriction')),
}
# https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
@@ -3284,7 +3330,7 @@ class InfoExtractor:
def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
urls = []
urls = set()
formats = []
for source in jwplayer_sources_data:
if not isinstance(source, dict):
@@ -3293,14 +3339,14 @@ class InfoExtractor:
base_url, self._proto_relative_url(source.get('file')))
if not source_url or source_url in urls:
continue
urls.append(source_url)
urls.add(source_url)
source_type = source.get('type') or ''
ext = mimetype2ext(source_type) or determine_ext(source_url)
if source_type == 'hls' or ext == 'm3u8':
if source_type == 'hls' or ext == 'm3u8' or 'format=m3u8-aapl' in source_url:
formats.extend(self._extract_m3u8_formats(
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
m3u8_id=m3u8_id, fatal=False))
elif source_type == 'dash' or ext == 'mpd':
elif source_type == 'dash' or ext == 'mpd' or 'format=mpd-time-csf' in source_url:
formats.extend(self._extract_mpd_formats(
source_url, video_id, mpd_id=mpd_id, fatal=False))
elif ext == 'smil':
@@ -3315,13 +3361,12 @@ class InfoExtractor:
'ext': ext,
})
else:
format_id = str_or_none(source.get('label'))
height = int_or_none(source.get('height'))
if height is None:
if height is None and format_id:
# Often no height is provided but there is a label in
# format like "1080p", "720p SD", or 1080.
height = int_or_none(self._search_regex(
r'^(\d{3,4})[pP]?(?:\b|$)', str(source.get('label') or ''),
'height', default=None))
height = parse_resolution(format_id).get('height')
a_format = {
'url': source_url,
'width': int_or_none(source.get('width')),
@@ -3329,6 +3374,7 @@ class InfoExtractor:
'tbr': int_or_none(source.get('bitrate'), scale=1000),
'filesize': int_or_none(source.get('filesize')),
'ext': ext,
'format_id': format_id
}
if source_url.startswith('rtmp'):
a_format['ext'] = 'flv'
@@ -3422,13 +3468,17 @@ class InfoExtractor:
continue
t['name'] = cls.ie_key()
yield t
if getattr(cls, '__wrapped__', None):
yield from cls.__wrapped__.get_testcases(include_onlymatching)
@classmethod
def get_webpage_testcases(cls):
tests = vars(cls).get('_WEBPAGE_TESTS', [])
for t in tests:
t['name'] = cls.ie_key()
return tests
yield t
if getattr(cls, '__wrapped__', None):
yield from cls.__wrapped__.get_webpage_testcases()
@classproperty(cache=True)
def age_limit(cls):
@@ -3474,7 +3524,7 @@ class InfoExtractor:
elif cls.IE_DESC:
desc += f' {cls.IE_DESC}'
if cls.SEARCH_KEY:
desc += f'; "{cls.SEARCH_KEY}:" prefix'
desc += f'{";" if cls.IE_DESC else ""} "{cls.SEARCH_KEY}:" prefix'
if search_examples:
_COUNTS = ('', '5', '10', 'all')
desc += f' (e.g. "{cls.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(search_examples)}")'
@@ -3690,10 +3740,12 @@ class InfoExtractor:
if plugin_name:
mro = inspect.getmro(cls)
super_class = cls.__wrapped__ = mro[mro.index(cls) + 1]
cls.IE_NAME, cls.ie_key = f'{super_class.IE_NAME}+{plugin_name}', super_class.ie_key
cls.PLUGIN_NAME, cls.ie_key = plugin_name, super_class.ie_key
cls.IE_NAME = f'{super_class.IE_NAME}+{plugin_name}'
while getattr(super_class, '__wrapped__', None):
super_class = super_class.__wrapped__
setattr(sys.modules[super_class.__module__], super_class.__name__, cls)
_PLUGIN_OVERRIDES[super_class].append(cls)
return super().__init_subclass__(**kwargs)
@@ -3750,3 +3802,6 @@ class UnsupportedURLIE(InfoExtractor):
def _real_extract(self, url):
raise UnsupportedError(url)
_PLUGIN_OVERRIDES = collections.defaultdict(list)
yt_dlp/extractor/crunchyroll.py
+3 -2
View File
@@ -182,7 +182,7 @@ class CrunchyrollBetaIE(CrunchyrollBaseIE):
self.to_screen(
'To get all formats of a hardsub language, use '
'"--extractor-args crunchyrollbeta:hardsub=<language_code or all>". '
'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta for more info',
'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info',
only_once=True)
else:
full_format_langs = set(map(str.lower, available_formats))
@@ -291,7 +291,8 @@ class CrunchyrollBetaShowIE(CrunchyrollBaseIE):
'season_id': episode.get('season_id'),
'season_number': episode.get('season_number'),
'episode': episode.get('title'),
'episode_number': episode.get('sequence_number')
'episode_number': episode.get('sequence_number'),
'language': episode.get('audio_locale'),
}
return self.playlist_result(entries(), internal_id, series_response.get('title'))
yt_dlp/extractor/curiositystream.py
+6 -2
View File
@@ -1,4 +1,5 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import compat_str
@@ -23,7 +24,7 @@ class CuriosityStreamBaseIE(InfoExtractor):
auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
if auth_cookie:
self.write_debug('Obtained auth_token cookie')
self._auth_token = auth_cookie.value
self._auth_token = urllib.parse.unquote(auth_cookie.value)
if self._auth_token:
headers['X-Auth-Token'] = self._auth_token
result = self._download_json(
@@ -54,8 +55,11 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
'channel': 'Curiosity Stream',
'categories': ['Technology', 'Interview'],
'average_rating': 96.79,
'average_rating': float,
'series_id': '2',
'thumbnail': r're:https://img.curiositystream.com/.+\.jpg',
'tags': [],
'duration': 158
},
'params': {
# m3u8 download
yt_dlp/extractor/discovery.py
+1 -1
View File
@@ -78,7 +78,7 @@ class DiscoveryIE(DiscoveryGoBaseIE):
'Downloading token JSON metadata', query={
'authRel': 'authorization',
'client_id': '3020a40c2356a645b4b4',
'nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
'nonce': ''.join(random.choices(string.ascii_letters, k=32)),
'redirectUri': 'https://www.discovery.com/',
})['access_token']
yt_dlp/extractor/drtv.py
+103 -7
View File
@@ -2,22 +2,24 @@ import binascii
import hashlib
import re
from .common import InfoExtractor
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
from ..compat import compat_urllib_parse_unquote
from ..utils import (
ExtractorError,
int_or_none,
float_or_none,
int_or_none,
mimetype2ext,
str_or_none,
traverse_obj,
try_get,
unified_timestamp,
update_url_query,
url_or_none,
)
SERIES_API = 'https://production-cdn.dr-massive.com/api/page?device=web_browser&item_detail_expand=all&lang=da&max_list_prefetch=3&path=%s'
class DRTVIE(InfoExtractor):
_VALID_URL = r'''(?x)
@@ -141,13 +143,13 @@ class DRTVIE(InfoExtractor):
}]
def _real_extract(self, url):
video_id = self._match_id(url)
raw_video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
webpage = self._download_webpage(url, raw_video_id)
if '>Programmet er ikke længere tilgængeligt' in webpage:
raise ExtractorError(
'Video %s is not available' % video_id, expected=True)
'Video %s is not available' % raw_video_id, expected=True)
video_id = self._search_regex(
(r'data-(?:material-identifier|episode-slug)="([^"]+)"',
@@ -182,6 +184,11 @@ class DRTVIE(InfoExtractor):
data = self._download_json(
programcard_url, video_id, 'Downloading video JSON', query=query)
supplementary_data = {}
if re.search(r'_\d+$', raw_video_id):
supplementary_data = self._download_json(
SERIES_API % f'/episode/{raw_video_id}', raw_video_id, fatal=False) or {}
title = str_or_none(data.get('Title')) or re.sub(
r'\s*\|\s*(?:TV\s*\|\s*DR|DRTV)$', '',
self._og_search_title(webpage))
@@ -313,8 +320,8 @@ class DRTVIE(InfoExtractor):
'season': str_or_none(data.get('SeasonTitle')),
'season_number': int_or_none(data.get('SeasonNumber')),
'season_id': str_or_none(data.get('SeasonUrn')),
'episode': str_or_none(data.get('EpisodeTitle')),
'episode_number': int_or_none(data.get('EpisodeNumber')),
'episode': traverse_obj(supplementary_data, ('entries', 0, 'item', 'contextualTitle')) or str_or_none(data.get('EpisodeTitle')),
'episode_number': traverse_obj(supplementary_data, ('entries', 0, 'item', 'episodeNumber')) or int_or_none(data.get('EpisodeNumber')),
'release_year': int_or_none(data.get('ProductionYear')),
}
@@ -372,3 +379,92 @@ class DRTVLiveIE(InfoExtractor):
'formats': formats,
'is_live': True,
}
class DRTVSeasonIE(InfoExtractor):
IE_NAME = 'drtv:season'
_VALID_URL = r'https?://(?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/saeson/(?P<display_id>[\w-]+)_(?P<id>\d+)'
_GEO_COUNTRIES = ['DK']
_TESTS = [{
'url': 'https://www.dr.dk/drtv/saeson/frank-and-kastaniegaarden_9008',
'info_dict': {
'id': '9008',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 8
}, {
'url': 'https://www.dr.dk/drtv/saeson/frank-and-kastaniegaarden_8761',
'info_dict': {
'id': '8761',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 19
}]
def _real_extract(self, url):
display_id, season_id = self._match_valid_url(url).group('display_id', 'id')
data = self._download_json(SERIES_API % f'/saeson/{display_id}_{season_id}', display_id)
entries = [{
'_type': 'url',
'url': f'https://www.dr.dk/drtv{episode["path"]}',
'ie_key': DRTVIE.ie_key(),
'title': episode.get('title'),
'episode': episode.get('episodeName'),
'description': episode.get('shortDescription'),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber')),
'episode_number': episode.get('episodeNumber'),
} for episode in traverse_obj(data, ('entries', 0, 'item', 'episodes', 'items'))]
return {
'_type': 'playlist',
'id': season_id,
'display_id': display_id,
'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'entries': entries,
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
}
class DRTVSeriesIE(InfoExtractor):
IE_NAME = 'drtv:series'
_VALID_URL = r'https?://(?:www\.)?(?:dr\.dk|dr-massive\.com)/drtv/serie/(?P<display_id>[\w-]+)_(?P<id>\d+)'
_GEO_COUNTRIES = ['DK']
_TESTS = [{
'url': 'https://www.dr.dk/drtv/serie/frank-and-kastaniegaarden_6954',
'info_dict': {
'id': '6954',
'display_id': 'frank-and-kastaniegaarden',
'title': 'Frank & Kastaniegaarden',
'series': 'Frank & Kastaniegaarden',
},
'playlist_mincount': 15
}]
def _real_extract(self, url):
display_id, series_id = self._match_valid_url(url).group('display_id', 'id')
data = self._download_json(SERIES_API % f'/serie/{display_id}_{series_id}', display_id)
entries = [{
'_type': 'url',
'url': f'https://www.dr.dk/drtv{season.get("path")}',
'ie_key': DRTVSeasonIE.ie_key(),
'title': season.get('title'),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'season_number': traverse_obj(data, ('entries', 0, 'item', 'seasonNumber'))
} for season in traverse_obj(data, ('entries', 0, 'item', 'show', 'seasons', 'items'))]
return {
'_type': 'playlist',
'id': series_id,
'display_id': display_id,
'title': traverse_obj(data, ('entries', 0, 'item', 'title')),
'series': traverse_obj(data, ('entries', 0, 'item', 'title')),
'entries': entries
}
yt_dlp/extractor/embedly.py
+63 -7
View File
@@ -1,24 +1,80 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from .youtube import YoutubeTabIE
from ..utils import parse_qs, smuggle_url, traverse_obj
class EmbedlyIE(InfoExtractor):
_VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
_VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?(?:src|url)=(?:[^#&]+)'
_TESTS = [{
'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
'info_dict': {
'id': 'UUGLim4T2loE5rwCMdpCIPVg',
'modified_date': '20221225',
'view_count': int,
'uploader_url': 'https://www.youtube.com/@TraciHinesMusic',
'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'uploader': 'TraciJHines',
'channel_url': 'https://www.youtube.com/@TraciHinesMusic',
'channel': 'TraciJHines',
'availability': 'public',
'uploader_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'description': '',
'tags': [],
'title': 'Uploads from TraciJHines',
},
'playlist_mincount': 10,
}, {
'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
'params': {'noplaylist': True},
'info_dict': {
'id': 'SU4fj_aEMVw',
'ext': 'mp4',
'title': 'I\'m on Patreon!',
'age_limit': 0,
'categories': ['Entertainment'],
'thumbnail': 'https://i.ytimg.com/vi_webp/SU4fj_aEMVw/maxresdefault.webp',
'live_status': 'not_live',
'playable_in_embed': True,
'channel': 'TraciJHines',
'uploader_id': 'TraciJHines',
'channel_url': 'https://www.youtube.com/channel/UCGLim4T2loE5rwCMdpCIPVg',
'uploader_url': 'http://www.youtube.com/user/TraciJHines',
'upload_date': '20150211',
'duration': 282,
'availability': 'public',
'channel_follower_count': int,
'tags': 'count:39',
'view_count': int,
'comment_count': int,
'channel_id': 'UCGLim4T2loE5rwCMdpCIPVg',
'like_count': int,
'uploader': 'TraciJHines',
'description': 'md5:8af6425f50bd46fbf29f3db0fc3a8364',
'chapters': list,
},
}, {
'url': 'https://cdn.embedly.com/widgets/media.html?src=https://player.vimeo.com/video/1234567?h=abcdefgh',
'only_matching': True,
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
# Bypass suitable check
def _extract_from_webpage(cls, url, webpage):
# Bypass "ie=cls" and suitable check
for mobj in re.finditer(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage):
yield mobj.group('url')
yield cls.url_result(mobj.group('url'))
for mobj in re.finditer(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage):
yield urllib.parse.unquote(mobj.group('url'))
yield cls.url_result(urllib.parse.unquote(mobj.group('url')))
def _real_extract(self, url):
return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))
qs = parse_qs(url)
src = urllib.parse.unquote(traverse_obj(qs, ('url', 0)) or '')
if src and YoutubeTabIE.suitable(src):
return self.url_result(src, YoutubeTabIE)
return self.url_result(smuggle_url(
urllib.parse.unquote(traverse_obj(qs, ('src', 0), ('url', 0))),
{'http_headers': {'Referer': url}}))
yt_dlp/extractor/europa.py
+84
View File
@@ -3,6 +3,7 @@ from ..utils import (
int_or_none,
orderedSet,
parse_duration,
parse_iso8601,
parse_qs,
qualities,
unified_strdate,
@@ -87,3 +88,86 @@ class EuropaIE(InfoExtractor):
'view_count': view_count,
'formats': formats
}
class EuroParlWebstreamIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://(?:multimedia|webstreaming)\.europarl\.europa\.eu/[^/#?]+/
(?:embed/embed\.html\?event=|(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
'''
_TESTS = [{
'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
'info_dict': {
'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
'ext': 'mp4',
'release_timestamp': 1663137900,
'title': 'Plenary session',
'release_date': '20220914',
},
'params': {
'skip_download': True,
}
}, {
'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/eu-cop27-un-climate-change-conference-in-sharm-el-sheikh-egypt-ep-delegation-meets-with-ngo-represen_20221114-1600-SPECIAL-OTHER',
'info_dict': {
'id': 'a8428de8-b9cd-6a2e-11e4-3805d9c9ff5c',
'ext': 'mp4',
'release_timestamp': 1668434400,
'release_date': '20221114',
'title': 'md5:d3550280c33cc70e0678652e3d52c028',
},
'params': {
'skip_download': True,
}
}, {
# embed webpage
'url': 'https://webstreaming.europarl.europa.eu/ep/embed/embed.html?event=20220914-0900-PLENARY&language=en&autoplay=true&logo=true',
'info_dict': {
'id': 'bcaa1db4-76ef-7e06-8da7-839bd0ad1dbe',
'ext': 'mp4',
'title': 'Plenary session',
'release_date': '20220914',
'release_timestamp': 1663137900,
},
'params': {
'skip_download': True,
}
}, {
# live webstream
'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/euroscola_20221115-1000-SPECIAL-EUROSCOLA',
'info_dict': {
'ext': 'mp4',
'id': '510eda7f-ba72-161b-7ee7-0e836cd2e715',
'release_timestamp': 1668502800,
'title': 'Euroscola 2022-11-15 19:21',
'release_date': '20221115',
'live_status': 'is_live',
},
'skip': 'not live anymore'
}]
def _real_extract(self, url):
display_id = self._match_id(url)
json_info = self._download_json(
'https://vis-api.vuplay.co.uk/event/external', display_id,
query={
'player_key': 'europarl|718f822c-a48c-4841-9947-c9cb9bb1743c',
'external_id': display_id,
})
formats, subtitles = self._extract_mpd_formats_and_subtitles(json_info['streaming_url'], display_id)
fmts, subs = self._extract_m3u8_formats_and_subtitles(
json_info['streaming_url'].replace('.mpd', '.m3u8'), display_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
return {
'id': json_info['id'],
'title': json_info.get('title'),
'formats': formats,
'subtitles': subtitles,
'release_timestamp': parse_iso8601(json_info.get('published_start')),
'is_live': 'LIVE' in json_info.get('state', '')
}
yt_dlp/extractor/extractors.py
+4 -2
View File
@@ -1,10 +1,10 @@
import contextlib
import os
from ..utils import load_plugins
from ..plugins import load_plugins
# NB: Must be before other imports so that plugins can be correctly injected
_PLUGIN_CLASSES = load_plugins('extractor', 'IE', {})
_PLUGIN_CLASSES = load_plugins('extractor', 'IE')
_LAZY_LOADER = False
if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
@@ -24,3 +24,5 @@ if not _LAZY_LOADER:
globals().update(_PLUGIN_CLASSES)
_ALL_CLASSES[:0] = _PLUGIN_CLASSES.values()
from .common import _PLUGIN_OVERRIDES # noqa: F401
yt_dlp/extractor/fifa.py
+6 -17
View File
@@ -17,8 +17,10 @@ class FifaIE(InfoExtractor):
'description': 'md5:f4520d0ee80529c8ba4134a7d692ff8b',
'ext': 'mp4',
'categories': ['FIFA Tournaments'],
'thumbnail': 'https://digitalhub.fifa.com/transform/fa6f0b3e-a2e9-4cf7-9f32-53c57bcb7360/2006_Final_ITA_FRA',
'thumbnail': 'https://digitalhub.fifa.com/transform/135e2656-3a51-407b-8810-6c34bec5b59b/FMR_2006_Italy_France_Final_Hero',
'duration': 8165,
'release_timestamp': 1152403200,
'release_date': '20060709',
},
'params': {'skip_download': 'm3u8'},
}, {
@@ -54,7 +56,7 @@ class FifaIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
preconnect_link = self._search_regex(
r'<link[^>]+rel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
r'<link\b[^>]+\brel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
video_details = self._download_json(
f'{preconnect_link}/sections/videoDetails/{video_id}', video_id, 'Downloading Video Details', fatal=False)
@@ -62,22 +64,9 @@ class FifaIE(InfoExtractor):
preplay_parameters = self._download_json(
f'{preconnect_link}/videoPlayerData/{video_id}', video_id, 'Downloading Preplay Parameters')['preplayParameters']
cid = preplay_parameters['contentId']
content_data = self._download_json(
f'https://content.uplynk.com/preplay/{cid}/multiple.json', video_id, 'Downloading Content Data', query={
'v': preplay_parameters['preplayAPIVersion'],
'tc': preplay_parameters['tokenCheckAlgorithmVersion'],
'rn': preplay_parameters['randomNumber'],
'exp': preplay_parameters['tokenExpirationDate'],
'ct': preplay_parameters['contentType'],
'cid': cid,
'mbtracks': preplay_parameters['tracksAssetNumber'],
'ad': preplay_parameters['adConfiguration'],
'ad.preroll': int(preplay_parameters['adPreroll']),
'ad.cmsid': preplay_parameters['adCMSSourceId'],
'ad.vid': preplay_parameters['adSourceVideoID'],
'sig': preplay_parameters['signature'],
})
'https://content.uplynk.com/preplay/{contentId}/multiple.json?{queryStr}&sig={signature}'.format(**preplay_parameters),
video_id, 'Downloading Content Data')
formats, subtitles = self._extract_m3u8_formats_and_subtitles(content_data['playURL'], video_id)
yt_dlp/extractor/foxsports.py
+38 -18
View File
@@ -1,31 +1,51 @@
from .common import InfoExtractor
from .uplynk import UplynkPreplayIE
from ..utils import HEADRequest, float_or_none, make_archive_id, smuggle_url
class FoxSportsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*video/(?P<id>\d+)'
_TEST = {
'url': 'http://www.foxsports.com/tennessee/video/432609859715',
'md5': 'b49050e955bebe32c301972e4012ac17',
_VALID_URL = r'https?://(?:www\.)?foxsports\.com/watch/(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://www.foxsports.com/watch/play-612168c6700004b',
'info_dict': {
'id': '432609859715',
'id': 'b72f5bd8658140baa5791bb676433733',
'ext': 'mp4',
'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
'description': 'Courtney Lee talks about Memphis being focused.',
# TODO: fix timestamp
'upload_date': '19700101', # '20150423',
# 'timestamp': 1429761109,
'uploader': 'NEWA-FNG-FOXSPORTS',
'display_id': 'play-612168c6700004b',
'title': 'md5:e0c4ecac3a1f25295b4fae22fb5c126a',
'description': 'md5:371bc43609708ae2b9e1a939229762af',
'uploader_id': '06b4a36349624051a9ba52ac3a91d268',
'upload_date': '20221205',
'timestamp': 1670262586,
'duration': 31.7317,
'thumbnail': r're:^https?://.*\.jpg$',
'extra_param_to_segment_url': str,
},
'params': {
# m3u8 download
'skip_download': True,
'skip_download': 'm3u8',
},
'add_ie': ['ThePlatform'],
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
json_ld = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={})
data = self._download_json(
f'https://api3.fox.com/v2.0/vodplayer/sportsclip/{video_id}',
video_id, note='Downloading API JSON', headers={
'x-api-key': 'cf289e299efdfa39fb6316f259d1de93',
})
preplay_url = self._request_webpage(
HEADRequest(data['url']), video_id, 'Fetching preplay URL').geturl()
return self.url_result(
'https://feed.theplatform.com/f/BKQ29B/foxsports-all?byId=' + video_id, 'ThePlatformFeed')
return {
'_type': 'url_transparent',
'ie_key': UplynkPreplayIE.ie_key(),
'url': smuggle_url(preplay_url, {'Origin': 'https://www.foxsports.com'}),
'display_id': video_id,
'title': data.get('name') or json_ld.get('title'),
'description': data.get('description') or json_ld.get('description'),
'duration': float_or_none(data.get('durationInSeconds')),
'timestamp': json_ld.get('timestamp'),
'thumbnails': json_ld.get('thumbnails'),
'_old_archive_ids': [make_archive_id(self, video_id)],
}
yt_dlp/extractor/freesound.py
+1
View File
@@ -52,6 +52,7 @@ class FreesoundIE(InfoExtractor):
tags_str = get_element_by_class('tags', webpage)
tags = re.findall(r'<a[^>]+>([^<]+)', tags_str) if tags_str else None
audio_url = re.sub(r'^https?://freesound\.org(https?://)', r'\1', audio_url)
audio_urls = [audio_url]
LQ_FORMAT = '-lq.mp3'
yt_dlp/extractor/funimation.py
+1 -1
View File
@@ -210,7 +210,7 @@ class FunimationIE(FunimationBaseIE):
page = self._download_json(
'https://www.funimation.com/api/showexperience/%s/' % experience_id,
display_id, headers=headers, expected_status=403, query={
'pinst_id': ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(8)]),
'pinst_id': ''.join(random.choices(string.digits + string.ascii_letters, k=8)),
}, note=f'Downloading {format_name} JSON')
sources = page.get('items') or []
if not sources:
yt_dlp/extractor/generic.py
+163 -143
View File
@@ -32,6 +32,7 @@ from ..utils import (
unified_timestamp,
unsmuggle_url,
url_or_none,
urljoin,
variadic,
xpath_attr,
xpath_text,
@@ -886,20 +887,6 @@ class GenericIE(InfoExtractor):
'thumbnail': r're:^https?://.*\.jpg$',
},
},
{
# JWPlayer config passed as variable
'url': 'http://www.txxx.com/videos/3326530/ariele/',
'info_dict': {
'id': '3326530_hq',
'ext': 'mp4',
'title': 'ARIELE | Tube Cup',
'uploader': 'www.txxx.com',
'age_limit': 18,
},
'params': {
'skip_download': True,
}
},
{
# Video.js embed, multiple formats
'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
@@ -1570,19 +1557,6 @@ class GenericIE(InfoExtractor):
},
'add_ie': ['WashingtonPost'],
},
{
# Mediaset embed
'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
'info_dict': {
'id': '720642',
'ext': 'mp4',
'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
},
'params': {
'skip_download': True,
},
'add_ie': ['Mediaset'],
},
{
# JOJ.sk embeds
'url': 'https://www.noviny.sk/slovensko/238543-slovenskom-sa-prehnala-vlna-silnych-burok',
@@ -1887,11 +1861,6 @@ class GenericIE(InfoExtractor):
'title': 'I AM BIO Podcast | BIO',
},
'playlist_mincount': 52,
},
{
# Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed)
'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
'only_matching': True,
}, {
# WimTv embed player
'url': 'http://www.msmotor.tv/wearefmi-pt-2-2021/',
@@ -1908,11 +1877,13 @@ class GenericIE(InfoExtractor):
'display_id': 'kelis-4th-of-july',
'ext': 'mp4',
'title': 'Kelis - 4th Of July',
'thumbnail': 'https://kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
'description': 'Kelis - 4th Of July',
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
},
'params': {
'skip_download': True,
},
'expected_warnings': ['Untested major version'],
}, {
# KVS Player
'url': 'https://www.kvs-demo.com/embed/105/',
@@ -1921,35 +1892,12 @@ class GenericIE(InfoExtractor):
'display_id': 'kelis-4th-of-july',
'ext': 'mp4',
'title': 'Kelis - 4th Of July / Embed Player',
'thumbnail': 'https://kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
},
'params': {
'skip_download': True,
},
}, {
# KVS Player
'url': 'https://thisvid.com/videos/french-boy-pantsed/',
'md5': '3397979512c682f6b85b3b04989df224',
'info_dict': {
'id': '2400174',
'display_id': 'french-boy-pantsed',
'ext': 'mp4',
'title': 'French Boy Pantsed - ThisVid.com',
'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
}
}, {
# KVS Player
'url': 'https://thisvid.com/embed/2400174/',
'md5': '3397979512c682f6b85b3b04989df224',
'info_dict': {
'id': '2400174',
'display_id': 'french-boy-pantsed',
'ext': 'mp4',
'title': 'French Boy Pantsed - ThisVid.com',
'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/2400000/2400174/preview.mp4.jpg',
}
}, {
# KVS Player
'url': 'https://youix.com/video/leningrad-zoj/',
'md5': '94f96ba95706dc3880812b27b7d8a2b8',
'info_dict': {
@@ -1957,8 +1905,8 @@ class GenericIE(InfoExtractor):
'display_id': 'leningrad-zoj',
'ext': 'mp4',
'title': 'Клип: Ленинград - ЗОЖ скачать, смотреть онлайн | Youix.com',
'thumbnail': 'https://youix.com/contents/videos_screenshots/18000/18485/preview_480x320_youix_com.mp4.jpg',
}
'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
},
}, {
# KVS Player
'url': 'https://youix.com/embed/18485',
@@ -1968,19 +1916,20 @@ class GenericIE(InfoExtractor):
'display_id': 'leningrad-zoj',
'ext': 'mp4',
'title': 'Ленинград - ЗОЖ',
'thumbnail': 'https://youix.com/contents/videos_screenshots/18000/18485/preview_480x320_youix_com.mp4.jpg',
}
'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
},
}, {
# KVS Player
'url': 'https://bogmedia.org/videos/21217/40-nochey-40-nights-2016/',
'md5': '94166bdb26b4cb1fb9214319a629fc51',
'info_dict': {
'id': '21217',
'display_id': '40-nochey-40-nights-2016',
'display_id': '40-nochey-2016',
'ext': 'mp4',
'title': '40 ночей (2016) - BogMedia.org',
'description': 'md5:4e6d7d622636eb7948275432eb256dc3',
'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
}
},
},
{
# KVS Player (for sites that serve kt_player.js via non-https urls)
@@ -1990,9 +1939,9 @@ class GenericIE(InfoExtractor):
'id': '389508',
'display_id': 'syren-de-mer-onlyfans-05-07-2020have-a-happy-safe-holiday5f014e68a220979bdb8cd-source',
'ext': 'mp4',
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
'thumbnail': 'http://www.camhub.world/contents/videos_screenshots/389000/389508/preview.mp4.jpg',
}
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
'thumbnail': r're:https?://www\.camhub\.world/contents/videos_screenshots/389000/389508/preview\.mp4\.jpg',
},
},
{
# Reddit-hosted video that will redirect and be processed by RedditIE
@@ -2195,7 +2144,52 @@ class GenericIE(InfoExtractor):
'age_limit': 0,
'direct': True,
}
}
},
{
'note': 'server returns data in brotli compression by default if `accept-encoding: *` is specified.',
'url': 'https://www.extra.cz/cauky-lidi-70-dil-babis-predstavil-pohadky-prymulanek-nebo-andrejovy-nove-saty-ac867',
'info_dict': {
'id': 'cauky-lidi-70-dil-babis-predstavil-pohadky-prymulanek-nebo-andrejovy-nove-saty-ac867',
'ext': 'mp4',
'title': 'čauky lidi 70 finall',
'description': 'čauky lidi 70 finall',
'thumbnail': 'h',
'upload_date': '20220606',
'timestamp': 1654513791,
'duration': 318.0,
'direct': True,
'age_limit': 0,
},
},
{
'note': 'JW Player embed with unicode-escape sequences in URL',
'url': 'https://www.medici.tv/en/concerts/lahav-shani-mozart-mahler-israel-philharmonic-abu-dhabi-classics',
'info_dict': {
'id': 'm',
'ext': 'mp4',
'title': 'Lahav Shani conducts the Israel Philharmonic\'s first-ever concert in Abu Dhabi',
'description': 'Mahler\'s ',
'uploader': 'www.medici.tv',
'age_limit': 0,
'thumbnail': r're:^https?://.+\.jpg',
},
'params': {
'skip_download': True,
},
},
{
'url': 'https://shooshtime.com/videos/284002/just-out-of-the-shower-joi/',
'md5': 'e2f0a4c329f7986280b7328e24036d60',
'info_dict': {
'id': '284002',
'display_id': 'just-out-of-the-shower-joi',
'ext': 'mp4',
'title': 'Just Out Of The Shower JOI - Shooshtime',
'thumbnail': 'https://i.shoosh.co/contents/videos_screenshots/284000/284002/preview.mp4.jpg',
'height': 720,
'age_limit': 18,
},
},
]
def report_following_redirect(self, new_url):
@@ -2212,6 +2206,13 @@ class GenericIE(InfoExtractor):
self._downloader.write_debug(f'Identified {num} {name}{format_field(note, None, "; %s")}')
def _fragment_query(self, url):
if self._configuration_arg('fragment_query'):
query_string = urllib.parse.urlparse(url).query
if query_string:
return {'extra_param_to_segment_url': query_string}
return {}
def _extract_rss(self, url, video_id, doc):
NS_MAP = {
'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
@@ -2259,43 +2260,87 @@ class GenericIE(InfoExtractor):
'entries': entries,
}
def _kvs_getrealurl(self, video_url, license_code):
@classmethod
def _kvs_get_real_url(cls, video_url, license_code):
if not video_url.startswith('function/0/'):
return video_url # not obfuscated
url_path, _, url_query = video_url.partition('?')
urlparts = url_path.split('/')[2:]
license = self._kvs_getlicensetoken(license_code)
newmagic = urlparts[5][:32]
parsed = urllib.parse.urlparse(video_url[len('function/0/'):])
license = cls._kvs_get_license_token(license_code)
urlparts = parsed.path.split('/')
for o in range(len(newmagic) - 1, -1, -1):
new = ''
l = (o + sum(int(n) for n in license[o:])) % 32
HASH_LENGTH = 32
hash = urlparts[3][:HASH_LENGTH]
indices = list(range(HASH_LENGTH))
for i in range(0, len(newmagic)):
if i == o:
new += newmagic[l]
elif i == l:
new += newmagic[o]
else:
new += newmagic[i]
newmagic = new
# Swap indices of hash according to the destination calculated from the license token
accum = 0
for src in reversed(range(HASH_LENGTH)):
accum += license[src]
dest = (src + accum) % HASH_LENGTH
indices[src], indices[dest] = indices[dest], indices[src]
urlparts[5] = newmagic + urlparts[5][32:]
return '/'.join(urlparts) + '?' + url_query
urlparts[3] = ''.join(hash[index] for index in indices) + urlparts[3][HASH_LENGTH:]
return urllib.parse.urlunparse(parsed._replace(path='/'.join(urlparts)))
def _kvs_getlicensetoken(self, license):
modlicense = license.replace('$', '').replace('0', '1')
center = int(len(modlicense) / 2)
@staticmethod
def _kvs_get_license_token(license):
license = license.replace('$', '')
license_values = [int(char) for char in license]
modlicense = license.replace('0', '1')
center = len(modlicense) // 2
fronthalf = int(modlicense[:center + 1])
backhalf = int(modlicense[center:])
modlicense = str(4 * abs(fronthalf - backhalf))[:center + 1]
modlicense = str(4 * abs(fronthalf - backhalf))
retval = ''
for o in range(0, center + 1):
for i in range(1, 5):
retval += str((int(license[o + i]) + int(modlicense[o])) % 10)
return retval
return [
(license_values[index + offset] + current) % 10
for index, current in enumerate(map(int, modlicense))
for offset in range(4)
]
def _extract_kvs(self, url, webpage, video_id):
flashvars = self._search_json(
r'(?s:<script\b[^>]*>.*?var\s+flashvars\s*=)',
webpage, 'flashvars', video_id, transform_source=js_to_json)
# extract the part after the last / as the display_id from the
# canonical URL.
display_id = self._search_regex(
r'(?:<link href="https?://[^"]+/(.+?)/?" rel="canonical"\s*/?>'
r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
webpage, 'display_id', fatal=False)
title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
thumbnail = flashvars['preview_url']
if thumbnail.startswith('//'):
protocol, _, _ = url.partition('/')
thumbnail = protocol + thumbnail
url_keys = list(filter(re.compile(r'^video_(?:url|alt_url\d*)$').match, flashvars.keys()))
formats = []
for key in url_keys:
if '/get_file/' not in flashvars[key]:
continue
format_id = flashvars.get(f'{key}_text', key)
formats.append({
'url': urljoin(url, self._kvs_get_real_url(flashvars[key], flashvars['license_code'])),
'format_id': format_id,
'ext': 'mp4',
**(parse_resolution(format_id) or parse_resolution(flashvars[key])),
'http_headers': {'Referer': url},
})
if not formats[-1].get('height'):
formats[-1]['quality'] = 1
return {
'id': flashvars['video_id'],
'display_id': display_id,
'title': title,
'thumbnail': thumbnail,
'formats': formats,
}
def _real_extract(self, url):
if url.startswith('//'):
@@ -2351,7 +2396,7 @@ class GenericIE(InfoExtractor):
# It may probably better to solve this by checking Content-Type for application/octet-stream
# after a HEAD request, but not sure if we can rely on this.
full_response = self._request_webpage(url, video_id, headers={
'Accept-Encoding': '*',
'Accept-Encoding': 'identity',
**smuggled_data.get('http_headers', {})
})
new_url = full_response.geturl()
@@ -2379,8 +2424,10 @@ class GenericIE(InfoExtractor):
subtitles = {}
if format_id.endswith('mpegurl'):
formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
info_dict.update(self._fragment_query(url))
elif format_id.endswith('mpd') or format_id.endswith('dash+xml'):
formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id, headers=headers)
info_dict.update(self._fragment_query(url))
elif format_id == 'f4m':
formats = self._extract_f4m_formats(url, video_id, headers=headers)
else:
@@ -2393,7 +2440,7 @@ class GenericIE(InfoExtractor):
info_dict.update({
'formats': formats,
'subtitles': subtitles,
'http_headers': headers,
'http_headers': headers or None,
})
return info_dict
@@ -2407,6 +2454,7 @@ class GenericIE(InfoExtractor):
if first_bytes.startswith(b'#EXTM3U'):
self.report_detected('M3U playlist')
info_dict['formats'], info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
info_dict.update(self._fragment_query(url))
return info_dict
# Maybe it's a direct link to a video?
@@ -2457,6 +2505,7 @@ class GenericIE(InfoExtractor):
doc,
mpd_base_url=full_response.geturl().rpartition('/')[0],
mpd_url=url)
info_dict.update(self._fragment_query(url))
self.report_detected('DASH manifest')
return info_dict
elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
@@ -2569,7 +2618,10 @@ class GenericIE(InfoExtractor):
m3u8_id='hls', fatal=False)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
for fmt in formats:
fmt.update(self._fragment_query(src))
if not formats:
formats.append({
'url': src,
'ext': (mimetype2ext(src_type)
@@ -2597,6 +2649,17 @@ class GenericIE(InfoExtractor):
self.report_detected('video.js embed')
return [{'formats': formats, 'subtitles': subtitles}]
# Look for generic KVS player (before json-ld bc of some urls that break otherwise)
found = self._search_regex((
r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:(?!\1)[^?#])+/kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:(?!\2)[^?#])+/kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,',
), webpage, 'KVS player', group='ver', default=False)
if found:
self.report_detected('KVS Player')
if found.split('.')[0] not in ('4', '5', '6'):
self.report_warning(f'Untested major version ({found}) in player engine - download may fail.')
return [self._extract_kvs(url, webpage, video_id)]
# Looking for http://schema.org/VideoObject
json_ld = self._search_json_ld(webpage, video_id, default={})
if json_ld.get('url') not in (url, None):
@@ -2639,52 +2702,6 @@ class GenericIE(InfoExtractor):
['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
if found:
self.report_detected('JW Player embed')
if not found:
# Look for generic KVS player
found = re.search(r'<script [^>]*?src="https?://.+?/kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)".*?>', webpage)
if found:
self.report_detected('KWS Player')
if found.group('maj_ver') not in ['4', '5']:
self.report_warning('Untested major version (%s) in player engine--Download may fail.' % found.group('ver'))
flashvars = re.search(r'(?ms)<script.*?>.*?var\s+flashvars\s*=\s*(\{.*?\});.*?</script>', webpage)
flashvars = self._parse_json(flashvars.group(1), video_id, transform_source=js_to_json)
# extract the part after the last / as the display_id from the
# canonical URL.
display_id = self._search_regex(
r'(?:<link href="https?://[^"]+/(.+?)/?" rel="canonical"\s*/?>'
r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
webpage, 'display_id', fatal=False
)
title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
thumbnail = flashvars['preview_url']
if thumbnail.startswith('//'):
protocol, _, _ = url.partition('/')
thumbnail = protocol + thumbnail
url_keys = list(filter(re.compile(r'video_url|video_alt_url\d*').fullmatch, flashvars.keys()))
formats = []
for key in url_keys:
if '/get_file/' not in flashvars[key]:
continue
format_id = flashvars.get(f'{key}_text', key)
formats.append({
'url': self._kvs_getrealurl(flashvars[key], flashvars['license_code']),
'format_id': format_id,
'ext': 'mp4',
**(parse_resolution(format_id) or parse_resolution(flashvars[key]))
})
if not formats[-1].get('height'):
formats[-1]['quality'] = 1
return [{
'id': flashvars['video_id'],
'display_id': display_id,
'title': title,
'thumbnail': thumbnail,
'formats': formats,
}]
if not found:
# Broaden the search a little bit
found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
@@ -2765,6 +2782,7 @@ class GenericIE(InfoExtractor):
entries = []
for video_url in orderedSet(found):
video_url = video_url.encode().decode('unicode-escape')
video_url = unescapeHTML(video_url)
video_url = video_url.replace('\\/', '/')
video_url = urllib.parse.urljoin(url, video_url)
@@ -2804,8 +2822,10 @@ class GenericIE(InfoExtractor):
return [self._extract_xspf_playlist(video_url, video_id)]
elif ext == 'm3u8':
entry_info_dict['formats'], entry_info_dict['subtitles'] = self._extract_m3u8_formats_and_subtitles(video_url, video_id, ext='mp4', headers=headers)
entry_info_dict.update(self._fragment_query(video_url))
elif ext == 'mpd':
entry_info_dict['formats'], entry_info_dict['subtitles'] = self._extract_mpd_formats_and_subtitles(video_url, video_id, headers=headers)
entry_info_dict.update(self._fragment_query(video_url))
elif ext == 'f4m':
entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id, headers=headers)
elif re.search(r'(?i)\.(?:ism|smil)/manifest', video_url) and video_url != url:
yt_dlp/extractor/goplay.py
+3 -3
View File
@@ -76,11 +76,11 @@ class GoPlayIE(InfoExtractor):
}
api = self._download_json(
f'https://api.viervijfzes.be/content/{video_id}',
video_id, headers={'Authorization': self._id_token})
f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',
video_id, headers={'Authorization': 'Bearer %s' % self._id_token})
formats, subs = self._extract_m3u8_formats_and_subtitles(
api['video']['S'], video_id, ext='mp4', m3u8_id='HLS')
api['manifestUrls']['hls'], video_id, ext='mp4', m3u8_id='HLS')
info_dict.update({
'id': video_id,
yt_dlp/extractor/gronkh.py
+13 -2
View File
@@ -9,15 +9,26 @@ from ..utils import (
class GronkhIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?gronkh\.tv/(?:watch/)?stream/(?P<id>\d+)'
_VALID_URL = r'https?://(?:www\.)?gronkh\.tv/(?:watch/)?streams?/(?P<id>\d+)'
_TESTS = [{
'url': 'https://gronkh.tv/streams/657',
'info_dict': {
'id': '657',
'ext': 'mp4',
'title': 'H.O.R.D.E. - DAS ZWEiTE ZEiTALTER 🎲 Session 1',
'view_count': int,
'thumbnail': 'https://01.cdn.vod.farm/preview/9e2555d3a23bf4e5c5b7c6b3b70a9d84.jpg',
'upload_date': '20221111'
},
'params': {'skip_download': True}
}, {
'url': 'https://gronkh.tv/stream/536',
'info_dict': {
'id': '536',
'ext': 'mp4',
'title': 'GTV0536, 2021-10-01 - MARTHA IS DEAD #FREiAB1830 !FF7 !horde !archiv',
'view_count': 19491,
'view_count': int,
'thumbnail': 'https://01.cdn.vod.farm/preview/6436746cce14e25f751260a692872b9b.jpg',
'upload_date': '20211001'
},
yt_dlp/extractor/hotstar.py
+28 -12
View File
@@ -148,6 +148,12 @@ class HotStarIE(HotStarBaseIE):
'dr': 'dynamic_range',
}
_TAG_FIELDS = {
'language': 'language',
'acodec': 'audio_codec',
'vcodec': 'video_codec',
}
@classmethod
def _video_url(cls, video_id, video_type=None, *, slug='ignore_me', root=None):
assert None in (video_type, root)
@@ -182,24 +188,22 @@ class HotStarIE(HotStarBaseIE):
for key, prefix in self._IGNORE_MAP.items()
for ignore in self._configuration_arg(key)):
continue
tag_dict = dict((t.split(':', 1) + [None])[:2] for t in tags.split(';'))
format_url = url_or_none(playback_set.get('playbackUrl'))
if not format_url:
continue
format_url = re.sub(r'(?<=//staragvod)(\d)', r'web\1', format_url)
dr = re.search(r'dynamic_range:(?P<dr>[a-z]+)', playback_set.get('tagsCombination')).group('dr')
ext = determine_ext(format_url)
current_formats, current_subs = [], {}
try:
if 'package:hls' in tags or ext == 'm3u8':
current_formats, current_subs = self._extract_m3u8_formats_and_subtitles(
format_url, video_id, 'mp4',
entry_protocol='m3u8_native',
m3u8_id=f'{dr}-hls', headers=headers)
format_url, video_id, ext='mp4', headers=headers)
elif 'package:dash' in tags or ext == 'mpd':
current_formats, current_subs = self._extract_mpd_formats_and_subtitles(
format_url, video_id, mpd_id=f'{dr}-dash', headers=headers)
format_url, video_id, headers=headers)
elif ext == 'f4m':
pass # XXX: produce broken files
else:
@@ -213,20 +217,32 @@ class HotStarIE(HotStarBaseIE):
geo_restricted = True
continue
if tags and 'encryption:plain' not in tags:
if tag_dict.get('encryption') not in ('plain', None):
for f in current_formats:
f['has_drm'] = True
if tags and 'language' in tags:
lang = re.search(r'language:(?P<lang>[a-z]+)', tags).group('lang')
for f in current_formats:
if not f.get('langauge'):
f['language'] = lang
for f in current_formats:
for k, v in self._TAG_FIELDS.items():
if not f.get(k):
f[k] = tag_dict.get(v)
if f.get('vcodec') != 'none' and not f.get('dynamic_range'):
f['dynamic_range'] = tag_dict.get('dynamic_range')
if f.get('acodec') != 'none' and not f.get('audio_channels'):
f['audio_channels'] = {
'stereo': 2,
'dolby51': 6,
}.get(tag_dict.get('audio_channel'))
f['format_note'] = join_nonempty(
tag_dict.get('ladder'),
tag_dict.get('audio_channel') if f.get('acodec') != 'none' else None,
f.get('format_note'),
delim=', ')
formats.extend(current_formats)
subs = self._merge_subtitles(subs, current_subs)
if not formats and geo_restricted:
self.raise_geo_restricted(countries=['IN'], metadata_available=True)
self._remove_duplicate_formats(formats)
for f in formats:
f.setdefault('http_headers', {}).update(headers)
@@ -235,7 +251,7 @@ class HotStarIE(HotStarBaseIE):
'title': video_data.get('title'),
'description': video_data.get('description'),
'duration': int_or_none(video_data.get('duration')),
'timestamp': int_or_none(video_data.get('broadcastDate') or video_data.get('startDate')),
'timestamp': int_or_none(traverse_obj(video_data, 'broadcastDate', 'startDate')),
'formats': formats,
'subtitles': subs,
'channel': video_data.get('channelName'),
yt_dlp/extractor/huya.py
+7 -2
View File
@@ -1,5 +1,6 @@
import hashlib
import random
import re
from ..compat import compat_urlparse, compat_b64decode
@@ -37,7 +38,7 @@ class HuyaLiveIE(InfoExtractor):
}]
_RESOLUTION = {
'蓝光4M': {
'蓝光': {
'width': 1920,
'height': 1080,
},
@@ -76,11 +77,15 @@ class HuyaLiveIE(InfoExtractor):
if re_secret:
fm, ss = self.encrypt(params, stream_info, stream_name)
for si in stream_data.get('vMultiStreamInfo'):
display_name, bitrate = re.fullmatch(
r'(.+?)(?:(\d+)M)?', si.get('sDisplayName')).groups()
rate = si.get('iBitRate')
if rate:
params['ratio'] = rate
else:
params.pop('ratio', None)
if bitrate:
rate = int(bitrate) * 1000
if re_secret:
params['wsSecret'] = hashlib.md5(
'_'.join([fm, params['u'], stream_name, ss, params['wsTime']]))
@@ -90,7 +95,7 @@ class HuyaLiveIE(InfoExtractor):
'tbr': rate,
'url': update_url_query(f'{stream_url}/{stream_name}.{stream_info.get("sFlvUrlSuffix")}',
query=params),
**self._RESOLUTION.get(si.get('sDisplayName'), {}),
**self._RESOLUTION.get(display_name, {}),
})
return {
yt_dlp/extractor/iqiyi.py
+12 -9
View File
@@ -527,11 +527,14 @@ class IqIE(InfoExtractor):
webpack_js_url = self._proto_relative_url(self._search_regex(
r'<script src="((?:https?)?//stc.iqiyipic.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
webpack_map1, webpack_map2 = [self._parse_json(js_map, video_id, transform_source=js_to_json) for js_map in self._search_regex(
r'\(({[^}]*})\[\w+\][^\)]*\)\s*\+\s*["\']\.["\']\s*\+\s*({[^}]*})\[\w+\]\+["\']\.js', webpack_js, 'JS locations', group=(1, 2))]
for module_index in reversed(list(webpack_map2.keys())):
webpack_map = self._search_json(
r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\da-f]+["\']\s*,?\s*)+}',
end_pattern=r'\[\w+\]\+["\']\.js', transform_source=js_to_json)
for module_index in reversed(webpack_map):
module_js = self._download_webpage(
f'https://stc.iqiyipic.com/_next/static/chunks/{webpack_map1.get(module_index, module_index)}.{webpack_map2[module_index]}.js',
f'https://stc.iqiyipic.com/_next/static/chunks/{module_index}.{webpack_map[module_index]}.js',
video_id, note=f'Downloading #{module_index} module JS', errnote='Unable to download module JS', fatal=False) or ''
if 'vms request' in module_js:
self.cache.store('iq', 'player_js', module_js)
@@ -543,11 +546,11 @@ class IqIE(InfoExtractor):
self._extract_vms_player_js(webpage, video_id), 'signature function')
def _update_bid_tags(self, webpage, video_id):
extracted_bid_tags = self._parse_json(
self._search_regex(
r'arguments\[1\][^,]*,\s*function\s*\([^\)]*\)\s*{\s*"use strict";?\s*var \w=({.+}})\s*,\s*\w\s*=\s*{\s*getNewVd',
self._extract_vms_player_js(webpage, video_id), 'video tags', default=''),
video_id, transform_source=js_to_json, fatal=False)
extracted_bid_tags = self._search_json(
r'function\s*\([^)]*\)\s*\{\s*"use strict";?\s*var \w\s*=\s*',
self._extract_vms_player_js(webpage, video_id), 'video tags', video_id,
contains_pattern=r'{\s*\d+\s*:\s*\{\s*nbid\s*:.+}\s*}',
end_pattern=r'\s*,\s*\w\s*=\s*\{\s*getNewVd', fatal=False, transform_source=js_to_json)
if not extracted_bid_tags:
return
self._BID_TAGS = {
yt_dlp/extractor/joj.py
+19 -7
View File
@@ -23,9 +23,19 @@ class JojIE(InfoExtractor):
'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
'ext': 'mp4',
'title': 'NOVÉ BÝVANIE',
'thumbnail': r're:^https?://.*\.jpg$',
'thumbnail': r're:^https?://.*?$',
'duration': 3118,
}
}, {
'url': 'https://media.joj.sk/embed/CSM0Na0l0p1',
'info_dict': {
'id': 'CSM0Na0l0p1',
'ext': 'mp4',
'height': 576,
'title': 'Extrémne rodiny 2 - POKRAČOVANIE (2012/04/09 21:30:00)',
'duration': 3937,
'thumbnail': r're:^https?://.*?$',
}
}, {
'url': 'https://media.joj.sk/embed/9i1cxv',
'only_matching': True,
@@ -43,10 +53,10 @@ class JojIE(InfoExtractor):
webpage = self._download_webpage(
'https://media.joj.sk/embed/%s' % video_id, video_id)
title = self._search_regex(
(r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
r'<title>(?P<title>[^<]+)'), webpage, 'title',
default=None, group='title') or self._og_search_title(webpage)
title = (self._search_json(r'videoTitle\s*:', webpage, 'title', video_id,
contains_pattern=r'["\'].+["\']', default=None)
or self._html_extract_title(webpage, default=None)
or self._og_search_title(webpage))
bitrates = self._parse_json(
self._search_regex(
@@ -58,11 +68,13 @@ class JojIE(InfoExtractor):
for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
if isinstance(format_url, compat_str):
height = self._search_regex(
r'(\d+)[pP]\.', format_url, 'height', default=None)
r'(\d+)[pP]|(pal)\.', format_url, 'height', default=None)
if height == 'pal':
height = 576
formats.append({
'url': format_url,
'format_id': format_field(height, None, '%sp'),
'height': int(height),
'height': int_or_none(height),
})
if not formats:
playlist = self._download_xml(
yt_dlp/extractor/kanal2.py
+66
View File
@@ -0,0 +1,66 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
join_nonempty,
traverse_obj,
unified_timestamp,
update_url_query,
)
class Kanal2IE(InfoExtractor):
_VALID_URL = r'https?://kanal2\.postimees\.ee/[^?#]+\?([^#]+&)?id=(?P<id>\d+)'
_TESTS = [{
'note': 'Test standard url (#5575)',
'url': 'https://kanal2.postimees.ee/pluss/video/?id=40792',
'md5': '7ea7b16266ec1798743777df241883dd',
'info_dict': {
'id': '40792',
'ext': 'mp4',
'title': 'Aedniku aabits / Osa 53 (05.08.2016 20:00)',
'thumbnail': r're:https?://.*\.jpg$',
'description': 'md5:53cabf3c5d73150d594747f727431248',
'upload_date': '20160805',
'timestamp': 1470420000,
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
playlist = self._download_json(
f'https://kanal2.postimees.ee/player/playlist/{video_id}',
video_id, query={'type': 'episodes'},
headers={'X-Requested-With': 'XMLHttpRequest'})
return {
'id': video_id,
'title': join_nonempty(*traverse_obj(playlist, ('info', ('title', 'subtitle'))), delim=' / '),
'description': traverse_obj(playlist, ('info', 'description')),
'thumbnail': traverse_obj(playlist, ('data', 'image')),
'formats': self.get_formats(playlist, video_id),
'timestamp': unified_timestamp(self._search_regex(
r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$',
traverse_obj(playlist, ('info', 'subtitle')), 'timestamp', default='') + ' +0200'),
}
def get_formats(self, playlist, video_id):
path = traverse_obj(playlist, ('data', 'path'))
if not path:
raise ExtractorError('Path value not found in playlist JSON response')
session = self._download_json(
'https://sts.postimees.ee/session/register',
video_id, note='Creating session', errnote='Error creating session',
headers={
'X-Original-URI': path,
'Accept': 'application/json',
})
if session.get('reason') != 'OK' or not session.get('session'):
reason = session.get('reason', 'unknown error')
raise ExtractorError(f'Unable to obtain session: {reason}')
formats = []
for stream in traverse_obj(playlist, ('data', 'streams', ..., 'file')):
formats.extend(self._extract_m3u8_formats(
update_url_query(stream, {'s': session['session']}), video_id, 'mp4'))
return formats
yt_dlp/extractor/kankanews.py
+48
View File
@@ -0,0 +1,48 @@
import time
import random
import string
import hashlib
import urllib.parse
from .common import InfoExtractor
class KankaNewsIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?kankanews\.com/a/\d+\-\d+\-\d+/(?P<id>\d+)\.shtml'
_TESTS = [{
'url': 'https://www.kankanews.com/a/2022-11-08/00310276054.shtml?appid=1088227',
'md5': '05e126513c74b1258d657452a6f4eef9',
'info_dict': {
'id': '4485057',
'url': 'http://mediaplay.kksmg.com/2022/11/08/h264_450k_mp4_1a388ad771e0e4cc28b0da44d245054e_ncm.mp4',
'ext': 'mp4',
'title': '视频｜第23个中国记者节，我们在进博切蛋糕',
'thumbnail': r're:^https?://.*\.jpg*',
}
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
video_id = self._search_regex(r'omsid\s*=\s*"(\d+)"', webpage, 'video id')
params = {
'nonce': ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)),
'omsid': video_id,
'platform': 'pc',
'timestamp': int(time.time()),
'version': '1.0',
}
params['sign'] = hashlib.md5((hashlib.md5((
urllib.parse.urlencode(params) + '&28c8edde3d61a0411511d3b1866f0636'
).encode()).hexdigest()).encode()).hexdigest()
meta = self._download_json('https://api-app.kankanews.com/kankan/pc/getvideo',
video_id, query=params)['result']['video']
return {
'id': video_id,
'url': meta['videourl'],
'title': self._search_regex(r'g\.title\s*=\s*"([^"]+)"', webpage, 'title'),
'thumbnail': meta.get('titlepic'),
}
yt_dlp/extractor/kick.py
+127
View File
@@ -0,0 +1,127 @@
from .common import InfoExtractor
from ..utils import (
HEADRequest,
UserNotLive,
float_or_none,
merge_dicts,
str_or_none,
traverse_obj,
unified_timestamp,
url_or_none,
)
class KickBaseIE(InfoExtractor):
def _real_initialize(self):
self._request_webpage(HEADRequest('https://kick.com/'), None, 'Setting up session')
xsrf_token = self._get_cookies('https://kick.com/').get('XSRF-TOKEN')
if not xsrf_token:
self.write_debug('kick.com did not set XSRF-TOKEN cookie')
KickBaseIE._API_HEADERS = {
'Authorization': f'Bearer {xsrf_token.value}',
'X-XSRF-TOKEN': xsrf_token.value,
} if xsrf_token else {}
def _call_api(self, path, display_id, note='Downloading API JSON', headers={}, **kwargs):
return self._download_json(
f'https://kick.com/api/v1/{path}', display_id, note=note,
headers=merge_dicts(headers, self._API_HEADERS), **kwargs)
class KickIE(KickBaseIE):
_VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w_]+)'
_TESTS = [{
'url': 'https://kick.com/yuppy',
'info_dict': {
'id': '6cde1-kickrp-joe-flemmingskick-info-heremust-knowmust-see21',
'ext': 'mp4',
'title': str,
'description': str,
'channel': 'yuppy',
'channel_id': '33538',
'uploader': 'Yuppy',
'uploader_id': '33793',
'upload_date': str,
'live_status': 'is_live',
'timestamp': int,
'thumbnail': r're:^https?://.*\.jpg',
'categories': list,
},
'skip': 'livestream',
}, {
'url': 'https://kick.com/kmack710',
'only_matching': True,
}]
def _real_extract(self, url):
channel = self._match_id(url)
response = self._call_api(f'channels/{channel}', channel)
if not traverse_obj(response, 'livestream', expected_type=dict):
raise UserNotLive(video_id=channel)
return {
'id': str(traverse_obj(
response, ('livestream', ('slug', 'id')), get_all=False, default=channel)),
'formats': self._extract_m3u8_formats(
response['playback_url'], channel, 'mp4', live=True),
'title': traverse_obj(
response, ('livestream', ('session_title', 'slug')), get_all=False, default=''),
'description': traverse_obj(response, ('user', 'bio')),
'channel': channel,
'channel_id': str_or_none(traverse_obj(response, 'id', ('livestream', 'channel_id'))),
'uploader': traverse_obj(response, 'name', ('user', 'username')),
'uploader_id': str_or_none(traverse_obj(response, 'user_id', ('user', 'id'))),
'is_live': True,
'timestamp': unified_timestamp(traverse_obj(response, ('livestream', 'created_at'))),
'thumbnail': traverse_obj(
response, ('livestream', 'thumbnail', 'url'), expected_type=url_or_none),
'categories': traverse_obj(response, ('recent_categories', ..., 'name')),
}
class KickVODIE(KickBaseIE):
_VALID_URL = r'https?://(?:www\.)?kick\.com/video/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
_TESTS = [{
'url': 'https://kick.com/video/54244b5e-050a-4df4-a013-b2433dafbe35',
'md5': '73691206a6a49db25c5aa1588e6538fc',
'info_dict': {
'id': '54244b5e-050a-4df4-a013-b2433dafbe35',
'ext': 'mp4',
'title': 'Making 710-carBoosting. Kinda No Pixel inspired. !guilded - !links',
'description': 'md5:a0d3546bf7955d0a8252ffe0fd6f518f',
'channel': 'kmack710',
'channel_id': '16278',
'uploader': 'Kmack710',
'uploader_id': '16412',
'upload_date': '20221206',
'timestamp': 1670318289,
'duration': 40104.0,
'thumbnail': r're:^https?://.*\.jpg',
'categories': ['Grand Theft Auto V'],
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
response = self._call_api(f'video/{video_id}', video_id)
return {
'id': video_id,
'formats': self._extract_m3u8_formats(response['source'], video_id, 'mp4'),
'title': traverse_obj(
response, ('livestream', ('session_title', 'slug')), get_all=False, default=''),
'description': traverse_obj(response, ('livestream', 'channel', 'user', 'bio')),
'channel': traverse_obj(response, ('livestream', 'channel', 'slug')),
'channel_id': str_or_none(traverse_obj(response, ('livestream', 'channel', 'id'))),
'uploader': traverse_obj(response, ('livestream', 'channel', 'user', 'username')),
'uploader_id': str_or_none(traverse_obj(response, ('livestream', 'channel', 'user_id'))),
'timestamp': unified_timestamp(response.get('created_at')),
'duration': float_or_none(traverse_obj(response, ('livestream', 'duration')), scale=1000),
'thumbnail': traverse_obj(
response, ('livestream', 'thumbnail'), expected_type=url_or_none),
'categories': traverse_obj(response, ('livestream', 'categories', ..., 'name')),
}
yt_dlp/extractor/la7.py
+34 -20
View File
@@ -2,7 +2,6 @@ import re
from .common import InfoExtractor
from ..utils import (
determine_ext,
float_or_none,
HEADRequest,
int_or_none,
@@ -13,13 +12,13 @@ from ..utils import (
class LA7IE(InfoExtractor):
IE_NAME = 'la7.it'
_VALID_URL = r'''(?x)(https?://)?(?:
(?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video)/|
_VALID_URL = r'''(?x)https?://(?:
(?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video|news)/|
tg\.la7\.it/repliche-tgla7\?id=
)(?P<id>.+)'''
_TESTS = [{
# 'src' is a plain URL
# single quality video
'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
'info_dict': {
@@ -29,6 +28,20 @@ class LA7IE(InfoExtractor):
'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
'thumbnail': 're:^https?://.*',
'upload_date': '20151002',
'formats': 'count:4',
},
}, {
# multiple quality video
'url': 'https://www.la7.it/calcio-femminile/news/il-gol-di-lindsey-thomas-fiorentina-vs-milan-serie-a-calcio-femminile-26-11-2022-461736',
'md5': 'd2370e78f75e8d1238cb3a0db9a2eda3',
'info_dict': {
'id': 'il-gol-di-lindsey-thomas-fiorentina-vs-milan-serie-a-calcio-femminile-26-11-2022-461736',
'ext': 'mp4',
'title': 'Il gol di Lindsey Thomas | Fiorentina vs Milan | Serie A Calcio Femminile',
'description': 'Il gol di Lindsey Thomas | Fiorentina vs Milan | Serie A Calcio Femminile',
'thumbnail': 're:^https?://.*',
'upload_date': '20221126',
'formats': 'count:8',
},
}, {
'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
@@ -39,7 +52,7 @@ class LA7IE(InfoExtractor):
def _generate_mp4_url(self, quality, m3u8_formats):
for f in m3u8_formats:
if f['vcodec'] != 'none' and quality in f['url']:
http_url = '%s%s.mp4' % (self._HOST, quality)
http_url = f'{self._HOST}{quality}.mp4'
urlh = self._request_webpage(
HEADRequest(http_url), quality,
@@ -58,12 +71,13 @@ class LA7IE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
if not url.startswith('http'):
url = '%s//%s' % (self.http_scheme(), url)
webpage = self._download_webpage(url, video_id)
video_path = self._search_regex(r'(/content/.*?).mp4', webpage, 'video_path')
if re.search(r'(?i)(drmsupport\s*:\s*true)\s*', webpage):
self.report_drm(video_id)
video_path = self._search_regex(
r'(/content/[\w/,]+?)\.mp4(?:\.csmil)?/master\.m3u8', webpage, 'video_path')
formats = self._extract_mpd_formats(
f'{self._HOST}/local/dash/,{video_path}.mp4.urlset/manifest.mpd',
@@ -90,8 +104,7 @@ class LA7IE(InfoExtractor):
class LA7PodcastEpisodeIE(InfoExtractor):
IE_NAME = 'la7.it:pod:episode'
_VALID_URL = r'''(?x)(https?://)?
(?:www\.)?la7\.it/[^/]+/podcast/([^/]+-)?(?P<id>\d+)'''
_VALID_URL = r'https?://(?:www\.)?la7\.it/[^/]+/podcast/([^/]+-)?(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.la7.it/voicetown/podcast/la-carezza-delle-memoria-di-carlo-verdone-23-03-2021-371497',
@@ -125,14 +138,15 @@ class LA7PodcastEpisodeIE(InfoExtractor):
webpage, 'video_id', group='vid')
media_url = self._search_regex(
(r'src:\s*([\'"])(?P<url>.+?mp3.+?)\1',
r'data-podcast=([\'"])(?P<url>.+?mp3.+?)\1'),
(r'src\s*:\s*([\'"])(?P<url>\S+?mp3.+?)\1',
r'data-podcast\s*=\s*([\'"])(?P<url>\S+?mp3.+?)\1'),
webpage, 'media_url', group='url')
ext = determine_ext(media_url)
formats = [{
'url': media_url,
'format_id': ext,
'ext': ext,
'format_id': 'http-mp3',
'ext': 'mp3',
'acodec': 'mp3',
'vcodec': 'none',
}]
title = self._html_search_regex(
@@ -173,7 +187,7 @@ class LA7PodcastEpisodeIE(InfoExtractor):
# and title is the same as the show_title
# add the date to the title
if date and not date_alt and ppn and ppn.lower() == title.lower():
title += ' del %s' % date
title = f'{title} del {date}'
return {
'id': video_id,
'title': title,
@@ -193,7 +207,7 @@ class LA7PodcastEpisodeIE(InfoExtractor):
class LA7PodcastIE(LA7PodcastEpisodeIE): # XXX: Do not subclass from concrete IE
IE_NAME = 'la7.it:podcast'
_VALID_URL = r'(https?://)?(www\.)?la7\.it/(?P<id>[^/]+)/podcast/?(?:$|[#?])'
_VALID_URL = r'https?://(?:www\.)?la7\.it/(?P<id>[^/]+)/podcast/?(?:$|[#?])'
_TESTS = [{
'url': 'https://www.la7.it/propagandalive/podcast',
@@ -201,7 +215,7 @@ class LA7PodcastIE(LA7PodcastEpisodeIE): # XXX: Do not subclass from concrete I
'id': 'propagandalive',
'title': "Propaganda Live",
},
'playlist_count': 10,
'playlist_count_min': 10,
}]
def _real_extract(self, url):
yt_dlp/extractor/linuxacademy.py
+2 -3
View File
@@ -75,9 +75,8 @@ class LinuxAcademyIE(InfoExtractor):
def _perform_login(self, username, password):
def random_string():
return ''.join([
random.choice('0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~')
for _ in range(32)])
return ''.join(random.choices(
'0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~', k=32))
webpage, urlh = self._download_webpage_handle(
self._AUTHORIZE_URL, None, 'Downloading authorize page', query={
yt_dlp/extractor/mediaset.py
+81 -120
View File
@@ -7,7 +7,6 @@ from ..utils import (
GeoRestrictedError,
int_or_none,
OnDemandPagedList,
parse_qs,
try_get,
urljoin,
update_url_query,
@@ -16,20 +15,25 @@ from ..utils import (
class MediasetIE(ThePlatformBaseIE):
_TP_TLD = 'eu'
_VALID_URL = r'''(?x)
_GUID_RE = r'F[0-9A-Z]{15}'
_VALID_URL = rf'''(?x)
(?:
mediaset:|
https?://
(?:\w+\.)+mediaset\.it/
(?:
(?:video|on-demand|movie)/(?:[^/]+/)+[^/]+_|
player/(?:v\d+/)?index\.html\?.*?\bprogramGuid=
player/(?:v\d+/)?index\.html\?\S*?\bprogramGuid=
)
)(?P<id>[0-9A-Z]{16,})
)(?P<id>{_GUID_RE})
'''
_EMBED_REGEX = [
rf'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//(?:\w+\.)+mediaset\.it/player/(?:v\d+/)?index\.html\?\S*?programGuid={_GUID_RE})[\'"&]'
]
_TESTS = [{
# full episode
'url': 'https://www.mediasetplay.mediaset.it/video/mrwronglezionidamore/episodio-1_F310575103000102',
'url': 'https://mediasetinfinity.mediaset.it/video/mrwronglezionidamore/episodio-1_F310575103000102',
'md5': 'a7e75c6384871f322adb781d3bd72c26',
'info_dict': {
'id': 'F310575103000102',
@@ -50,7 +54,7 @@ class MediasetIE(ThePlatformBaseIE):
'chapters': [{'start_time': 0.0, 'end_time': 439.88}, {'start_time': 439.88, 'end_time': 1685.84}, {'start_time': 1685.84, 'end_time': 2682.0}],
},
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/matrix/puntata-del-25-maggio_F309013801000501',
'url': 'https://mediasetinfinity.mediaset.it/video/matrix/puntata-del-25-maggio_F309013801000501',
'md5': '1276f966ac423d16ba255ce867de073e',
'info_dict': {
'id': 'F309013801000501',
@@ -71,51 +75,8 @@ class MediasetIE(ThePlatformBaseIE):
'chapters': [{'start_time': 0.0, 'end_time': 3409.08}, {'start_time': 3409.08, 'end_time': 6565.008}],
},
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/cameracafe5/episodio-69-pezzo-di-luna_F303843101017801',
'md5': 'd1650ac9ff944f185556126a736df148',
'info_dict': {
'id': 'F303843101017801',
'ext': 'mp4',
'title': 'Episodio 69 - Pezzo di luna',
'description': 'md5:7c32c8ec4118b72588b9412f11353f73',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 263.008,
'upload_date': '20200902',
'series': 'Camera Café 5',
'timestamp': 1599064700,
'uploader': 'Italia 1',
'uploader_id': 'I1',
'season': 'Season 5',
'episode': 'Episode 178',
'season_number': 5,
'episode_number': 178,
'chapters': [{'start_time': 0.0, 'end_time': 261.88}, {'start_time': 261.88, 'end_time': 263.008}],
},
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/cameracafe5/episodio-51-tu-chi-sei_F303843107000601',
'md5': '567e9ad375b7a27a0e370650f572a1e3',
'info_dict': {
'id': 'F303843107000601',
'ext': 'mp4',
'title': 'Episodio 51 - Tu chi sei?',
'description': 'md5:42ef006e56824cc31787a547590923f4',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 367.021,
'upload_date': '20200902',
'series': 'Camera Café 5',
'timestamp': 1599069817,
'uploader': 'Italia 1',
'uploader_id': 'I1',
'season': 'Season 5',
'episode': 'Episode 6',
'season_number': 5,
'episode_number': 6,
'chapters': [{'start_time': 0.0, 'end_time': 358.68}, {'start_time': 358.68, 'end_time': 367.021}],
},
}, {
# movie
'url': 'https://www.mediasetplay.mediaset.it/movie/selvaggi/selvaggi_F006474501000101',
'md5': '720440187a2ae26af8148eb9e6b901ed',
# DRM
'url': 'https://mediasetinfinity.mediaset.it/movie/selvaggi/selvaggi_F006474501000101',
'info_dict': {
'id': 'F006474501000101',
'ext': 'mp4',
@@ -129,70 +90,69 @@ class MediasetIE(ThePlatformBaseIE):
'uploader_id': 'B6',
'chapters': [{'start_time': 0.0, 'end_time': 1938.56}, {'start_time': 1938.56, 'end_time': 5233.01}],
},
'params': {
'ignore_no_formats_error': True,
},
'expected_warnings': [
'None of the available releases match the specified AssetType, ProtectionScheme, and/or Format preferences',
'Content behind paywall and DRM',
],
'skip': True,
}, {
# clip
'url': 'https://www.mediasetplay.mediaset.it/video/gogglebox/un-grande-classico-della-commedia-sexy_FAFU000000661680',
# old domain
'url': 'https://www.mediasetplay.mediaset.it/video/mrwronglezionidamore/episodio-1_F310575103000102',
'only_matching': True,
}, {
# iframe simple
# iframe
'url': 'https://static3.mediasetplay.mediaset.it/player/index.html?appKey=5ad3966b1de1c4000d5cec48&programGuid=FAFU000000665924&id=665924',
'only_matching': True,
}, {
# iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
'url': 'https://static3.mediasetplay.mediaset.it/player/index.html?appKey=5ad3966b1de1c4000d5cec48&programGuid=FAFU000000665104&id=665104',
'only_matching': True,
}, {
# embedUrl (from https://www.wittytv.it/amici/est-ce-que-tu-maimes-gabriele-5-dicembre-copia/)
'url': 'https://static3.mediasetplay.mediaset.it/player/v2/index.html?partnerId=wittytv&configId=&programGuid=FD00000000153323&autoplay=true&purl=http://www.wittytv.it/amici/est-ce-que-tu-maimes-gabriele-5-dicembre-copia/',
'only_matching': True,
}, {
'url': 'mediaset:FAFU000000665924',
'only_matching': True,
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/mediasethaacuoreilfuturo/palmieri-alicudi-lisola-dei-tre-bambini-felici--un-decreto-per-alicudi-e-tutte-le-microscuole_FD00000000102295',
'only_matching': True,
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/cherryseason/anticipazioni-degli-episodi-del-23-ottobre_F306837101005C02',
'only_matching': True,
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/tg5/ambiente-onda-umana-per-salvare-il-pianeta_F309453601079D01',
'only_matching': True,
}, {
'url': 'https://www.mediasetplay.mediaset.it/video/grandefratellovip/benedetta-una-doccia-gelata_F309344401044C135',
'only_matching': True,
}, {
'url': 'https://www.mediasetplay.mediaset.it/movie/herculeslaleggendahainizio/hercules-la-leggenda-ha-inizio_F305927501000102',
'only_matching': True,
}, {
'url': 'https://mediasetinfinity.mediaset.it/video/braveandbeautiful/episodio-113_F310948005000402',
'only_matching': True,
}, {
'url': 'https://static3.mediasetplay.mediaset.it/player/v2/index.html?partnerId=wittytv&configId=&programGuid=FD00000000153323',
'only_matching': True,
}]
def _extract_from_webpage(self, url, webpage):
def _program_guid(qs):
return qs.get('programGuid', [None])[0]
for mobj in re.finditer(
r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml.*?)\1',
webpage):
embed_url = mobj.group('url')
embed_qs = parse_qs(embed_url)
program_guid = _program_guid(embed_qs)
if program_guid:
yield self.url_result(embed_url)
continue
video_id = embed_qs.get('id', [None])[0]
if not video_id:
continue
urlh = self._request_webpage(embed_url, video_id, note='Following embed URL redirect')
embed_url = urlh.geturl()
program_guid = _program_guid(parse_qs(embed_url))
if program_guid:
yield self.url_result(embed_url)
_WEBPAGE_TESTS = [{
# Mediaset embed
'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
'info_dict': {
'id': 'FD00000000004929',
'ext': 'mp4',
'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
'duration': 67.013,
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Mediaset Play',
'uploader_id': 'QY',
'upload_date': '20201005',
'timestamp': 1601866168,
'chapters': [],
},
'params': {
'skip_download': True,
}
}, {
# WittyTV embed
'url': 'https://www.wittytv.it/mauriziocostanzoshow/ultima-puntata-venerdi-25-novembre/',
'info_dict': {
'id': 'F312172801000801',
'ext': 'mp4',
'title': 'Ultima puntata - Venerdì 25 novembre',
'description': 'Una serata all\'insegna della musica e del buonumore ma non priva di spunti di riflessione',
'duration': 6203.01,
'thumbnail': r're:^https?://.*\.jpg$',
'uploader': 'Canale 5',
'uploader_id': 'C5',
'upload_date': '20221126',
'timestamp': 1669428689,
'chapters': list,
'series': 'Maurizio Costanzo Show',
'season': 'Season 12',
'season_number': 12,
'episode': 'Episode 8',
'episode_number': 8,
},
'params': {
'skip_download': True,
}
}]
def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
for video in smil.findall(self._xpath_ns('.//video', namespace)):
@@ -217,7 +177,7 @@ class MediasetIE(ThePlatformBaseIE):
def _real_extract(self, url):
guid = self._match_id(url)
tp_path = 'PR1GhC/media/guid/2702976343/' + guid
tp_path = f'PR1GhC/media/guid/2702976343/{guid}'
info = self._extract_theplatform_metadata(tp_path, guid)
formats = []
@@ -225,15 +185,17 @@ class MediasetIE(ThePlatformBaseIE):
first_e = geo_e = None
asset_type = 'geoNo:HD,browser,geoIT|geoNo:HD,geoIT|geoNo:SD,browser,geoIT|geoNo:SD,geoIT|geoNo|HD|SD'
# TODO: fixup ISM+none manifest URLs
for f in ('MPEG4', 'M3U'):
for f in ('MPEG4', 'MPEG-DASH', 'M3U'):
try:
tp_formats, tp_subtitles = self._extract_theplatform_smil(
update_url_query('http://link.theplatform.%s/s/%s' % (self._TP_TLD, tp_path), {
update_url_query(f'http://link.theplatform.{self._TP_TLD}/s/{tp_path}', {
'mbr': 'true',
'formats': f,
'assetTypes': asset_type,
}), guid, 'Downloading %s SMIL data' % (f.split('+')[0]))
}), guid, f'Downloading {f.split("+")[0]} SMIL data')
except ExtractorError as e:
if e.orig_msg == 'None of the available releases match the specified AssetType, ProtectionScheme, and/or Format preferences':
e.orig_msg = 'This video is DRM protected'
if not geo_e and isinstance(e, GeoRestrictedError):
geo_e = e
if not first_e:
@@ -248,7 +210,7 @@ class MediasetIE(ThePlatformBaseIE):
raise geo_e or first_e
feed_data = self._download_json(
'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2/guid/-/' + guid,
f'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2/guid/-/{guid}',
guid, fatal=False)
if feed_data:
publish_info = feed_data.get('mediasetprogram$publishInfo') or {}
@@ -299,23 +261,23 @@ class MediasetShowIE(MediasetIE): # XXX: Do not subclass from concrete IE
'''
_TESTS = [{
# TV Show webpage (general webpage)
'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061',
'url': 'https://mediasetinfinity.mediaset.it/programmi-tv/leiene/leiene_SE000000000061',
'info_dict': {
'id': '000000000061',
'title': 'Le Iene',
'title': 'Le Iene 2022/2023',
},
'playlist_mincount': 7,
'playlist_mincount': 6,
}, {
# TV Show webpage (specific season)
'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061,ST000000002763',
'url': 'https://mediasetinfinity.mediaset.it/programmi-tv/leiene/leiene_SE000000000061,ST000000002763',
'info_dict': {
'id': '000000002763',
'title': 'Le Iene',
'title': 'Le Iene 2021/2022',
},
'playlist_mincount': 7,
}, {
# TV Show specific playlist (with multiple pages)
'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/iservizi_SE000000000061,ST000000002763,sb100013375',
'url': 'https://mediasetinfinity.mediaset.it/programmi-tv/leiene/iservizi_SE000000000061,ST000000002763,sb100013375',
'info_dict': {
'id': '100013375',
'title': 'I servizi',
@@ -340,10 +302,9 @@ class MediasetShowIE(MediasetIE): # XXX: Do not subclass from concrete IE
playlist_id, st, sb = self._match_valid_url(url).group('id', 'st', 'sb')
if not sb:
page = self._download_webpage(url, st or playlist_id)
entries = [self.url_result(urljoin('https://www.mediasetplay.mediaset.it', url))
entries = [self.url_result(urljoin('https://mediasetinfinity.mediaset.it', url))
for url in re.findall(r'href="([^<>=]+SE\d{12},ST\d{12},sb\d{9})">[^<]+<', page)]
title = (self._html_search_regex(r'(?s)<h1[^>]*>(.+?)</h1>', page, 'title', default=None)
or self._og_search_title(page))
title = self._html_extract_title(page).split('|')[0].strip()
return self.playlist_result(entries, st or playlist_id, title)
entries = OnDemandPagedList(
yt_dlp/extractor/mediastream.py
+155
View File
@@ -0,0 +1,155 @@
import re
from .common import InfoExtractor
from ..utils import clean_html, get_element_html_by_class
class MediaStreamIE(InfoExtractor):
_VALID_URL = r'https?://mdstrm.com/(?:embed|live-stream)/(?P<id>\w+)'
_TESTS = [{
'url': 'https://mdstrm.com/embed/6318e3f1d1d316083ae48831',
'md5': '97b4f2634b8e8612cc574dfcd504df05',
'info_dict': {
'id': '6318e3f1d1d316083ae48831',
'title': 'Video: Así fue el despido de Thomas Tuchel del Chelsea',
'description': 'md5:358ce1e1396010d50a1ece1be3633c95',
'thumbnail': r're:^https?://[^?#]+6318e3f1d1d316083ae48831',
'ext': 'mp4',
},
}]
_WEBPAGE_TESTS = [{
'url': 'https://www.multimedios.com/video/costa-rica-tv-en-vivo/v2616',
'info_dict': {
'id': '5a7b1e63a8da282c34d65445',
'title': 're:mmtv-costarica',
'description': 'mmtv-costarica',
'thumbnail': 're:^https?://[^?#]+5a7b1e63a8da282c34d65445',
'ext': 'mp4',
'live_status': 'is_live',
},
'params': {
'skip_download': 'Livestream'
},
}, {
'url': 'https://www.multimedios.com/television/clases-de-llaves-y-castigos-quien-sabe-mas',
'md5': 'de31f0b1ecc321fb35bf22d58734ea40',
'info_dict': {
'id': '63731bab8ec9b308a2c9ed28',
'title': 'Clases de llaves y castigos ¿Quién sabe más?',
'description': 'md5:1b49aa1ee5a4b32fbd66104b2d629e9d',
'thumbnail': 're:^https?://[^?#]+63731bab8ec9b308a2c9ed28',
'ext': 'mp4',
},
}, {
'url': 'https://www.americatv.com.pe/videos/esto-es-guerra/facundo-gonzalez-sufrio-fuerte-golpe-durante-competencia-frente-hugo-garcia-eeg-noticia-139120',
'info_dict': {
'id': '63756df1c638b008a5659dec',
'title': 'Facundo González sufrió fuerte golpe durante competencia frente a Hugo García en EEG',
'description': 'md5:9490c034264afd756eef7b2c3adee69e',
'thumbnail': 're:^https?://[^?#]+63756df1c638b008a5659dec',
'ext': 'mp4',
},
}, {
'url': 'https://www.americatv.com.pe/videos/al-fondo-hay-sitio/nuevas-lomas-town-bernardo-mata-se-enfrento-sujeto-luchar-amor-macarena-noticia-139083',
'info_dict': {
'id': '637307669609130f74cd3a6e',
'title': 'Las Nuevas Lomas Town: Bernardo De La Mata se enfrentó a sujeto para luchar por el amor de Macarena',
'description': 'md5:60d71772f1e1496923539ae58aa17124',
'thumbnail': 're:^https?://[^?#]+637307669609130f74cd3a6e',
'ext': 'mp4',
},
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
for mobj in re.finditer(r'<script[^>]+>[^>]*playerMdStream.mdstreamVideo\(\s*[\'"](?P<video_id>\w+)', webpage):
yield f'https://mdstrm.com/embed/{mobj.group("video_id")}'
yield from re.findall(
r'<iframe[^>]src\s*=\s*"(https://mdstrm.com/[\w-]+/\w+)', webpage)
for mobj in re.finditer(
r'''(?x)
<(?:div|ps-mediastream)[^>]+
class\s*=\s*"[^"]*MediaStreamVideoPlayer[^"]*"[^>]+
data-video-id\s*=\s*"(?P<video_id>\w+)\s*"
(?:\s*data-video-type\s*=\s*"(?P<video_type>[^"]+))?
''', webpage):
video_type = 'live-stream' if mobj.group('video_type') == 'live' else 'embed'
yield f'https://mdstrm.com/{video_type}/{mobj.group("video_id")}'
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
if 'Debido a tu ubicación no puedes ver el contenido' in webpage:
self.raise_geo_restricted()
player_config = self._search_json(r'window.MDSTRM.OPTIONS\s*=', webpage, 'metadata', video_id)
formats, subtitles = [], {}
for video_format in player_config['src']:
if video_format == 'hls':
fmts, subs = self._extract_m3u8_formats_and_subtitles(player_config['src'][video_format], video_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif video_format == 'mpd':
fmts, subs = self._extract_mpd_formats_and_subtitles(player_config['src'][video_format], video_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
formats.append({
'url': player_config['src'][video_format],
})
return {
'id': video_id,
'title': self._og_search_title(webpage) or player_config.get('title'),
'description': self._og_search_description(webpage),
'formats': formats,
'subtitles': subtitles,
'is_live': player_config.get('type') == 'live',
'thumbnail': self._og_search_thumbnail(webpage),
}
class WinSportsVideoIE(InfoExtractor):
_VALID_URL = r'https?://www\.winsports\.co/videos/(?P<display_id>[\w-]+)-(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.winsports.co/videos/siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco-60536',
'info_dict': {
'id': '62dc8357162c4b0821fcfb3c',
'display_id': 'siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco',
'title': '¡Siempre Castellanos! Gran atajada del portero \'cardenal\' para evitar la caída de su arco',
'description': 'md5:eb811b2b2882bdc59431732c06b905f2',
'thumbnail': r're:^https?://[^?#]+62dc8357162c4b0821fcfb3c',
'ext': 'mp4',
},
}, {
'url': 'https://www.winsports.co/videos/observa-aqui-los-goles-del-empate-entre-tolima-y-nacional-60548',
'info_dict': {
'id': '62dcb875ef12a5526790b552',
'display_id': 'observa-aqui-los-goles-del-empate-entre-tolima-y-nacional',
'title': 'Observa aquí los goles del empate entre Tolima y Nacional',
'description': 'md5:b19402ba6e46558b93fd24b873eea9c9',
'thumbnail': r're:^https?://[^?#]+62dcb875ef12a5526790b552',
'ext': 'mp4',
},
}]
def _real_extract(self, url):
display_id, video_id = self._match_valid_url(url).group('display_id', 'id')
webpage = self._download_webpage(url, display_id)
media_setting_json = self._search_json(
r'<script\s*[^>]+data-drupal-selector="drupal-settings-json">', webpage, 'drupal-setting-json', display_id)
mediastream_id = media_setting_json['settings']['mediastream_formatter'][video_id]['mediastream_id']
return self.url_result(
f'https://mdstrm.com/embed/{mediastream_id}', MediaStreamIE, video_id, url_transparent=True,
display_id=display_id, video_title=clean_html(get_element_html_by_class('title-news', webpage)))
yt_dlp/extractor/mixch.py
+4 -6
View File
@@ -1,8 +1,5 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
traverse_obj,
)
from ..utils import UserNotLive, traverse_obj
class MixchIE(InfoExtractor):
@@ -33,7 +30,7 @@ class MixchIE(InfoExtractor):
initial_js_state = self._parse_json(self._search_regex(
r'(?m)^\s*window\.__INITIAL_JS_STATE__\s*=\s*(\{.+?\});\s*$', webpage, 'initial JS state'), video_id)
if not initial_js_state.get('liveInfo'):
raise ExtractorError('Livestream has ended.', expected=True)
raise UserNotLive(video_id=video_id)
return {
'id': video_id,
@@ -45,7 +42,8 @@ class MixchIE(InfoExtractor):
'uploader_id': video_id,
'formats': [{
'format_id': 'hls',
'url': traverse_obj(initial_js_state, ('liveInfo', 'hls')) or 'https://d1hd0ww6piyb43.cloudfront.net/hls/torte_%s.m3u8' % video_id,
'url': (traverse_obj(initial_js_state, ('liveInfo', 'hls'))
or f'https://d1hd0ww6piyb43.cloudfront.net/hls/torte_{video_id}.m3u8'),
'ext': 'mp4',
'protocol': 'm3u8',
}],
yt_dlp/extractor/naver.py
+17 -9
View File
@@ -8,6 +8,7 @@ from ..utils import (
clean_html,
dict_get,
int_or_none,
join_nonempty,
merge_dicts,
parse_duration,
traverse_obj,
@@ -72,13 +73,11 @@ class NaverBaseIE(InfoExtractor):
def get_subs(caption_url):
if re.search(self._CAPTION_EXT_RE, caption_url):
return [{
'url': replace_ext(caption_url, 'ttml'),
}, {
'url': replace_ext(caption_url, 'vtt'),
}]
else:
return [{'url': caption_url}]
return [
replace_ext(caption_url, 'ttml'),
replace_ext(caption_url, 'vtt'),
]
return [caption_url]
automatic_captions = {}
subtitles = {}
@@ -87,7 +86,13 @@ class NaverBaseIE(InfoExtractor):
if not caption_url:
continue
sub_dict = automatic_captions if caption.get('type') == 'auto' else subtitles
sub_dict.setdefault(dict_get(caption, ('locale', 'language')), []).extend(get_subs(caption_url))
lang = caption.get('locale') or join_nonempty('language', 'country', from_dict=caption) or 'und'
if caption.get('type') == 'fan':
lang += '_fan%d' % next(i for i in itertools.count(1) if f'{lang}_fan{i}' not in sub_dict)
sub_dict.setdefault(lang, []).extend({
'url': sub_url,
'name': join_nonempty('label', 'fanName', from_dict=caption, delim=' - '),
} for sub_url in get_subs(caption_url))
user = meta.get('user', {})
@@ -254,7 +259,7 @@ class NaverLiveIE(InfoExtractor):
class NaverNowIE(NaverBaseIE):
IE_NAME = 'navernow'
_VALID_URL = r'https?://now\.naver\.com/s/now\.(?P<id>[0-9]+)'
_VALID_URL = r'https?://now\.naver\.com/s/now\.(?P<id>\w+)'
_API_URL = 'https://apis.naver.com/now_web/oldnow_web/v4'
_TESTS = [{
'url': 'https://now.naver.com/s/now.4759?shareReplayId=26331132#replay=',
@@ -313,6 +318,9 @@ class NaverNowIE(NaverBaseIE):
'title': '아이키의 떰즈업',
},
'playlist_mincount': 101,
}, {
'url': 'https://now.naver.com/s/now.kihyunplay?shareReplayId=30573291#replay',
'only_matching': True,
}]
def _extract_replay(self, show_id, replay_id):
yt_dlp/extractor/nbc.py
+149 -103
View File
@@ -3,29 +3,31 @@ import json
import re
from .common import InfoExtractor
from .theplatform import ThePlatformIE
from .theplatform import ThePlatformIE, default_ns
from .adobepass import AdobePassIE
from ..compat import compat_urllib_parse_unquote
from ..utils import (
ExtractorError,
HEADRequest,
RegexNotFoundError,
UserNotLive,
clean_html,
int_or_none,
parse_age_limit,
parse_duration,
RegexNotFoundError,
smuggle_url,
str_or_none,
traverse_obj,
try_get,
unified_strdate,
unescapeHTML,
unified_timestamp,
update_url_query,
url_basename,
variadic,
xpath_attr,
)
class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
_VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>n?\d+))'
_VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>(?:NBCE|n)?\d+))'
_TESTS = [
{
@@ -38,10 +40,18 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'timestamp': 1424246400,
'upload_date': '20150218',
'uploader': 'NBCU-COM',
'episode': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
'episode_number': 86,
'season': 'Season 2',
'season_number': 2,
'series': 'Tonight Show: Jimmy Fallon',
'duration': 237.0,
'chapters': 'count:1',
'tags': 'count:4',
'thumbnail': r're:https?://.+\.jpg',
},
'params': {
# m3u8 download
'skip_download': True,
'skip_download': 'm3u8',
},
},
{
@@ -55,11 +65,7 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'upload_date': '20141206',
'uploader': 'NBCU-COM',
},
'params': {
# m3u8 download
'skip_download': True,
},
'skip': 'Only works from US',
'skip': 'page not found',
},
{
# HLS streams requires the 'hdnea3' cookie
@@ -73,10 +79,59 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
'upload_date': '20090315',
'uploader': 'NBCU-COM',
},
'skip': 'page not found',
},
{
# manifest url does not have extension
'url': 'https://www.nbc.com/the-golden-globe-awards/video/oprah-winfrey-receives-cecil-b-de-mille-award-at-the-2018-golden-globes/3646439',
'info_dict': {
'id': '3646439',
'ext': 'mp4',
'title': 'Oprah Winfrey Receives Cecil B. de Mille Award at the 2018 Golden Globes',
'episode': 'Oprah Winfrey Receives Cecil B. de Mille Award at the 2018 Golden Globes',
'episode_number': 1,
'season': 'Season 75',
'season_number': 75,
'series': 'The Golden Globe Awards',
'description': 'Oprah Winfrey receives the Cecil B. de Mille Award at the 75th Annual Golden Globe Awards.',
'uploader': 'NBCU-COM',
'upload_date': '20180107',
'timestamp': 1515312000,
'duration': 570.0,
'tags': 'count:8',
'thumbnail': r're:https?://.+\.jpg',
'chapters': 'count:1',
},
'params': {
'skip_download': True,
'skip_download': 'm3u8',
},
},
{
# new video_id format
'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978',
'info_dict': {
'id': 'NBCE125189978',
'ext': 'mp4',
'title': 'Ben\'s First Leap | NBC\'s Quantum Leap',
'description': 'md5:a82762449b7ec4bb83291a7b355ebf8e',
'uploader': 'NBCU-COM',
'series': 'Quantum Leap',
'season': 'Season 1',
'season_number': 1,
'episode': 'Ben\'s First Leap | NBC\'s Quantum Leap',
'episode_number': 1,
'duration': 170.171,
'chapters': [],
'timestamp': 1663956155,
'upload_date': '20220923',
'tags': 'count:10',
'age_limit': 0,
'thumbnail': r're:https?://.+\.jpg',
},
'expected_warnings': ['Ignoring subtitle tracks'],
'params': {
'skip_download': 'm3u8',
},
'skip': 'Only works from US',
},
{
'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
@@ -136,6 +191,7 @@ class NBCIE(ThePlatformIE): # XXX: Do not subclass from concrete IE
query = {
'mbr': 'true',
'manifest': 'm3u',
'switch': 'HLSServiceSecure',
}
video_id = video_data['mpxGuid']
tp_path = 'NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id)
@@ -599,32 +655,36 @@ class NBCStationsIE(InfoExtractor):
_TESTS = [{
'url': 'https://www.nbclosangeles.com/news/local/large-structure-fire-in-downtown-la-prompts-smoke-odor-advisory/2968618/',
'md5': '462041d91bd762ef5a38b7d85d6dc18f',
'info_dict': {
'id': '2968618',
'ext': 'mp4',
'title': 'Large Structure Fire in Downtown LA Prompts Smoke Odor Advisory',
'description': None,
'description': 'md5:417ed3c2d91fe9d301e6db7b0942f182',
'timestamp': 1661135892,
'upload_date': '20220821',
'upload_date': '20220822',
'uploader': 'NBC 4',
'uploader_id': 'KNBC',
'channel_id': 'KNBC',
'channel': 'nbclosangeles',
},
'params': {
'skip_download': 'm3u8',
},
}, {
'url': 'https://www.telemundoarizona.com/responde/huracan-complica-reembolso-para-televidente-de-tucson/2247002/',
'md5': '0917dcf7885be1023a9220630d415f67',
'info_dict': {
'id': '2247002',
'ext': 'mp4',
'title': 'Huracán complica que televidente de Tucson reciba reembolso',
'title': 'Huracán complica que televidente de Tucson reciba reembolso',
'description': 'md5:af298dc73aab74d4fca6abfb12acb6cf',
'timestamp': 1660886507,
'upload_date': '20220819',
'uploader': 'Telemundo Arizona',
'uploader_id': 'KTAZ',
'channel_id': 'KTAZ',
'channel': 'telemundoarizona',
},
'params': {
'skip_download': 'm3u8',
},
}]
_RESOLUTIONS = {
@@ -640,51 +700,42 @@ class NBCStationsIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
nbc_data = self._search_json(
r'<script>var\s*nbc\s*=', webpage, 'NBC JSON data', video_id)
r'<script>\s*var\s+nbc\s*=', webpage, 'NBC JSON data', video_id)
pdk_acct = nbc_data.get('pdkAcct') or 'Yh1nAC'
fw_ssid = traverse_obj(nbc_data, ('video', 'fwSSID'))
fw_network_id = traverse_obj(nbc_data, ('video', 'fwNetworkID'), default='382114')
video_data = self._parse_json(self._html_search_regex(
r'data-videos="([^"]*)"', webpage, 'video data', default='{}'), video_id)
video_data = variadic(video_data)[0]
video_data.update(self._parse_json(self._html_search_regex(
r'data-meta="([^"]*)"', webpage, 'metadata', default='{}'), video_id))
video_data = self._search_json(
r'data-videos="\[', webpage, 'video data', video_id, default={}, transform_source=unescapeHTML)
video_data.update(self._search_json(
r'data-meta="', webpage, 'metadata', video_id, default={}, transform_source=unescapeHTML))
if not video_data:
raise ExtractorError('No video metadata found in webpage', expected=True)
formats = []
info, formats, subtitles = {}, [], {}
is_live = int_or_none(video_data.get('mpx_is_livestream')) == 1
query = {
'formats': 'MPEG-DASH none,M3U none,MPEG-DASH none,MPEG4,MP3',
'format': 'SMIL',
'fwsitesection': fw_ssid,
'fwNetworkID': traverse_obj(nbc_data, ('video', 'fwNetworkID'), default='382114'),
'pprofile': 'ots_desktop_html',
'sensitive': 'false',
'w': '1920',
'h': '1080',
'mode': 'LIVE' if is_live else 'on-demand',
'vpaid': 'script',
'schema': '2.0',
'sdk': 'PDK 6.1.3',
}
if video_data.get('mpx_is_livestream') == '1':
live = True
player_id = traverse_obj(
video_data, 'mpx_m3upid', ('video', 'meta', 'mpx_m3upid'), 'mpx_pid',
('video', 'meta', 'mpx_pid'), 'pid_streaming_web_medium')
query = {
'mbr': 'true',
'assetTypes': 'LegacyRelease',
'fwsitesection': fw_ssid,
'fwNetworkID': fw_network_id,
'pprofile': 'ots_desktop_html',
'sensitive': 'false',
'w': '1920',
'h': '1080',
'rnd': '1660303',
'mode': 'LIVE',
'format': 'SMIL',
'tracking': 'true',
'formats': 'M3U+none,MPEG-DASH+none,MPEG4,MP3',
'vpaid': 'script',
'schema': '2.0',
'SDK': 'PDK+6.1.3',
}
info = {
'title': f'{channel} livestream',
}
if is_live:
player_id = traverse_obj(video_data, ((None, ('video', 'meta')), (
'mpx_m3upid', 'mpx_pid', 'pid_streaming_web_medium')), get_all=False)
info['title'] = f'{channel} livestream'
else:
live = False
player_id = traverse_obj(
video_data, ('video', 'meta', 'pid_streaming_web_high'), 'pid_streaming_web_high',
('video', 'meta', 'mpx_pid'), 'mpx_pid')
player_id = traverse_obj(video_data, (
(None, ('video', 'meta')), ('pid_streaming_web_high', 'mpx_pid')), get_all=False)
date_string = traverse_obj(video_data, 'date_string', 'date_gmt')
if date_string:
@@ -692,63 +743,58 @@ class NBCStationsIE(InfoExtractor):
r'datetime="([^"]+)"', date_string, 'date string', fatal=False)
else:
date_string = traverse_obj(
nbc_data, ('dataLayer', 'adobe', 'prop70'), ('dataLayer', 'adobe', 'eVar70'),
('dataLayer', 'adobe', 'eVar59'))
nbc_data, ('dataLayer', 'adobe', ('prop70', 'eVar70', 'eVar59')), get_all=False)
video_url = traverse_obj(video_data, ('video', 'meta', 'mp4_url'), 'mp4_url')
video_url = traverse_obj(video_data, ((None, ('video', 'meta')), 'mp4_url'), get_all=False)
if video_url:
height = url_basename(video_url).split('-')[1].split('p')[0]
height = self._search_regex(r'\d+-(\d+)p', url_basename(video_url), 'height', default=None)
formats.append({
'url': video_url,
'ext': 'mp4',
'width': int_or_none(self._RESOLUTIONS.get(height)),
'height': int_or_none(height),
'format_id': f'http-{height}',
'format_id': 'http-mp4',
})
query = {
'mbr': 'true',
'assetTypes': 'LegacyRelease',
'fwsitesection': fw_ssid,
'fwNetworkID': fw_network_id,
'format': 'redirect',
'manifest': 'm3u',
'Tracking': 'true',
'Embedded': 'true',
'formats': 'MPEG4',
}
info = {
'title': video_data.get('title') or traverse_obj(
nbc_data, ('dataLayer', 'contenttitle'), ('dataLayer', 'title'),
('dataLayer', 'adobe', 'prop22'), ('dataLayer', 'id')),
'description': traverse_obj(video_data, 'summary', 'excerpt', 'video_hero_text'),
'upload_date': str_or_none(unified_strdate(date_string)),
'timestamp': int_or_none(unified_timestamp(date_string)),
}
info.update({
'title': video_data.get('title') or traverse_obj(nbc_data, (
'dataLayer', (None, 'adobe'), ('contenttitle', 'title', 'prop22')), get_all=False),
'description':
traverse_obj(video_data, 'summary', 'excerpt', 'video_hero_text')
or clean_html(traverse_obj(nbc_data, ('dataLayer', 'summary'))),
'timestamp': unified_timestamp(date_string),
})
if not player_id:
raise ExtractorError(
'No video player ID or livestream player ID found in webpage', expected=True)
smil = None
if player_id and fw_ssid:
smil = self._download_xml(
f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
note='Downloading SMIL data', query=query, fatal=is_live)
if smil:
manifest_url = xpath_attr(smil, f'.//{{{default_ns}}}video', 'src', fatal=is_live)
subtitles = self._parse_smil_subtitles(smil, default_ns)
fmts, subs = self._extract_m3u8_formats_and_subtitles(
manifest_url, video_id, 'mp4', m3u8_id='hls', fatal=is_live,
live=is_live, errnote='No HLS formats found')
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
headers = {'Origin': f'https://www.{channel}.com'}
manifest, urlh = self._download_webpage_handle(
f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
headers=headers, query=query, note='Downloading manifest')
if live:
manifest_url = self._search_regex(r'<video src="([^"]*)', manifest, 'manifest URL')
else:
manifest_url = urlh.geturl()
formats.extend(self._extract_m3u8_formats(
manifest_url, video_id, 'mp4', headers=headers, m3u8_id='hls',
fatal=live, live=live, errnote='No HLS formats found'))
if not formats:
self.raise_no_formats('No video content found in webpage', expected=True)
elif is_live:
try:
self._request_webpage(
HEADRequest(formats[0]['url']), video_id, note='Checking live status')
except ExtractorError:
raise UserNotLive(video_id=channel)
return {
'id': str_or_none(video_id),
'id': video_id,
'channel': channel,
'uploader': str_or_none(nbc_data.get('on_air_name')),
'uploader_id': str_or_none(nbc_data.get('callLetters')),
'channel_id': nbc_data.get('callLetters'),
'uploader': nbc_data.get('on_air_name'),
'formats': formats,
'is_live': live,
'subtitles': subtitles,
'is_live': is_live,
**info,
}
yt_dlp/extractor/netverse.py
+110 -5
View File
@@ -1,4 +1,6 @@
from .common import InfoExtractor
import itertools
from .common import InfoExtractor, SearchInfoExtractor
from .dailymotion import DailymotionIE
from ..utils import smuggle_url, traverse_obj
@@ -16,6 +18,26 @@ class NetverseBaseIE(InfoExtractor):
f'https://api.netverse.id/medias/api/v2/{self._ENDPOINTS[endpoint]}/{slug}/{season_id}',
display_id or slug, query=query)
def _get_comments(self, video_id):
last_page_number = None
for i in itertools.count(1):
comment_data = self._download_json(
f'https://api.netverse.id/mediadetails/api/v3/videos/comments/{video_id}',
video_id, data=b'', fatal=False, query={'page': i},
note=f'Downloading JSON comment metadata page {i}') or {}
yield from traverse_obj(comment_data, ('response', 'comments', 'data', ..., {
'id': '_id',
'text': 'comment',
'author_id': 'customer_id',
'author': ('customer', 'name'),
'author_thumbnail': ('customer', 'profile_picture'),
}))
if not last_page_number:
last_page_number = traverse_obj(comment_data, ('response', 'comments', 'last_page'))
if i >= (last_page_number or 0):
break
class NetverseIE(NetverseBaseIE):
_VALID_URL = r'https?://(?:\w+\.)?netverse\.id/(?P<type>watch|video)/(?P<display_id>[^/?#&]+)'
@@ -28,7 +50,7 @@ class NetverseIE(NetverseBaseIE):
'ext': 'mp4',
'season': 'Season 2016',
'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/T7aV31Y0eGRWBbwkK/x1080',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'episode_number': 22,
'episode': 'Episode 22',
'uploader_id': 'x2ir3vq',
@@ -51,7 +73,7 @@ class NetverseIE(NetverseBaseIE):
'ext': 'mp4',
'season': 'Season 2',
'description': 'md5:8a74f70812cca267e19ee0635f0af835',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/Thwuy1YURicFmGu0v/x1080',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'episode_number': 2,
'episode': 'Episode 2',
'view_count': int,
@@ -75,7 +97,7 @@ class NetverseIE(NetverseBaseIE):
'title': 'Tetangga Baru',
'season': 'Season 1',
'description': 'md5:23fcf70e97d461d3029d25d59b2ccfb9',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/T3Ogm1YEnnyjVKAFF/x1080',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'episode_number': 1,
'episode': 'Episode 1',
'timestamp': 1624538169,
@@ -96,7 +118,7 @@ class NetverseIE(NetverseBaseIE):
'info_dict': {
'id': 'x887jzz',
'ext': 'mp4',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TfuZ_1Y6PboJ5An_s/x1080',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'season': 'Season 1',
'episode_number': 1,
'description': 'md5:d4f627b3e7a3f9acdc55f6cdd5ea41d5',
@@ -114,6 +136,60 @@ class NetverseIE(NetverseBaseIE):
'upload_date': '20220225',
},
'skip': 'This video get Geo-blocked for some country'
}, {
# video with comments
'url': 'https://netverse.id/video/episode-1-season-2016-ok-food',
'info_dict': {
'id': 'k6hetBPiQMljSxxvAy7',
'ext': 'mp4',
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'display_id': 'episode-1-season-2016-ok-food',
'like_count': int,
'description': '',
'duration': 1471,
'age_limit': 0,
'timestamp': 1642405848,
'episode_number': 1,
'season': 'Season 2016',
'uploader_id': 'x2ir3vq',
'title': 'Episode 1 - Season 2016 - Ok Food',
'upload_date': '20220117',
'tags': [],
'view_count': int,
'episode': 'Episode 1',
'uploader': 'Net Prime',
'comment_count': int,
},
'params':{
'getcomments': True
}
}, {
# video with multiple page comment
'url': 'https://netverse.id/video/match-island-eps-1-fix',
'info_dict': {
'id': 'x8aznjc',
'ext': 'mp4',
'like_count': int,
'tags': ['Match-Island', 'Pd00111'],
'display_id': 'match-island-eps-1-fix',
'view_count': int,
'episode': 'Episode 1',
'uploader': 'Net Prime',
'duration': 4070,
'timestamp': 1653068165,
'description': 'md5:e9cf3b480ad18e9c33b999e3494f223f',
'age_limit': 0,
'title': 'Welcome To Match Island',
'upload_date': '20220520',
'episode_number': 1,
'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/[^/]+/x1080',
'uploader_id': 'x2ir3vq',
'season': 'Season 1',
'comment_count': int,
},
'params':{
'getcomments': True
}
}]
def _real_extract(self, url):
@@ -131,6 +207,7 @@ class NetverseIE(NetverseBaseIE):
'thumbnail': traverse_obj(videos, ('program_detail', 'thumbnail_image')),
'description': traverse_obj(videos, ('program_detail', 'description')),
'episode_number': videos.get('episode_order'),
'__post_extractor': self.extract_comments(display_id),
}
@@ -174,3 +251,31 @@ class NetversePlaylistIE(NetverseBaseIE):
self.parse_playlist(playlist_data['response'], playlist_id),
traverse_obj(playlist_data, ('response', 'webseries_info', 'slug')),
traverse_obj(playlist_data, ('response', 'webseries_info', 'title')))
class NetverseSearchIE(SearchInfoExtractor):
_SEARCH_KEY = 'netsearch'
_TESTS = [{
'url': 'netsearch10:tetangga',
'info_dict': {
'id': 'tetangga',
'title': 'tetangga',
},
'playlist_count': 10,
}]
def _search_results(self, query):
last_page = None
for i in itertools.count(1):
search_data = self._download_json(
'https://api.netverse.id/search/elastic/search', query,
query={'q': query, 'page': i}, note=f'Downloading page {i}')
videos = traverse_obj(search_data, ('response', 'data', ...))
for video in videos:
yield self.url_result(f'https://netverse.id/video/{video["slug"]}', NetverseIE)
last_page = last_page or traverse_obj(search_data, ('response', 'lastpage'))
if not videos or i >= (last_page or 0):
break
yt_dlp/extractor/niconico.py
+16 -10
View File
@@ -675,8 +675,8 @@ class NiconicoSeriesIE(InfoExtractor):
class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
IE_NAME = 'niconico:history'
IE_DESC = 'NicoNico user history. Requires cookies.'
_VALID_URL = r'https?://(?:www\.|sp\.)?nicovideo\.jp/my/history'
IE_DESC = 'NicoNico user history or likes. Requires cookies.'
_VALID_URL = r'https?://(?:www\.|sp\.)?nicovideo\.jp/my/(?P<id>history(?:/like)?)'
_TESTS = [{
'note': 'PC page, with /video',
@@ -694,23 +694,29 @@ class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
'note': 'mobile page, without /video',
'url': 'https://sp.nicovideo.jp/my/history',
'only_matching': True,
}, {
'note': 'PC page',
'url': 'https://www.nicovideo.jp/my/history/like',
'only_matching': True,
}, {
'note': 'Mobile page',
'url': 'https://sp.nicovideo.jp/my/history/like',
'only_matching': True,
}]
def _call_api(self, list_id, resource, query):
path = 'likes' if list_id == 'history/like' else 'watch/history'
return self._download_json(
'https://nvapi.nicovideo.jp/v1/users/me/watch/history', 'history',
f'Downloading {resource}', query=query,
headers=self._API_HEADERS)['data']
f'https://nvapi.nicovideo.jp/v1/users/me/{path}', list_id,
f'Downloading {resource}', query=query, headers=self._API_HEADERS)['data']
def _real_extract(self, url):
list_id = 'history'
list_id = self._match_id(url)
try:
mylist = self._call_api(list_id, 'list', {
'pageSize': 1,
})
mylist = self._call_api(list_id, 'list', {'pageSize': 1})
except ExtractorError as e:
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
self.raise_login_required('You have to be logged in to get your watch history')
self.raise_login_required('You have to be logged in to get your history')
raise
return self.playlist_result(self._entries(list_id), list_id, **self._parse_owner(mylist))
yt_dlp/extractor/noice.py
+116
View File
@@ -0,0 +1,116 @@
from .common import InfoExtractor
from ..utils import (
clean_html,
determine_ext,
int_or_none,
parse_iso8601,
traverse_obj,
variadic,
)
class NoicePodcastIE(InfoExtractor):
_VALID_URL = r'https?://open\.noice\.id/content/(?P<id>[a-fA-F0-9-]+)'
_TESTS = [{
'url': 'https://open.noice.id/content/7694bb04-ff0f-40fa-a60b-5b39f29584b2',
'info_dict': {
'id': '7694bb04-ff0f-40fa-a60b-5b39f29584b2',
'ext': 'm4a',
'season': 'Season 1',
'description': 'md5:58d1274e6857b6fbbecf47075885380d',
'release_date': '20221115',
'timestamp': 1668496642,
'season_number': 1,
'upload_date': '20221115',
'release_timestamp': 1668496642,
'title': 'Eps 1. Belajar dari Wishnutama: Kreatif Bukan Followers! (bersama Wishnutama)',
'modified_date': '20221121',
'categories': ['Bisnis dan Keuangan'],
'duration': 3567,
'modified_timestamp': 1669030647,
'thumbnail': 'https://images.noiceid.cc/catalog/content-1668496302560',
'channel_id': '9dab1024-5b92-4265-ae1c-63da87359832',
'like_count': int,
'channel': 'Noice Space Talks',
'comment_count': int,
'dislike_count': int,
'channel_follower_count': int,
}
}, {
'url': 'https://open.noice.id/content/222134e4-99f2-456f-b8a2-b8be404bf063',
'info_dict': {
'id': '222134e4-99f2-456f-b8a2-b8be404bf063',
'ext': 'm4a',
'release_timestamp': 1653488220,
'description': 'md5:35074f6190cef52b05dd133bb2ef460e',
'upload_date': '20220525',
'timestamp': 1653460637,
'release_date': '20220525',
'thumbnail': 'https://images.noiceid.cc/catalog/content-1653460337625',
'title': 'Eps 1: Dijodohin Sama Anak Pak RT',
'modified_timestamp': 1669030647,
'season_number': 1,
'modified_date': '20221121',
'categories': ['Cerita dan Drama'],
'duration': 1830,
'season': 'Season 1',
'channel_id': '60193f6b-d24d-4b23-913b-ceed5a731e74',
'dislike_count': int,
'like_count': int,
'comment_count': int,
'channel': 'Dear Jerome',
'channel_follower_count': int,
}
}]
def _get_formats_and_subtitles(self, media_url, video_id):
formats, subtitles = [], {}
for url in variadic(media_url):
ext = determine_ext(url)
if ext == 'm3u8':
fmts, subs = self._extract_m3u8_formats_and_subtitles(url, video_id)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
formats.append({
'url': url,
'ext': 'mp3',
'vcodec': 'none',
'acodec': 'mp3',
})
return formats, subtitles
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
nextjs_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['contentDetails']
media_url_list = traverse_obj(nextjs_data, (('rawContentUrl', 'url'), ))
formats, subtitles = self._get_formats_and_subtitles(media_url_list, display_id)
return {
'id': nextjs_data.get('id') or display_id,
'title': nextjs_data.get('title') or self._html_search_meta('og:title', webpage),
'formats': formats,
'subtitles': subtitles,
'description': (nextjs_data.get('description') or clean_html(nextjs_data.get('htmlDescription'))
or self._html_search_meta(['description', 'og:description'], webpage)),
'thumbnail': nextjs_data.get('image') or self._html_search_meta('og:image', webpage),
'timestamp': parse_iso8601(nextjs_data.get('createdAt')),
'release_timestamp': parse_iso8601(nextjs_data.get('publishedAt')),
'modified_timestamp': parse_iso8601(
nextjs_data.get('updatedAt') or self._html_search_meta('og:updated_time', webpage)),
'duration': int_or_none(nextjs_data.get('duration')),
'categories': traverse_obj(nextjs_data, ('genres', ..., 'name')),
'season': nextjs_data.get('seasonName'),
'season_number': int_or_none(nextjs_data.get('seasonNumber')),
'channel': traverse_obj(nextjs_data, ('catalog', 'title')),
'channel_id': traverse_obj(nextjs_data, ('catalog', 'id'), 'catalogId'),
**traverse_obj(nextjs_data, ('meta', 'aggregations', {
'like_count': 'likes',
'dislike_count': 'dislikes',
'comment_count': 'comments',
'channel_follower_count': 'followers',
}))
}
yt_dlp/extractor/nosnl.py
+27 -7
View File
@@ -3,7 +3,7 @@ from ..utils import parse_duration, parse_iso8601, traverse_obj
class NOSNLArticleIE(InfoExtractor):
_VALID_URL = r'https?://nos\.nl/((?!video)(\w+/)?\w+/)\d+-(?P<display_id>[\w-]+)'
_VALID_URL = r'https?://nos\.nl/(?P<type>video|(\w+/)?\w+)/?\d+-(?P<display_id>[\w-]+)'
_TESTS = [
{
# only 1 video
@@ -22,13 +22,14 @@ class NOSNLArticleIE(InfoExtractor):
'info_dict': {
'id': '2440409',
'title': 'Vannacht sliepen weer enkele honderden asielzoekers in Ter Apel buiten',
'description': 'Er werd wel geprobeerd om kwetsbare migranten onderdak te bieden, zegt het COA.',
'description': 'md5:72b1e1674d798460e79d78fa37e9f56d',
'tags': ['aanmeldcentrum', 'Centraal Orgaan opvang asielzoekers', 'COA', 'asielzoekers', 'Ter Apel'],
'modified_timestamp': 1660452773,
'modified_date': '20220814',
'upload_date': '20220813',
'thumbnail': 'https://cdn.nos.nl/image/2022/07/18/880346/1024x576a.jpg',
'timestamp': 1660401384,
'categories': ['Regionaal nieuws', 'Binnenland'],
},
'playlist_count': 2,
}, {
@@ -37,20 +38,37 @@ class NOSNLArticleIE(InfoExtractor):
'info_dict': {
'id': '2440789',
'title': 'Wekdienst 16/8: Groningse acties tien jaar na zware aardbeving • Femke Bol in actie op EK atletiek ',
'description': 'Nieuws, weer, verkeer: met dit overzicht begin je geïnformeerd aan de dag.',
'description': 'md5:0bd277ed7a44fc15cb12a9d27d8f6641',
'tags': ['wekdienst'],
'modified_date': '20220816',
'modified_timestamp': 1660625449,
'timestamp': 1660625449,
'upload_date': '20220816',
'thumbnail': 'https://cdn.nos.nl/image/2022/08/16/888178/1024x576a.jpg',
'categories': ['Binnenland', 'Buitenland'],
},
'playlist_count': 2,
}, {
# video url
'url': 'https://nos.nl/video/2452718-xi-en-trudeau-botsen-voor-de-camera-op-g20-top-je-hebt-gelekt',
'info_dict': {
'id': '2452718',
'title': 'Xi en Trudeau botsen voor de camera op G20-top: \'Je hebt gelekt\'',
'modified_date': '20221117',
'description': 'md5:61907dac576f75c11bf8ffffd4a3cc0f',
'tags': ['Xi', 'Trudeau', 'G20', 'indonesié'],
'upload_date': '20221117',
'thumbnail': 'https://cdn.nos.nl/image/2022/11/17/916155/1024x576a.jpg',
'modified_timestamp': 1668663388,
'timestamp': 1668663388,
'categories': ['Buitenland'],
},
'playlist_mincount': 1,
}
]
def _entries(self, nextjs_json, display_id):
for item in nextjs_json['items']:
for item in nextjs_json:
if item.get('type') == 'video':
formats, subtitle = self._extract_m3u8_formats_and_subtitles(
traverse_obj(item, ('source', 'url')), display_id, ext='mp4')
@@ -77,13 +95,14 @@ class NOSNLArticleIE(InfoExtractor):
}
def _real_extract(self, url):
display_id = self._match_valid_url(url).group('display_id')
site_type, display_id = self._match_valid_url(url).group('type', 'display_id')
webpage = self._download_webpage(url, display_id)
nextjs_json = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['data']
return {
'_type': 'playlist',
'entries': self._entries(nextjs_json, display_id),
'entries': self._entries(
[nextjs_json['video']] if site_type == 'video' else nextjs_json['items'], display_id),
'id': str(nextjs_json['id']),
'title': nextjs_json.get('title') or self._html_search_meta(['title', 'og:title', 'twitter:title'], webpage),
'description': (nextjs_json.get('description')
@@ -91,5 +110,6 @@ class NOSNLArticleIE(InfoExtractor):
'tags': nextjs_json.get('keywords'),
'modified_timestamp': parse_iso8601(nextjs_json.get('modifiedAt')),
'thumbnail': nextjs_json.get('shareImageSrc') or self._html_search_meta(['og:image', 'twitter:image'], webpage),
'timestamp': parse_iso8601(nextjs_json.get('publishedAt'))
'timestamp': parse_iso8601(nextjs_json.get('publishedAt')),
'categories': traverse_obj(nextjs_json, ('categories', ..., 'label')),
}
yt_dlp/extractor/odnoklassniki.py
+33
View File
@@ -11,6 +11,7 @@ from ..utils import (
int_or_none,
qualities,
smuggle_url,
traverse_obj,
unescapeHTML,
unified_strdate,
unsmuggle_url,
@@ -153,6 +154,26 @@ class OdnoklassnikiIE(InfoExtractor):
'title': 'Быковское крещение',
'duration': 3038.181,
},
'skip': 'HTTP Error 400',
}, {
'note': 'subtitles',
'url': 'https://ok.ru/video/4249587550747',
'info_dict': {
'id': '4249587550747',
'ext': 'mp4',
'title': 'Small Country An African Childhood (2020) (1080p) +subtitle',
'uploader': 'Sunflower Movies',
'uploader_id': '595802161179',
'upload_date': '20220816',
'duration': 6728,
'age_limit': 0,
'thumbnail': r're:^https?://i\.mycdn\.me/videoPreview\?.+',
'like_count': int,
'subtitles': dict,
},
'params': {
'skip_download': True,
},
}, {
'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
'only_matching': True,
@@ -202,6 +223,7 @@ class OdnoklassnikiIE(InfoExtractor):
'like_count': 0,
'duration': 10444,
},
'skip': 'Site no longer embeds',
}]
@classmethod
@@ -294,6 +316,16 @@ class OdnoklassnikiIE(InfoExtractor):
like_count = int_or_none(metadata.get('likeCount'))
subtitles = {}
for sub in traverse_obj(metadata, ('movie', 'subtitleTracks', ...), expected_type=dict):
sub_url = sub.get('url')
if not sub_url:
continue
subtitles.setdefault(sub.get('language') or 'en', []).append({
'url': sub_url,
'ext': 'vtt',
})
info = {
'id': video_id,
'title': title,
@@ -305,6 +337,7 @@ class OdnoklassnikiIE(InfoExtractor):
'like_count': like_count,
'age_limit': age_limit,
'start_time': start_time,
'subtitles': subtitles,
}
# pladform
yt_dlp/extractor/oneplace.py
+43
View File
@@ -0,0 +1,43 @@
from .common import InfoExtractor
class OnePlacePodcastIE(InfoExtractor):
_VALID_URL = r'https?://www\.oneplace\.com/[\w]+/[^/]+/listen/[\w-]+-(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.oneplace.com/ministries/a-daily-walk/listen/living-in-the-last-days-part-2-958461.html',
'info_dict': {
'id': '958461',
'ext': 'mp3',
'title': 'Living in the Last Days Part 2 | A Daily Walk with John Randall',
'description': 'md5:fbb8f1cf21447ac54ecaa2887fc20c6e',
}
}, {
'url': 'https://www.oneplace.com/ministries/ankerberg-show/listen/ep-3-relying-on-the-constant-companionship-of-the-holy-spirit-part-2-922513.html',
'info_dict': {
'id': '922513',
'ext': 'mp3',
'description': 'md5:8b810b4349aa40a5d033b4536fe428e1',
'title': 'md5:ce10f7d8d5ddcf485ed8905ef109659d',
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
return {
'id': video_id,
'url': self._search_regex((
r'mp3-url\s*=\s*"([^"]+)',
r'<div[^>]+id\s*=\s*"player"[^>]+data-media-url\s*=\s*"(?P<media_url>[^"]+)',
), webpage, 'media url'),
'ext': 'mp3',
'vcodec': 'none',
'title': self._html_search_regex((
r'<div[^>]class\s*=\s*"details"[^>]+>[^<]<h2[^>]+>(?P<content>[^>]+)>',
self._meta_regex('og:title'), self._meta_regex('title'),
), webpage, 'title', group='content', default=None),
'description': self._html_search_regex(
r'<div[^>]+class="[^"]+epDesc"[^>]*>\s*(?P<desc>.+?)\s*</div>',
webpage, 'description', default=None),
}
yt_dlp/extractor/peekvids.py
+152 -38
View File
@@ -1,71 +1,128 @@
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
get_element_by_class,
int_or_none,
merge_dicts,
url_or_none,
)
class PeekVidsIE(InfoExtractor):
class PeekVidsBaseIE(InfoExtractor):
def _real_extract(self, url):
domain, video_id = self._match_valid_url(url).group('domain', 'id')
webpage = self._download_webpage(url, video_id, expected_status=429)
if '>Rate Limit Exceeded' in webpage:
raise ExtractorError(
f'You are suspected as a bot. Wait, or pass the captcha on the site and provide cookies. {self._login_hint()}',
video_id=video_id, expected=True)
title = self._html_search_regex(r'(?s)<h1\b[^>]*>(.+?)</h1>', webpage, 'title')
display_id = video_id
video_id = self._search_regex(r'(?s)<video\b[^>]+\bdata-id\s*=\s*["\']?([\w-]+)', webpage, 'short video ID')
srcs = self._download_json(
f'https://www.{domain}/v-alt/{video_id}', video_id,
note='Downloading list of source files')
formats = []
for k, v in srcs.items():
f_url = url_or_none(v)
if not f_url:
continue
height = self._search_regex(r'^data-src(\d{3,})$', k, 'height', default=None)
if not height:
continue
formats.append({
'url': f_url,
'format_id': height,
'height': int_or_none(height),
})
if not formats:
formats = [{'url': url} for url in srcs.values()]
info = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={})
info.pop('url', None)
# may not have found the thumbnail if it was in a list in the ld+json
info.setdefault('thumbnail', self._og_search_thumbnail(webpage))
detail = (get_element_by_class('detail-video-block', webpage)
or get_element_by_class('detail-block', webpage) or '')
info['description'] = self._html_search_regex(
rf'(?s)(.+?)(?:{re.escape(info.get("description", ""))}\s*<|<ul\b)',
detail, 'description', default=None) or None
info['title'] = re.sub(r'\s*[,-][^,-]+$', '', info.get('title') or title) or self._generic_title(url)
def cat_tags(name, html):
l = self._html_search_regex(
rf'(?s)<span\b[^>]*>\s*{re.escape(name)}\s*:\s*</span>(.+?)</li>',
html, name, default='')
return list(filter(None, re.split(r'\s+', l)))
return merge_dicts({
'id': video_id,
'display_id': display_id,
'age_limit': 18,
'formats': formats,
'categories': cat_tags('Categories', detail),
'tags': cat_tags('Tags', detail),
'uploader': self._html_search_regex(r'[Uu]ploaded\s+by\s(.+?)"', webpage, 'uploader', default=None),
}, info)
class PeekVidsIE(PeekVidsBaseIE):
_VALID_URL = r'''(?x)
https?://(?:www\.)?peekvids\.com/
https?://(?:www\.)?(?P<domain>peekvids\.com)/
(?:(?:[^/?#]+/){2}|embed/?\?(?:[^#]*&)?v=)
(?P<id>[^/?&#]*)
'''
_TESTS = [{
'url': 'https://peekvids.com/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp/BSyLMbN0YCd',
'md5': 'a00940646c428e232407e3e62f0e8ef5',
'md5': '2ff6a357a9717dc9dc9894b51307e9a2',
'info_dict': {
'id': 'BSyLMbN0YCd',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp, SEXYhub',
'id': '1262717',
'display_id': 'BSyLMbN0YCd',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
'ext': 'mp4',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'Watch Dane Jones - Cute redhead with perfect tits with Mini Vamp (7 min), uploaded by SEXYhub.com',
'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
'timestamp': 1642579329,
'upload_date': '20220119',
'duration': 416,
'view_count': int,
'age_limit': 18,
'uploader': 'SEXYhub.com',
'categories': list,
'tags': list,
},
}]
_DOMAIN = 'www.peekvids.com'
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
short_video_id = self._html_search_regex(r'<video [^>]*data-id="(.+?)"', webpage, 'short video ID')
srcs = self._download_json(
f'https://{self._DOMAIN}/v-alt/{short_video_id}', video_id,
note='Downloading list of source files')
formats = [{
'url': url,
'ext': 'mp4',
'format_id': name[8:],
} for name, url in srcs.items() if len(name) > 8 and name.startswith('data-src')]
if not formats:
formats = [{'url': url} for url in srcs.values()]
info = self._search_json_ld(webpage, video_id, expected_type='VideoObject')
info.update({
'id': video_id,
'age_limit': 18,
'formats': formats,
})
return info
class PlayVidsIE(PeekVidsIE): # XXX: Do not subclass from concrete IE
_VALID_URL = r'https?://(?:www\.)?playvids\.com/(?:embed/|[^/]{2}/)?(?P<id>[^/?#]*)'
class PlayVidsIE(PeekVidsBaseIE):
_VALID_URL = r'https?://(?:www\.)?(?P<domain>playvids\.com)/(?:embed/|\w\w?/)?(?P<id>[^/?#]*)'
_TESTS = [{
'url': 'https://www.playvids.com/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
'md5': 'cd7dfd8a2e815a45402369c76e3c1825',
'md5': '2f12e50213dd65f142175da633c4564c',
'info_dict': {
'id': 'U3pBrYhsjXM',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp, SEXYhub',
'id': '1978030',
'display_id': 'U3pBrYhsjXM',
'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
'ext': 'mp4',
'thumbnail': r're:^https?://.*\.jpg$',
'description': 'Watch Dane Jones - Cute redhead with perfect tits with Mini Vamp video in HD, uploaded by SEXYhub.com',
'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
'timestamp': 1640435839,
'upload_date': '20211225',
'duration': 416,
'view_count': int,
'age_limit': 18,
'uploader': 'SEXYhub.com',
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/es/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
@@ -73,5 +130,62 @@ class PlayVidsIE(PeekVidsIE): # XXX: Do not subclass from concrete IE
}, {
'url': 'https://www.playvids.com/embed/U3pBrYhsjXM',
'only_matching': True,
}, {
'url': 'https://www.playvids.com/bKmGLe3IwjZ/sv/brazzers-800-phone-sex-madison-ivy-always-on-the-line',
'md5': 'e783986e596cafbf46411a174ab42ba6',
'info_dict': {
'id': '762385',
'display_id': 'bKmGLe3IwjZ',
'ext': 'mp4',
'title': 'Brazzers - 1 800 Phone Sex: Madison Ivy Always On The Line 6',
'description': 'md5:bdcd2db2b8ad85831a491d7c8605dcef',
'timestamp': 1516958544,
'upload_date': '20180126',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 480,
'uploader': 'Brazzers',
'age_limit': 18,
'view_count': int,
'age_limit': 18,
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/v/47iUho33toY',
'md5': 'b056b5049d34b648c1e86497cf4febce',
'info_dict': {
'id': '700621',
'display_id': '47iUho33toY',
'ext': 'mp4',
'title': 'KATEE OWEN STRIPTIASE IN SEXY RED LINGERIE',
'description': None,
'timestamp': 1507052209,
'upload_date': '20171003',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 332,
'uploader': 'Cacerenele',
'age_limit': 18,
'view_count': int,
'categories': list,
'tags': list,
},
}, {
'url': 'https://www.playvids.com/z3_7iwWCmqt/sexy-teen-filipina-striptease-beautiful-pinay-bargirl-strips-and-dances',
'md5': 'efa09be9f031314b7b7e3bc6510cd0df',
'info_dict': {
'id': '1523518',
'display_id': 'z3_7iwWCmqt',
'ext': 'mp4',
'title': 'SEXY TEEN FILIPINA STRIPTEASE - Beautiful Pinay Bargirl Strips and Dances',
'description': None,
'timestamp': 1607470323,
'upload_date': '20201208',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 593,
'uploader': 'yorours',
'age_limit': 18,
'view_count': int,
'categories': list,
'tags': list,
},
}]
_DOMAIN = 'www.playvids.com'
yt_dlp/extractor/pinterest.py
+102 -51
View File
@@ -1,19 +1,24 @@
import json
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
determine_ext,
float_or_none,
int_or_none,
try_get,
str_or_none,
strip_or_none,
traverse_obj,
unified_timestamp,
url_or_none,
)
class PinterestBaseIE(InfoExtractor):
_VALID_URL_BASE = r'https?://(?:[^/]+\.)?pinterest\.(?:com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)'
_VALID_URL_BASE = r'''(?x)
https?://(?:[^/]+\.)?pinterest\.(?:
com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|
dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|
co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)'''
def _call_api(self, resource, video_id, options):
return self._download_json(
@@ -24,14 +29,53 @@ class PinterestBaseIE(InfoExtractor):
def _extract_video(self, data, extract_formats=True):
video_id = data['id']
thumbnails = []
images = data.get('images')
if isinstance(images, dict):
for thumbnail_id, thumbnail in images.items():
if not isinstance(thumbnail, dict):
continue
thumbnail_url = url_or_none(thumbnail.get('url'))
if not thumbnail_url:
continue
thumbnails.append({
'url': thumbnail_url,
'width': int_or_none(thumbnail.get('width')),
'height': int_or_none(thumbnail.get('height')),
})
title = (data.get('title') or data.get('grid_title') or video_id).strip()
info = {
'title': strip_or_none(traverse_obj(data, 'title', 'grid_title', default='')),
'description': traverse_obj(data, 'seo_description', 'description'),
'timestamp': unified_timestamp(data.get('created_at')),
'thumbnails': thumbnails,
'uploader': traverse_obj(data, ('closeup_attribution', 'full_name')),
'uploader_id': str_or_none(traverse_obj(data, ('closeup_attribution', 'id'))),
'repost_count': int_or_none(data.get('repin_count')),
'comment_count': int_or_none(data.get('comment_count')),
'categories': traverse_obj(data, ('pin_join', 'visual_annotation'), expected_type=list),
'tags': traverse_obj(data, 'hashtags', expected_type=list),
}
urls = []
formats = []
duration = None
if extract_formats:
for format_id, format_dict in data['videos']['video_list'].items():
domain = data.get('domain', '')
if domain.lower() != 'uploaded by user' and traverse_obj(data, ('embed', 'src')):
if not info['title']:
info['title'] = None
return {
'_type': 'url_transparent',
'url': data['embed']['src'],
**info,
}
elif extract_formats:
video_list = traverse_obj(
data, ('videos', 'video_list'),
('story_pin_data', 'pages', ..., 'blocks', ..., 'video', 'video_list'),
expected_type=dict, get_all=False, default={})
for format_id, format_dict in video_list.items():
if not isinstance(format_dict, dict):
continue
format_url = url_or_none(format_dict.get('url'))
@@ -53,72 +97,79 @@ class PinterestBaseIE(InfoExtractor):
'duration': duration,
})
description = data.get('description') or data.get('description_html') or data.get('seo_description')
timestamp = unified_timestamp(data.get('created_at'))
def _u(field):
return try_get(data, lambda x: x['closeup_attribution'][field], compat_str)
uploader = _u('full_name')
uploader_id = _u('id')
repost_count = int_or_none(data.get('repin_count'))
comment_count = int_or_none(data.get('comment_count'))
categories = try_get(data, lambda x: x['pin_join']['visual_annotation'], list)
tags = data.get('hashtags')
thumbnails = []
images = data.get('images')
if isinstance(images, dict):
for thumbnail_id, thumbnail in images.items():
if not isinstance(thumbnail, dict):
continue
thumbnail_url = url_or_none(thumbnail.get('url'))
if not thumbnail_url:
continue
thumbnails.append({
'url': thumbnail_url,
'width': int_or_none(thumbnail.get('width')),
'height': int_or_none(thumbnail.get('height')),
})
return {
'id': video_id,
'title': title,
'description': description,
'duration': duration,
'timestamp': timestamp,
'thumbnails': thumbnails,
'uploader': uploader,
'uploader_id': uploader_id,
'repost_count': repost_count,
'comment_count': comment_count,
'categories': categories,
'tags': tags,
'formats': formats,
'duration': duration,
'webpage_url': f'https://www.pinterest.com/pin/{video_id}/',
'extractor_key': PinterestIE.ie_key(),
'extractor': PinterestIE.IE_NAME,
**info,
}
class PinterestIE(PinterestBaseIE):
_VALID_URL = r'%s/pin/(?P<id>\d+)' % PinterestBaseIE._VALID_URL_BASE
_TESTS = [{
# formats found in data['videos']
'url': 'https://www.pinterest.com/pin/664281013778109217/',
'md5': '6550c2af85d6d9f3fe3b88954d1577fc',
'info_dict': {
'id': '664281013778109217',
'ext': 'mp4',
'title': 'Origami',
'description': 'md5:b9d90ddf7848e897882de9e73344f7dd',
'description': 'md5:e29801cab7d741ea8c741bc50c8d00ab',
'duration': 57.7,
'timestamp': 1593073622,
'upload_date': '20200625',
'uploader': 'Love origami -I am Dafei',
'uploader_id': '586523688879454212',
'repost_count': 50,
'comment_count': 0,
'repost_count': int,
'comment_count': int,
'categories': list,
'tags': list,
'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
},
}, {
# formats found in data['story_pin_data']
'url': 'https://www.pinterest.com/pin/1084663891475263837/',
'md5': '069ac19919ab9e1e13fa60de46290b03',
'info_dict': {
'id': '1084663891475263837',
'ext': 'mp4',
'title': 'Gadget, Cool products, Amazon product, technology, Kitchen gadgets',
'description': 'md5:d0a4b6ae996ff0c6eed83bc869598d13',
'uploader': 'CoolCrazyGadgets',
'uploader_id': '1084664028912989237',
'upload_date': '20211003',
'timestamp': 1633246654.0,
'duration': 14.9,
'comment_count': int,
'repost_count': int,
'categories': 'count:9',
'tags': list,
'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
},
}, {
# vimeo.com embed
'url': 'https://www.pinterest.ca/pin/441282463481903715/',
'info_dict': {
'id': '111691128',
'ext': 'mp4',
'title': 'Tonite Let\'s All Make Love In London (1967)',
'description': 'md5:8190f37b3926807809ec57ec21aa77b2',
'uploader': 'Vimeo',
'uploader_id': '473792960706651251',
'upload_date': '20180120',
'timestamp': 1516409040,
'duration': 3404,
'comment_count': int,
'repost_count': int,
'categories': 'count:9',
'tags': [],
'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
'uploader_url': 'https://vimeo.com/willardandrade',
},
'params': {
'skip_download': 'm3u8',
},
}, {
'url': 'https://co.pinterest.com/pin/824721750502199491/',
yt_dlp/extractor/plutotv.py
+12 -1
View File
@@ -84,6 +84,17 @@ class PlutoTVIE(InfoExtractor):
}, {
'url': 'https://pluto.tv/it/on-demand/series/csi-vegas/episode/legacy-2021-1-1',
'only_matching': True,
},
{
'url': 'https://pluto.tv/en/on-demand/movies/attack-of-the-killer-tomatoes-1977-1-1-ptv1',
'md5': '7db56369c0da626a32d505ec6eb3f89f',
'info_dict': {
'id': '5b190c7bb0875c36c90c29c4',
'ext': 'mp4',
'title': 'Attack of the Killer Tomatoes',
'description': 'A group of scientists band together to save the world from mutated tomatoes that KILL! (1978)',
'duration': 5700,
}
}
]
@@ -103,7 +114,7 @@ class PlutoTVIE(InfoExtractor):
compat_urlparse.urljoin(first_segment_url.group(1), '0-end/master.m3u8'))
continue
first_segment_url = re.search(
r'^(https?://.*/).+\-0+\.ts$', res,
r'^(https?://.*/).+\-0+[0-1]0\.ts$', res,
re.MULTILINE)
if first_segment_url:
m3u8_urls.add(
yt_dlp/extractor/polskieradio.py
+165 -48
View File
@@ -10,6 +10,7 @@ from ..compat import (
compat_urlparse
)
from ..utils import (
determine_ext,
extract_attributes,
ExtractorError,
InAdvancePagedList,
@@ -17,6 +18,7 @@ from ..utils import (
js_to_json,
parse_iso8601,
strip_or_none,
traverse_obj,
unified_timestamp,
unescapeHTML,
url_or_none,
@@ -48,28 +50,11 @@ class PolskieRadioBaseExtractor(InfoExtractor):
yield entry
class PolskieRadioIE(PolskieRadioBaseExtractor):
_VALID_URL = r'https?://(?:www\.)?polskieradio(?:24)?\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
_TESTS = [{ # Old-style single broadcast.
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
'info_dict': {
'id': '1587943',
'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
},
'playlist': [{
'md5': '2984ee6ce9046d91fc233bc1a864a09a',
'info_dict': {
'id': '1540576',
'ext': 'mp3',
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
'timestamp': 1456594200,
'upload_date': '20160227',
'duration': 2364,
'thumbnail': r're:^https?://static\.prsa\.pl/images/.*\.jpg$'
},
}],
}, { # New-style single broadcast.
class PolskieRadioLegacyIE(PolskieRadioBaseExtractor):
# legacy sites
IE_NAME = 'polskieradio:legacy'
_VALID_URL = r'https?://(?:www\.)?polskieradio(?:24)?\.pl/\d+/\d+/[Aa]rtykul/(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.polskieradio.pl/8/2382/Artykul/2534482,Zagarysci-Poezja-jak-spoiwo',
'info_dict': {
'id': '2534482',
@@ -96,16 +81,6 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
'ext': 'mp3',
'title': 'Pogłos 29 października godz. 23:01',
},
}, {
'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
'only_matching': True,
}, {
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
'only_matching': True,
}, {
# with mp4 video
'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
'only_matching': True,
}, {
'url': 'https://polskieradio24.pl/130/4503/Artykul/2621876,Narusza-nasza-suwerennosc-Publicysci-o-uzaleznieniu-funduszy-UE-od-praworzadnosci',
'only_matching': True,
@@ -114,7 +89,9 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
webpage, urlh = self._download_webpage_handle(url, playlist_id)
if PolskieRadioIE.suitable(urlh.url):
return self.url_result(urlh.url, PolskieRadioIE, playlist_id)
content = self._search_regex(
r'(?s)<div[^>]+class="\s*this-article\s*"[^>]*>(.+?)<div[^>]+class="tags"[^>]*>',
@@ -153,23 +130,160 @@ class PolskieRadioIE(PolskieRadioBaseExtractor):
return self.playlist_result(entries, playlist_id, title, description)
class PolskieRadioCategoryIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
class PolskieRadioIE(InfoExtractor):
# new next.js sites, excluding radiokierowcow.pl
_VALID_URL = r'https?://(?:[^/]+\.)?polskieradio(?:24)?\.pl/artykul/(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
'url': 'https://jedynka.polskieradio.pl/artykul/1587943',
'info_dict': {
'id': '1587943',
'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
},
'playlist': [{
'md5': '2984ee6ce9046d91fc233bc1a864a09a',
'info_dict': {
'id': '7a85d429-5356-4def-a347-925e4ae7406b',
'ext': 'mp3',
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
},
}],
}, {
'url': 'https://trojka.polskieradio.pl/artykul/1632955',
'only_matching': True,
}, {
# with mp4 video
'url': 'https://trojka.polskieradio.pl/artykul/1634903',
'only_matching': True,
}, {
'url': 'https://jedynka.polskieradio.pl/artykul/3042436,Polityka-wschodnia-ojca-i-syna-Wladyslawa-Lokietka-i-Kazimierza-Wielkiego',
'only_matching': True,
}]
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
article_data = traverse_obj(
self._search_nextjs_data(webpage, playlist_id), ('props', 'pageProps', 'data', 'articleData'))
title = strip_or_none(article_data['title'])
description = strip_or_none(article_data.get('lead'))
entries = [{
'url': entry['file'],
'ext': determine_ext(entry.get('fileName')),
'id': self._search_regex(
r'([a-f\d]{8}-(?:[a-f\d]{4}-){3}[a-f\d]{12})', entry['file'], 'entry id'),
'title': strip_or_none(entry.get('description')) or title,
} for entry in article_data.get('attachments') or () if entry['fileType'] in ('Audio', )]
return self.playlist_result(entries, playlist_id, title, description)
class PolskieRadioAuditionIE(InfoExtractor):
# new next.js sites
IE_NAME = 'polskieradio:audition'
_VALID_URL = r'https?://(?:[^/]+\.)?polskieradio\.pl/audycj[ae]/(?P<id>\d+)'
_TESTS = [{
# articles, PR1
'url': 'https://jedynka.polskieradio.pl/audycje/5102',
'info_dict': {
'id': '5102',
'title': 'HISTORIA ŻYWA',
'title': 'Historia żywa',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 38,
}, {
'url': 'http://www.polskieradio.pl/7/4807',
# episodes, PR1
'url': 'https://jedynka.polskieradio.pl/audycje/5769',
'info_dict': {
'id': '4807',
'title': 'Vademecum 1050. rocznicy Chrztu Polski'
'id': '5769',
'title': 'AgroFakty',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 5
'playlist_mincount': 269,
}, {
# both episodes and articles, PR3
'url': 'https://trojka.polskieradio.pl/audycja/8906',
'info_dict': {
'id': '8906',
'title': 'Trójka budzi',
'thumbnail': r're:https://static\.prsa\.pl/images/.+',
},
'playlist_mincount': 722,
}]
def _call_lp3(self, path, query, video_id, note):
return self._download_json(
f'https://lp3test.polskieradio.pl/{path}', video_id, note,
query=query, headers={'x-api-key': '9bf6c5a2-a7d0-4980-9ed7-a3f7291f2a81'})
def _entries(self, playlist_id, has_episodes, has_articles):
for i in itertools.count(1) if has_episodes else []:
page = self._call_lp3(
'AudioArticle/GetListByCategoryId', {
'categoryId': playlist_id,
'PageSize': 10,
'skip': i,
'format': 400,
}, playlist_id, f'Downloading episode list page {i}')
if not traverse_obj(page, 'data'):
break
for episode in page['data']:
yield {
'id': str(episode['id']),
'url': episode['file'],
'title': episode.get('title'),
'duration': int_or_none(episode.get('duration')),
'timestamp': parse_iso8601(episode.get('datePublic')),
}
for i in itertools.count(1) if has_articles else []:
page = self._call_lp3(
'Article/GetListByCategoryId', {
'categoryId': playlist_id,
'PageSize': 9,
'skip': i,
'format': 400,
}, playlist_id, f'Downloading article list page {i}')
if not traverse_obj(page, 'data'):
break
for article in page['data']:
yield {
'_type': 'url_transparent',
'ie_key': PolskieRadioIE.ie_key(),
'id': str(article['id']),
'url': article['url'],
'title': article.get('shortTitle'),
'description': traverse_obj(article, ('description', 'lead')),
'timestamp': parse_iso8601(article.get('datePublic')),
}
def _real_extract(self, url):
playlist_id = self._match_id(url)
page_props = traverse_obj(
self._search_nextjs_data(self._download_webpage(url, playlist_id), playlist_id),
('props', 'pageProps', ('data', None)), get_all=False)
has_episodes = bool(traverse_obj(page_props, 'episodes', 'audios'))
has_articles = bool(traverse_obj(page_props, 'articles'))
return self.playlist_result(
self._entries(playlist_id, has_episodes, has_articles), playlist_id,
title=traverse_obj(page_props, ('details', 'name')),
description=traverse_obj(page_props, ('details', 'description', 'lead')),
thumbnail=traverse_obj(page_props, ('details', 'photo')))
class PolskieRadioCategoryIE(InfoExtractor):
# legacy sites
IE_NAME = 'polskieradio:category'
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
'only_matching': True
}, {
@@ -186,9 +300,6 @@ class PolskieRadioCategoryIE(InfoExtractor):
'title': 'Muzyka',
},
'playlist_mincount': 61
}, {
'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
'only_matching': True,
}, {
'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
'only_matching': True,
@@ -196,7 +307,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
@classmethod
def suitable(cls, url):
return False if PolskieRadioIE.suitable(url) else super(PolskieRadioCategoryIE, cls).suitable(url)
return False if PolskieRadioLegacyIE.suitable(url) else super().suitable(url)
def _entries(self, url, page, category_id):
content = page
@@ -209,7 +320,7 @@ class PolskieRadioCategoryIE(InfoExtractor):
if not href:
continue
yield self.url_result(
compat_urlparse.urljoin(url, href), PolskieRadioIE.ie_key(),
compat_urlparse.urljoin(url, href), PolskieRadioLegacyIE,
entry_id, entry.get('title'))
mobj = re.search(
r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
@@ -222,7 +333,9 @@ class PolskieRadioCategoryIE(InfoExtractor):
def _real_extract(self, url):
category_id = self._match_id(url)
webpage = self._download_webpage(url, category_id)
webpage, urlh = self._download_webpage_handle(url, category_id)
if PolskieRadioAuditionIE.suitable(urlh.url):
return self.url_result(urlh.url, PolskieRadioAuditionIE, category_id)
title = self._html_search_regex(
r'<title>([^<]+) - [^<]+ - [^<]+</title>',
webpage, 'title', fatal=False)
@@ -358,7 +471,7 @@ class PolskieRadioPodcastListIE(PolskieRadioPodcastBaseExtractor):
'entries': InAdvancePagedList(
get_page, math.ceil(data['itemCount'] / self._PAGE_SIZE), self._PAGE_SIZE),
'id': str(data['id']),
'title': data['title'],
'title': data.get('title'),
'description': data.get('description'),
'uploader': data.get('announcer'),
}
@@ -374,6 +487,10 @@ class PolskieRadioPodcastIE(PolskieRadioPodcastBaseExtractor):
'ext': 'mp3',
'title': 'Theresa May rezygnuje. Co dalej z brexitem?',
'description': 'md5:e41c409a29d022b70ef0faa61dbded60',
'episode': 'Theresa May rezygnuje. Co dalej z brexitem?',
'duration': 2893,
'thumbnail': 'https://static.prsa.pl/images/58649376-c8a0-4ba2-a714-78b383285f5f.jpg',
'series': 'Raport o stanie świata',
},
}]
yt_dlp/extractor/reddit.py
+88 -12
View File
@@ -1,20 +1,20 @@
import random
from urllib.parse import urlparse
import urllib.parse
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
float_or_none,
int_or_none,
traverse_obj,
try_get,
unescapeHTML,
url_or_none,
traverse_obj
)
class RedditIE(InfoExtractor):
_VALID_URL = r'https?://(?P<subdomain>[^/]+\.)?reddit(?:media)?\.com/r/(?P<slug>[^/]+/comments/(?P<id>[^/?#&]+))'
_VALID_URL = r'https?://(?P<subdomain>[^/]+\.)?reddit(?:media)?\.com/(?P<slug>(?:r|user)/[^/]+/comments/(?P<id>[^/?#&]+))'
_TESTS = [{
'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
'info_dict': {
@@ -32,6 +32,7 @@ class RedditIE(InfoExtractor):
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'videos',
},
'params': {
'skip_download': True,
@@ -55,6 +56,58 @@ class RedditIE(InfoExtractor):
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'aww',
},
}, {
# User post
'url': 'https://www.reddit.com/user/creepyt0es/comments/nip71r/i_plan_to_make_more_stickers_and_prints_check/',
'info_dict': {
'id': 'zasobba6wp071',
'ext': 'mp4',
'display_id': 'nip71r',
'title': 'I plan to make more stickers and prints! Check them out on my Etsy! Or get them through my Patreon. Links below.',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:5',
'timestamp': 1621709093,
'upload_date': '20210522',
'uploader': 'creepyt0es',
'duration': 6,
'like_count': int,
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'u_creepyt0es',
},
'params': {
'skip_download': True,
},
}, {
# videos embedded in reddit text post
'url': 'https://www.reddit.com/r/KamenRider/comments/wzqkxp/finale_kamen_rider_revice_episode_50_family_to/',
'playlist_count': 2,
'info_dict': {
'id': 'wzqkxp',
'title': 'md5:72d3d19402aa11eff5bd32fc96369b37',
},
}, {
# crossposted reddit-hosted media
'url': 'https://www.reddit.com/r/dumbfuckers_club/comments/zjjw82/cringe/',
'md5': '746180895c7b75a9d6b05341f507699a',
'info_dict': {
'id': 'a1oneun6pa5a1',
'ext': 'mp4',
'display_id': 'zjjw82',
'title': 'Cringe',
'uploader': 'Otaku-senpai69420',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'upload_date': '20221212',
'timestamp': 1670812309,
'duration': 16,
'like_count': int,
'dislike_count': int,
'comment_count': int,
'age_limit': 0,
'channel_id': 'dumbfuckers_club',
},
}, {
'url': 'https://www.reddit.com/r/videos/comments/6rrwyj',
@@ -95,17 +148,13 @@ class RedditIE(InfoExtractor):
self._set_cookie('.reddit.com', 'reddit_session', self._gen_session_id())
self._set_cookie('.reddit.com', '_options', '%7B%22pref_quarantine_optin%22%3A%20true%7D')
data = self._download_json(f'https://{subdomain}reddit.com/r/{slug}/.json', video_id, fatal=False)
data = self._download_json(f'https://{subdomain}reddit.com/{slug}/.json', video_id, fatal=False)
if not data:
# Fall back to old.reddit.com in case the requested subdomain fails
data = self._download_json(f'https://old.reddit.com/r/{slug}/.json', video_id)
data = self._download_json(f'https://old.reddit.com/{slug}/.json', video_id)
data = data[0]['data']['children'][0]['data']
video_url = data['url']
# Avoid recursing into the same reddit URL
if 'reddit.com/' in video_url and '/%s/' % video_id in video_url:
raise ExtractorError('No media found', expected=True)
over_18 = data.get('over_18')
if over_18 is True:
age_limit = 18
@@ -142,14 +191,42 @@ class RedditIE(InfoExtractor):
'thumbnails': thumbnails,
'timestamp': float_or_none(data.get('created_utc')),
'uploader': data.get('author'),
'channel_id': data.get('subreddit'),
'like_count': int_or_none(data.get('ups')),
'dislike_count': int_or_none(data.get('downs')),
'comment_count': int_or_none(data.get('num_comments')),
'age_limit': age_limit,
}
parsed_url = urllib.parse.urlparse(video_url)
# Check for embeds in text posts, or else raise to avoid recursing into the same reddit URL
if 'reddit.com' in parsed_url.netloc and f'/{video_id}/' in parsed_url.path:
entries = []
for media in traverse_obj(data, ('media_metadata', ...), expected_type=dict):
if not media.get('id') or media.get('e') != 'RedditVideo':
continue
formats = []
if media.get('hlsUrl'):
formats.extend(self._extract_m3u8_formats(
unescapeHTML(media['hlsUrl']), video_id, 'mp4', m3u8_id='hls', fatal=False))
if media.get('dashUrl'):
formats.extend(self._extract_mpd_formats(
unescapeHTML(media['dashUrl']), video_id, mpd_id='dash', fatal=False))
if formats:
entries.append({
'id': media['id'],
'display_id': video_id,
'formats': formats,
**info,
})
if entries:
return self.playlist_result(entries, video_id, info.get('title'))
raise ExtractorError('No media found', expected=True)
# Check if media is hosted on reddit:
reddit_video = traverse_obj(data, (('media', 'secure_media'), 'reddit_video'), get_all=False)
reddit_video = traverse_obj(data, (
(None, ('crosspost_parent_list', ...)), ('secure_media', 'media'), 'reddit_video'), get_all=False)
if reddit_video:
playlist_urls = [
try_get(reddit_video, lambda x: unescapeHTML(x[y]))
@@ -189,7 +266,6 @@ class RedditIE(InfoExtractor):
'duration': int_or_none(reddit_video.get('duration')),
}
parsed_url = urlparse(video_url)
if parsed_url.netloc == 'v.redd.it':
self.raise_no_formats('This video is processing', expected=True, video_id=video_id)
return {
yt_dlp/extractor/rozhlas.py
+136 -4
View File
@@ -1,8 +1,5 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
remove_start,
)
from ..utils import extract_attributes, int_or_none, remove_start, traverse_obj
class RozhlasIE(InfoExtractor):
@@ -45,3 +42,138 @@ class RozhlasIE(InfoExtractor):
'duration': duration,
'vcodec': 'none',
}
class RozhlasVltavaIE(InfoExtractor):
_VALID_URL = r'https?://(?:\w+\.rozhlas|english\.radio)\.cz/[\w-]+-(?P<id>\d+)'
_TESTS = [{
'url': 'https://wave.rozhlas.cz/papej-masicko-porcujeme-a-bilancujeme-filmy-a-serialy-ktere-letos-zabily-8891337',
'md5': 'ba2fdbc1242fc16771c7695d271ec355',
'info_dict': {
'id': 8891337,
'title': 'md5:21f99739d04ab49d8c189ec711eef4ec',
},
'playlist_count': 1,
'playlist': [{
'md5': 'ba2fdbc1242fc16771c7695d271ec355',
'info_dict': {
'id': '10520988',
'ext': 'mp3',
'title': 'Papej masíčko! Porcujeme a bilancujeme filmy a seriály, které to letos zabily',
'description': 'md5:1c6d29fb9564e1f17fc1bb83ae7da0bc',
'duration': 1574,
'artist': 'Aleš Stuchlý',
'channel_id': 'radio-wave',
},
}]
}, {
'url': 'https://wave.rozhlas.cz/poslechnete-si-neklid-podcastovy-thriller-o-vine-strachu-a-vztahu-ktery-zasel-8554744',
'info_dict': {
'id': 8554744,
'title': 'Poslechněte si Neklid. Podcastový thriller o vině, strachu a vztahu, který zašel příliš daleko',
},
'playlist_count': 5,
'playlist': [{
'md5': '93d4109cf8f40523699ae9c1d4600bdd',
'info_dict': {
'id': '9890713',
'ext': 'mp3',
'title': 'Neklid #1',
'description': '1. díl: Neklid: 1. díl',
'duration': 1025,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #1',
'chapter_number': 1,
},
}, {
'md5': 'e9763235be4a6dcf94bc8a5bac1ca126',
'info_dict': {
'id': '9890716',
'ext': 'mp3',
'title': 'Neklid #2',
'description': '2. díl: Neklid: 2. díl',
'duration': 768,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #2',
'chapter_number': 2,
},
}, {
'md5': '00b642ea94b78cc949ac84da09f87895',
'info_dict': {
'id': '9890722',
'ext': 'mp3',
'title': 'Neklid #3',
'description': '3. díl: Neklid: 3. díl',
'duration': 607,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #3',
'chapter_number': 3,
},
}, {
'md5': 'faef97b1b49da7df874740f118c19dea',
'info_dict': {
'id': '9890728',
'ext': 'mp3',
'title': 'Neklid #4',
'description': '4. díl: Neklid: 4. díl',
'duration': 621,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #4',
'chapter_number': 4,
},
}, {
'md5': '6e729fa39b647325b868d419c76f3efa',
'info_dict': {
'id': '9890734',
'ext': 'mp3',
'title': 'Neklid #5',
'description': '5. díl: Neklid: 5. díl',
'duration': 908,
'artist': 'Josef Kokta',
'channel_id': 'radio-wave',
'chapter': 'Neklid #5',
'chapter_number': 5,
},
}]
}]
def _extract_video(self, entry):
chapter_number = int_or_none(traverse_obj(entry, ('meta', 'ga', 'contentSerialPart')))
return {
'id': entry['meta']['ga']['contentId'],
'title': traverse_obj(entry, ('meta', 'ga', 'contentName')),
'description': entry.get('title'),
'duration': entry.get('duration'),
'artist': traverse_obj(entry, ('meta', 'ga', 'contentAuthor')),
'channel_id': traverse_obj(entry, ('meta', 'ga', 'contentCreator')),
'chapter': traverse_obj(entry, ('meta', 'ga', 'contentNameShort')) if chapter_number else None,
'chapter_number': chapter_number,
'formats': [{
'url': audio_link['url'],
'ext': audio_link.get('variant'),
'format_id': audio_link.get('variant'),
'abr': audio_link.get('bitrate'),
'acodec': audio_link.get('variant'),
'vcodec': 'none',
} for audio_link in entry['audioLinks']],
}
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
# FIXME: Use get_element_text_and_html_by_tag when it accepts less strict html
data = self._parse_json(extract_attributes(self._search_regex(
r'(<div class="mujRozhlasPlayer" data-player=\'[^\']+\'>)',
webpage, 'player'))['data-player'], video_id)['data']
return {
'_type': 'playlist',
'id': data.get('embedId'),
'title': traverse_obj(data, ('series', 'title')),
'entries': map(self._extract_video, data['playlist']),
}
yt_dlp/extractor/rumble.py
+84 -20
View File
@@ -4,11 +4,15 @@ import re
from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
ExtractorError,
UnsupportedError,
clean_html,
get_element_by_class,
int_or_none,
parse_count,
parse_iso8601,
traverse_obj,
unescapeHTML,
ExtractorError,
)
@@ -111,24 +115,6 @@ class RumbleEmbedIE(InfoExtractor):
}]
_WEBPAGE_TESTS = [
{
'note': 'Rumble embed',
'url': 'https://rumble.com/vdmum1-moose-the-dog-helps-girls-dig-a-snow-fort.html',
'md5': '53af34098a7f92c4e51cf0bd1c33f009',
'info_dict': {
'id': 'vb0ofn',
'ext': 'mp4',
'timestamp': 1612662578,
'uploader': 'LovingMontana',
'channel': 'LovingMontana',
'upload_date': '20210207',
'title': 'Winter-loving dog helps girls dig a snow fort ',
'channel_url': 'https://rumble.com/c/c-546523',
'thumbnail': 'https://sp.rmbl.ws/s8/1/5/f/x/x/5fxxb.OvCc.1-small-Moose-The-Dog-Helps-Girls-D.jpg',
'duration': 103,
'live_status': 'not_live',
}
},
{
'note': 'Rumble JS embed',
'url': 'https://therightscoop.com/what-does-9-plus-1-plus-1-equal-listen-to-this-audio-of-attempted-kavanaugh-assassins-call-and-youll-get-it',
@@ -200,7 +186,7 @@ class RumbleEmbedIE(InfoExtractor):
'filesize': 'size',
'width': 'w',
'height': 'h',
}, default={})
}, expected_type=lambda x: int(x) or None)
})
subtitles = {
@@ -235,6 +221,84 @@ class RumbleEmbedIE(InfoExtractor):
}
class RumbleIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?rumble\.com/(?P<id>v(?!ideos)[\w.-]+)[^/]*$'
_EMBED_REGEX = [r'<a class=video-item--a href=(?P<url>/v[\w.-]+\.html)>']
_TESTS = [{
'add_ie': ['RumbleEmbed'],
'url': 'https://rumble.com/vdmum1-moose-the-dog-helps-girls-dig-a-snow-fort.html',
'md5': '53af34098a7f92c4e51cf0bd1c33f009',
'info_dict': {
'id': 'vb0ofn',
'ext': 'mp4',
'timestamp': 1612662578,
'uploader': 'LovingMontana',
'channel': 'LovingMontana',
'upload_date': '20210207',
'title': 'Winter-loving dog helps girls dig a snow fort ',
'description': 'Moose the dog is more than happy to help with digging out this epic snow fort. Great job, Moose!',
'channel_url': 'https://rumble.com/c/c-546523',
'thumbnail': r're:https://.+\.jpg',
'duration': 103,
'like_count': int,
'view_count': int,
'live_status': 'not_live',
}
}, {
'url': 'http://www.rumble.com/vDMUM1?key=value',
'only_matching': True,
}]
_WEBPAGE_TESTS = [{
'url': 'https://rumble.com/videos?page=2',
'playlist_count': 25,
'info_dict': {
'id': 'videos?page=2',
'title': 'All videos',
'description': 'Browse videos uploaded to Rumble.com',
'age_limit': 0,
},
}, {
'url': 'https://rumble.com/live-videos',
'playlist_mincount': 19,
'info_dict': {
'id': 'live-videos',
'title': 'Live Videos',
'description': 'Live videos on Rumble.com',
'age_limit': 0,
},
}, {
'url': 'https://rumble.com/search/video?q=rumble&sort=views',
'playlist_count': 24,
'info_dict': {
'id': 'video?q=rumble&sort=views',
'title': 'Search results for: rumble',
'age_limit': 0,
},
}]
def _real_extract(self, url):
page_id = self._match_id(url)
webpage = self._download_webpage(url, page_id)
url_info = next(RumbleEmbedIE.extract_from_webpage(self._downloader, url, webpage), None)
if not url_info:
raise UnsupportedError(url)
release_ts_str = self._search_regex(
r'(?:Livestream begins|Streamed on):\s+<time datetime="([^"]+)',
webpage, 'release date', fatal=False, default=None)
view_count_str = self._search_regex(r'<span class="media-heading-info">([\d,]+) Views',
webpage, 'view count', fatal=False, default=None)
return self.url_result(
url_info['url'], ie_key=url_info['ie_key'], url_transparent=True,
view_count=parse_count(view_count_str),
release_timestamp=parse_iso8601(release_ts_str),
like_count=parse_count(get_element_by_class('rumbles-count', webpage)),
description=clean_html(get_element_by_class('media-description', webpage)),
)
class RumbleChannelIE(InfoExtractor):
_VALID_URL = r'(?P<url>https?://(?:www\.)?rumble\.com/(?:c|user)/(?P<id>[^&?#$/]+))'
yt_dlp/extractor/rutube.py
+27 -4
View File
@@ -91,12 +91,12 @@ class RutubeBaseIE(InfoExtractor):
class RutubeIE(RutubeBaseIE):
IE_NAME = 'rutube'
IE_DESC = 'Rutube videos'
_VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/(?P<id>[\da-z]{32})'
_VALID_URL = r'https?://rutube\.ru/(?:video(?:/private)?|(?:play/)?embed)/(?P<id>[\da-z]{32})'
_EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//rutube\.ru/(?:play/)?embed/[\da-z]{32}.*?)\1']
_TESTS = [{
'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
'md5': '1d24f180fac7a02f3900712e5a5764d6',
'md5': 'e33ac625efca66aba86cbec9851f2692',
'info_dict': {
'id': '3eac3b4561676c17df9132a9a1e62e3e',
'ext': 'mp4',
@@ -108,6 +108,10 @@ class RutubeIE(RutubeBaseIE):
'timestamp': 1381943602,
'upload_date': '20131016',
'age_limit': 0,
'view_count': int,
'thumbnail': 'http://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg',
'category': ['Новости и СМИ'],
},
}, {
'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
@@ -121,6 +125,24 @@ class RutubeIE(RutubeBaseIE):
}, {
'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_type=source',
'only_matching': True,
}, {
'url': 'https://rutube.ru/video/private/884fb55f07a97ab673c7d654553e0f48/?p=x2QojCumHTS3rsKHWXN8Lg',
'md5': 'd106225f15d625538fe22971158e896f',
'info_dict': {
'id': '884fb55f07a97ab673c7d654553e0f48',
'ext': 'mp4',
'title': 'Яцуноками, Nioh2',
'description': 'Nioh2: финал сражения с боссом Яцуноками',
'duration': 15,
'uploader': 'mexus',
'uploader_id': '24222106',
'timestamp': 1670646232,
'upload_date': '20221210',
'age_limit': 0,
'view_count': int,
'thumbnail': 'http://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg',
'category': ['Видеоигры'],
},
}]
@classmethod
@@ -129,8 +151,9 @@ class RutubeIE(RutubeBaseIE):
def _real_extract(self, url):
video_id = self._match_id(url)
info = self._download_and_extract_info(video_id)
info['formats'] = self._download_and_extract_formats(video_id)
query = parse_qs(url)
info = self._download_and_extract_info(video_id, query)
info['formats'] = self._download_and_extract_formats(video_id, query)
return info
yt_dlp/extractor/screencastify.py
+52
View File
@@ -0,0 +1,52 @@
import urllib.parse
from .common import InfoExtractor
from ..utils import traverse_obj, update_url_query
class ScreencastifyIE(InfoExtractor):
_VALID_URL = r'https?://watch\.screencastify\.com/v/(?P<id>[^/?#]+)'
_TESTS = [{
'url': 'https://watch.screencastify.com/v/sYVkZip3quLKhHw4Ybk8',
'info_dict': {
'id': 'sYVkZip3quLKhHw4Ybk8',
'ext': 'mp4',
'title': 'Inserting and Aligning the Case Top and Bottom',
'description': '',
'uploader': 'Paul Gunn',
'extra_param_to_segment_url': str,
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
info = self._download_json(
f'https://umbrella.svc.screencastify.com/api/umbrellaService/watch/{video_id}', video_id)
query_string = traverse_obj(info, ('manifest', 'auth', 'query'))
query = urllib.parse.parse_qs(query_string)
formats = []
dash_manifest_url = traverse_obj(info, ('manifest', 'url'))
if dash_manifest_url:
formats.extend(
self._extract_mpd_formats(
dash_manifest_url, video_id, mpd_id='dash', query=query, fatal=False))
hls_manifest_url = traverse_obj(info, ('manifest', 'hlsUrl'))
if hls_manifest_url:
formats.extend(
self._extract_m3u8_formats(
hls_manifest_url, video_id, ext='mp4', m3u8_id='hls', query=query, fatal=False))
for f in formats:
f['url'] = update_url_query(f['url'], query)
return {
'id': video_id,
'title': info.get('title'),
'description': info.get('description'),
'uploader': info.get('userName'),
'formats': formats,
'extra_param_to_segment_url': query_string,
}
yt_dlp/extractor/sibnet.py
+17
View File
@@ -0,0 +1,17 @@
from .common import InfoExtractor
class SibnetEmbedIE(InfoExtractor):
# Ref: https://help.sibnet.ru/?sibnet_video_embed
_VALID_URL = False
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//video\.sibnet\.ru/shell\.php\?.*?\bvideoid=\d+.*?)\1']
_WEBPAGE_TESTS = [{
'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
'info_dict': {
'id': 'shell', # FIXME?
'ext': 'mp4',
'age_limit': 0,
'thumbnail': 'https://video.sibnet.ru/upload/cover/video_1887072_0.jpg',
'title': 'КВН Москва не сразу строилась - Девушка впервые играет в Mortal Kombat',
}
}]
yt_dlp/extractor/slideslive.py
+520 -56
View File
@@ -1,103 +1,567 @@
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
bool_or_none,
ExtractorError,
int_or_none,
parse_qs,
smuggle_url,
try_get,
traverse_obj,
unified_timestamp,
update_url_query,
url_or_none,
xpath_text,
)
class SlidesLiveIE(InfoExtractor):
_VALID_URL = r'https?://slideslive\.com/(?P<id>[0-9]+)'
_WORKING = False
_VALID_URL = r'https?://slideslive\.com/(?:embed/(?:presentation/)?)?(?P<id>[0-9]+)'
_TESTS = [{
# video_service_name = YOUTUBE
# service_name = yoda, only XML slides info
'url': 'https://slideslive.com/38902413/gcc-ia16-backend',
'md5': 'b29fcd6c6952d0c79c5079b0e7a07e6f',
'info_dict': {
'id': 'LMtgR8ba0b0',
'id': '38902413',
'ext': 'mp4',
'title': 'GCC IA16 backend',
'description': 'Watch full version of this video at https://slideslive.com/38902413.',
'uploader': 'SlidesLive Videos - A',
'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
'timestamp': 1597615266,
'upload_date': '20170925',
}
}, {
# video_service_name = yoda
'url': 'https://slideslive.com/38935785',
'md5': '575cd7a6c0acc6e28422fe76dd4bcb1a',
'info_dict': {
'id': 'RMraDYN5ozA_',
'ext': 'mp4',
'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges',
'timestamp': 1648189972,
'upload_date': '20220325',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:42',
'chapters': 'count:41',
'duration': 1638,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# video_service_name = youtube
# service_name = yoda, /v7/ slides
'url': 'https://slideslive.com/38935785',
'info_dict': {
'id': '38935785',
'ext': 'mp4',
'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges',
'upload_date': '20211115',
'timestamp': 1636996003,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:640',
'chapters': 'count:639',
'duration': 9832,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = yoda, /v1/ slides
'url': 'https://slideslive.com/38973182/how-should-a-machine-learning-researcher-think-about-ai-ethics',
'info_dict': {
'id': '38973182',
'ext': 'mp4',
'title': 'How Should a Machine Learning Researcher Think About AI Ethics?',
'upload_date': '20220201',
'thumbnail': r're:^https?://.*\.jpg',
'timestamp': 1643728135,
'thumbnails': 'count:3',
'chapters': 'count:2',
'duration': 5889,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = youtube, only XML slides info
'url': 'https://slideslive.com/38897546/special-metaprednaska-petra-ludwiga-hodnoty-pro-lepsi-spolecnost',
'md5': '8a79b5e3d700837f40bd2afca3c8fa01',
'info_dict': {
'id': 'jmg02wCJD5M',
'display_id': '38897546',
'ext': 'mp4',
'title': 'SPECIÁL: Meta-přednáška Petra Ludwiga - Hodnoty pro lepší společnost',
'description': 'Watch full version of this video at https://slideslive.com/38897546.',
'channel_url': 'https://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw',
'channel': 'SlidesLive Videos - G1',
'channel_id': 'UCZWdAkNYFncuX0khyvhqnxw',
'uploader_id': 'UCZWdAkNYFncuX0khyvhqnxw',
'uploader': 'SlidesLive Videos - G1',
'uploader_url': 'http://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw',
'live_status': 'not_live',
'upload_date': '20160710',
'timestamp': 1618786715,
'duration': 6827,
'like_count': int,
'view_count': int,
'comment_count': int,
'channel_follower_count': int,
'age_limit': 0,
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:169',
'playable_in_embed': True,
'availability': 'unlisted',
'tags': [],
'categories': ['People & Blogs'],
'chapters': 'count:168',
},
}, {
# embed-only presentation, only XML slides info
'url': 'https://slideslive.com/embed/presentation/38925850',
'info_dict': {
'id': '38925850',
'ext': 'mp4',
'title': 'Towards a Deep Network Architecture for Structured Smoothness',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:8',
'timestamp': 1629671508,
'upload_date': '20210822',
'chapters': 'count:7',
'duration': 326,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# embed-only presentation, only JSON slides info, /v5/ slides (.png)
'url': 'https://slideslive.com/38979920/',
'info_dict': {
'id': '38979920',
'ext': 'mp4',
'title': 'MoReL: Multi-omics Relational Learning',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:7',
'timestamp': 1654714970,
'upload_date': '20220608',
'chapters': 'count:6',
'duration': 171,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v2/ slides (.jpg)
'url': 'https://slideslive.com/38954074',
'info_dict': {
'id': '38954074',
'ext': 'mp4',
'title': 'Decentralized Attribution of Generative Models',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:16',
'timestamp': 1622806321,
'upload_date': '20210604',
'chapters': 'count:15',
'duration': 306,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v4/ slides (.png)
'url': 'https://slideslive.com/38979570/',
'info_dict': {
'id': '38979570',
'ext': 'mp4',
'title': 'Efficient Active Search for Combinatorial Optimization Problems',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:9',
'timestamp': 1654714896,
'upload_date': '20220608',
'chapters': 'count:8',
'duration': 295,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v10/ slides
'url': 'https://slideslive.com/embed/presentation/38979880?embed_parent_url=https%3A%2F%2Fedit.videoken.com%2F',
'info_dict': {
'id': '38979880',
'ext': 'mp4',
'title': 'The Representation Power of Neural Networks',
'timestamp': 1654714962,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:22',
'upload_date': '20220608',
'chapters': 'count:21',
'duration': 294,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v7/ slides, 2 video slides
'url': 'https://slideslive.com/embed/presentation/38979682?embed_container_origin=https%3A%2F%2Fedit.videoken.com',
'playlist_count': 3,
'info_dict': {
'id': '38979682-playlist',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models',
},
'playlist': [{
'info_dict': {
'id': '38979682',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models',
'timestamp': 1654714920,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:30',
'upload_date': '20220608',
'chapters': 'count:31',
'duration': 272,
},
}, {
'info_dict': {
'id': '38979682-021',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 021',
'duration': 3,
'timestamp': 1654714920,
'upload_date': '20220608',
},
}, {
'info_dict': {
'id': '38979682-024',
'ext': 'mp4',
'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 024',
'duration': 4,
'timestamp': 1654714920,
'upload_date': '20220608',
},
}],
'params': {
'skip_download': 'm3u8',
},
}, {
# /v6/ slides, 1 video slide, edit.videoken.com embed
'url': 'https://slideslive.com/38979481/',
'playlist_count': 2,
'info_dict': {
'id': '38979481-playlist',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification',
},
'playlist': [{
'info_dict': {
'id': '38979481',
'ext': 'mp4',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification',
'timestamp': 1654714877,
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:43',
'upload_date': '20220608',
'chapters': 'count:43',
'duration': 315,
},
}, {
'info_dict': {
'id': '38979481-013',
'ext': 'mp4',
'title': 'How to Train Your MAML to Excel in Few-Shot Classification - Slide 013',
'duration': 3,
'timestamp': 1654714877,
'upload_date': '20220608',
},
}],
'params': {
'skip_download': 'm3u8',
},
}, {
# /v3/ slides, .jpg and .png, service_name = youtube
'url': 'https://slideslive.com/embed/38932460/',
'info_dict': {
'id': 'RTPdrgkyTiE',
'display_id': '38932460',
'ext': 'mp4',
'title': 'Active Learning for Hierarchical Multi-Label Classification',
'description': 'Watch full version of this video at https://slideslive.com/38932460.',
'channel': 'SlidesLive Videos - A',
'channel_id': 'UC62SdArr41t_-_fX40QCLRw',
'channel_url': 'https://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw',
'uploader': 'SlidesLive Videos - A',
'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
'uploader_url': 'http://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw',
'upload_date': '20200903',
'timestamp': 1602599092,
'duration': 942,
'age_limit': 0,
'live_status': 'not_live',
'playable_in_embed': True,
'availability': 'unlisted',
'categories': ['People & Blogs'],
'tags': [],
'channel_follower_count': int,
'like_count': int,
'view_count': int,
'thumbnail': r're:^https?://.*\.(?:jpg|png|webp)',
'thumbnails': 'count:21',
'chapters': 'count:20',
},
'params': {
'skip_download': 'm3u8',
},
}, {
# /v3/ slides, .png only, service_name = yoda
'url': 'https://slideslive.com/38983994',
'info_dict': {
'id': '38983994',
'ext': 'mp4',
'title': 'Zero-Shot AutoML with Pretrained Models',
'timestamp': 1662384834,
'upload_date': '20220905',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:23',
'chapters': 'count:22',
'duration': 295,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# service_name = yoda
'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend',
'only_matching': True,
}, {
# video_service_name = url
# dead link, service_name = url
'url': 'https://slideslive.com/38922070/learning-transferable-skills-1',
'only_matching': True,
}, {
# video_service_name = vimeo
# dead link, service_name = vimeo
'url': 'https://slideslive.com/38921896/retrospectives-a-venue-for-selfreflection-in-ml-research-3',
'only_matching': True,
}]
_WEBPAGE_TESTS = [{
# only XML slides info
'url': 'https://iclr.cc/virtual_2020/poster_Hklr204Fvr.html',
'info_dict': {
'id': '38925850',
'ext': 'mp4',
'title': 'Towards a Deep Network Architecture for Structured Smoothness',
'thumbnail': r're:^https?://.*\.jpg',
'thumbnails': 'count:8',
'timestamp': 1629671508,
'upload_date': '20210822',
'chapters': 'count:7',
'duration': 326,
},
'params': {
'skip_download': 'm3u8',
},
}]
@classmethod
def _extract_embed_urls(cls, url, webpage):
# Reference: https://slideslive.com/embed_presentation.js
for embed_id in re.findall(r'(?s)new\s+SlidesLiveEmbed\s*\([^)]+\bpresentationId:\s*["\'](\d+)["\']', webpage):
url_parsed = urllib.parse.urlparse(url)
origin = f'{url_parsed.scheme}://{url_parsed.netloc}'
yield update_url_query(
f'https://slideslive.com/embed/presentation/{embed_id}', {
'embed_parent_url': url,
'embed_container_origin': origin,
})
def _download_embed_webpage_handle(self, video_id, headers):
return self._download_webpage_handle(
f'https://slideslive.com/embed/presentation/{video_id}', video_id,
headers=headers, query=traverse_obj(headers, {
'embed_parent_url': 'Referer',
'embed_container_origin': 'Origin',
}))
def _extract_custom_m3u8_info(self, m3u8_data):
m3u8_dict = {}
lookup = {
'PRESENTATION-TITLE': 'title',
'PRESENTATION-UPDATED-AT': 'timestamp',
'PRESENTATION-THUMBNAIL': 'thumbnail',
'PLAYLIST-TYPE': 'playlist_type',
'VOD-VIDEO-SERVICE-NAME': 'service_name',
'VOD-VIDEO-ID': 'service_id',
'VOD-VIDEO-SERVERS': 'video_servers',
'VOD-SUBTITLES': 'subtitles',
'VOD-SLIDES-JSON-URL': 'slides_json_url',
'VOD-SLIDES-XML-URL': 'slides_xml_url',
}
for line in m3u8_data.splitlines():
if not line.startswith('#EXT-SL-'):
continue
tag, _, value = line.partition(':')
key = lookup.get(tag.lstrip('#EXT-SL-'))
if not key:
continue
m3u8_dict[key] = value
# Some values are stringified JSON arrays
for key in ('video_servers', 'subtitles'):
if key in m3u8_dict:
m3u8_dict[key] = self._parse_json(m3u8_dict[key], None, fatal=False) or []
return m3u8_dict
def _extract_formats_and_duration(self, cdn_hostname, path, video_id, skip_duration=False):
formats, duration = [], None
hls_formats = self._extract_m3u8_formats(
f'https://{cdn_hostname}/{path}/master.m3u8',
video_id, 'mp4', m3u8_id='hls', fatal=False, live=True)
if hls_formats:
if not skip_duration:
duration = self._extract_m3u8_vod_duration(
hls_formats[0]['url'], video_id, note='Extracting duration from HLS manifest')
formats.extend(hls_formats)
dash_formats = self._extract_mpd_formats(
f'https://{cdn_hostname}/{path}/master.mpd', video_id, mpd_id='dash', fatal=False)
if dash_formats:
if not duration and not skip_duration:
duration = self._extract_mpd_vod_duration(
f'https://{cdn_hostname}/{path}/master.mpd', video_id,
note='Extracting duration from DASH manifest')
formats.extend(dash_formats)
return formats, duration
def _real_extract(self, url):
video_id = self._match_id(url)
video_data = self._download_json(
'https://ben.slideslive.com/player/' + video_id, video_id)
service_name = video_data['video_service_name'].lower()
webpage, urlh = self._download_embed_webpage_handle(
video_id, headers=traverse_obj(parse_qs(url), {
'Referer': ('embed_parent_url', -1),
'Origin': ('embed_container_origin', -1)}))
redirect_url = urlh.geturl()
if 'domain_not_allowed' in redirect_url:
domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False)
if not domain:
raise ExtractorError(
'This is an embed-only presentation. Try passing --referer', expected=True)
webpage, _ = self._download_embed_webpage_handle(video_id, headers={
'Referer': f'https://{domain}/',
'Origin': f'https://{domain}',
})
player_token = self._search_regex(r'data-player-token="([^"]+)"', webpage, 'player token')
player_data = self._download_webpage(
f'https://ben.slideslive.com/player/{video_id}', video_id,
note='Downloading player info', query={'player_token': player_token})
player_info = self._extract_custom_m3u8_info(player_data)
service_name = player_info['service_name'].lower()
assert service_name in ('url', 'yoda', 'vimeo', 'youtube')
service_id = video_data['video_service_id']
service_id = player_info['service_id']
slide_url_template = 'https://slides.slideslive.com/%s/slides/original/%s%s'
slides, slides_info = {}, []
if player_info.get('slides_json_url'):
slides = self._download_json(
player_info['slides_json_url'], video_id, fatal=False,
note='Downloading slides JSON', errnote=False) or {}
slide_ext_default = '.png'
slide_quality = traverse_obj(slides, ('slide_qualities', 0))
if slide_quality:
slide_ext_default = '.jpg'
slide_url_template = f'https://cdn.slideslive.com/data/presentations/%s/slides/{slide_quality}/%s%s'
for slide_id, slide in enumerate(traverse_obj(slides, ('slides', ...), expected_type=dict), 1):
slides_info.append((
slide_id, traverse_obj(slide, ('image', 'name')),
traverse_obj(slide, ('image', 'extname'), default=slide_ext_default),
int_or_none(slide.get('time'), scale=1000)))
if not slides and player_info.get('slides_xml_url'):
slides = self._download_xml(
player_info['slides_xml_url'], video_id, fatal=False,
note='Downloading slides XML', errnote='Failed to download slides info')
slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s%s'
for slide_id, slide in enumerate(slides.findall('./slide') if slides else [], 1):
slides_info.append((
slide_id, xpath_text(slide, './slideName', 'name'), '.jpg',
int_or_none(xpath_text(slide, './timeSec', 'time'))))
chapters, thumbnails = [], []
if url_or_none(player_info.get('thumbnail')):
thumbnails.append({'id': 'cover', 'url': player_info['thumbnail']})
for slide_id, slide_path, slide_ext, start_time in slides_info:
if slide_path:
thumbnails.append({
'id': f'{slide_id:03d}',
'url': slide_url_template % (video_id, slide_path, slide_ext),
})
chapters.append({
'title': f'Slide {slide_id:03d}',
'start_time': start_time,
})
subtitles = {}
for sub in try_get(video_data, lambda x: x['subtitles'], list) or []:
if not isinstance(sub, dict):
continue
for sub in traverse_obj(player_info, ('subtitles', ...), expected_type=dict):
webvtt_url = url_or_none(sub.get('webvtt_url'))
if not webvtt_url:
continue
lang = sub.get('language') or 'en'
subtitles.setdefault(lang, []).append({
subtitles.setdefault(sub.get('language') or 'en', []).append({
'url': webvtt_url,
'ext': 'vtt',
})
info = {
'id': video_id,
'thumbnail': video_data.get('thumbnail'),
'is_live': bool_or_none(video_data.get('is_live')),
'title': player_info.get('title') or self._html_search_meta('title', webpage, default=''),
'timestamp': unified_timestamp(player_info.get('timestamp')),
'is_live': player_info.get('playlist_type') != 'vod',
'thumbnails': thumbnails,
'chapters': chapters,
'subtitles': subtitles,
}
if service_name in ('url', 'yoda'):
info['title'] = video_data['title']
if service_name == 'url':
info['url'] = service_id
else:
formats = []
_MANIFEST_PATTERN = 'https://01.cdn.yoda.slideslive.com/%s/master.%s'
# use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
formats.extend(self._extract_m3u8_formats(
_MANIFEST_PATTERN % (service_id, 'm3u8'),
service_id, 'mp4', m3u8_id='hls', fatal=False))
formats.extend(self._extract_mpd_formats(
_MANIFEST_PATTERN % (service_id, 'mpd'), service_id,
mpd_id='dash', fatal=False))
info.update({
'id': service_id,
'formats': formats,
})
if service_name == 'url':
info['url'] = service_id
elif service_name == 'yoda':
formats, duration = self._extract_formats_and_duration(
player_info['video_servers'][0], service_id, video_id)
info.update({
'duration': duration,
'formats': formats,
})
else:
info.update({
'_type': 'url_transparent',
'url': service_id,
'ie_key': service_name.capitalize(),
'title': video_data.get('title'),
'display_id': video_id,
})
if service_name == 'vimeo':
info['url'] = smuggle_url(
'https://player.vimeo.com/video/' + service_id,
f'https://player.vimeo.com/video/{service_id}',
{'http_headers': {'Referer': url}})
return info
video_slides = traverse_obj(slides, ('slides', ..., 'video', 'id'))
if not video_slides:
return info
def entries():
yield info
service_data = self._download_json(
f'https://ben.slideslive.com/player/{video_id}/slides_video_service_data',
video_id, fatal=False, query={
'player_token': player_token,
'videos': ','.join(video_slides),
}, note='Downloading video slides info', errnote='Failed to download video slides info') or {}
for slide_id, slide in enumerate(traverse_obj(slides, ('slides', ...)), 1):
if not traverse_obj(slide, ('video', 'service')) == 'yoda':
continue
video_path = traverse_obj(slide, ('video', 'id'))
cdn_hostname = traverse_obj(service_data, (
video_path, 'video_servers', ...), get_all=False)
if not cdn_hostname or not video_path:
continue
formats, _ = self._extract_formats_and_duration(
cdn_hostname, video_path, video_id, skip_duration=True)
if not formats:
continue
yield {
'id': f'{video_id}-{slide_id:03d}',
'title': f'{info["title"]} - Slide {slide_id:03d}',
'timestamp': info['timestamp'],
'duration': int_or_none(traverse_obj(slide, ('video', 'duration_ms')), scale=1000),
'formats': formats,
}
return self.playlist_result(entries(), f'{video_id}-playlist', info['title'])
yt_dlp/extractor/soundcloud.py
+21
View File
@@ -782,6 +782,27 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
'%s (%s)' % (user['username'], resource.capitalize()))
class SoundcloudUserPermalinkIE(SoundcloudPagedPlaylistBaseIE):
_VALID_URL = r'https?://api\.soundcloud\.com/users/(?P<id>\d+)'
IE_NAME = 'soundcloud:user:permalink'
_TESTS = [{
'url': 'https://api.soundcloud.com/users/30909869',
'info_dict': {
'id': '30909869',
'title': 'neilcic',
},
'playlist_mincount': 23,
}]
def _real_extract(self, url):
user_id = self._match_id(url)
user = self._download_json(
self._resolv_url(url), user_id, 'Downloading user info', headers=self._HEADERS)
return self._extract_playlist(
f'{self._API_V2_BASE}stream/users/{user["id"]}', str(user['id']), user.get('username'))
class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
_VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
IE_NAME = 'soundcloud:trackstation'
yt_dlp/extractor/spankbang.py
+3 -4
View File
@@ -177,7 +177,6 @@ class SpankBangPlaylistIE(InfoExtractor):
def _real_extract(self, url):
mobj = self._match_valid_url(url)
playlist_id = mobj.group('id')
display_id = mobj.group('display_id')
webpage = self._download_webpage(
url, playlist_id, headers={'Cookie': 'country=US; mobile=on'})
@@ -186,11 +185,11 @@ class SpankBangPlaylistIE(InfoExtractor):
urljoin(url, mobj.group('path')),
ie=SpankBangIE.ie_key(), video_id=mobj.group('id'))
for mobj in re.finditer(
r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/%s(?:(?!\1).)*)\1'
% re.escape(display_id), webpage)]
r'<a[^>]+\bhref=(["\'])(?P<path>/?[\da-z]+-(?P<id>[\da-z]+)/playlist/[^"\'](?:(?!\1).)*)\1',
webpage)]
title = self._html_search_regex(
r'<h1>([^<]+)\s+playlist\s*<', webpage, 'playlist title',
r'<em>([^<]+)</em>\s+playlist\s*<', webpage, 'playlist title',
fatal=False)
return self.playlist_result(entries, playlist_id, title)
yt_dlp/extractor/stv.py
+2
View File
@@ -73,6 +73,8 @@ class STVPlayerIE(InfoExtractor):
})
programme = result.get('programme') or {}
if programme.get('drmEnabled'):
self.report_drm(video_id)
return {
'_type': 'url_transparent',
yt_dlp/extractor/swearnet.py
+1 -1
View File
@@ -62,7 +62,7 @@ class SwearnetEpisodeIE(InfoExtractor):
'id': str(json_data['videoId']),
'title': json_data.get('name') or self._html_search_meta(['og:title', 'twitter:title'], webpage),
'description': (json_data.get('description')
or self._html_search_meta(['og:description', 'twitter:description'])),
or self._html_search_meta(['og:description', 'twitter:description'], webpage)),
'duration': int_or_none(json_data.get('seconds')),
'formats': formats,
'subtitles': subtitles,
yt_dlp/extractor/tencent.py
+2 -2
View File
@@ -32,7 +32,7 @@ class TencentBaseIE(InfoExtractor):
padding_mode='whitespace').hex().upper()
def _get_video_api_response(self, video_url, video_id, series_id, subtitle_format, video_format, video_quality):
guid = ''.join([random.choice(string.digits + string.ascii_lowercase) for _ in range(16)])
guid = ''.join(random.choices(string.digits + string.ascii_lowercase, k=16))
ckey = self._get_ckey(video_id, video_url, guid)
query = {
'vid': video_id,
@@ -55,7 +55,7 @@ class TencentBaseIE(InfoExtractor):
'platform': self._PLATFORM,
# For VQQ
'guid': guid,
'flowid': ''.join(random.choice(string.digits + string.ascii_lowercase) for _ in range(32)),
'flowid': ''.join(random.choices(string.digits + string.ascii_lowercase, k=32)),
}
return self._search_json(r'QZOutputJson=', self._download_webpage(
yt_dlp/extractor/testurl.py
+6 -5
View File
@@ -23,11 +23,12 @@ class TestURLIE(InfoExtractor):
if len(matching_extractors) == 0:
raise ExtractorError(f'No extractors matching {extractor_id!r} found', expected=True)
elif len(matching_extractors) > 1:
try: # Check for exact match
extractor = next(
ie for ie in matching_extractors
if ie.IE_NAME.lower() == extractor_id.lower())
except StopIteration:
extractor = next(( # Check for exact match
ie for ie in matching_extractors if ie.IE_NAME.lower() == extractor_id.lower()
), None) or next(( # Check for exact match without plugin suffix
ie for ie in matching_extractors if ie.IE_NAME.split('+')[0].lower() == extractor_id.lower()
), None)
if not extractor:
raise ExtractorError(
'Found multiple matching extractors: %s' % ' '.join(ie.IE_NAME for ie in matching_extractors),
expected=True)
yt_dlp/extractor/thisvid.py
+226
View File
@@ -0,0 +1,226 @@
import itertools
import re
import urllib.parse
from .common import InfoExtractor
from ..utils import (
clean_html,
get_element_by_class,
int_or_none,
url_or_none,
urljoin,
)
class ThisVidIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+)'
_TESTS = [{
'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
'md5': '839becb572995687e11a69dc4358a386',
'info_dict': {
'id': '3533241',
'ext': 'mp4',
'title': 'Sitting on ball tight jeans',
'description': 'md5:372353bb995883d1b65fddf507489acd',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
'uploader_id': '150629',
'uploader': 'jeanslevisjeans',
'display_id': 'sitting-on-ball-tight-jeans',
'age_limit': 18,
}
}, {
'url': 'https://thisvid.com/embed/3533241/',
'md5': '839becb572995687e11a69dc4358a386',
'info_dict': {
'id': '3533241',
'ext': 'mp4',
'title': 'Sitting on ball tight jeans',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
'uploader_id': '150629',
'uploader': 'jeanslevisjeans',
'display_id': 'sitting-on-ball-tight-jeans',
'age_limit': 18,
}
}]
def _real_extract(self, url):
main_id, type_ = re.match(self._VALID_URL, url).group('id', 'type')
webpage = self._download_webpage(url, main_id)
title = self._html_search_regex(
r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>',
webpage, 'title')
if type_ == 'embed':
# look for more metadata
video_alt_url = url_or_none(self._search_regex(
rf'''video_alt_url\s*:\s+'({self._VALID_URL}/)',''',
webpage, 'video_alt_url', default=None))
if video_alt_url and video_alt_url != url:
webpage = self._download_webpage(
video_alt_url, main_id,
note='Redirecting embed to main page', fatal=False) or webpage
video_holder = get_element_by_class('video-holder', webpage) or ''
if '>This video is a private video' in video_holder:
self.raise_login_required(
(clean_html(video_holder) or 'Private video').partition('\n')[0])
uploader = self._html_search_regex(
r'''(?s)<span\b[^>]*>Added by:\s*</span><a\b[^>]+\bclass\s*=\s*["']author\b[^>]+\bhref\s*=\s*["']https://thisvid\.com/members/([0-9]+/.{3,}?)\s*</a>''',
webpage, 'uploader', default='')
uploader = re.split(r'''/["'][^>]*>\s*''', uploader)
if len(uploader) == 2:
# id must be non-empty, uploader could be ''
uploader_id, uploader = uploader
uploader = uploader or None
else:
uploader_id = uploader = None
return self.url_result(
url, ie='Generic', url_transparent=True,
title=title,
age_limit=18,
uploader=uploader,
uploader_id=uploader_id)
class ThisVidPlaylistBaseIE(InfoExtractor):
_PLAYLIST_URL_RE = None
@classmethod
def _find_urls(cls, html):
for m in re.finditer(rf'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>{cls._PLAYLIST_URL_RE}\b)[^>]+>''', html):
yield m.group('url')
def _generate_playlist_entries(self, url, playlist_id, html=None):
page_url = url
for page in itertools.count(1):
if not html:
html = self._download_webpage(
page_url, playlist_id, note=f'Downloading page {page}',
fatal=False) or ''
yield from self._find_urls(html)
next_page = get_element_by_class('pagination-next', html) or ''
if next_page:
# member list page
next_page = urljoin(url, self._search_regex(
r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?!#)(?:(?!\1).)+)''',
next_page, 'next page link', group='url', default=None))
# in case a member page should have pagination-next with empty link, not just `else:`
if next_page is None:
# playlist page
parsed_url = urllib.parse.urlparse(page_url)
base_path, _, num = parsed_url.path.rpartition('/')
num = int_or_none(num)
if num is None:
base_path, num = parsed_url.path.rstrip('/'), 1
parsed_url = parsed_url._replace(path=f'{base_path}/{num + 1}')
next_page = urllib.parse.urlunparse(parsed_url)
if page_url == next_page:
next_page = None
if not next_page:
return
page_url, html = next_page, None
def _make_playlist_result(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
title = re.split(
r'(?i)\s*\|\s*ThisVid\.com\s*$',
self._og_search_title(webpage, default=None)
or self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title', webpage, 'title', fatal=False) or '', 1)[0] or None
return self.playlist_from_matches(
self._generate_playlist_entries(url, playlist_id, webpage),
playlist_id=playlist_id, playlist_title=title, ie=ThisVidIE)
class ThisVidMemberIE(ThisVidPlaylistBaseIE):
_VALID_URL = r'https?://thisvid\.com/members/(?P<id>\d+)'
_TESTS = [{
'url': 'https://thisvid.com/members/2140501/',
'info_dict': {
'id': '2140501',
'title': 'Rafflesia\'s Profile',
},
'playlist_mincount': 16,
}, {
'url': 'https://thisvid.com/members/2140501/favourite_videos/',
'info_dict': {
'id': '2140501',
'title': 'Rafflesia\'s Favourite Videos',
},
'playlist_mincount': 15,
}, {
'url': 'https://thisvid.com/members/636468/public_videos/',
'info_dict': {
'id': '636468',
'title': 'Happymouth\'s Public Videos',
},
'playlist_mincount': 196,
}]
_PLAYLIST_URL_RE = ThisVidIE._VALID_URL
def _real_extract(self, url):
return self._make_playlist_result(url)
class ThisVidPlaylistIE(ThisVidPlaylistBaseIE):
_VALID_URL = r'https?://thisvid\.com/playlist/(?P<id>\d+)/video/(?P<video_id>[A-Za-z0-9-]+)'
_TESTS = [{
'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
'info_dict': {
'id': '6615',
'title': 'Underwear Stuff',
},
'playlist_mincount': 200,
}, {
'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
'info_dict': {
'id': '1072387',
'ext': 'mp4',
'title': 'Big Italian Booty 28',
'description': 'md5:1bccf7b13765e18fb27bf764dba7ede2',
'uploader_id': '367912',
'uploader': 'Jcmusclefun',
'age_limit': 18,
'display_id': 'big-italian-booty-28',
'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+1072387/preview\.jpg',
},
'params': {
'noplaylist': True,
},
}]
_PLAYLIST_URL_RE = _VALID_URL
def _generate_playlist_entries(self, url, playlist_id, html=None):
for wrapped_url in super()._generate_playlist_entries(url, playlist_id, html):
video_id = re.match(self._VALID_URL, wrapped_url).group('video_id')
yield urljoin(url, f'/videos/{video_id}/')
def _real_extract(self, url):
playlist_id, video_id = self._match_valid_url(url).group('id', 'video_id')
if not self._yes_playlist(playlist_id, video_id):
redirect_url = urljoin(url, f'/videos/{video_id}/')
return self.url_result(redirect_url, ThisVidIE)
result = self._make_playlist_result(url)
# Fix duplicated title (`the title - the title` => `the title`)
title = result['title']
t_len = len(title)
if t_len > 5 and t_len % 2 != 0:
t_len = t_len // 2
if title[t_len] == '-':
first, second = map(str.strip, (title[:t_len], title[t_len + 1:]))
if first and first == second:
result['title'] = first
return result
yt_dlp/extractor/tiktok.py
+169 -60
View File
@@ -11,11 +11,13 @@ from ..utils import (
HEADRequest,
LazyList,
UnsupportedError,
UserNotLive,
get_element_by_id,
get_first,
int_or_none,
join_nonempty,
qualities,
remove_start,
srt_subtitles_timecode,
str_or_none,
traverse_obj,
@@ -29,11 +31,15 @@ class TikTokBaseIE(InfoExtractor):
_WORKING_APP_VERSION = None
_APP_NAME = 'trill'
_AID = 1180
_API_HOSTNAME = 'api-h2.tiktokv.com'
_UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s'
_WEBPAGE_HOST = 'https://www.tiktok.com/'
QUALITIES = ('360p', '540p', '720p', '1080p')
@property
def _API_HOSTNAME(self):
return self._configuration_arg(
'api_hostname', ['api16-normal-c-useast1a.tiktokv.com'], ie_key=TikTokIE)[0]
@staticmethod
def _create_url(user_id, video_id):
return f'https://www.tiktok.com/@{user_id or "_"}/video/{video_id}'
@@ -44,14 +50,14 @@ class TikTokBaseIE(InfoExtractor):
def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True,
note='Downloading API JSON', errnote='Unable to download API page'):
self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160)))
self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choices('0123456789abcdef', k=160)))
webpage_cookies = self._get_cookies(self._WEBPAGE_HOST)
if webpage_cookies.get('sid_tt'):
self._set_cookie(self._API_HOSTNAME, 'sid_tt', webpage_cookies['sid_tt'].value)
return self._download_json(
'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
fatal=fatal, note=note, errnote=errnote, headers={
'User-Agent': f'com.ss.android.ugc.trill/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
'User-Agent': f'com.ss.android.ugc.{self._APP_NAME}/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
'Accept': 'application/json',
}, query=query)
@@ -63,8 +69,8 @@ class TikTokBaseIE(InfoExtractor):
'build_number': app_version,
'manifest_version_code': manifest_app_version,
'update_version_code': manifest_app_version,
'openudid': ''.join(random.choice('0123456789abcdef') for _ in range(16)),
'uuid': ''.join([random.choice(string.digits) for _ in range(16)]),
'openudid': ''.join(random.choices('0123456789abcdef', k=16)),
'uuid': ''.join(random.choices(string.digits, k=16)),
'_rticket': int(time.time() * 1000),
'ts': int(time.time()),
'device_brand': 'Google',
@@ -126,11 +132,21 @@ class TikTokBaseIE(InfoExtractor):
continue
raise e
def _extract_aweme_app(self, aweme_id):
feed_list = self._call_api(
'feed', {'aweme_id': aweme_id}, aweme_id, note='Downloading video feed',
errnote='Unable to download video feed').get('aweme_list') or []
aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
if not aweme_detail:
raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
return self._parse_aweme_video_app(aweme_detail)
def _get_subtitles(self, aweme_detail, aweme_id):
# TODO: Extract text positioning info
subtitles = {}
# aweme/detail endpoint subs
captions_info = traverse_obj(
aweme_detail, ('interaction_stickers', ..., 'auto_video_caption_info', 'auto_captions', ...), expected_type=dict, default=[])
aweme_detail, ('interaction_stickers', ..., 'auto_video_caption_info', 'auto_captions', ...), expected_type=dict)
for caption in captions_info:
caption_url = traverse_obj(caption, ('url', 'url_list', ...), expected_type=url_or_none, get_all=False)
if not caption_url:
@@ -145,6 +161,24 @@ class TikTokBaseIE(InfoExtractor):
f'{i + 1}\n{srt_subtitles_timecode(line["start_time"] / 1000)} --> {srt_subtitles_timecode(line["end_time"] / 1000)}\n{line["text"]}'
for i, line in enumerate(caption_json['utterances']) if line.get('text'))
})
# feed endpoint subs
if not subtitles:
for caption in traverse_obj(aweme_detail, ('video', 'cla_info', 'caption_infos', ...), expected_type=dict):
if not caption.get('url'):
continue
subtitles.setdefault(caption.get('lang') or 'en', []).append({
'ext': remove_start(caption.get('caption_format'), 'web'),
'url': caption['url'],
})
# webpage subs
if not subtitles:
for caption in traverse_obj(aweme_detail, ('video', 'subtitleInfos', ...), expected_type=dict):
if not caption.get('Url'):
continue
subtitles.setdefault(caption.get('LanguageCodeName') or 'en', []).append({
'ext': remove_start(caption.get('Format'), 'web'),
'url': caption['Url'],
})
return subtitles
def _parse_aweme_video_app(self, aweme_detail):
@@ -354,7 +388,7 @@ class TikTokBaseIE(InfoExtractor):
'timestamp': int_or_none(aweme_detail.get('createTime')),
'creator': str_or_none(author_info.get('nickname')),
'uploader': str_or_none(author_info.get('uniqueId') or aweme_detail.get('author')),
'uploader_id': str_or_none(author_info.get('id') or aweme_detail.get('authorId')),
'uploader_id': str_or_none(traverse_obj(author_info, 'id', 'uid', 'authorId')),
'uploader_url': user_url,
'track': str_or_none(music_info.get('title')),
'album': str_or_none(music_info.get('album')) or None,
@@ -369,7 +403,7 @@ class TikTokBaseIE(InfoExtractor):
class TikTokIE(TikTokBaseIE):
_VALID_URL = r'https?://www\.tiktok\.com/(?:embed|@(?P<user_id>[\w\.-]+)/video)/(?P<id>\d+)'
_VALID_URL = r'https?://www\.tiktok\.com/(?:embed|@(?P<user_id>[\w\.-]+)?/video)/(?P<id>\d+)'
_EMBED_REGEX = [rf'<(?:script|iframe)[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL})']
_TESTS = [{
@@ -521,14 +555,6 @@ class TikTokIE(TikTokBaseIE):
'only_matching': True
}]
def _extract_aweme_app(self, aweme_id):
feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id,
note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or []
aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
if not aweme_detail:
raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
return self._parse_aweme_video_app(aweme_detail)
def _real_extract(self, url):
video_id, user_id = self._match_valid_url(url).group('id', 'user_id')
try:
@@ -613,7 +639,7 @@ class TikTokUserIE(TikTokBaseIE):
'max_cursor': 0,
'min_cursor': 0,
'retry_type': 'no_retry',
'device_id': ''.join(random.choice(string.digits) for _ in range(19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
'device_id': ''.join(random.choices(string.digits, k=19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
}
for page in itertools.count(1):
@@ -661,7 +687,7 @@ class TikTokBaseListIE(TikTokBaseIE): # XXX: Conventionally, base classes shoul
'cursor': 0,
'count': 20,
'type': 5,
'device_id': ''.join(random.choice(string.digits) for i in range(19))
'device_id': ''.join(random.choices(string.digits, k=19))
}
for page in itertools.count(1):
@@ -763,56 +789,68 @@ class TikTokTagIE(TikTokBaseListIE):
return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id)
class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
class DouyinIE(TikTokBaseIE):
_VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://www.douyin.com/video/6961737553342991651',
'md5': '10523312c8b8100f353620ac9dc8f067',
'md5': 'a97db7e3e67eb57bf40735c022ffa228',
'info_dict': {
'id': '6961737553342991651',
'ext': 'mp4',
'title': '#杨超越 小小水手带你去远航❤️',
'uploader': '杨超越',
'upload_date': '20210513',
'timestamp': 1620905839,
'description': '#杨超越 小小水手带你去远航❤️',
'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 19782,
'timestamp': 1620905839,
'upload_date': '20210513',
'track': '@杨超越创作的原声',
'view_count': int,
'like_count': int,
'repost_count': int,
'comment_count': int,
}
},
}, {
'url': 'https://www.douyin.com/video/6982497745948921092',
'md5': 'd78408c984b9b5102904cf6b6bc2d712',
'md5': '34a87ebff3833357733da3fe17e37c0e',
'info_dict': {
'id': '6982497745948921092',
'ext': 'mp4',
'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
'uploader': '杨超越工作室',
'upload_date': '20210708',
'timestamp': 1625739481,
'description': '这个夏日和小羊@杨超越 一起遇见白色幻想',
'uploader_id': '408654318141572',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA',
'creator': '杨超越工作室',
'duration': 42608,
'timestamp': 1625739481,
'upload_date': '20210708',
'track': '@杨超越工作室创作的原声',
'view_count': int,
'like_count': int,
'repost_count': int,
'comment_count': int,
}
},
}, {
'url': 'https://www.douyin.com/video/6953975910773099811',
'md5': '72e882e24f75064c218b76c8b713c185',
'md5': 'dde3302460f19db59c47060ff013b902',
'info_dict': {
'id': '6953975910773099811',
'ext': 'mp4',
'title': '#一起看海 出现在你的夏日里',
'uploader': '杨超越',
'upload_date': '20210422',
'timestamp': 1619098692,
'description': '#一起看海 出现在你的夏日里',
'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 17228,
'timestamp': 1619098692,
'upload_date': '20210422',
'track': '@杨超越创作的原声',
'view_count': int,
'like_count': int,
'repost_count': int,
'comment_count': int,
}
},
}, {
'url': 'https://www.douyin.com/video/6950251282489675042',
'md5': 'b4db86aec367ef810ddd38b1737d2fed',
@@ -828,25 +866,30 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
'like_count': int,
'repost_count': int,
'comment_count': int,
}
},
'skip': 'No longer available',
}, {
'url': 'https://www.douyin.com/video/6963263655114722595',
'md5': '1abe1c477d05ee62efb40bf2329957cf',
'md5': 'cf9f11f0ec45d131445ec2f06766e122',
'info_dict': {
'id': '6963263655114722595',
'ext': 'mp4',
'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
'uploader': '杨超越',
'upload_date': '20210517',
'timestamp': 1621261163,
'description': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 15115,
'timestamp': 1621261163,
'upload_date': '20210517',
'track': '@杨超越创作的原声',
'view_count': int,
'like_count': int,
'repost_count': int,
'comment_count': int,
}
},
}]
_APP_VERSIONS = [('9.6.0', '960')]
_APP_VERSIONS = [('23.3.0', '230300')]
_APP_NAME = 'aweme'
_AID = 1128
_API_HOSTNAME = 'aweme.snssdk.com'
@@ -859,7 +902,8 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
try:
return self._extract_aweme_app(video_id)
except ExtractorError as e:
self.report_warning(f'{e}; trying with webpage')
e.expected = True
self.to_screen(f'{e}; trying with webpage')
webpage = self._download_webpage(url, video_id)
render_data_json = self._search_regex(
@@ -867,7 +911,10 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
webpage, 'render data', default=None)
if not render_data_json:
# TODO: Run verification challenge code to generate signature cookies
raise ExtractorError('Fresh cookies (not necessarily logged in) are needed')
cookies = self._get_cookies(self._WEBPAGE_HOST)
expected = not cookies.get('s_v_web_id') or not cookies.get('ttwid')
raise ExtractorError(
'Fresh cookies (not necessarily logged in) are needed', expected=expected)
render_data = self._parse_json(
render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
@@ -875,31 +922,54 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
class TikTokVMIE(InfoExtractor):
_VALID_URL = r'https?://(?:vm|vt)\.tiktok\.com/(?P<id>\w+)'
_VALID_URL = r'https?://(?:(?:vm|vt)\.tiktok\.com|(?:www\.)tiktok\.com/t)/(?P<id>\w+)'
IE_NAME = 'vm.tiktok'
_TESTS = [{
'url': 'https://vm.tiktok.com/ZSe4FqkKd',
'url': 'https://www.tiktok.com/t/ZTRC5xgJp',
'info_dict': {
'id': '7023491746608712966',
'id': '7170520270497680683',
'ext': 'mp4',
'title': 'md5:5607564db90271abbbf8294cca77eddd',
'description': 'md5:5607564db90271abbbf8294cca77eddd',
'duration': 11,
'upload_date': '20211026',
'uploader_id': '7007385080558846981',
'creator': 'Memes',
'artist': 'Memes',
'track': 'original sound',
'uploader': 'susmandem',
'timestamp': 1635284105,
'thumbnail': r're:https://.+\.webp.*',
'like_count': int,
'title': 'md5:c64f6152330c2efe98093ccc8597871c',
'uploader_id': '6687535061741700102',
'upload_date': '20221127',
'view_count': int,
'like_count': int,
'comment_count': int,
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAObqu3WCTXxmw2xwZ3iLEHnEecEIw7ks6rxWqOqOhaPja9BI7gqUQnjw8_5FSoDXX',
'album': 'Wave of Mutilation: Best of Pixies',
'thumbnail': r're:https://.+\.webp.*',
'duration': 5,
'timestamp': 1669516858,
'repost_count': int,
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAXcNoOEOxVyBzuII_E--T0MeCrLP0ay1Sm6x_n3dluiWEoWZD0VlQOytwad4W0i0n',
}
'artist': 'Pixies',
'track': 'Where Is My Mind?',
'description': 'md5:c64f6152330c2efe98093ccc8597871c',
'uploader': 'sigmachaddeus',
'creator': 'SigmaChad',
},
}, {
'url': 'https://vm.tiktok.com/ZTR45GpSF/',
'info_dict': {
'id': '7106798200794926362',
'ext': 'mp4',
'title': 'md5:edc3e7ea587847f8537468f2fe51d074',
'uploader_id': '6997695878846268418',
'upload_date': '20220608',
'view_count': int,
'like_count': int,
'comment_count': int,
'thumbnail': r're:https://.+\.webp.*',
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAdZ_NcPPgMneaGrW0hN8O_J_bwLshwNNERRF5DxOw2HKIzk0kdlLrR8RkVl1ksrMO',
'duration': 29,
'timestamp': 1654680400,
'repost_count': int,
'artist': 'Akihitoko',
'track': 'original sound',
'description': 'md5:edc3e7ea587847f8537468f2fe51d074',
'uploader': 'akihitoko1',
'creator': 'Akihitoko',
},
}, {
'url': 'https://vt.tiktok.com/ZSe4FqkKd',
'only_matching': True,
@@ -911,3 +981,42 @@ class TikTokVMIE(InfoExtractor):
if self.suitable(new_url): # Prevent infinite loop in case redirect fails
raise UnsupportedError(new_url)
return self.url_result(new_url)
class TikTokLiveIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/live'
IE_NAME = 'tiktok:live'
_TESTS = [{
'url': 'https://www.tiktok.com/@iris04201/live',
'only_matching': True,
}]
def _real_extract(self, url):
uploader = self._match_id(url)
webpage = self._download_webpage(url, uploader, headers={'User-Agent': 'User-Agent:Mozilla/5.0'})
room_id = self._html_search_regex(r'snssdk\d*://live\?room_id=(\d+)', webpage, 'room ID', default=None)
if not room_id:
raise UserNotLive(video_id=uploader)
live_info = traverse_obj(self._download_json(
'https://www.tiktok.com/api/live/detail/', room_id, query={
'aid': '1988',
'roomID': room_id,
}), 'LiveRoomInfo', expected_type=dict, default={})
if 'status' not in live_info:
raise ExtractorError('Unexpected response from TikTok API')
# status = 2 if live else 4
if not int_or_none(live_info['status']) == 2:
raise UserNotLive(video_id=uploader)
return {
'id': room_id,
'title': live_info.get('title') or self._html_search_meta(['og:title', 'twitter:title'], webpage, default=''),
'uploader': uploader,
'uploader_id': traverse_obj(live_info, ('ownerInfo', 'id')),
'creator': traverse_obj(live_info, ('ownerInfo', 'nickname')),
'concurrent_view_count': traverse_obj(live_info, ('liveRoomStats', 'userCount'), expected_type=int),
'formats': self._extract_m3u8_formats(live_info['liveUrl'], room_id, 'mp4', live=True),
'is_live': True,
}
yt_dlp/extractor/trtcocuk.py
+48
View File
@@ -0,0 +1,48 @@
from .common import InfoExtractor
from ..utils import ExtractorError, int_or_none, parse_iso8601, traverse_obj
class TrtCocukVideoIE(InfoExtractor):
_VALID_URL = r'https?://www\.trtcocuk\.net\.tr/video/(?P<id>[\w-]+)'
_TESTS = [{
'url': 'https://www.trtcocuk.net.tr/video/kaptan-pengu-ve-arkadaslari-1',
'info_dict': {
'id': '3789738',
'ext': 'mp4',
'season_number': 1,
'series': '"Kaptan Pengu ve Arkadaşları"',
'season': 'Season 1',
'title': 'Kaptan Pengu ve Arkadaşları 1 Bölüm İzle TRT Çocuk',
'release_date': '20201209',
'release_timestamp': 1607513774,
}
}, {
'url': 'https://www.trtcocuk.net.tr/video/sef-rokanin-lezzet-dunyasi-17',
'info_dict': {
'id': '10260842',
'ext': 'mp4',
'series': '"Şef Roka\'nın Lezzet Dünyası"',
'title': 'Şef Roka\'nın Lezzet Dünyası 17 Bölüm İzle TRT Çocuk',
}
}]
def _real_extract(self, url):
display_id = self._match_id(url)
webpage = self._download_webpage(url, display_id)
nuxtjs_data = self._search_nuxt_data(webpage, display_id)['data']
try:
video_url = self._parse_json(nuxtjs_data['video'], display_id)
except ExtractorError:
video_url = nuxtjs_data['video']
formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, display_id)
return {
'id': str(nuxtjs_data['id']),
'formats': formats,
'subtitles': subtitles,
'season_number': int_or_none(nuxtjs_data.get('season')),
'release_timestamp': parse_iso8601(nuxtjs_data.get('publishedDate')),
'series': traverse_obj(nuxtjs_data, ('show', 0, 'title')),
'title': self._html_extract_title(webpage) # TODO: get better title
}
yt_dlp/extractor/twitcasting.py
+22 -12
View File
@@ -38,7 +38,7 @@ class TwitCastingIE(InfoExtractor):
'description': 'Twitter Oficial da cantora brasileira Ivete Sangalo.',
'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20110822',
'timestamp': 1314010824,
'timestamp': 1313978424,
'duration': 32,
'view_count': int,
},
@@ -52,10 +52,10 @@ class TwitCastingIE(InfoExtractor):
'ext': 'mp4',
'title': 'Live playing something #3689740',
'uploader_id': 'mttbernardini',
'description': 'Salve, io sono Matto (ma con la e). Questa è la mia presentazione, in quanto sono letteralmente matto (nel senso di strano), con qualcosa in più.',
'description': 'md5:1dc7efa2f1ab932fcd119265cebeec69',
'thumbnail': r're:^https?://.*\.jpg$',
'upload_date': '20120212',
'timestamp': 1329028024,
'upload_date': '20120211',
'timestamp': 1328995624,
'duration': 681,
'view_count': int,
},
@@ -64,15 +64,22 @@ class TwitCastingIE(InfoExtractor):
'videopassword': 'abc',
},
}, {
'note': 'archive is split in 2 parts',
'url': 'https://twitcasting.tv/loft_heaven/movie/685979292',
'info_dict': {
'id': '685979292',
'ext': 'mp4',
'title': '南波一海のhear_here “ナタリー望月哲さんに聞く編集と「渋谷系狂騒曲」”',
'duration': 6964.599334,
'title': '【無料配信】南波一海のhear/here “ナタリー望月哲さんに聞く編集と「渋谷系狂騒曲」”',
'uploader_id': 'loft_heaven',
'description': 'md5:3a0c7b53019df987ce545c935538bacf',
'upload_date': '20210604',
'timestamp': 1622802114,
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 6964,
'view_count': int,
},
'params': {
'skip_download': True,
},
'playlist_mincount': 2,
}]
def _parse_data_movie_playlist(self, dmp, video_id):
@@ -88,15 +95,18 @@ class TwitCastingIE(InfoExtractor):
def _real_extract(self, url):
uploader_id, video_id = self._match_valid_url(url).groups()
webpage, urlh = self._download_webpage_handle(url, video_id)
video_password = self.get_param('videopassword')
request_data = None
if video_password:
request_data = urlencode_postdata({
'password': video_password,
**self._hidden_inputs(webpage),
}, encoding='utf-8')
webpage, urlh = self._download_webpage_handle(
url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'})
webpage, urlh = self._download_webpage_handle(
url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'},
note='Trying video password')
if urlh.geturl() != url and request_data:
webpage = self._download_webpage(
urlh.geturl(), video_id, data=request_data,
@@ -122,7 +132,7 @@ class TwitCastingIE(InfoExtractor):
duration = (try_get(video_js_data, lambda x: sum(float_or_none(y.get('duration')) for y in x) / 1000)
or parse_duration(clean_html(get_element_by_class('tw-player-duration-time', webpage))))
view_count = str_to_int(self._search_regex(
(r'Total\s*:\s*([\d,]+)\s*Views', r'総視聴者\s*:\s*([\d,]+)\s*</'), webpage, 'views', None))
(r'Total\s*:\s*Views\s*([\d,]+)', r'総視聴者\s*:\s*([\d,]+)\s*</'), webpage, 'views', None))
timestamp = unified_timestamp(self._search_regex(
r'data-toggle="true"[^>]+datetime="([^"]+)"',
webpage, 'datetime', None))
yt_dlp/extractor/twitter.py
+80 -5
View File
@@ -293,7 +293,7 @@ class TwitterCardIE(InfoExtractor):
class TwitterIE(TwitterBaseIE):
IE_NAME = 'twitter'
_VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)'
_VALID_URL = TwitterBaseIE._BASE_REGEX + r'(?:(?:i/web|[^/]+)/status|statuses)/(?P<id>\d+)(?:/video/(?P<index>\d+))?'
_TESTS = [{
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -336,7 +336,7 @@ class TwitterIE(TwitterBaseIE):
'id': '665052190608723968',
'display_id': '665052190608723968',
'ext': 'mp4',
'title': 'md5:55fef1d5b811944f1550e91b44abb82e',
'title': 'md5:e99588f17b3dd0503814ffb560e64731',
'description': 'A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens. https://t.co/OkSqT2fjWJ',
'uploader_id': 'starwars',
'uploader': r're:Star Wars.*',
@@ -648,7 +648,7 @@ class TwitterIE(TwitterBaseIE):
'uploader_url': 'https://twitter.com/Rizdraws',
'upload_date': '20220928',
'timestamp': 1664391723,
'thumbnail': 're:^https?://.*\\.jpg',
'thumbnail': r're:^https?://.+\.jpg',
'like_count': int,
'repost_count': int,
'comment_count': int,
@@ -727,6 +727,71 @@ class TwitterIE(TwitterBaseIE):
},
'add_ie': ['TwitterSpaces'],
'params': {'skip_download': 'm3u8'},
}, {
# URL specifies video number but --yes-playlist
'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/1',
'playlist_mincount': 2,
'info_dict': {
'id': '1600649710662213632',
'title': 'md5:be05989b0722e114103ed3851a0ffae2',
'timestamp': 1670459604.0,
'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
'comment_count': int,
'uploader_id': 'CTVJLaidlaw',
'repost_count': int,
'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
'upload_date': '20221208',
'age_limit': 0,
'uploader': 'Jocelyn Laidlaw',
'uploader_url': 'https://twitter.com/CTVJLaidlaw',
'like_count': int,
},
}, {
# URL specifies video number and --no-playlist
'url': 'https://twitter.com/CTVJLaidlaw/status/1600649710662213632/video/2',
'info_dict': {
'id': '1600649511827013632',
'ext': 'mp4',
'title': 'md5:be05989b0722e114103ed3851a0ffae2',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1670459604.0,
'uploader_id': 'CTVJLaidlaw',
'uploader': 'Jocelyn Laidlaw',
'repost_count': int,
'comment_count': int,
'tags': ['colorectalcancer', 'cancerjourney', 'imnotaquitter'],
'duration': 102.226,
'uploader_url': 'https://twitter.com/CTVJLaidlaw',
'display_id': '1600649710662213632',
'like_count': int,
'description': 'md5:591c19ce66fadc2359725d5cd0d1052c',
'upload_date': '20221208',
'age_limit': 0,
},
'params': {'noplaylist': True},
}, {
# id pointing to TweetWithVisibilityResults type entity which wraps the actual Tweet over
# note the id different between extraction and url
'url': 'https://twitter.com/s2FAKER/status/1621117700482416640',
'info_dict': {
'id': '1621117577354424321',
'display_id': '1621117700482416640',
'ext': 'mp4',
'title': '뽀 - 아 최우제 이동속도 봐',
'description': '아 최우제 이동속도 봐 https://t.co/dxu2U5vXXB',
'duration': 24.598,
'uploader': '뽀',
'uploader_id': 's2FAKER',
'uploader_url': 'https://twitter.com/s2FAKER',
'upload_date': '20230202',
'timestamp': 1675339553.0,
'thumbnail': r're:https?://pbs\.twimg\.com/.+',
'age_limit': 18,
'tags': [],
'like_count': int,
'repost_count': int,
'comment_count': int,
},
}, {
# onion route
'url': 'https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/TwitterBlue/status/1484226494708662273',
@@ -769,9 +834,12 @@ class TwitterIE(TwitterBaseIE):
result = traverse_obj(data, (
'threaded_conversation_with_injections_v2', 'instructions', 0, 'entries',
lambda _, v: v['entryId'] == f'tweet-{twid}', 'content', 'itemContent',
'tweet_results', 'result'
'tweet_results', 'result', ('tweet', None),
), expected_type=dict, default={}, get_all=False)
if result.get('__typename') not in ('Tweet', None):
self.report_warning(f'Unknown typename: {result.get("__typename")}', twid, only_once=True)
if 'tombstone' in result:
cause = traverse_obj(result, ('tombstone', 'text', 'text'), expected_type=str)
raise ExtractorError(f'Twitter API says: {cause or "Unknown error"}', expected=True)
@@ -828,7 +896,7 @@ class TwitterIE(TwitterBaseIE):
}
def _real_extract(self, url):
twid = self._match_id(url)
twid, selected_index = self._match_valid_url(url).group('id', 'index')
if self.is_logged_in or self._configuration_arg('force_graphql'):
self.write_debug(f'Using GraphQL API (Auth = {self.is_logged_in})')
result = self._call_graphql_api('zZXycP0V6H7m-2r0mOnFcA/TweetDetail', twid)
@@ -998,6 +1066,13 @@ class TwitterIE(TwitterBaseIE):
entries[0]['_old_archive_ids'] = [make_archive_id(self, twid)]
if not self._yes_playlist(twid, selected_index, video_label='URL-specified video number'):
index = int(selected_index) - 1
if index >= len(entries):
raise ExtractorError(f'Video #{selected_index} is unavailable', expected=True)
return entries[index]
if len(entries) == 1:
return entries[0]
yt_dlp/extractor/txxx.py
+418
View File
@@ -0,0 +1,418 @@
import base64
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
js_to_json,
merge_dicts,
parse_duration,
traverse_obj,
try_call,
urljoin,
variadic,
)
def decode_base64(text):
return base64.b64decode(text.translate(text.maketrans({
'\u0405': 'S',
'\u0406': 'I',
'\u0408': 'J',
'\u0410': 'A',
'\u0412': 'B',
'\u0415': 'E',
'\u041a': 'K',
'\u041c': 'M',
'\u041d': 'H',
'\u041e': 'O',
'\u0420': 'P',
'\u0421': 'C',
'\u0425': 'X',
',': '/',
'.': '+',
'~': '=',
}))).decode()
def get_formats(host, video_file):
return [{
'url': urljoin(f'https://{host}', decode_base64(video['video_url'])),
'format_id': try_call(lambda: variadic(video['format'])[0].lstrip('_')),
'quality': index,
} for index, video in enumerate(video_file) if video.get('video_url')]
class TxxxIE(InfoExtractor):
_DOMAINS = (
'hclips.com',
'hdzog.com',
'hdzog.tube',
'hotmovs.com',
'hotmovs.tube',
'inporn.com',
'privatehomeclips.com',
'tubepornclassic.com',
'txxx.com',
'txxx.tube',
'upornia.com',
'upornia.tube',
'vjav.com',
'vjav.tube',
'vxxx.com',
'voyeurhit.com',
'voyeurhit.tube',
)
_VALID_URL = rf'''(?x)
https?://(?:www\.)?(?P<host>{"|".join(map(re.escape, _DOMAINS))})/
(?:videos?[/-]|embed/)(?P<id>\d+)(?:/(?P<display_id>[^/?#]+))?
'''
_EMBED_REGEX = [rf'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?(?:{"|".join(map(re.escape, _DOMAINS))})/embed/[^"\']*)\1']
_TESTS = [{
'url': 'https://txxx.com/videos/16574965/digital-desire-malena-morgan/',
'md5': 'c54e4ace54320aaf8e2a72df87859391',
'info_dict': {
'id': '16574965',
'display_id': 'digital-desire-malena-morgan',
'ext': 'mp4',
'title': 'Digital Desire - Malena Morgan',
'uploader': 'Lois Argentum',
'duration': 694,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://txxx.tube/videos/16574965/digital-desire-malena-morgan/',
'md5': 'c54e4ace54320aaf8e2a72df87859391',
'info_dict': {
'id': '16574965',
'display_id': 'digital-desire-malena-morgan',
'ext': 'mp4',
'title': 'Digital Desire - Malena Morgan',
'uploader': 'Lois Argentum',
'duration': 694,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://vxxx.com/video-68925/',
'md5': '1fcff3748b0c5b41fe41d0afa22409e1',
'info_dict': {
'id': '68925',
'display_id': '68925',
'ext': 'mp4',
'title': 'Malena Morgan',
'uploader': 'Huge Hughes',
'duration': 694,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://hclips.com/videos/6291073/malena-morgan-masturbates-her-sweet/',
'md5': 'a5dd4f83363972ee043313cff85e7e26',
'info_dict': {
'id': '6291073',
'display_id': 'malena-morgan-masturbates-her-sweet',
'ext': 'mp4',
'title': 'Malena Morgan masturbates her sweet',
'uploader': 'John Salt',
'duration': 426,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://hdzog.com/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
'md5': 'f8bdedafd45d1ec2875c43fe33a846d3',
'info_dict': {
'id': '67063',
'display_id': 'gorgeous-malena-morgan-will-seduce-you-at-the-first-glance',
'ext': 'mp4',
'title': 'Gorgeous Malena Morgan will seduce you at the first glance',
'uploader': 'momlesson',
'duration': 601,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://hdzog.tube/videos/67063/gorgeous-malena-morgan-will-seduce-you-at-the-first-glance/',
'md5': 'f8bdedafd45d1ec2875c43fe33a846d3',
'info_dict': {
'id': '67063',
'display_id': 'gorgeous-malena-morgan-will-seduce-you-at-the-first-glance',
'ext': 'mp4',
'title': 'Gorgeous Malena Morgan will seduce you at the first glance',
'uploader': 'momlesson',
'duration': 601,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://hotmovs.com/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
'md5': '71d32c51584876472db87e561171a386',
'info_dict': {
'id': '8789287',
'display_id': 'unbelievable-malena-morgan-performing-in-incredible-masturantion',
'ext': 'mp4',
'title': 'Unbelievable Malena Morgan performing in incredible masturantion',
'uploader': 'Davit Sanchez',
'duration': 940,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://hotmovs.tube/videos/8789287/unbelievable-malena-morgan-performing-in-incredible-masturantion/',
'md5': '71d32c51584876472db87e561171a386',
'info_dict': {
'id': '8789287',
'display_id': 'unbelievable-malena-morgan-performing-in-incredible-masturantion',
'ext': 'mp4',
'title': 'Unbelievable Malena Morgan performing in incredible masturantion',
'uploader': 'Davit Sanchez',
'duration': 940,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://inporn.com/video/517897/malena-morgan-solo/',
'md5': '344db467481edf78f193cdf5820a7cfb',
'info_dict': {
'id': '517897',
'display_id': 'malena-morgan-solo',
'ext': 'mp4',
'title': 'Malena Morgan - Solo',
'uploader': 'Ashley Oxy',
'duration': 480,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://privatehomeclips.com/videos/3630599/malena-morgan-cam-show/',
'md5': 'ea657273e352493c5fb6357fbfa4f126',
'info_dict': {
'id': '3630599',
'display_id': 'malena-morgan-cam-show',
'ext': 'mp4',
'title': 'malena morgan cam show',
'uploader': 'Member9915',
'duration': 290,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://tubepornclassic.com/videos/1015455/mimi-rogers-full-body-massage-nude-compilation/',
'md5': '2e9a6cf610c9862e86e0ce24f08f4427',
'info_dict': {
'id': '1015455',
'display_id': 'mimi-rogers-full-body-massage-nude-compilation',
'ext': 'mp4',
'title': 'Mimi Rogers - Full Body Massage (Nude) compilation',
'uploader': '88bhuto',
'duration': 286,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://upornia.com/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
'md5': '7ff7033340bc88a173198b7c22600e4f',
'info_dict': {
'id': '1498858',
'display_id': 'twistys-malena-morgan-starring-at-dr-morgan-baller',
'ext': 'mp4',
'title': 'Twistys - Malena Morgan starring at Dr. Morgan-Baller',
'uploader': 'mindgeek',
'duration': 480,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://upornia.tube/videos/1498858/twistys-malena-morgan-starring-at-dr-morgan-baller/',
'md5': '7ff7033340bc88a173198b7c22600e4f',
'info_dict': {
'id': '1498858',
'display_id': 'twistys-malena-morgan-starring-at-dr-morgan-baller',
'ext': 'mp4',
'title': 'Twistys - Malena Morgan starring at Dr. Morgan-Baller',
'uploader': 'mindgeek',
'duration': 480,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://vjav.com/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
'md5': '6de5bc1f13bdfc3491a77f23edb1676f',
'info_dict': {
'id': '11761',
'display_id': 'yui-hatano-in-if-yui-was-my-girlfriend2',
'ext': 'mp4',
'title': 'Yui Hatano in If Yui Was My Girlfriend',
'uploader': 'Matheus69',
'duration': 3310,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://vjav.tube/videos/11761/yui-hatano-in-if-yui-was-my-girlfriend2/',
'md5': '6de5bc1f13bdfc3491a77f23edb1676f',
'info_dict': {
'id': '11761',
'display_id': 'yui-hatano-in-if-yui-was-my-girlfriend2',
'ext': 'mp4',
'title': 'Yui Hatano in If Yui Was My Girlfriend',
'uploader': 'Matheus69',
'duration': 3310,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://voyeurhit.com/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
'md5': '12b4666e9c3e60dafe9182e5d12aae33',
'info_dict': {
'id': '332875',
'display_id': 'charlotte-stokely-elle-alexandra-malena-morgan-lingerie',
'ext': 'mp4',
'title': 'Charlotte Stokely, Elle Alexandra, Malena Morgan-Lingerie',
'uploader': 'Kyle Roberts',
'duration': 655,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}, {
'url': 'https://voyeurhit.tube/videos/332875/charlotte-stokely-elle-alexandra-malena-morgan-lingerie/',
'md5': '12b4666e9c3e60dafe9182e5d12aae33',
'info_dict': {
'id': '332875',
'display_id': 'charlotte-stokely-elle-alexandra-malena-morgan-lingerie',
'ext': 'mp4',
'title': 'Charlotte Stokely, Elle Alexandra, Malena Morgan-Lingerie',
'uploader': 'Kyle Roberts',
'duration': 655,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}]
_WEBPAGE_TESTS = [{
'url': 'https://pornzog.com/video/9125519/michelle-malone-dreamgirls-wild-wet-3/',
'info_dict': {
'id': '5119660',
'display_id': '5119660',
'ext': 'mp4',
'title': 'Michelle Malone - Dreamgirls - Wild Wet 3',
'uploader': 'FallenAngel12',
'duration': 402,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
}
}]
def _call_api(self, url, video_id, fatal=False, **kwargs):
content = self._download_json(url, video_id, fatal=fatal, **kwargs)
if traverse_obj(content, 'error'):
raise self._error_or_warning(ExtractorError(
f'Txxx said: {content["error"]}', expected=True), fatal=fatal)
return content or {}
def _real_extract(self, url):
video_id, host, display_id = self._match_valid_url(url).group('id', 'host', 'display_id')
headers = {'Referer': url, 'X-Requested-With': 'XMLHttpRequest'}
video_file = self._call_api(
f'https://{host}/api/videofile.php?video_id={video_id}&lifetime=8640000',
video_id, fatal=True, note='Downloading video file info', headers=headers)
slug = f'{int(1E6 * (int(video_id) // 1E6))}/{1000 * (int(video_id) // 1000)}'
video_info = self._call_api(
f'https://{host}/api/json/video/86400/{slug}/{video_id}.json',
video_id, note='Downloading video info', headers=headers)
return {
'id': video_id,
'display_id': display_id,
'title': traverse_obj(video_info, ('video', 'title')),
'uploader': traverse_obj(video_info, ('video', 'user', 'username')),
'duration': parse_duration(traverse_obj(video_info, ('video', 'duration'))),
'view_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'viewed'))),
'like_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'likes'))),
'dislike_count': int_or_none(traverse_obj(video_info, ('video', 'statistics', 'dislikes'))),
'age_limit': 18,
'formats': get_formats(host, video_file),
}
class PornTopIE(InfoExtractor):
_VALID_URL = r'https?://(?P<host>(?:www\.)?porntop\.com)/video/(?P<id>\d+)(?:/(?P<display_id>[^/?]+))?'
_TESTS = [{
'url': 'https://porntop.com/video/101569/triple-threat-with-lia-lor-malena-morgan-and-dani-daniels/',
'md5': '612ba7b3cb99455b382972948e200b08',
'info_dict': {
'id': '101569',
'display_id': 'triple-threat-with-lia-lor-malena-morgan-and-dani-daniels',
'ext': 'mp4',
'title': 'Triple Threat With Lia Lor, Malena Morgan And Dani Daniels',
'description': 'md5:285357d9d3a00ce5acb29f39f826dbf6',
'uploader': 'PatrickBush',
'duration': 480,
'view_count': int,
'like_count': int,
'dislike_count': int,
'age_limit': 18,
'timestamp': 1609455029,
'upload_date': '20201231',
'thumbnail': 'https://tn.porntop.com/media/tn/sources/101569_1.jpg',
}
}]
def _real_extract(self, url):
video_id, host, display_id = self._match_valid_url(url).group('id', 'host', 'display_id')
webpage = self._download_webpage(url, video_id)
json_ld = self._json_ld(self._search_json(
r'\bschemaJson\s*=', webpage, 'JSON-LD', video_id, transform_source=js_to_json,
contains_pattern='{[^<]+?VideoObject[^<]+};'), video_id, fatal=True)
video_file = self._parse_json(decode_base64(self._search_regex(
r"window\.initPlayer\(.*}}},\s*'(?P<json_b64c>[^']+)'",
webpage, 'json_urls', group='json_b64c')), video_id)
return merge_dicts({
'id': video_id,
'display_id': display_id,
'age_limit': 18,
'formats': get_formats(host, video_file),
}, json_ld)
yt_dlp/extractor/udemy.py
+15 -5
View File
@@ -11,8 +11,10 @@ from ..utils import (
int_or_none,
js_to_json,
sanitized_Request,
smuggle_url,
try_get,
unescapeHTML,
unsmuggle_url,
url_or_none,
urlencode_postdata,
)
@@ -106,7 +108,7 @@ class UdemyIE(InfoExtractor):
% (course_id, lecture_id),
lecture_id, 'Downloading lecture JSON', query={
'fields[lecture]': 'title,description,view_html,asset',
'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data',
'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data,course_is_drmed',
})
def _handle_error(self, response):
@@ -199,16 +201,19 @@ class UdemyIE(InfoExtractor):
def _real_extract(self, url):
lecture_id = self._match_id(url)
course_id = unsmuggle_url(url, {})[1].get('course_id')
webpage = self._download_webpage(url, lecture_id)
course_id, _ = self._extract_course_info(webpage, lecture_id)
webpage = None
if not course_id:
webpage = self._download_webpage(url, lecture_id)
course_id, _ = self._extract_course_info(webpage, lecture_id)
try:
lecture = self._download_lecture(course_id, lecture_id)
except ExtractorError as e:
# Error could possibly mean we are not enrolled in the course
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
webpage = webpage or self._download_webpage(url, lecture_id)
self._enroll_course(url, webpage, course_id)
lecture = self._download_lecture(course_id, lecture_id)
else:
@@ -391,6 +396,9 @@ class UdemyIE(InfoExtractor):
if f.get('url'):
formats.append(f)
if not formats and asset.get('course_is_drmed'):
self.report_drm(video_id)
return {
'id': video_id,
'title': title,
@@ -449,7 +457,9 @@ class UdemyCourseIE(UdemyIE): # XXX: Do not subclass from concrete IE
if lecture_id:
entry = {
'_type': 'url_transparent',
'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
'url': smuggle_url(
f'https://www.udemy.com/{course_path}/learn/v4/t/lecture/{entry["id"]}',
{'course_id': course_id}),
'title': entry.get('title'),
'ie_key': UdemyIE.ie_key(),
}
yt_dlp/extractor/uplynk.py
+51 -29
View File
@@ -2,40 +2,42 @@ import re
from .common import InfoExtractor
from ..utils import (
float_or_none,
ExtractorError,
float_or_none,
smuggle_url,
traverse_obj,
unsmuggle_url,
update_url_query,
)
class UplynkIE(InfoExtractor):
IE_NAME = 'uplynk'
_VALID_URL = r'https?://.*?\.uplynk\.com/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.(?:m3u8|json)(?:.*?\bpbs=(?P<session_id>[^&]+))?'
_TEST = {
'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8',
'info_dict': {
'id': 'e89eaf2ce9054aa89d92ddb2d817a52e',
'ext': 'mp4',
'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4',
'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa',
},
'params': {
# m3u8 download
'skip_download': True,
},
}
class UplynkBaseIE(InfoExtractor):
_UPLYNK_URL_RE = r'''(?x)
https?://[\w-]+\.uplynk\.com/(?P<path>
ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|
(?P<id>[0-9a-f]{32})
)\.(?:m3u8|json)
(?:.*?\bpbs=(?P<session_id>[^&]+))?'''
def _extract_uplynk_info(self, uplynk_content_url):
path, external_id, video_id, session_id = re.match(UplynkIE._VALID_URL, uplynk_content_url).groups()
def _extract_uplynk_info(self, url):
uplynk_content_url, smuggled_data = unsmuggle_url(url, {})
mobj = re.match(self._UPLYNK_URL_RE, uplynk_content_url)
if not mobj:
raise ExtractorError('Necessary parameters not found in Uplynk URL')
path, external_id, video_id, session_id = mobj.group('path', 'external_id', 'id', 'session_id')
display_id = video_id or external_id
headers = traverse_obj(
smuggled_data, {'Referer': 'Referer', 'Origin': 'Origin'}, casesense=False)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
'http://content.uplynk.com/%s.m3u8' % path,
display_id, 'mp4', 'm3u8_native')
f'http://content.uplynk.com/{path}.m3u8', display_id, 'mp4', headers=headers)
if session_id:
for f in formats:
f['extra_param_to_segment_url'] = 'pbs=' + session_id
asset = self._download_json('http://content.uplynk.com/player/assetinfo/%s.json' % path, display_id)
f['extra_param_to_segment_url'] = f'pbs={session_id}'
asset = self._download_json(
f'http://content.uplynk.com/player/assetinfo/{path}.json', display_id)
if asset.get('error') == 1:
raise ExtractorError('% said: %s' % (self.IE_NAME, asset['msg']), expected=True)
msg = asset.get('msg') or 'unknown error'
raise ExtractorError(f'{self.IE_NAME} said: {msg}', expected=True)
return {
'id': asset['asset'],
@@ -47,20 +49,40 @@ class UplynkIE(InfoExtractor):
'subtitles': subtitles,
}
class UplynkIE(UplynkBaseIE):
IE_NAME = 'uplynk'
_VALID_URL = UplynkBaseIE._UPLYNK_URL_RE
_TEST = {
'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8',
'info_dict': {
'id': 'e89eaf2ce9054aa89d92ddb2d817a52e',
'ext': 'mp4',
'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4',
'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa',
'duration': 530.2739166666679,
'thumbnail': r're:^https?://.*\.jpg$',
},
'params': {
'skip_download': 'm3u8',
},
}
def _real_extract(self, url):
return self._extract_uplynk_info(url)
class UplynkPreplayIE(UplynkIE): # XXX: Do not subclass from concrete IE
class UplynkPreplayIE(UplynkBaseIE):
IE_NAME = 'uplynk:preplay'
_VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
_VALID_URL = r'https?://[\w-]+\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
path, external_id, video_id = self._match_valid_url(url).groups()
display_id = video_id or external_id
preplay = self._download_json(url, display_id)
content_url = 'http://content.uplynk.com/%s.m3u8' % path
content_url = f'http://content.uplynk.com/{path}.m3u8'
session_id = preplay.get('sid')
if session_id:
content_url += '?pbs=' + session_id
return self._extract_uplynk_info(content_url)
content_url = update_url_query(content_url, {'pbs': session_id})
return self._extract_uplynk_info(smuggle_url(content_url, smuggled_data))
yt_dlp/extractor/urplay.py
+26 -2
View File
@@ -14,12 +14,13 @@ class URPlayIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
'md5': 'ff5b0c89928f8083c74bbd5099c9292d',
'md5': '5ba36643c77cc3d34ffeadad89937d1e',
'info_dict': {
'id': '203704',
'ext': 'mp4',
'title': 'UR Samtiden - Livet, universum och rymdens märkliga musik : Om vetenskap, kritiskt tänkande och motstånd',
'description': 'md5:5344508a52aa78c1ced6c1b8b9e44e9a',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1513292400,
'upload_date': '20171214',
'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
@@ -29,6 +30,24 @@ class URPlayIE(InfoExtractor):
'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
'age_limit': 15,
},
}, {
'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
'info_dict': {
'id': '222967',
'ext': 'mp4',
'title': 'En förälders dagbok : Mitt barn skadar sig själv',
'description': 'md5:9f771eef03a732a213b367b52fe826ca',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1629676800,
'upload_date': '20210823',
'series': 'En förälders dagbok',
'duration': 1740,
'age_limit': 15,
'episode_number': 3,
'categories': 'count:2',
'tags': 'count:7',
'episode': 'Mitt barn skadar sig själv',
},
}, {
'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
'info_dict': {
@@ -36,12 +55,17 @@ class URPlayIE(InfoExtractor):
'ext': 'mp4',
'title': 'Tripp, Trapp, Träd : Sovkudde',
'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
'thumbnail': r're:^https?://.+\.jpg',
'timestamp': 1440086400,
'upload_date': '20150820',
'series': 'Tripp, Trapp, Träd',
'duration': 865,
'age_limit': 1,
'episode_number': 1,
'categories': [],
'tags': ['Sova'],
'episode': 'Sovkudde',
'season': 'Säsong 1',
},
}, {
'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
@@ -69,7 +93,7 @@ class URPlayIE(InfoExtractor):
urplayer_streams = urplayer_data.get('streamingInfo', {})
for k, v in urplayer_streams.get('raw', {}).items():
if not (k in ('sd', 'hd') and isinstance(v, dict)):
if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)):
continue
file_http = v.get('location')
if file_http:
yt_dlp/extractor/videa.py
+1 -1
View File
@@ -119,7 +119,7 @@ class VideaIE(InfoExtractor):
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
query = parse_qs(player_url)
random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
random_seed = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
query['_s'] = random_seed
query['_t'] = result[:16]
yt_dlp/extractor/videoken.py
+336
View File
@@ -0,0 +1,336 @@
import base64
import functools
import math
import re
import time
import urllib.parse
from .common import InfoExtractor
from .slideslive import SlidesLiveIE
from ..utils import (
ExtractorError,
InAdvancePagedList,
int_or_none,
traverse_obj,
update_url_query,
url_or_none,
)
class VideoKenBaseIE(InfoExtractor):
_ORGANIZATIONS = {
'videos.icts.res.in': 'icts',
'videos.cncf.io': 'cncf',
'videos.neurips.cc': 'neurips',
}
_BASE_URL_RE = rf'https?://(?P<host>{"|".join(map(re.escape, _ORGANIZATIONS))})/'
_PAGE_SIZE = 12
def _get_org_id_and_api_key(self, org, video_id):
details = self._download_json(
f'https://analytics.videoken.com/api/videolake/{org}/details', video_id,
note='Downloading organization ID and API key', headers={
'Accept': 'application/json',
})
return details['id'], details['apikey']
def _create_slideslive_url(self, video_url, video_id, referer):
if not video_url and not video_id:
return
elif not video_url or 'embed/sign-in' in video_url:
video_url = f'https://slideslive.com/embed/{video_id.lstrip("slideslive-")}'
if url_or_none(referer):
return update_url_query(video_url, {
'embed_parent_url': referer,
'embed_container_origin': f'https://{urllib.parse.urlparse(referer).netloc}',
})
return video_url
def _extract_videos(self, videos, url):
for video in traverse_obj(videos, (('videos', 'results'), ...)):
video_id = traverse_obj(video, 'youtube_id', 'videoid')
if not video_id:
continue
ie_key = None
if traverse_obj(video, 'type', 'source') == 'youtube':
video_url = video_id
ie_key = 'Youtube'
else:
video_url = traverse_obj(video, 'embed_url', 'embeddableurl')
if urllib.parse.urlparse(video_url).netloc == 'slideslive.com':
ie_key = SlidesLiveIE
video_url = self._create_slideslive_url(video_url, video_id, url)
if not video_url:
continue
yield self.url_result(video_url, ie_key, video_id)
class VideoKenIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'(?:(?:topic|category)/[^/#?]+/)?video/(?P<id>[\w-]+)'
_TESTS = [{
# neurips -> videoken -> slideslive
'url': 'https://videos.neurips.cc/video/slideslive-38922815',
'info_dict': {
'id': '38922815',
'ext': 'mp4',
'title': 'Efficient Processing of Deep Neural Network: from Algorithms to Hardware Architectures',
'timestamp': 1630939331,
'upload_date': '20210906',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:330',
'chapters': 'count:329',
},
'params': {
'skip_download': 'm3u8',
},
'expected_warnings': ['Failed to download VideoKen API JSON'],
}, {
# neurips -> videoken -> slideslive -> youtube
'url': 'https://videos.neurips.cc/topic/machine%20learning/video/slideslive-38923348',
'info_dict': {
'id': '2Xa_dt78rJE',
'ext': 'mp4',
'display_id': '38923348',
'title': 'Machine Education',
'description': 'Watch full version of this video at https://slideslive.com/38923348.',
'channel': 'SlidesLive Videos - G2',
'channel_id': 'UCOExahQQ588Da8Nft_Ltb9w',
'channel_url': 'https://www.youtube.com/channel/UCOExahQQ588Da8Nft_Ltb9w',
'uploader': 'SlidesLive Videos - G2',
'uploader_id': 'UCOExahQQ588Da8Nft_Ltb9w',
'uploader_url': 'http://www.youtube.com/channel/UCOExahQQ588Da8Nft_Ltb9w',
'duration': 2504,
'timestamp': 1618922125,
'upload_date': '20200131',
'age_limit': 0,
'channel_follower_count': int,
'view_count': int,
'availability': 'unlisted',
'live_status': 'not_live',
'playable_in_embed': True,
'categories': ['People & Blogs'],
'tags': [],
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:78',
'chapters': 'count:77',
},
'params': {
'skip_download': 'm3u8',
},
'expected_warnings': ['Failed to download VideoKen API JSON'],
}, {
# icts -> videoken -> youtube
'url': 'https://videos.icts.res.in/topic/random%20variable/video/zysIsojYdvc',
'info_dict': {
'id': 'zysIsojYdvc',
'ext': 'mp4',
'title': 'Small-worlds, complex networks and random graphs (Lecture 3) by Remco van der Hofstad',
'description': 'md5:87433069d79719eeadc1962cc2ace00b',
'channel': 'International Centre for Theoretical Sciences',
'channel_id': 'UCO3xnVTHzB7l-nc8mABUJIQ',
'channel_url': 'https://www.youtube.com/channel/UCO3xnVTHzB7l-nc8mABUJIQ',
'uploader': 'International Centre for Theoretical Sciences',
'uploader_id': 'ICTStalks',
'uploader_url': 'http://www.youtube.com/user/ICTStalks',
'duration': 3372,
'upload_date': '20191004',
'age_limit': 0,
'live_status': 'not_live',
'availability': 'public',
'playable_in_embed': True,
'channel_follower_count': int,
'like_count': int,
'view_count': int,
'categories': ['Science & Technology'],
'tags': [],
'thumbnail': r're:^https?://.*\.(?:jpg|webp)',
'thumbnails': 'count:42',
'chapters': 'count:20',
},
'params': {
'skip_download': 'm3u8',
},
}, {
'url': 'https://videos.cncf.io/category/478/video/IL4nxbmUIX8',
'only_matching': True,
}, {
'url': 'https://videos.cncf.io/topic/kubernetes/video/YAM2d7yTrrI',
'only_matching': True,
}, {
'url': 'https://videos.icts.res.in/video/d7HuP_abpKU',
'only_matching': True,
}]
def _real_extract(self, url):
hostname, video_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], video_id)
details = self._download_json(
'https://analytics.videoken.com/api/videoinfo_private', video_id, query={
'videoid': video_id,
'org_id': org_id,
}, headers={'Accept': 'application/json'}, note='Downloading VideoKen API JSON',
errnote='Failed to download VideoKen API JSON', fatal=False)
if details:
return next(self._extract_videos({'videos': [details]}, url))
# fallback for API error 400 response
elif video_id.startswith('slideslive-'):
return self.url_result(
self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id)
elif re.match(r'^[\w-]{11}$', video_id):
self.url_result(video_id, 'Youtube', video_id)
else:
raise ExtractorError('Unable to extract without VideoKen API response')
class VideoKenPlayerIE(VideoKenBaseIE):
_VALID_URL = r'https?://player\.videoken\.com/embed/slideslive-(?P<id>\d+)'
_TESTS = [{
'url': 'https://player.videoken.com/embed/slideslive-38968434',
'info_dict': {
'id': '38968434',
'ext': 'mp4',
'title': 'Deep Learning with Label Differential Privacy',
'timestamp': 1643377020,
'upload_date': '20220128',
'thumbnail': r're:^https?://.*\.(?:jpg|png)',
'thumbnails': 'count:30',
'chapters': 'count:29',
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
return self.url_result(
self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id)
class VideoKenPlaylistIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'(?:category/\d+/)?playlist/(?P<id>\d+)'
_TESTS = [{
'url': 'https://videos.icts.res.in/category/1822/playlist/381',
'playlist_mincount': 117,
'info_dict': {
'id': '381',
'title': 'Cosmology - The Next Decade',
},
}]
def _real_extract(self, url):
hostname, playlist_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], playlist_id)
videos = self._download_json(
f'https://analytics.videoken.com/api/{org_id}/playlistitems/{playlist_id}/',
playlist_id, headers={'Accept': 'application/json'}, note='Downloading API JSON')
return self.playlist_result(self._extract_videos(videos, url), playlist_id, videos.get('title'))
class VideoKenCategoryIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'category/(?P<id>\d+)/?(?:$|[?#])'
_TESTS = [{
'url': 'https://videos.icts.res.in/category/1822/',
'playlist_mincount': 500,
'info_dict': {
'id': '1822',
'title': 'Programs',
},
}, {
'url': 'https://videos.neurips.cc/category/350/',
'playlist_mincount': 34,
'info_dict': {
'id': '350',
'title': 'NeurIPS 2018',
},
}, {
'url': 'https://videos.cncf.io/category/479/',
'playlist_mincount': 328,
'info_dict': {
'id': '479',
'title': 'KubeCon + CloudNativeCon Europe\'19',
},
}]
def _get_category_page(self, category_id, org_id, page=1, note=None):
return self._download_json(
f'https://analytics.videoken.com/api/videolake/{org_id}/category_videos', category_id,
fatal=False, note=note if note else f'Downloading category page {page}',
query={
'category_id': category_id,
'page_number': page,
'length': self._PAGE_SIZE,
}, headers={'Accept': 'application/json'}) or {}
def _entries(self, category_id, org_id, url, page):
videos = self._get_category_page(category_id, org_id, page + 1)
yield from self._extract_videos(videos, url)
def _real_extract(self, url):
hostname, category_id = self._match_valid_url(url).group('host', 'id')
org_id, _ = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], category_id)
category_info = self._get_category_page(category_id, org_id, note='Downloading category info')
category = category_info['category_name']
total_pages = math.ceil(int(category_info['recordsTotal']) / self._PAGE_SIZE)
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, category_id, org_id, url),
total_pages, self._PAGE_SIZE), category_id, category)
class VideoKenTopicIE(VideoKenBaseIE):
_VALID_URL = VideoKenBaseIE._BASE_URL_RE + r'topic/(?P<id>[^/#?]+)/?(?:$|[?#])'
_TESTS = [{
'url': 'https://videos.neurips.cc/topic/machine%20learning/',
'playlist_mincount': 500,
'info_dict': {
'id': 'machine_learning',
'title': 'machine learning',
},
}, {
'url': 'https://videos.icts.res.in/topic/gravitational%20waves/',
'playlist_mincount': 77,
'info_dict': {
'id': 'gravitational_waves',
'title': 'gravitational waves'
},
}, {
'url': 'https://videos.cncf.io/topic/prometheus/',
'playlist_mincount': 134,
'info_dict': {
'id': 'prometheus',
'title': 'prometheus',
},
}]
def _get_topic_page(self, topic, org_id, search_id, api_key, page=1, note=None):
return self._download_json(
'https://es.videoken.com/api/v1.0/get_results', topic, fatal=False, query={
'orgid': org_id,
'size': self._PAGE_SIZE,
'query': topic,
'page': page,
'sort': 'upload_desc',
'filter': 'all',
'token': api_key,
'is_topic': 'true',
'category': '',
'searchid': search_id,
}, headers={'Accept': 'application/json'},
note=note if note else f'Downloading topic page {page}') or {}
def _entries(self, topic, org_id, search_id, api_key, url, page):
videos = self._get_topic_page(topic, org_id, search_id, api_key, page + 1)
yield from self._extract_videos(videos, url)
def _real_extract(self, url):
hostname, topic_id = self._match_valid_url(url).group('host', 'id')
topic = urllib.parse.unquote(topic_id)
topic_id = topic.replace(' ', '_')
org_id, api_key = self._get_org_id_and_api_key(self._ORGANIZATIONS[hostname], topic)
search_id = base64.b64encode(f':{topic}:{int(time.time())}:transient'.encode()).decode()
total_pages = int_or_none(self._get_topic_page(
topic, org_id, search_id, api_key, note='Downloading topic info')['total_no_of_pages'])
return self.playlist_result(InAdvancePagedList(
functools.partial(self._entries, topic, org_id, search_id, api_key, url),
total_pages, self._PAGE_SIZE), topic_id, topic)
yt_dlp/extractor/vimeo.py
+89 -43
View File
@@ -2,6 +2,7 @@ import base64
import functools
import re
import itertools
import urllib.error
from .common import InfoExtractor
from ..compat import (
@@ -311,7 +312,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
)
\.
)?
vimeo(?:pro)?\.com/
vimeo\.com/
(?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
(?:[^/]+/)*?
(?:
@@ -355,31 +356,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
},
'skip': 'No longer available'
},
{
'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
'note': 'Vimeo Pro video (#1197)',
'info_dict': {
'id': '68093876',
'ext': 'mp4',
'uploader_url': r're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
'uploader_id': 'openstreetmapus',
'uploader': 'OpenStreetMap US',
'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
'description': 'md5:2c362968038d4499f4d79f88458590c1',
'duration': 1595,
'upload_date': '20130610',
'timestamp': 1370893156,
'license': 'by',
'thumbnail': 'https://i.vimeocdn.com/video/440260469-19b0d92fca3bd84066623b53f1eb8aaa3980c6c809e2d67b6b39ab7b4a77a344-d_960',
'view_count': int,
'comment_count': int,
'like_count': int,
},
'params': {
'format': 'best[protocol=https]',
},
},
{
'url': 'http://player.vimeo.com/video/54469442',
'md5': 'b3e7f4d2cbb53bd7dc3bb6ff4ed5cfbd',
@@ -837,15 +813,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
if unlisted_hash:
return self._extract_from_api(video_id, unlisted_hash)
orig_url = url
is_pro = 'vimeopro.com/' in url
if is_pro:
# some videos require portfolio_id to be present in player url
# https://github.com/ytdl-org/youtube-dl/issues/20070
url = self._extract_url(url, self._download_webpage(url, video_id))
if not url:
url = 'https://vimeo.com/' + video_id
elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
if any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
url = 'https://vimeo.com/' + video_id
self._try_album_password(url)
@@ -947,14 +915,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
video_description = self._html_search_meta(
['description', 'og:description', 'twitter:description'],
webpage, default=None)
if not video_description and is_pro:
orig_webpage = self._download_webpage(
orig_url, video_id,
note='Downloading webpage for description',
fatal=False)
if orig_webpage:
video_description = self._html_search_meta(
'description', orig_webpage, default=None)
if not video_description:
self.report_warning('Cannot find video description')
@@ -1393,3 +1353,89 @@ class VHXEmbedIE(VimeoBaseInfoExtractor):
info = self._parse_config(config, video_id)
info['id'] = video_id
return info
class VimeoProIE(VimeoBaseInfoExtractor):
IE_NAME = 'vimeo:pro'
_VALID_URL = r'https?://(?:www\.)?vimeopro\.com/[^/?#]+/(?P<slug>[^/?#]+)(?:(?:/videos?/(?P<id>[0-9]+)))?'
_TESTS = [{
# Vimeo URL derived from video_id
'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
'note': 'Vimeo Pro video (#1197)',
'info_dict': {
'id': '68093876',
'ext': 'mp4',
'uploader_url': r're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
'uploader_id': 'openstreetmapus',
'uploader': 'OpenStreetMap US',
'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
'description': 'md5:2c362968038d4499f4d79f88458590c1',
'duration': 1595,
'upload_date': '20130610',
'timestamp': 1370893156,
'license': 'by',
'thumbnail': 'https://i.vimeocdn.com/video/440260469-19b0d92fca3bd84066623b53f1eb8aaa3980c6c809e2d67b6b39ab7b4a77a344-d_960',
'view_count': int,
'comment_count': int,
'like_count': int,
'tags': 'count:1',
},
'params': {
'format': 'best[protocol=https]',
},
}, {
# password-protected VimeoPro page with Vimeo player embed
'url': 'https://vimeopro.com/cadfem/simulation-conference-mechanische-systeme-in-perfektion',
'info_dict': {
'id': '764543723',
'ext': 'mp4',
'title': 'Mechanische Systeme in Perfektion: Realität erfassen, Innovation treiben',
'thumbnail': 'https://i.vimeocdn.com/video/1543784598-a1a750494a485e601110136b9fe11e28c2131942452b3a5d30391cb3800ca8fd-d_1280',
'description': 'md5:2a9d195cd1b0f6f79827107dc88c2420',
'uploader': 'CADFEM',
'uploader_id': 'cadfem',
'uploader_url': 'https://vimeo.com/cadfem',
'duration': 12505,
'chapters': 'count:10',
},
'params': {
'videopassword': 'Conference2022',
'skip_download': True,
},
}]
def _real_extract(self, url):
display_id, video_id = self._match_valid_url(url).group('slug', 'id')
if video_id:
display_id = video_id
webpage = self._download_webpage(url, display_id)
password_form = self._search_regex(
r'(?is)<form[^>]+?method=["\']post["\'][^>]*>(.+?password.+?)</form>',
webpage, 'password form', default=None)
if password_form:
try:
webpage = self._download_webpage(url, display_id, data=urlencode_postdata({
'password': self._get_video_password(),
**self._hidden_inputs(password_form),
}), note='Logging in with video password')
except ExtractorError as e:
if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 418:
raise ExtractorError('Wrong video password', expected=True)
raise
description = None
# even if we have video_id, some videos require player URL with portfolio_id query param
# https://github.com/ytdl-org/youtube-dl/issues/20070
vimeo_url = VimeoIE._extract_url(url, webpage)
if vimeo_url:
description = self._html_search_meta('description', webpage, default=None)
elif video_id:
vimeo_url = f'https://vimeo.com/{video_id}'
else:
raise ExtractorError(
'No Vimeo embed or video ID could be found in VimeoPro page', expected=True)
return self.url_result(vimeo_url, VimeoIE, video_id, url_transparent=True,
description=description)
yt_dlp/extractor/viu.py
+1 -1
View File
@@ -251,7 +251,7 @@ class ViuOTTIE(InfoExtractor):
return self._user_token
def _get_token(self, country_code, video_id):
rand = ''.join(random.choice('0123456789') for _ in range(10))
rand = ''.join(random.choices('0123456789', k=10))
return self._download_json(
f'https://api-gateway-global.viu.com/api/auth/token?v={rand}000', video_id,
headers={'Content-Type': 'application/json'}, note='Getting bearer token',
yt_dlp/extractor/vk.py
+3 -3
View File
@@ -6,6 +6,7 @@ from .common import InfoExtractor
from .dailymotion import DailymotionIE
from .odnoklassniki import OdnoklassnikiIE
from .pladform import PladformIE
from .sibnet import SibnetEmbedIE
from .vimeo import VimeoIE
from .youtube import YoutubeIE
from ..compat import compat_urlparse
@@ -101,8 +102,7 @@ class VKIE(VKBaseIE):
(?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>([\da-f]+)|(ln-[\da-zA-Z]+)))?
)
'''
# https://help.sibnet.ru/?sibnet_video_embed
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//video\.sibnet\.ru/shell\.php\?.*?\bvideoid=\d+.*?)\1']
_TESTS = [
{
'url': 'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
@@ -455,7 +455,7 @@ class VKIE(VKBaseIE):
if odnoklassniki_url:
return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
sibnet_url = next(self._extract_embed_urls(url, info_page), None)
sibnet_url = next(SibnetEmbedIE._extract_embed_urls(url, info_page), None)
if sibnet_url:
return self.url_result(sibnet_url)
yt_dlp/extractor/volejtv.py
+40
View File
@@ -0,0 +1,40 @@
from .common import InfoExtractor
class VolejTVIE(InfoExtractor):
_VALID_URL = r'https?://volej\.tv/video/(?P<id>\d+)'
_TESTS = [{
'url': 'https://volej.tv/video/725742/',
'info_dict': {
'id': '725742',
'ext': 'mp4',
'description': 'Zápas VK Královo Pole vs VK Prostějov 10.12.2022 v 19:00 na Volej.TV',
'thumbnail': 'https://volej.tv/images/og/16/17186/og.png',
'title': 'VK Královo Pole vs VK Prostějov',
}
}, {
'url': 'https://volej.tv/video/725605/',
'info_dict': {
'id': '725605',
'ext': 'mp4',
'thumbnail': 'https://volej.tv/images/og/15/17185/og.png',
'title': 'VK Lvi Praha vs VK Euro Sitex Příbram',
'description': 'Zápas VK Lvi Praha vs VK Euro Sitex Příbram 11.12.2022 v 19:00 na Volej.TV',
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
json_data = self._search_json(
r'<\s*!\[CDATA[^=]+=', webpage, 'CDATA', video_id)
formats, subtitle = self._extract_m3u8_formats_and_subtitles(
json_data['urls']['hls'], video_id)
return {
'id': video_id,
'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage),
'description': self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage),
'formats': formats,
'subtitles': subtitle,
}
yt_dlp/extractor/vrv.py
+1 -1
View File
@@ -30,7 +30,7 @@ class VRVBaseIE(InfoExtractor):
base_url = self._API_DOMAIN + '/core/' + path
query = [
('oauth_consumer_key', self._API_PARAMS['oAuthKey']),
('oauth_nonce', ''.join([random.choice(string.ascii_letters) for _ in range(32)])),
('oauth_nonce', ''.join(random.choices(string.ascii_letters, k=32))),
('oauth_signature_method', 'HMAC-SHA1'),
('oauth_timestamp', int(time.time())),
]
yt_dlp/extractor/webcamerapl.py
+44
View File
@@ -0,0 +1,44 @@
import codecs
from .common import InfoExtractor
class WebcameraplIE(InfoExtractor):
_VALID_URL = r'https?://(?P<id>[\w-]+)\.webcamera\.pl'
_TESTS = [{
'url': 'https://warszawa-plac-zamkowy.webcamera.pl',
'info_dict': {
'id': 'warszawa-plac-zamkowy',
'ext': 'mp4',
'title': r're:WIDOK NA PLAC ZAMKOWY W WARSZAWIE \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
'live_status': 'is_live',
}
}, {
'url': 'https://gdansk-stare-miasto.webcamera.pl/',
'info_dict': {
'id': 'gdansk-stare-miasto',
'ext': 'mp4',
'title': r're:GDAŃSK - widok na Stare Miasto \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
'live_status': 'is_live',
}
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
rot13_m3u8_url = self._search_regex(r'data-src\s*=\s*"(uggc[^"]+\.z3h8)"',
webpage, 'm3u8 url', default=None)
if not rot13_m3u8_url:
self.raise_no_formats('No video/audio found at the provided url', expected=True)
m3u8_url = codecs.decode(rot13_m3u8_url, 'rot-13')
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, live=True)
return {
'id': video_id,
'title': self._html_search_regex(r'<h1\b[^>]*>([^>]+)</h1>', webpage, 'title'),
'formats': formats,
'subtitles': subtitles,
'is_live': True,
}
yt_dlp/extractor/wistia.py
+27 -14
View File
@@ -6,12 +6,15 @@ from base64 import b64decode
from .common import InfoExtractor
from ..utils import (
ExtractorError,
HEADRequest,
determine_ext,
float_or_none,
int_or_none,
parse_qs,
traverse_obj,
try_get,
update_url_query,
urlhandle_detect_ext,
)
@@ -34,6 +37,16 @@ class WistiaBaseIE(InfoExtractor):
return embed_config
def _get_real_ext(self, url):
ext = determine_ext(url, default_ext='bin')
if ext == 'bin':
urlh = self._request_webpage(
HEADRequest(url), None, note='Checking media extension',
errnote='HEAD request returned error', fatal=False)
if urlh:
ext = urlhandle_detect_ext(urlh, default='bin')
return 'mp4' if ext == 'mov' else ext
def _extract_media(self, embed_config):
data = embed_config['media']
video_id = data['hashedId']
@@ -51,13 +64,13 @@ class WistiaBaseIE(InfoExtractor):
continue
elif atype in ('still', 'still_image'):
thumbnails.append({
'url': aurl,
'url': aurl.replace('.bin', f'.{self._get_real_ext(aurl)}'),
'width': int_or_none(a.get('width')),
'height': int_or_none(a.get('height')),
'filesize': int_or_none(a.get('size')),
})
else:
aext = a.get('ext')
aext = a.get('ext') or self._get_real_ext(aurl)
display_name = a.get('display_name')
format_id = atype
if atype and atype.endswith('_video') and display_name:
@@ -169,26 +182,26 @@ class WistiaIE(WistiaBaseIE):
'md5': '10c1ce9c4dde638202513ed17a3767bd',
'info_dict': {
'id': 'a6ndpko1wg',
'ext': 'bin',
'ext': 'mp4',
'title': 'Episode 2: Boxed Water\'s retention is thirsty',
'upload_date': '20210324',
'description': 'md5:da5994c2c2d254833b412469d9666b7a',
'duration': 966.0,
'timestamp': 1616614369,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/53dc60239348dc9b9fba3755173ea4c2.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/53dc60239348dc9b9fba3755173ea4c2.png',
}
}, {
'url': 'wistia:5vd7p4bct5',
'md5': 'b9676d24bf30945d97060638fbfe77f0',
'info_dict': {
'id': '5vd7p4bct5',
'ext': 'bin',
'ext': 'mp4',
'title': 'md5:eaa9f64c4efd7b5f098b9b6118597679',
'description': 'md5:a9bea0315f0616aa5df2dc413ddcdd0f',
'upload_date': '20220915',
'timestamp': 1663258727,
'duration': 623.019,
'thumbnail': r're:https?://embed(?:-ssl)?.wistia.com/.+\.(?:jpg|bin)$',
'thumbnail': r're:https?://embed(?:-ssl)?.wistia.com/.+\.jpg$',
},
}, {
'url': 'wistia:sh7fpupwlt',
@@ -208,25 +221,25 @@ class WistiaIE(WistiaBaseIE):
'url': 'https://www.weidert.com/blog/wistia-channels-video-marketing-tool',
'info_dict': {
'id': 'cqwukac3z1',
'ext': 'bin',
'ext': 'mp4',
'title': 'How Wistia Channels Can Help Capture Inbound Value From Your Video Content',
'duration': 158.125,
'timestamp': 1618974400,
'description': 'md5:27abc99a758573560be72600ef95cece',
'upload_date': '20210421',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/6c551820ae950cdee2306d6cbe9ef742.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/6c551820ae950cdee2306d6cbe9ef742.jpg',
}
}, {
'url': 'https://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
'md5': 'b9676d24bf30945d97060638fbfe77f0',
'info_dict': {
'id': '5vd7p4bct5',
'ext': 'bin',
'ext': 'mp4',
'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
'upload_date': '20220915',
'timestamp': 1663258727,
'duration': 623.019,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/83e6ec693e2c05a0ce65809cbaead86a.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/83e6ec693e2c05a0ce65809cbaead86a.jpg',
'description': 'a Paywall Videos video',
},
}]
@@ -302,9 +315,9 @@ class WistiaChannelIE(WistiaBaseIE):
'url': 'https://fast.wistia.net/embed/channel/3802iirk0l?wchannelid=3802iirk0l&wmediaid=sp5dqjzw3n',
'info_dict': {
'id': 'sp5dqjzw3n',
'ext': 'bin',
'ext': 'mp4',
'title': 'The Roof S2: The Modern CRO',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/dadfa9233eaa505d5e0c85c23ff70741.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/dadfa9233eaa505d5e0c85c23ff70741.png',
'duration': 86.487,
'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season.\n',
'timestamp': 1619790290,
@@ -334,12 +347,12 @@ class WistiaChannelIE(WistiaBaseIE):
'info_dict': {
'id': 'pz0m0l0if3',
'title': 'A Framework for Improving Product Team Performance',
'ext': 'bin',
'ext': 'mp4',
'timestamp': 1653935275,
'upload_date': '20220530',
'description': 'Learn how to help your company improve and achieve your product related goals.',
'duration': 1854.39,
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/12fd19e56413d9d6f04e2185c16a6f8854e25226.bin',
'thumbnail': 'https://embed-ssl.wistia.com/deliveries/12fd19e56413d9d6f04e2185c16a6f8854e25226.png',
},
'params': {'noplaylist': True, 'skip_download': True},
}]
yt_dlp/extractor/xanimu.py
+51
View File
@@ -0,0 +1,51 @@
import re
from .common import InfoExtractor
from ..utils import int_or_none
class XanimuIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?xanimu\.com/(?P<id>[^/]+)/?'
_TESTS = [{
'url': 'https://xanimu.com/51944-the-princess-the-frog-hentai/',
'md5': '899b88091d753d92dad4cb63bbf357a7',
'info_dict': {
'id': '51944-the-princess-the-frog-hentai',
'ext': 'mp4',
'title': 'The Princess + The Frog Hentai',
'thumbnail': 'https://xanimu.com/storage/2020/09/the-princess-and-the-frog-hentai.jpg',
'description': r're:^Enjoy The Princess \+ The Frog Hentai',
'duration': 207.0,
'age_limit': 18
}
}, {
'url': 'https://xanimu.com/huge-expansion/',
'only_matching': True
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
formats = []
for format in ['videoHigh', 'videoLow']:
format_url = self._search_json(r'var\s+%s\s*=' % re.escape(format), webpage, format,
video_id, default=None, contains_pattern=r'[\'"]([^\'"]+)[\'"]')
if format_url:
formats.append({
'url': format_url,
'format_id': format,
'quality': -2 if format.endswith('Low') else None,
})
return {
'id': video_id,
'formats': formats,
'title': self._search_regex(r'[\'"]headline[\'"]:\s*[\'"]([^"]+)[\'"]', webpage,
'title', default=None) or self._html_extract_title(webpage),
'thumbnail': self._html_search_meta('thumbnailUrl', webpage, default=None),
'description': self._html_search_meta('description', webpage, default=None),
'duration': int_or_none(self._search_regex(r'duration:\s*[\'"]([^\'"]+?)[\'"]',
webpage, 'duration', fatal=False)),
'age_limit': 18
}
yt_dlp/extractor/xiami.py
-198
View File
@@ -1,198 +0,0 @@
from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote
from ..utils import int_or_none
class XiamiBaseIE(InfoExtractor):
_API_BASE_URL = 'https://emumo.xiami.com/song/playlist/cat/json/id'
def _download_webpage_handle(self, *args, **kwargs):
webpage = super(XiamiBaseIE, self)._download_webpage_handle(*args, **kwargs)
if '>Xiami is currently not available in your country.<' in webpage:
self.raise_geo_restricted('Xiami is currently not available in your country')
return webpage
def _extract_track(self, track, track_id=None):
track_name = track.get('songName') or track.get('name') or track['subName']
artist = track.get('artist') or track.get('artist_name') or track.get('singers')
title = '%s - %s' % (artist, track_name) if artist else track_name
track_url = self._decrypt(track['location'])
subtitles = {}
lyrics_url = track.get('lyric_url') or track.get('lyric')
if lyrics_url and lyrics_url.startswith('http'):
subtitles['origin'] = [{'url': lyrics_url}]
return {
'id': track.get('song_id') or track_id,
'url': track_url,
'title': title,
'thumbnail': track.get('pic') or track.get('album_pic'),
'duration': int_or_none(track.get('length')),
'creator': track.get('artist', '').split(';')[0],
'track': track_name,
'track_number': int_or_none(track.get('track')),
'album': track.get('album_name') or track.get('title'),
'artist': artist,
'subtitles': subtitles,
}
def _extract_tracks(self, item_id, referer, typ=None):
playlist = self._download_json(
'%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''),
item_id, headers={
'Referer': referer,
})
return [
self._extract_track(track, item_id)
for track in playlist['data']['trackList']]
@staticmethod
def _decrypt(origin):
n = int(origin[0])
origin = origin[1:]
short_length = len(origin) // n
long_num = len(origin) - short_length * n
l = tuple()
for i in range(0, n):
length = short_length
if i < long_num:
length += 1
l += (origin[0:length], )
origin = origin[length:]
ans = ''
for i in range(0, short_length + 1):
for j in range(0, n):
if len(l[j]) > i:
ans += l[j][i]
return compat_urllib_parse_unquote(ans).replace('^', '0')
class XiamiSongIE(XiamiBaseIE):
IE_NAME = 'xiami:song'
IE_DESC = '虾米音乐'
_VALID_URL = r'https?://(?:www\.)?xiami\.com/song/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'http://www.xiami.com/song/1775610518',
'md5': '521dd6bea40fd5c9c69f913c232cb57e',
'info_dict': {
'id': '1775610518',
'ext': 'mp3',
'title': 'HONNE - Woman',
'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
'duration': 265,
'creator': 'HONNE',
'track': 'Woman',
'album': 'Woman',
'artist': 'HONNE',
'subtitles': {
'origin': [{
'ext': 'lrc',
}],
},
},
'skip': 'Georestricted',
}, {
'url': 'http://www.xiami.com/song/1775256504',
'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
'info_dict': {
'id': '1775256504',
'ext': 'mp3',
'title': '戴荃 - 悟空',
'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
'duration': 200,
'creator': '戴荃',
'track': '悟空',
'album': '悟空',
'artist': '戴荃',
'subtitles': {
'origin': [{
'ext': 'lrc',
}],
},
},
'skip': 'Georestricted',
}, {
'url': 'http://www.xiami.com/song/1775953850',
'info_dict': {
'id': '1775953850',
'ext': 'mp3',
'title': 'До Скону - Чума Пожирает Землю',
'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
'duration': 683,
'creator': 'До Скону',
'track': 'Чума Пожирает Землю',
'track_number': 7,
'album': 'Ад',
'artist': 'До Скону',
},
'params': {
'skip_download': True,
},
}, {
'url': 'http://www.xiami.com/song/xLHGwgd07a1',
'only_matching': True,
}]
def _real_extract(self, url):
return self._extract_tracks(self._match_id(url), url)[0]
class XiamiPlaylistBaseIE(XiamiBaseIE):
def _real_extract(self, url):
item_id = self._match_id(url)
return self.playlist_result(self._extract_tracks(item_id, url, self._TYPE), item_id)
class XiamiAlbumIE(XiamiPlaylistBaseIE):
IE_NAME = 'xiami:album'
IE_DESC = '虾米音乐 - 专辑'
_VALID_URL = r'https?://(?:www\.)?xiami\.com/album/(?P<id>[^/?#&]+)'
_TYPE = '1'
_TESTS = [{
'url': 'http://www.xiami.com/album/2100300444',
'info_dict': {
'id': '2100300444',
},
'playlist_count': 10,
'skip': 'Georestricted',
}, {
'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
'only_matching': True,
}, {
'url': 'http://www.xiami.com/album/URVDji2a506',
'only_matching': True,
}]
class XiamiArtistIE(XiamiPlaylistBaseIE):
IE_NAME = 'xiami:artist'
IE_DESC = '虾米音乐 - 歌手'
_VALID_URL = r'https?://(?:www\.)?xiami\.com/artist/(?P<id>[^/?#&]+)'
_TYPE = '2'
_TESTS = [{
'url': 'http://www.xiami.com/artist/2132?spm=0.0.0.0.dKaScp',
'info_dict': {
'id': '2132',
},
'playlist_count': 20,
'skip': 'Georestricted',
}, {
'url': 'http://www.xiami.com/artist/bC5Tk2K6eb99',
'only_matching': True,
}]
class XiamiCollectionIE(XiamiPlaylistBaseIE):
IE_NAME = 'xiami:collection'
IE_DESC = '虾米音乐 - 精选集'
_VALID_URL = r'https?://(?:www\.)?xiami\.com/collect/(?P<id>[^/?#&]+)'
_TYPE = '3'
_TEST = {
'url': 'http://www.xiami.com/collect/156527391?spm=a1z1s.2943601.6856193.12.4jpBnr',
'info_dict': {
'id': '156527391',
},
'playlist_mincount': 29,
'skip': 'Georestricted',
}
yt_dlp/extractor/yandexvideo.py
+2 -2
View File
@@ -270,9 +270,9 @@ class ZenYandexIE(InfoExtractor):
for s_url in stream_urls:
ext = determine_ext(s_url)
if ext == 'mpd':
formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
formats.extend(self._extract_mpd_formats(s_url, video_id, mpd_id='dash'))
elif ext == 'm3u8':
formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
formats.extend(self._extract_m3u8_formats(s_url, video_id, 'mp4'))
return {
'id': video_id,
'title': video_json.get('title') or self._og_search_title(webpage),
yt_dlp/extractor/yle_areena.py
+69 -27
View File
@@ -1,40 +1,79 @@
from .common import InfoExtractor
from .kaltura import KalturaIE
from ..utils import int_or_none, traverse_obj, url_or_none
from ..utils import (
int_or_none,
smuggle_url,
traverse_obj,
unified_strdate,
url_or_none,
)
class YleAreenaIE(InfoExtractor):
_VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[\d-]+)'
_TESTS = [{
'url': 'https://areena.yle.fi/1-4371942',
'md5': '932edda0ecf5dfd6423804182d32f8ac',
'info_dict': {
'id': '0_a3tjk92c',
'ext': 'mp4',
'title': 'Pouchit',
'description': 'md5:d487309c3abbe5650265bbd1742d2f82',
'series': 'Modernit miehet',
'season': 'Season 1',
'season_number': 1,
'episode': 'Episode 2',
'episode_number': 2,
'thumbnail': 'http://cfvod.kaltura.com/p/1955031/sp/195503100/thumbnail/entry_id/0_a3tjk92c/version/100061',
'uploader_id': 'ovp@yle.fi',
'duration': 1435,
'view_count': int,
'upload_date': '20181204',
'timestamp': 1543916210,
'subtitles': {'fin': [{'url': r're:^https?://', 'ext': 'srt'}]},
'age_limit': 7,
_TESTS = [
{
'url': 'https://areena.yle.fi/1-4371942',
'md5': '932edda0ecf5dfd6423804182d32f8ac',
'info_dict': {
'id': '0_a3tjk92c',
'ext': 'mp4',
'title': 'Pouchit',
'description': 'md5:d487309c3abbe5650265bbd1742d2f82',
'series': 'Modernit miehet',
'season': 'Season 1',
'season_number': 1,
'episode': 'Episode 2',
'episode_number': 2,
'thumbnail': 'http://cfvod.kaltura.com/p/1955031/sp/195503100/thumbnail/entry_id/0_a3tjk92c/version/100061',
'uploader_id': 'ovp@yle.fi',
'duration': 1435,
'view_count': int,
'upload_date': '20181204',
'release_date': '20190106',
'timestamp': 1543916210,
'subtitles': {'fin': [{'url': r're:^https?://', 'ext': 'srt'}]},
'age_limit': 7,
'webpage_url': 'https://areena.yle.fi/1-4371942'
}
},
{
'url': 'https://areena.yle.fi/1-2158940',
'md5': 'cecb603661004e36af8c5188b5212b12',
'info_dict': {
'id': '1_l38iz9ur',
'ext': 'mp4',
'title': 'Albi haluaa vessan',
'description': 'md5:15236d810c837bed861fae0e88663c33',
'series': 'Albi Lumiukko',
'season': None,
'season_number': None,
'episode': None,
'episode_number': None,
'thumbnail': 'http://cfvod.kaltura.com/p/1955031/sp/195503100/thumbnail/entry_id/1_l38iz9ur/version/100021',
'uploader_id': 'ovp@yle.fi',
'duration': 319,
'view_count': int,
'upload_date': '20211202',
'release_date': '20211215',
'timestamp': 1638448202,
'subtitles': {},
'age_limit': 0,
'webpage_url': 'https://areena.yle.fi/1-2158940'
}
}
}]
]
def _real_extract(self, url):
video_id = self._match_id(url)
info = self._search_json_ld(self._download_webpage(url, video_id), video_id, default={})
video_data = self._download_json(
f'https://player.api.yle.fi/v1/preview/{video_id}.json?app_id=player_static_prod&app_key=8930d72170e48303cf5f3867780d549b',
video_id)
video_id, headers={
'origin': 'https://areena.yle.fi',
'referer': 'https://areena.yle.fi/',
'content-type': 'application/json'
})
# Example title: 'K1, J2: Pouchit | Modernit miehet'
series, season_number, episode_number, episode = self._search_regex(
@@ -54,7 +93,9 @@ class YleAreenaIE(InfoExtractor):
return {
'_type': 'url_transparent',
'url': 'kaltura:1955031:%s' % traverse_obj(video_data, ('data', 'ongoing_ondemand', 'kaltura', 'id')),
'url': smuggle_url(
f'kaltura:1955031:{video_data["data"]["ongoing_ondemand"]["kaltura"]["id"]}',
{'source_url': url}),
'ie_key': KalturaIE.ie_key(),
'title': (traverse_obj(video_data, ('data', 'ongoing_ondemand', 'title', 'fin'), expected_type=str)
or episode or info.get('title')),
@@ -62,10 +103,11 @@ class YleAreenaIE(InfoExtractor):
'series': (traverse_obj(video_data, ('data', 'ongoing_ondemand', 'series', 'title', 'fin'), expected_type=str)
or series),
'season_number': (int_or_none(self._search_regex(r'Kausi (\d+)', description, 'season number', default=None))
or int(season_number)),
or int_or_none(season_number)),
'episode_number': (traverse_obj(video_data, ('data', 'ongoing_ondemand', 'episode_number'), expected_type=int_or_none)
or int(episode_number)),
or int_or_none(episode_number)),
'thumbnails': traverse_obj(info, ('thumbnails', ..., {'url': 'url'})),
'age_limit': traverse_obj(video_data, ('data', 'ongoing_ondemand', 'content_rating', 'age_restriction'), expected_type=int_or_none),
'subtitles': subtitles,
'release_date': unified_strdate(traverse_obj(video_data, ('data', 'ongoing_ondemand', 'start_time'), expected_type=str)),
}
yt_dlp/extractor/youku.py
+23 -13
View File
@@ -96,31 +96,41 @@ class YoukuIE(InfoExtractor):
'thumbnail': r're:^https?://.*',
'uploader': '明月庄主moon',
'uploader_id': '38465621',
'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UMTUzODYyNDg0',
'tags': list,
},
}, {
'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
'url': 'https://v.youku.com/v_show/id_XNTA2NTA0MjA1Mg==.html',
'info_dict': {
'id': 'XMjIyNzAzMTQ4NA',
'id': 'XNTA2NTA0MjA1Mg',
'ext': 'mp4',
'title': '卡马乔国足开大脚长传冲吊集锦',
'duration': 289,
'title': 'Minecraft我的世界：建造超大巨型航空飞机，菜鸟vs高手vs黑客',
'duration': 542.13,
'thumbnail': r're:^https?://.*',
'uploader': '阿卜杜拉之星',
'uploader_id': '2382249',
'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==',
'uploader': '波哥游戏解说',
'uploader_id': '156688084',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UNjI2NzUyMzM2',
'tags': list,
},
}, {
'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
'only_matching': True,
'url': 'https://v.youku.com/v_show/id_XNTE1MzczOTg4MA==.html',
'info_dict': {
'id': 'XNTE1MzczOTg4MA',
'ext': 'mp4',
'title': '国产超A特工片',
'duration': 362.97,
'thumbnail': r're:^https?://.*',
'uploader': '陈晓娟说历史',
'uploader_id': '1640913339',
'uploader_url': 'https://www.youku.com/profile/index/?uid=UNjU2MzY1MzM1Ng==',
'tags': list,
},
}]
@staticmethod
def get_ysuid():
return '%d%s' % (int(time.time()), ''.join([
random.choice(string.ascii_letters) for i in range(3)]))
return '%d%s' % (int(time.time()), ''.join(
random.choices(string.ascii_letters, k=3)))
def get_format_name(self, fm):
_dict = {
@@ -151,7 +161,7 @@ class YoukuIE(InfoExtractor):
# request basic data
basic_data_params = {
'vid': video_id,
'ccode': '0532',
'ccode': '0524',
'client_ip': '192.168.1.1',
'utid': cna,
'client_ts': time.time() / 1000,
yt_dlp/extractor/youporn.py
+22 -2
View File
@@ -4,6 +4,7 @@ from .common import InfoExtractor
from ..utils import (
extract_attributes,
int_or_none,
merge_dicts,
str_to_int,
unified_strdate,
url_or_none,
@@ -64,6 +65,24 @@ class YouPornIE(InfoExtractor):
}, {
'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
'only_matching': True,
}, {
'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
'info_dict': {
'id': '16290308',
'age_limit': 18,
'categories': [],
'description': 'md5:00ea70f642f431c379763c17c2f396bc',
'display_id': 'tinderspecial-trailer1',
'duration': 298.0,
'ext': 'mp4',
'upload_date': '20201123',
'uploader': 'Ersties',
'tags': [],
'thumbnail': 'https://fi1.ypncdn.com/202011/23/16290308/original/8/tinderspecial-trailer1-8(m=eaAaaEPbaaaa).jpg',
'timestamp': 1606089600,
'title': 'Tinder In Real Life',
'view_count': int,
}
}]
def _real_extract(self, url):
@@ -159,7 +178,8 @@ class YouPornIE(InfoExtractor):
r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
'tags')
return {
data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False)
return merge_dicts(data, {
'id': video_id,
'display_id': display_id,
'title': title,
@@ -174,4 +194,4 @@ class YouPornIE(InfoExtractor):
'tags': tags,
'age_limit': age_limit,
'formats': formats,
}
})
yt_dlp/extractor/youtube.py
+204 -26
View File
@@ -292,7 +292,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors"""
_RESERVED_NAMES = (
r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
r'browse|oembed|get_video_info|iframe_api|s/player|source|'
r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
@@ -1012,7 +1012,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
youtube\.googleapis\.com)/ # the various hostnames, with wildcard subdomains
(?:.*?\#/)? # handle anchor (#/) redirect urls
(?: # the various things that can precede the ID:
(?:(?:v|embed|e|shorts)/(?!videoseries|live_stream)) # v/ or embed/ or e/ or shorts/
(?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream)) # v/ or embed/ or e/ or shorts/
|(?: # or the v= param in all its forms
(?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
(?:\?|\#!?) # the params delimiter ? or # or #!
@@ -2544,7 +2544,67 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'tags': [],
},
'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
}
}, {
'note': 'Audio formats with Dynamic Range Compression',
'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
'info_dict': {
'id': 'Tq92D6wQ1mg',
'ext': 'weba',
'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
'channel_follower_count': int,
'description': 'md5:17eccca93a786d51bc67646756894066',
'upload_date': '20191228',
'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
'playable_in_embed': True,
'like_count': int,
'categories': ['Entertainment'],
'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
'age_limit': 18,
'channel': 'Projekt Melody',
'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
'view_count': int,
'availability': 'needs_auth',
'comment_count': int,
'live_status': 'not_live',
'uploader': 'Projekt Melody',
'duration': 106,
},
'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
},
{
'url': 'https://www.youtube.com/live/qVv6vCqciTM',
'info_dict': {
'id': 'qVv6vCqciTM',
'ext': 'mp4',
'age_limit': 0,
'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
'comment_count': int,
'chapters': 'count:13',
'upload_date': '20221223',
'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
'like_count': int,
'release_date': '20221223',
'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
'view_count': int,
'playable_in_embed': True,
'duration': 4438,
'availability': 'public',
'channel_follower_count': int,
'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
'categories': ['Entertainment'],
'live_status': 'was_live',
'release_timestamp': 1671793345,
'channel': 'さなちゃんねる',
'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
'uploader': 'さなちゃんねる',
},
},
]
_WEBPAGE_TESTS = [
@@ -2621,18 +2681,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
"""
@returns (manifest_url, manifest_stream_number, is_live) or None
"""
with lock:
refetch_manifest(format_id, delay)
for retry in self.RetryManager(fatal=False):
with lock:
refetch_manifest(format_id, delay)
f = next((f for f in formats if f['format_id'] == format_id), None)
if not f:
if not is_live:
self.to_screen(f'{video_id}: Video is no longer live')
else:
self.report_warning(
f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
return None
return f['manifest_url'], f['manifest_stream_number'], is_live
f = next((f for f in formats if f['format_id'] == format_id), None)
if not f:
if not is_live:
retry.error = f'{video_id}: Video is no longer live'
else:
retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
continue
return f['manifest_url'], f['manifest_stream_number'], is_live
return None
for f in formats:
f['is_live'] = is_live
@@ -3553,7 +3614,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
itag = str_or_none(fmt.get('itag'))
audio_track = fmt.get('audioTrack') or {}
stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
if stream_id in stream_ids:
continue
@@ -3634,11 +3695,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
dct = {
'asr': int_or_none(fmt.get('audioSampleRate')),
'filesize': int_or_none(fmt.get('contentLength')),
'format_id': itag,
'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
'format_note': join_nonempty(
'%s%s' % (audio_track.get('displayName') or '',
' (default)' if language_preference > 0 else ''),
fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
'DRC' if fmt.get('isDrc') else None,
try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
@@ -3647,13 +3709,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'fps': int_or_none(fmt.get('fps')) or None,
'audio_channels': fmt.get('audioChannels'),
'height': height,
'quality': q(quality),
'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
'has_drm': bool(fmt.get('drmFamilies')),
'tbr': tbr,
'url': fmt_url,
'width': int_or_none(fmt.get('width')),
'language': join_nonempty(audio_track.get('id', '').split('.')[0],
'desc' if language_preference < -1 else ''),
'desc' if language_preference < -1 else '') or None,
'language_preference': language_preference,
# Strictly de-prioritize damaged and 3gp formats
'preference': -10 if is_damaged else -2 if itag == '17' else None,
@@ -4085,7 +4147,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if not trans_code:
continue
orig_trans_code = trans_code
if caption_track.get('kind') != 'asr':
if caption_track.get('kind') != 'asr' and trans_code != 'und':
if not get_translated_subs:
continue
trans_code += f'-{lang_code}'
@@ -4382,6 +4444,25 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
elif key.startswith('grid') and key.endswith('Renderer'):
return renderer
def _extract_channel_renderer(self, renderer):
channel_id = renderer['channelId']
title = self._get_text(renderer, 'title')
channel_url = f'https://www.youtube.com/channel/{channel_id}'
return {
'_type': 'url',
'url': channel_url,
'id': channel_id,
'ie_key': YoutubeTabIE.ie_key(),
'channel': title,
'channel_id': channel_id,
'channel_url': channel_url,
'title': title,
'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
'playlist_count': self._get_count(renderer, 'videoCountText'),
'description': self._get_text(renderer, 'descriptionSnippet'),
}
def _grid_entries(self, grid_renderer):
for item in grid_renderer['items']:
if not isinstance(item, dict):
@@ -4407,9 +4488,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
# channel
channel_id = renderer.get('channelId')
if channel_id:
yield self.url_result(
'https://www.youtube.com/channel/%s' % channel_id,
ie=YoutubeTabIE.ie_key(), video_title=title)
yield self._extract_channel_renderer(renderer)
continue
# generic endpoint URL support
ep_url = urljoin('https://www.youtube.com/', try_get(
@@ -5060,7 +5139,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
IE_DESC = 'YouTube Tabs'
_VALID_URL = r'''(?x:
https?://
(?:\w+\.)?
(?!consent\.)(?:\w+\.)?
(?:
youtube(?:kids)?\.com|
%(invidious)s
@@ -5762,7 +5841,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'uploader': 'cole-dlp-test-acc',
'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
'channel': 'cole-dlp-test-acc',
'channel_follower_count': int,
},
'playlist_mincount': 1,
'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
@@ -5930,7 +6008,6 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'title': 'cole-dlp-test-acc - Shorts',
'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
'channel': 'cole-dlp-test-acc',
'channel_follower_count': int,
'description': 'test description',
'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
@@ -5976,8 +6053,40 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
'channel': str,
}
}],
'params': {'extract_flat': True},
'params': {'extract_flat': True, 'playlist_items': '1'},
'playlist_mincount': 1
}, {
# Channel renderer metadata. Contains number of videos on the channel
'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
'info_dict': {
'id': 'UCiu-3thuViMebBjw_5nWYrA',
'title': 'cole-dlp-test-acc - Channels',
'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
'channel': 'cole-dlp-test-acc',
'description': 'test description',
'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
'tags': [],
'uploader': 'cole-dlp-test-acc',
'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
},
'playlist': [{
'info_dict': {
'_type': 'url',
'ie_key': 'YoutubeTab',
'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
'title': 'PewDiePie',
'channel': 'PewDiePie',
'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
'thumbnails': list,
'channel_follower_count': int,
'playlist_count': int
}
}],
'params': {'extract_flat': True},
}]
@classmethod
@@ -6531,6 +6640,30 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
# 'title': '#cats',
# }],
},
}, {
# Channel results
'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
'info_dict': {
'id': 'kurzgesagt',
'title': 'kurzgesagt',
},
'playlist': [{
'info_dict': {
'_type': 'url',
'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
'ie_key': 'YoutubeTab',
'channel': 'Kurzgesagt – In a Nutshell',
'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
'title': 'Kurzgesagt – In a Nutshell',
'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
'playlist_count': int, # XXX: should have a way of saying > 1
'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
'thumbnails': list
}
}],
'params': {'extract_flat': True, 'playlist_items': '1'},
'playlist_mincount': 1,
}, {
'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
'only_matching': True,
@@ -6816,6 +6949,51 @@ class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
}
class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
IE_NAME = 'youtube:consent'
IE_DESC = False # Do not list
_VALID_URL = r'https?://consent\.youtube\.com/m\?'
_TESTS = [{
'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
'info_dict': {
'id': 'qVv6vCqciTM',
'ext': 'mp4',
'age_limit': 0,
'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
'comment_count': int,
'chapters': 'count:13',
'upload_date': '20221223',
'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
'like_count': int,
'release_date': '20221223',
'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
'view_count': int,
'playable_in_embed': True,
'duration': 4438,
'availability': 'public',
'channel_follower_count': int,
'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
'categories': ['Entertainment'],
'live_status': 'was_live',
'release_timestamp': 1671793345,
'channel': 'さなちゃんねる',
'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
'uploader': 'さなちゃんねる',
},
'add_ie': ['Youtube'],
'params': {'skip_download': 'Youtube'},
}]
def _real_extract(self, url):
redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
if not redirect_url:
raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
return self.url_result(redirect_url)
class YoutubeTruncatedIDIE(InfoExtractor):
IE_NAME = 'youtube:truncated_id'
IE_DESC = False # Do not list
yt_dlp/extractor/zdf.py
+21 -7
View File
@@ -24,7 +24,7 @@ from ..utils import (
class ZDFBaseIE(InfoExtractor):
_GEO_COUNTRIES = ['DE']
_QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd')
_QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd', 'uhd')
def _call_api(self, url, video_id, item, api_token=None, referrer=None):
headers = {}
@@ -174,7 +174,8 @@ class ZDFIE(ZDFBaseIE):
'thumbnail': 'md5:e65f459f741be5455c952cd820eb188e',
'title': 'heute journal vom 30.12.2021',
'timestamp': 1640897100,
}
},
'skip': 'No longer available: "Diese Seite wurde leider nicht gefunden"',
}, {
'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
'info_dict': {
@@ -189,7 +190,7 @@ class ZDFIE(ZDFBaseIE):
},
}, {
'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
'md5': '1b93bdec7d02fc0b703c5e7687461628',
'md5': '57af4423db0455a3975d2dc4578536bc',
'info_dict': {
'ext': 'mp4',
'id': 'video_funk_1770473',
@@ -198,7 +199,7 @@ class ZDFIE(ZDFBaseIE):
'title': 'Alles ist verzaubert',
'timestamp': 1635520560,
'upload_date': '20211029',
'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-100~1920x1080?cb=1636466431799',
'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-102~1920x1080?cb=1663848412907',
},
}, {
# Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche
@@ -241,10 +242,23 @@ class ZDFIE(ZDFBaseIE):
'title': 'Das Geld anderer Leute',
'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
'duration': 2581.0,
'timestamp': 1654790700,
'upload_date': '20220609',
'timestamp': 1675160100,
'upload_date': '20230131',
'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350',
},
}, {
'url': 'https://www.zdf.de/dokumentation/terra-x/unser-gruener-planet-wuesten-doku-100.html',
'info_dict': {
'id': '220605_dk_gruener_planet_wuesten_tex',
'ext': 'mp4',
'title': 'Unser grüner Planet - Wüsten',
'description': 'md5:4fc647b6f9c3796eea66f4a0baea2862',
'duration': 2613.0,
'timestamp': 1654450200,
'upload_date': '20220605',
'format_note': 'uhd, main',
'thumbnail': 'https://www.zdf.de/assets/saguaro-kakteen-102~3840x2160?cb=1655910690796',
},
}]
def _extract_entry(self, url, player, content, video_id):
@@ -259,7 +273,7 @@ class ZDFIE(ZDFBaseIE):
raise ExtractorError('Could not extract ptmd_path')
info = self._extract_ptmd(
urljoin(url, ptmd_path.replace('{playerId}', 'ngplayer_2_4')), video_id, player['apiToken'], url)
urljoin(url, ptmd_path.replace('{playerId}', 'android_native_5')), video_id, player['apiToken'], url)
thumbnails = []
layouts = try_get(
yt_dlp/jsinterp.py
+17 -1
View File
@@ -343,7 +343,8 @@ class JSInterpreter:
inner, outer = self._separate(expr, expr[0], 1)
if expr[0] == '/':
flags, outer = self._regex_flags(outer)
inner = re.compile(inner[1:], flags=flags)
# Avoid https://github.com/python/cpython/issues/74534
inner = re.compile(inner[1:].replace('[[', r'[\['), flags=flags)
else:
inner = json.loads(js_to_json(f'{inner}{expr[0]}', strict=True))
if not outer:
@@ -402,10 +403,25 @@ class JSInterpreter:
m = re.match(r'''(?x)
(?P<try>try)\s*\{|
(?P<if>if)\s*\(|
(?P<switch>switch)\s*\(|
(?P<for>for)\s*\(
''', expr)
md = m.groupdict() if m else {}
if md.get('if'):
cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
if_expr, expr = self._separate_at_paren(expr.lstrip())
# TODO: "else if" is not handled
else_expr = None
m = re.match(r'else\s*{', expr)
if m:
else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
cndn = _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion))
ret, should_abort = self.interpret_statement(
if_expr if cndn else else_expr, local_vars, allow_recursion)
if should_abort:
return ret, True
if md.get('try'):
try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
err = None
yt_dlp/options.py
+83 -90
View File
@@ -29,6 +29,8 @@ from .utils import (
expand_path,
format_field,
get_executable_path,
get_system_config_dirs,
get_user_config_dirs,
join_nonempty,
orderedSet_from_options,
remove_end,
@@ -38,71 +40,55 @@ from .version import __version__
def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
PACKAGE_NAME = 'yt-dlp'
root = Config(create_parser())
if ignore_config_files == 'if_override':
ignore_config_files = overrideArguments is not None
def _readUserConf(package_name, default=[]):
# .config
xdg_config_home = os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config')
userConfFile = os.path.join(xdg_config_home, package_name, 'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
def read_config(*paths):
path = os.path.join(*paths)
conf = Config.read_file(path, default=None)
if conf is not None:
return conf, path
# appdata
appdata_dir = os.getenv('appdata')
if appdata_dir:
userConfFile = os.path.join(appdata_dir, package_name, 'config')
userConf = Config.read_file(userConfFile, default=None)
if userConf is None:
userConfFile += '.txt'
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
def _load_from_config_dirs(config_dirs):
for config_dir in config_dirs:
head, tail = os.path.split(config_dir)
assert tail == PACKAGE_NAME or config_dir == os.path.join(compat_expanduser('~'), f'.{PACKAGE_NAME}')
# home
userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
userConf = Config.read_file(userConfFile, default=None)
if userConf is None:
userConfFile += '.txt'
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
yield read_config(head, f'{PACKAGE_NAME}.conf')
if tail.startswith('.'): # ~/.PACKAGE_NAME
yield read_config(head, f'{PACKAGE_NAME}.conf.txt')
yield read_config(config_dir, 'config')
yield read_config(config_dir, 'config.txt')
return default, None
def add_config(label, path, user=False):
def add_config(label, path=None, func=None):
""" Adds config and returns whether to continue """
if root.parse_known_args()[0].ignoreconfig:
return False
# Multiple package names can be given here
# E.g. ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
# the configuration file of any of these three packages
for package in ('yt-dlp',):
if user:
args, current_path = _readUserConf(package, default=None)
else:
current_path = os.path.join(path, '%s.conf' % package)
args = Config.read_file(current_path, default=None)
if args is not None:
root.append_config(args, current_path, label=label)
return True
elif func:
assert path is None
args, current_path = next(
filter(None, _load_from_config_dirs(func(PACKAGE_NAME))), (None, None))
else:
current_path = os.path.join(path, 'yt-dlp.conf')
args = Config.read_file(current_path, default=None)
if args is not None:
root.append_config(args, current_path, label=label)
return True
def load_configs():
yield not ignore_config_files
yield add_config('Portable', get_executable_path())
yield add_config('Home', expand_path(root.parse_known_args()[0].paths.get('home', '')).strip())
yield add_config('User', None, user=True)
yield add_config('System', '/etc')
yield add_config('User', func=get_user_config_dirs)
yield add_config('System', func=get_system_config_dirs)
opts = optparse.Values({'verbose': True, 'print_help': False})
try:
try:
if overrideArguments:
if overrideArguments is not None:
root.append_config(overrideArguments, label='Override')
else:
root.append_config(sys.argv[1:], label='Command-line')
@@ -277,6 +263,20 @@ def create_parser():
out_dict[key] = out_dict.get(key, []) + [val] if append else val
setattr(parser.values, option.dest, out_dict)
def when_prefix(default):
return {
'default': {},
'type': 'str',
'action': 'callback',
'callback': _dict_from_options_callback,
'callback_kwargs': {
'allowed_keys': '|'.join(map(re.escape, POSTPROCESS_WHEN)),
'default_key': default,
'multiple_keys': False,
'append': True,
},
}
parser = _YoutubeDLOptionParser()
alias_group = optparse.OptionGroup(parser, 'Aliases')
Formatter = string.Formatter()
@@ -443,12 +443,14 @@ def create_parser():
'allowed_values': {
'filename', 'filename-sanitization', 'format-sort', 'abort-on-error', 'format-spec', 'no-playlist-metafiles',
'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge',
'no-attach-info-json', 'embed-metadata', 'embed-thumbnail-atomicparsley',
'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
'no-attach-info-json', 'embed-thumbnail-atomicparsley', 'no-external-downloader-progress',
'embed-metadata', 'seperate-video-versions', 'no-clean-infojson', 'no-keep-subs', 'no-certifi',
'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-youtube-prefer-utc-upload-date',
}, 'aliases': {
'youtube-dl': ['all', '-multistreams'],
'youtube-dlc': ['all', '-no-youtube-channel-redirect', '-no-live-chat'],
'2021': ['2022', 'no-certifi', 'filename-sanitization', 'no-youtube-prefer-utc-upload-date'],
'2022': ['no-external-downloader-progress'],
}
}, help=(
'Options that can help keep compatibility with youtube-dl or youtube-dlc '
@@ -493,6 +495,11 @@ def create_parser():
action='store_const', const='::', dest='source_address',
help='Make all connections via IPv6',
)
network.add_option(
'--enable-file-urls', action='store_true',
dest='enable_file_urls', default=False,
help='Enable file:// URLs. This is disabled by default for security reasons.'
)
geo = optparse.OptionGroup(parser, 'Geo-restriction')
geo.add_option(
@@ -535,10 +542,10 @@ def create_parser():
'-I', '--playlist-items',
dest='playlist_items', metavar='ITEM_SPEC', default=None,
help=(
'Comma separated playlist_index of the videos to download. '
'Comma separated playlist_index of the items to download. '
'You can specify a range using "[START]:[STOP][:STEP]". For backward compatibility, START-STOP is also supported. '
'Use negative indices to count from the right and negative STEP to download in reverse order. '
'E.g. "-I 1:3,7,-5::2" used on a playlist of size 15 will download the videos at index 1,2,3,7,11,13,15'))
'E.g. "-I 1:3,7,-5::2" used on a playlist of size 15 will download the items at index 1,2,3,7,11,13,15'))
selection.add_option(
'--match-title',
dest='matchtitle', metavar='REGEX',
@@ -554,13 +561,14 @@ def create_parser():
selection.add_option(
'--max-filesize',
metavar='SIZE', dest='max_filesize', default=None,
help='Abort download if filesize if larger than SIZE, e.g. 50k or 44.6M')
help='Abort download if filesize is larger than SIZE, e.g. 50k or 44.6M')
selection.add_option(
'--date',
metavar='DATE', dest='date', default=None,
help=(
'Download only videos uploaded on this date. The date can be "YYYYMMDD" or in the format '
'[now|today|yesterday][-N[day|week|month|year]]. E.g. --date today-2weeks'))
'Download only videos uploaded on this date. '
'The date can be "YYYYMMDD" or in the format [now|today|yesterday][-N[day|week|month|year]]. '
'E.g. "--date today-2weeks" downloads only videos uploaded on the same day two weeks ago'))
selection.add_option(
'--datebefore',
metavar='DATE', dest='datebefore', default=None,
@@ -635,7 +643,7 @@ def create_parser():
selection.add_option(
'--break-per-input',
action='store_true', dest='break_per_url', default=False,
help='--break-on-existing, --break-on-reject, --max-downloads, and autonumber resets per input URL')
help='Alters --max-downloads, --break-on-existing, --break-on-reject, and autonumber to reset per input URL')
selection.add_option(
'--no-break-per-input',
action='store_false', dest='break_per_url',
@@ -875,11 +883,11 @@ def create_parser():
'This option can be used multiple times to set the sleep for the different retry types, '
'e.g. --retry-sleep linear=1::2 --retry-sleep fragment:exp=1:20'))
downloader.add_option(
'--skip-unavailable-fragments', '--no-abort-on-unavailable-fragment',
'--skip-unavailable-fragments', '--no-abort-on-unavailable-fragments',
action='store_true', dest='skip_unavailable_fragments', default=True,
help='Skip unavailable fragments for DASH, hlsnative and ISM downloads (default) (Alias: --no-abort-on-unavailable-fragment)')
help='Skip unavailable fragments for DASH, hlsnative and ISM downloads (default) (Alias: --no-abort-on-unavailable-fragments)')
downloader.add_option(
'--abort-on-unavailable-fragment', '--no-skip-unavailable-fragments',
'--abort-on-unavailable-fragments', '--no-skip-unavailable-fragments',
action='store_false', dest='skip_unavailable_fragments',
help='Abort download if a fragment is unavailable (Alias: --no-skip-unavailable-fragments)')
downloader.add_option(
@@ -1086,28 +1094,16 @@ def create_parser():
help='Do not download the video but write all related files (Alias: --no-download)')
verbosity.add_option(
'-O', '--print',
metavar='[WHEN:]TEMPLATE', dest='forceprint', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': 'video|' + '|'.join(map(re.escape, POSTPROCESS_WHEN)),
'default_key': 'video',
'multiple_keys': False,
'append': True,
}, help=(
metavar='[WHEN:]TEMPLATE', dest='forceprint', **when_prefix('video'),
help=(
'Field name or output template to print to screen, optionally prefixed with when to print it, separated by a ":". '
'Supported values of "WHEN" are the same as that of --use-postprocessor, and "video" (default). '
'Supported values of "WHEN" are the same as that of --use-postprocessor (default: video). '
'Implies --quiet. Implies --simulate unless --no-simulate or later stages of WHEN are used. '
'This option can be used multiple times'))
verbosity.add_option(
'--print-to-file',
metavar='[WHEN:]TEMPLATE FILE', dest='print_to_file', default={}, type='str', nargs=2,
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': 'video|' + '|'.join(map(re.escape, POSTPROCESS_WHEN)),
'default_key': 'video',
'multiple_keys': False,
'append': True,
}, help=(
metavar='[WHEN:]TEMPLATE FILE', dest='print_to_file', nargs=2, **when_prefix('video'),
help=(
'Append given template to the file. The values of WHEN and TEMPLATE are same as that of --print. '
'FILE uses the same syntax as the output template. This option can be used multiple times'))
verbosity.add_option(
@@ -1584,14 +1580,16 @@ def create_parser():
help=optparse.SUPPRESS_HELP)
postproc.add_option(
'--parse-metadata',
metavar='FROM:TO', dest='parse_metadata', action='append',
metavar='[WHEN:]FROM:TO', dest='parse_metadata', **when_prefix('pre_process'),
help=(
'Parse additional metadata like title/artist from other fields; '
'see "MODIFYING METADATA" for details'))
'Parse additional metadata like title/artist from other fields; see "MODIFYING METADATA" for details. '
'Supported values of "WHEN" are the same as that of --use-postprocessor (default: pre_process)'))
postproc.add_option(
'--replace-in-metadata',
dest='parse_metadata', metavar='FIELDS REGEX REPLACE', action='append', nargs=3,
help='Replace text in a metadata field using the given regex. This option can be used multiple times')
dest='parse_metadata', metavar='[WHEN:]FIELDS REGEX REPLACE', nargs=3, **when_prefix('pre_process'),
help=(
'Replace text in a metadata field using the given regex. This option can be used multiple times. '
'Supported values of "WHEN" are the same as that of --use-postprocessor (default: pre_process)'))
postproc.add_option(
'--xattrs', '--xattr',
action='store_true', dest='xattrs', default=False,
@@ -1629,19 +1627,13 @@ def create_parser():
help='Location of the ffmpeg binary; either the path to the binary or its containing directory')
postproc.add_option(
'--exec',
metavar='[WHEN:]CMD', dest='exec_cmd', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': '|'.join(map(re.escape, POSTPROCESS_WHEN)),
'default_key': 'after_move',
'multiple_keys': False,
'append': True,
}, help=(
'Execute a command, optionally prefixed with when to execute it (after_move if unspecified), separated by a ":". '
'Supported values of "WHEN" are the same as that of --use-postprocessor. '
metavar='[WHEN:]CMD', dest='exec_cmd', **when_prefix('after_move'),
help=(
'Execute a command, optionally prefixed with when to execute it, separated by a ":". '
'Supported values of "WHEN" are the same as that of --use-postprocessor (default: after_move). '
'Same syntax as the output template can be used to pass any field as arguments to the command. '
'After download, an additional field "filepath" that contains the final path of the downloaded file '
'is also available, and if no fields are passed, %(filepath)q is appended to the end of the command. '
'is also available, and if no fields are passed, %(filepath,_filename|)q is appended to the end of the command. '
'This option can be used multiple times'))
postproc.add_option(
'--no-exec',
@@ -1714,7 +1706,8 @@ def create_parser():
'ARGS are a semicolon ";" delimited list of NAME=VALUE. '
'The "when" argument determines when the postprocessor is invoked. '
'It can be one of "pre_process" (after video extraction), "after_filter" (after video passes filter), '
'"before_dl" (before each video download), "post_process" (after each video download; default), '
'"video" (after --format; before --print/--output), "before_dl" (before each video download), '
'"post_process" (after each video download; default), '
'"after_move" (after moving video file to it\'s final locations), '
'"after_video" (after downloading and processing all formats of a video), '
'or "playlist" (at end of playlist). '
yt_dlp/plugins.py
+173
View File
@@ -0,0 +1,173 @@
import contextlib
import importlib
import importlib.abc
import importlib.machinery
import importlib.util
import inspect
import itertools
import pkgutil
import sys
import traceback
import zipimport
from pathlib import Path
from zipfile import ZipFile
from .compat import functools # isort: split
from .utils import (
get_executable_path,
get_system_config_dirs,
get_user_config_dirs,
orderedSet,
write_string,
)
PACKAGE_NAME = 'yt_dlp_plugins'
COMPAT_PACKAGE_NAME = 'ytdlp_plugins'
class PluginLoader(importlib.abc.Loader):
"""Dummy loader for virtual namespace packages"""
def exec_module(self, module):
return None
@functools.cache
def dirs_in_zip(archive):
try:
with ZipFile(archive) as zip_:
return set(itertools.chain.from_iterable(
Path(file).parents for file in zip_.namelist()))
except FileNotFoundError:
pass
except Exception as e:
write_string(f'WARNING: Could not read zip file {archive}: {e}\n')
return set()
class PluginFinder(importlib.abc.MetaPathFinder):
"""
This class provides one or multiple namespace packages.
It searches in sys.path and yt-dlp config folders for
the existing subdirectories from which the modules can be imported
"""
def __init__(self, *packages):
self._zip_content_cache = {}
self.packages = set(itertools.chain.from_iterable(
itertools.accumulate(name.split('.'), lambda a, b: '.'.join((a, b)))
for name in packages))
def search_locations(self, fullname):
candidate_locations = []
def _get_package_paths(*root_paths, containing_folder='plugins'):
for config_dir in orderedSet(map(Path, root_paths), lazy=True):
with contextlib.suppress(OSError):
yield from (config_dir / containing_folder).iterdir()
# Load from yt-dlp config folders
candidate_locations.extend(_get_package_paths(
*get_user_config_dirs('yt-dlp'),
*get_system_config_dirs('yt-dlp'),
containing_folder='plugins'))
# Load from yt-dlp-plugins folders
candidate_locations.extend(_get_package_paths(
get_executable_path(),
*get_user_config_dirs(''),
*get_system_config_dirs(''),
containing_folder='yt-dlp-plugins'))
candidate_locations.extend(map(Path, sys.path)) # PYTHONPATH
with contextlib.suppress(ValueError): # Added when running __main__.py directly
candidate_locations.remove(Path(__file__).parent)
parts = Path(*fullname.split('.'))
for path in orderedSet(candidate_locations, lazy=True):
candidate = path / parts
if candidate.is_dir():
yield candidate
elif path.suffix in ('.zip', '.egg', '.whl'):
if parts in dirs_in_zip(path):
yield candidate
def find_spec(self, fullname, path=None, target=None):
if fullname not in self.packages:
return None
search_locations = list(map(str, self.search_locations(fullname)))
if not search_locations:
return None
spec = importlib.machinery.ModuleSpec(fullname, PluginLoader(), is_package=True)
spec.submodule_search_locations = search_locations
return spec
def invalidate_caches(self):
dirs_in_zip.cache_clear()
for package in self.packages:
if package in sys.modules:
del sys.modules[package]
def directories():
spec = importlib.util.find_spec(PACKAGE_NAME)
return spec.submodule_search_locations if spec else []
def iter_modules(subpackage):
fullname = f'{PACKAGE_NAME}.{subpackage}'
with contextlib.suppress(ModuleNotFoundError):
pkg = importlib.import_module(fullname)
yield from pkgutil.iter_modules(path=pkg.__path__, prefix=f'{fullname}.')
def load_module(module, module_name, suffix):
return inspect.getmembers(module, lambda obj: (
inspect.isclass(obj)
and obj.__name__.endswith(suffix)
and obj.__module__.startswith(module_name)
and not obj.__name__.startswith('_')
and obj.__name__ in getattr(module, '__all__', [obj.__name__])))
def load_plugins(name, suffix):
classes = {}
for finder, module_name, _ in iter_modules(name):
if any(x.startswith('_') for x in module_name.split('.')):
continue
try:
if sys.version_info < (3, 10) and isinstance(finder, zipimport.zipimporter):
# zipimporter.load_module() is deprecated in 3.10 and removed in 3.12
# The exec_module branch below is the replacement for >= 3.10
# See: https://docs.python.org/3/library/zipimport.html#zipimport.zipimporter.exec_module
module = finder.load_module(module_name)
else:
spec = finder.find_spec(module_name)
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
except Exception:
write_string(f'Error while importing module {module_name!r}\n{traceback.format_exc(limit=-1)}')
continue
classes.update(load_module(module, module_name, suffix))
# Compat: old plugin system using __init__.py
# Note: plugins imported this way do not show up in directories()
# nor are considered part of the yt_dlp_plugins namespace package
with contextlib.suppress(FileNotFoundError):
spec = importlib.util.spec_from_file_location(
name, Path(get_executable_path(), COMPAT_PACKAGE_NAME, name, '__init__.py'))
plugins = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = plugins
spec.loader.exec_module(plugins)
classes.update(load_module(plugins, spec.name, suffix))
return classes
sys.meta_path.insert(0, PluginFinder(f'{PACKAGE_NAME}.extractor', f'{PACKAGE_NAME}.postprocessor'))
__all__ = ['directories', 'load_plugins', 'PACKAGE_NAME', 'COMPAT_PACKAGE_NAME']

Some files were not shown because too many files have changed in this diff Show More