From b23046bbc8e53f32a3853dc33138f2986f3aed06 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 13 Jun 2026 18:06:23 -0500 Subject: [PATCH] [ie/youtube:tab] Fix pagination (#16948) * Support `continuationViewModel` continuation tokens Closes #16692, Closes #16943 Authored by: bashonly --- yt_dlp/extractor/youtube/_base.py | 6 +- yt_dlp/extractor/youtube/_tab.py | 158 +++++++++++++++--------------- 2 files changed, 85 insertions(+), 79 deletions(-) diff --git a/yt_dlp/extractor/youtube/_base.py b/yt_dlp/extractor/youtube/_base.py index d3e4ab1176..59a470ff6d 100644 --- a/yt_dlp/extractor/youtube/_base.py +++ b/yt_dlp/extractor/youtube/_base.py @@ -1037,8 +1037,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor): return next_continuation return traverse_obj(renderer, ( - ('contents', 'items', 'rows', 'subThreads'), ..., 'continuationItemRenderer', - ('continuationEndpoint', ('button', 'buttonRenderer', 'command')), + ('contents', 'items', 'rows', 'subThreads'), ..., ( + ('continuationItemRenderer', ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))), + ('continuationItemViewModel', 'continuationCommand', 'innertubeCommand'), + ), ), get_all=False, expected_type=cls._extract_continuation_ep_data) @classmethod diff --git a/yt_dlp/extractor/youtube/_tab.py b/yt_dlp/extractor/youtube/_tab.py index 78d5e5b608..197231daff 100644 --- a/yt_dlp/extractor/youtube/_tab.py +++ b/yt_dlp/extractor/youtube/_tab.py @@ -1032,14 +1032,14 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'playlist_mincount': 94, 'info_dict': { 'id': 'UCqj7Cz7revf5maW9g5pgNcg', - 'title': 'Igor Kleiner - Playlists', - 'description': r're:(?s)Добро пожаловать на мой канал! Здесь вы найдете видео .{504}/a1/50b/10a$', - 'uploader': 'Igor Kleiner ', + 'title': 'Igor DS: ИИ, Наука и Творчество - Playlists', + 'description': r're:(?s)Добро пожаловать! Здесь сложные технологии встречаются.+\n$', + 'uploader': 'Igor DS: ИИ, Наука и Творчество ', 'uploader_id': '@IgorDataScience', 'uploader_url': 'https://www.youtube.com/@IgorDataScience', - 'channel': 'Igor Kleiner ', + 'channel': 'Igor DS: ИИ, Наука и Творчество ', 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg', - 'tags': 'count:23', + 'tags': 'count:19', 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg', 'channel_follower_count': int, }, @@ -1049,14 +1049,13 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'playlist_mincount': 94, 'info_dict': { 'id': 'UCqj7Cz7revf5maW9g5pgNcg', - 'title': 'Igor Kleiner - Playlists', - 'description': r're:(?s)Добро пожаловать на мой канал! Здесь вы найдете видео .{504}/a1/50b/10a$', - 'uploader': 'Igor Kleiner ', + 'title': 'Igor DS: ИИ, Наука и Творчество - Playlists', + 'description': r're:(?s)Добро пожаловать! Здесь сложные технологии встречаются.+\n$', 'uploader_id': '@IgorDataScience', 'uploader_url': 'https://www.youtube.com/@IgorDataScience', - 'tags': 'count:23', + 'tags': 'count:19', 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg', - 'channel': 'Igor Kleiner ', + 'channel': 'Igor DS: ИИ, Наука и Творчество ', 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg', 'channel_follower_count': int, }, @@ -1139,90 +1138,89 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'playlist_count': 0, }, { 'note': 'Home tab', - 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured', + 'url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/featured', 'info_dict': { - 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'title': 'lex will - Home', - 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488', - 'uploader': 'lex will', - 'uploader_id': '@lexwill718', - 'channel': 'lex will', - 'tags': ['bible', 'history', 'prophesy'], - 'uploader_url': 'https://www.youtube.com/@lexwill718', - 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w', - 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w', + 'id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'title': 'Creative Commons - Home', + 'description': 'md5:7cfc22824277588d26a66054f22d93c8', + 'uploader': 'Creative Commons', + 'uploader_id': '@creativecommons', + 'uploader_url': 'https://www.youtube.com/@creativecommons', + 'channel': 'Creative Commons', + 'channel_id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'channel_url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q', 'channel_follower_count': int, + 'tags': ['creative commons', 'remix', 'culture', 'nonprofit'], }, - 'playlist_mincount': 2, + 'playlist_mincount': 6, }, { 'note': 'Videos tab', - 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos', + 'url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/videos', 'info_dict': { - 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'title': 'lex will - Videos', - 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488', - 'uploader': 'lex will', - 'uploader_id': '@lexwill718', - 'tags': ['bible', 'history', 'prophesy'], - 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w', - 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'uploader_url': 'https://www.youtube.com/@lexwill718', - 'channel': 'lex will', + 'id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'title': 'Creative Commons - Videos', + 'description': 'md5:7cfc22824277588d26a66054f22d93c8', + 'uploader': 'Creative Commons', + 'uploader_id': '@creativecommons', + 'uploader_url': 'https://www.youtube.com/@creativecommons', + 'channel': 'Creative Commons', + 'channel_id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'channel_url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q', 'channel_follower_count': int, + 'tags': ['creative commons', 'remix', 'culture', 'nonprofit'], }, - 'playlist_mincount': 975, + 'playlist_mincount': 239, }, { 'note': 'Videos tab, sorted by popular', - 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid', + 'url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/videos?view=0&sort=p&flow=grid', 'info_dict': { - 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'title': 'lex will - Videos', - 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488', - 'uploader': 'lex will', - 'uploader_id': '@lexwill718', - 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'uploader_url': 'https://www.youtube.com/@lexwill718', - 'channel': 'lex will', - 'tags': ['bible', 'history', 'prophesy'], - 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w', + 'id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'title': 'Creative Commons - Videos', + 'description': 'md5:7cfc22824277588d26a66054f22d93c8', + 'uploader': 'Creative Commons', + 'uploader_id': '@creativecommons', + 'uploader_url': 'https://www.youtube.com/@creativecommons', + 'channel': 'Creative Commons', + 'channel_id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'channel_url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q', 'channel_follower_count': int, + 'tags': ['creative commons', 'remix', 'culture', 'nonprofit'], }, - 'playlist_mincount': 199, + 'playlist_mincount': 239, }, { 'note': 'Playlists tab', - 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists', + 'url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q/playlists', 'info_dict': { - 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'title': 'lex will - Playlists', - 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488', - 'uploader': 'lex will', - 'uploader_id': '@lexwill718', - 'uploader_url': 'https://www.youtube.com/@lexwill718', - 'channel': 'lex will', - 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w', - 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'tags': ['bible', 'history', 'prophesy'], + 'id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'title': 'Creative Commons - Playlists', + 'description': 'md5:7cfc22824277588d26a66054f22d93c8', + 'uploader': 'Creative Commons', + 'uploader_id': '@creativecommons', + 'uploader_url': 'https://www.youtube.com/@creativecommons', + 'channel': 'Creative Commons', + 'channel_id': 'UCTwECeGqMZee77BjdoYtI2Q', + 'channel_url': 'https://www.youtube.com/channel/UCTwECeGqMZee77BjdoYtI2Q', 'channel_follower_count': int, + 'tags': ['creative commons', 'remix', 'culture', 'nonprofit'], }, - 'playlist_mincount': 17, + 'playlist_mincount': 20, }, { 'note': 'Posts tab', - 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community', + 'url': 'https://www.youtube.com/channel/UCtS3BcCw-tITPFYSvkbP0Bg/posts', 'info_dict': { - 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'title': 'lex will - Posts', - 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488', - 'channel': 'lex will', - 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w', - 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w', - 'tags': ['bible', 'history', 'prophesy'], + 'id': 'UCtS3BcCw-tITPFYSvkbP0Bg', + 'title': 'Office Hours Live with Tim Heidecker - Posts', + 'description': 'md5:01ec1460ea6c6e2aa47d3be9c756559c', + 'uploader': 'Office Hours Live with Tim Heidecker', + 'uploader_id': '@OfficeHoursLive', + 'uploader_url': 'https://www.youtube.com/@OfficeHoursLive', + 'channel': 'Office Hours Live with Tim Heidecker', + 'channel_id': 'UCtS3BcCw-tITPFYSvkbP0Bg', + 'channel_url': 'https://www.youtube.com/channel/UCtS3BcCw-tITPFYSvkbP0Bg', 'channel_follower_count': int, - 'uploader_url': 'https://www.youtube.com/@lexwill718', - 'uploader_id': '@lexwill718', - 'uploader': 'lex will', + 'tags': 'count:17', }, - 'playlist_mincount': 18, - 'skip': 'This Community isn\'t available', + 'playlist_mincount': 145, }, { # TODO: fix channel_is_verified extraction 'note': 'Search tab', @@ -1272,6 +1270,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): }, 'playlist_count': 96, }, { + # TODO: fix availability extraction 'note': 'Large playlist', 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q', 'info_dict': { @@ -1296,6 +1295,8 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'url': 'http://www.youtube.com/user/NASAgovVideo/videos', 'only_matching': True, }, { + # TODO: fix availability extraction + # The 'note' below is outdated: there is no longer a "Load more" button 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos', 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA', 'info_dict': { @@ -1313,7 +1314,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'uploader': 'Interstellar Movie', 'uploader_url': 'https://www.youtube.com/@InterstellarMovie', }, - 'playlist_mincount': 21, + 'playlist_mincount': 10, }, { # TODO: fix availability extraction 'note': 'Playlist with "show unavailable videos" button', @@ -1336,6 +1337,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'playlist_mincount': 150, 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'], }, { + # TODO: fix availability extraction 'note': 'Playlist with unavailable videos in page 7', 'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w', 'info_dict': { @@ -1407,7 +1409,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): }, { 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live', 'info_dict': { - 'id': 'VFGoUmo74wE', # This will keep changing + 'id': 'ubIX-TwVqZI', # This will keep changing 'ext': 'mp4', 'title': str, 'upload_date': r're:\d{8}', @@ -1586,6 +1588,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'playlist_count': 50, 'expected_warnings': ['YouTube Music is not directly supported'], }, { + # TODO: investigate test failing on differing channel*/uploader*/view_count 'note': 'unlisted single video playlist', 'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQLfIN0MMgp0wVV6MP3bM4_', 'info_dict': { @@ -1809,7 +1812,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'title': 'Not Just Bikes - Shorts', 'tags': 'count:10', 'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A', - 'description': 'md5:295758591d0d43d8594277be54584da7', + 'description': 'md5:2cb3ccdafa58608fa016f1de4930ec54', 'channel_follower_count': int, 'channel_id': 'UC0intLFzLaudFG-xAvUEO-A', 'channel': 'Not Just Bikes', @@ -1831,8 +1834,8 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'channel': '中村悠一', 'channel_follower_count': int, 'description': 'md5:76b312b48a26c3b0e4d90e2dfc1b417d', - 'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura', - 'uploader_id': '@Yuichi-Nakamura', + 'uploader_url': 'https://www.youtube.com/@中村悠一のあそびば', + 'uploader_id': '@中村悠一のあそびば', 'uploader': '中村悠一', }, 'playlist_mincount': 60, @@ -2010,7 +2013,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): 'channel': '99% Invisible', 'uploader_id': '@99percentinvisiblepodcast', }, - 'playlist_count': 5, + 'playlist_mincount': 5, }, { # Releases tab, with rich entry playlistRenderers (same as Podcasts tab) # TODO: fix channel_is_verified extraction @@ -2034,6 +2037,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): # Playlist with only shorts, shown as reel renderers # FIXME: future: YouTube currently doesn't give continuation for this, # may do in future. + # TODO: fix availability extraction 'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg', 'info_dict': { 'id': 'UUxqPAgubo4coVn9Lx1FuKcg',