[ie/tiktok] Extract save_count (#15054)

Closes #15053
Authored by: pomtnp

README.md

@@ -1351,6 +1351,7 @@ The available fields are:
  - `repost_count` (numeric): Number of reposts of the video
  - `average_rating` (numeric): Average rating given by users, the scale used depends on the webpage
  - `comment_count` (numeric): Number of comments on the video (For some extractors, comments are only downloaded at the end, and so this field cannot be used)
+ - `save_count` (numeric): Number of times the video has been saved or bookmarked
  - `age_limit` (numeric): Age restriction for the video (years)
  - `live_status` (string): One of "not_live", "is_live", "is_upcoming", "was_live", "post_live" (was live, but VOD is not yet processed)
  - `is_live` (boolean): Whether this video is a live stream or a fixed-length video

test/helper.py

@@ -263,7 +263,7 @@ def expect_info_dict(self, got_dict, expected_dict):
         # NB: Keep in sync with the docstring of extractor/common.py
         'ie_key', 'url', 'id', 'ext', 'direct', 'display_id', 'title', 'alt_title', 'description', 'media_type',
         'uploader', 'uploader_id', 'uploader_url', 'channel', 'channel_id', 'channel_url', 'channel_is_verified',
-        'channel_follower_count', 'comment_count', 'view_count', 'concurrent_view_count',
+        'channel_follower_count', 'comment_count', 'view_count', 'concurrent_view_count', 'save_count',
         'like_count', 'dislike_count', 'repost_count', 'average_rating', 'age_limit', 'duration', 'thumbnail', 'heatmap',
         'chapters', 'chapter', 'chapter_number', 'chapter_id', 'start_time', 'end_time', 'section_start', 'section_end',
         'categories', 'tags', 'cast', 'composers', 'artists', 'album_artists', 'creators', 'genres',

test/test_utils.py

@@ -1280,6 +1280,9 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('[new Date("spam"), \'("eggs")\']')
         self.assertEqual(json.loads(on), ['spam', '("eggs")'], msg='Date regex should match a single string')
 
+        on = js_to_json('[0.077, 7.06, 29.064, 169.0072]')
+        self.assertEqual(json.loads(on), [0.077, 7.06, 29.064, 169.0072])
+
     def test_js_to_json_malformed(self):
         self.assertEqual(js_to_json('42a1'), '42"a1"')
         self.assertEqual(js_to_json('42a-1'), '42"a"-1')

yt_dlp/YoutubeDL.py

@@ -595,7 +595,7 @@ class YoutubeDL:
         'width', 'height', 'asr', 'audio_channels', 'fps',
         'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
         'timestamp', 'release_timestamp', 'available_at',
-        'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
+        'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', 'save_count',
         'average_rating', 'comment_count', 'age_limit',
         'start_time', 'end_time',
         'chapter_number', 'season_number', 'episode_number',

yt_dlp/extractor/common.py

@@ -348,6 +348,7 @@ class InfoExtractor:
     duration:       Length of the video in seconds, as an integer or float.
     view_count:     How many users have watched the video on the platform.
     concurrent_view_count: How many users are currently watching the video on the platform.
+    save_count:     Number of times the video has been saved or bookmarked
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     repost_count:   Number of reposts of the video

yt_dlp/extractor/tiktok.py

@@ -454,6 +454,7 @@ class TikTokBaseIE(InfoExtractor):
                 'like_count': 'digg_count',
                 'repost_count': 'share_count',
                 'comment_count': 'comment_count',
+                'save_count': 'collect_count',
             }, expected_type=int_or_none),
             **author_info,
             'channel_url': format_field(author_info, 'channel_id', self._UPLOADER_URL_FORMAT, default=None),
@@ -607,6 +608,7 @@ class TikTokBaseIE(InfoExtractor):
                 'like_count': 'diggCount',
                 'repost_count': 'shareCount',
                 'comment_count': 'commentCount',
+                'save_count': 'collectCount',
             }), expected_type=int_or_none),
             'thumbnails': [
                 {
@@ -646,6 +648,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'artist': 'Ysrbeats',
             'album': 'Lehanga',
             'track': 'Lehanga',
@@ -675,6 +678,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'artists': ['Evan Todd', 'Jessica Keenan Wynn', 'Alice Lee', 'Barrett Wilbert Weed', 'Jon Eidson'],
             'track': 'Big Fun',
         },
@@ -702,6 +706,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
         },
     }, {
         # Sponsored video, only available with feed workaround
@@ -725,6 +730,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
         },
         'skip': 'This video is unavailable',
     }, {
@@ -751,6 +757,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
         },
     }, {
         # hydration JSON is sent in a <script> element
@@ -773,6 +780,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
         },
         'skip': 'This video is unavailable',
     }, {
@@ -798,6 +806,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:^https://.+\.(?:webp|jpe?g)',
         },
     }, {
@@ -824,6 +833,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:^https://.+',
             'thumbnails': 'count:3',
         },
@@ -851,6 +861,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:^https://.+\.webp',
         },
         'skip': 'Unavailable via feed API, only audio available via web',
@@ -879,6 +890,7 @@ class TikTokIE(TikTokBaseIE):
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
+            'save_count': int,
             'thumbnail': r're:^https://.+\.(?:webp|jpe?g)',
         },
     }, {
@@ -1288,6 +1300,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
@@ -1312,6 +1325,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
@@ -1336,6 +1350,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:https?://.+\.jpe?g',
         },
     }, {
@@ -1353,6 +1368,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
         },
         'skip': 'No longer available',
     }, {
@@ -1377,6 +1393,7 @@ class DouyinIE(TikTokBaseIE):
             'like_count': int,
             'repost_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:https?://.+\.jpe?g',
         },
     }]
@@ -1437,6 +1454,7 @@ class TikTokVMIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
             'comment_count': int,
+            'save_count': int,
             'thumbnail': r're:https://.+\.webp.*',
             'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAdZ_NcPPgMneaGrW0hN8O_J_bwLshwNNERRF5DxOw2HKIzk0kdlLrR8RkVl1ksrMO',
             'duration': 29,

yt_dlp/extractor/tumblr.py

@@ -20,6 +20,8 @@ class TumblrIE(InfoExtractor):
             'id': '54196191430',
             'ext': 'mp4',
             'title': 'md5:dfac39636969fe6bf1caa2d50405f069',
+            'timestamp': 1372531260,
+            'upload_date': '20130629',
             'description': 'md5:390ab77358960235b6937ab3b8528956',
             'uploader_id': 'tatianamaslanydaily',
             'uploader_url': 'https://tatianamaslanydaily.tumblr.com/',
@@ -39,6 +41,8 @@ class TumblrIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Mona\xa0“talking” in\xa0“english”',
             'description': 'md5:082a3a621530cb786ad2b7592a6d9e2c',
+            'timestamp': 1597865276,
+            'upload_date': '20200819',
             'uploader_id': 'maskofthedragon',
             'uploader_url': 'https://maskofthedragon.tumblr.com/',
             'thumbnail': r're:^https?://.*\.jpg',
@@ -76,6 +80,8 @@ class TumblrIE(InfoExtractor):
             'id': '159704441298',
             'ext': 'mp4',
             'title': 'md5:ba79365861101f4911452728d2950561',
+            'timestamp': 1492489550,
+            'upload_date': '20170418',
             'description': 'md5:773738196cea76b6996ec71e285bdabc',
             'uploader_id': 'jujanon',
             'uploader_url': 'https://jujanon.tumblr.com/',
@@ -93,6 +99,8 @@ class TumblrIE(InfoExtractor):
             'id': '180294460076',
             'ext': 'mp4',
             'title': 'duality of bird',
+            'timestamp': 1542651819,
+            'upload_date': '20181119',
             'description': 'duality of bird',
             'uploader_id': 'todaysbird',
             'uploader_url': 'https://todaysbird.tumblr.com/',
@@ -238,6 +246,8 @@ class TumblrIE(InfoExtractor):
         'info_dict': {
             'id': '730460905855467520',
             'uploader_id': 'felixcosm',
+            'upload_date': '20231006',
+            'timestamp': 1696621805,
             'repost_count': int,
             'tags': 'count:15',
             'description': 'md5:2eb3482a3c6987280cbefb6839068f32',
@@ -327,6 +337,8 @@ class TumblrIE(InfoExtractor):
         'url': 'https://www.tumblr.com/anyaboz/765332564457209856/my-music-video-for-selkie-by-nobodys-wolf-child',
         'info_dict': {
             'id': '765332564457209856',
+            'timestamp': 1729878010,
+            'upload_date': '20241025',
             'uploader_id': 'anyaboz',
             'repost_count': int,
             'age_limit': 0,
@@ -445,6 +457,8 @@ class TumblrIE(InfoExtractor):
             'uploader_id': uploader_id,
             'uploader_url': f'https://{uploader_id}.tumblr.com/' if uploader_id else None,
             **traverse_obj(post_json, {
+                # Try oldest post in reblog chain, fall back to timestamp of the post itself
+                'timestamp': ((('trail', 0, 'post'), None), 'timestamp', {int_or_none}, any),
                 'like_count': ('like_count', {int_or_none}),
                 'repost_count': ('reblog_count', {int_or_none}),
                 'tags': ('tags', ..., {str}),

yt_dlp/extractor/zoom.py

@@ -4,13 +4,15 @@ from ..utils import (
     int_or_none,
     js_to_json,
     parse_filesize,
+    parse_qs,
     parse_resolution,
     str_or_none,
-    traverse_obj,
+    update_url_query,
     url_basename,
     urlencode_postdata,
     urljoin,
 )
+from ..utils.traversal import traverse_obj
 
 
 class ZoomIE(InfoExtractor):
@@ -87,6 +89,7 @@ class ZoomIE(InfoExtractor):
     def _real_extract(self, url):
         base_url, url_type, video_id = self._match_valid_url(url).group('base_url', 'type', 'id')
         query = {}
+        start_params = traverse_obj(url, {'startTime': ({parse_qs}, 'startTime', -1)})
 
         if url_type == 'share':
             webpage = self._get_real_webpage(url, base_url, video_id, 'share')
@@ -94,7 +97,7 @@ class ZoomIE(InfoExtractor):
             redirect_path = self._download_json(
                 f'{base_url}nws/recording/1.0/play/share-info/{meeting_id}',
                 video_id, note='Downloading share info JSON')['result']['redirectUrl']
-            url = urljoin(base_url, redirect_path)
+            url = update_url_query(urljoin(base_url, redirect_path), start_params)
             query['continueMode'] = 'true'
 
         webpage = self._get_real_webpage(url, base_url, video_id, 'play')
@@ -103,6 +106,7 @@ class ZoomIE(InfoExtractor):
             # When things go wrong, file_id can be empty string
             raise ExtractorError('Unable to extract file ID')
 
+        query.update(start_params)
         data = self._download_json(
             f'{base_url}nws/recording/1.0/play/info/{file_id}', video_id, query=query,
             note='Downloading play info JSON')['result']

yt_dlp/utils/_utils.py

@@ -2830,7 +2830,7 @@ def js_to_json(code, vars={}, *, strict=False):
         {STRING_RE}|
         {COMMENT_RE}|,(?={SKIP_RE}[\]}}])|
         void\s0|(?:(?<![0-9])[eE]|[a-df-zA-DF-Z_$])[.a-zA-Z_$0-9]*|
-        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{SKIP_RE}:)?|
+        \b(?:0[xX][0-9a-fA-F]+|(?<!\.)0+[0-7]+)(?:{SKIP_RE}:)?|
         [0-9]+(?={SKIP_RE}:)|
         !+
         ''', fix_kv, code)