Merge branch 'yt-dlp:master' into viu-indonesia-fix-6482-partial

2026-08-26 05:44:18 +03:00 · 2024-02-14 21:21:46 +08:00
parent 1ba1fb1327 3dc9232e1a
commit 827821d6ca
29 changed files with 1450 additions and 704 deletions
@@ -257,6 +257,7 @@ from .blogger import BloggerIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE
+from .boosty import BoostyIE
 from .bostonglobe import BostonGlobeIE
 from .box import BoxIE
 from .boxcast import BoxCastVideoIE
@@ -78,14 +78,14 @@ class Ant1NewsGrArticleIE(AntennaBaseIE):

    _TESTS = [{
        'url': 'https://www.ant1news.gr/afieromata/article/549468/o-tzeims-mpont-sta-meteora-oi-apeiles-kai-o-xesikomos-ton-kalogeron',
-        'md5': '294f18331bb516539d72d85a82887dcc',
+        'md5': '57eb8d12181f0fa2b14b0b138e1de9b6',
        'info_dict': {
            'id': '_xvg/m_cmbatw=',
            'ext': 'mp4',
            'title': 'md5:a93e8ecf2e4073bfdffcb38f59945411',
-            'timestamp': 1603092840,
-            'upload_date': '20201019',
-            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/640/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
+            'timestamp': 1666166520,
+            'upload_date': '20221019',
+            'thumbnail': 'https://ant1media.azureedge.net/imgHandler/1920/756206d2-d640-40e2-b201-3555abdfc0db.jpg',
        },
    }, {
        'url': 'https://ant1news.gr/Society/article/620286/symmoria-anilikon-dikigoros-thymaton-ithelan-na-toys-apoteleiosoyn',
@@ -117,7 +117,7 @@ class Ant1NewsGrEmbedIE(AntennaBaseIE):
    _BASE_PLAYER_URL_RE = r'(?:https?:)?//(?:[a-zA-Z0-9\-]+\.)?(?:antenna|ant1news)\.gr/templates/pages/player'
    _VALID_URL = rf'{_BASE_PLAYER_URL_RE}\?([^#]+&)?cid=(?P<id>[^#&]+)'
    _EMBED_REGEX = [rf'<iframe[^>]+?src=(?P<_q1>["\'])(?P<url>{_BASE_PLAYER_URL_RE}\?(?:(?!(?P=_q1)).)+)(?P=_q1)']
-    _API_PATH = '/news/templates/data/jsonPlayer'
+    _API_PATH = '/templates/data/jsonPlayer'

    _TESTS = [{
        'url': 'https://www.antenna.gr/templates/pages/player?cid=3f_li_c_az_jw_y_u=&w=670&h=377',
@@ -0,0 +1,209 @@
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    qualities,
+    str_or_none,
+    url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class BoostyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?boosty\.to/(?P<user>[^/#?]+)/posts/(?P<post_id>[^/#?]+)'
+    _TESTS = [{
+        # single ok_video
+        'url': 'https://boosty.to/kuplinov/posts/e55d050c-e3bb-4873-a7db-ac7a49b40c38',
+        'info_dict': {
+            'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
+            'title': 'phasma_3',
+            'channel': 'Kuplinov',
+            'channel_id': '7958701',
+            'timestamp': 1655031975,
+            'upload_date': '20220612',
+            'release_timestamp': 1655049000,
+            'release_date': '20220612',
+            'modified_timestamp': 1668680993,
+            'modified_date': '20221117',
+            'tags': ['куплинов', 'phasmophobia'],
+            'like_count': int,
+            'ext': 'mp4',
+            'duration': 105,
+            'view_count': int,
+            'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+        },
+    }, {
+        # multiple ok_video
+        'url': 'https://boosty.to/maddyson/posts/0c652798-3b35-471f-8b48-a76a0b28736f',
+        'info_dict': {
+            'id': '0c652798-3b35-471f-8b48-a76a0b28736f',
+            'title': 'то что не пропустил юта6',
+            'channel': 'Илья Давыдов',
+            'channel_id': '6808257',
+            'timestamp': 1694017040,
+            'upload_date': '20230906',
+            'release_timestamp': 1694017040,
+            'release_date': '20230906',
+            'modified_timestamp': 1694071178,
+            'modified_date': '20230907',
+            'like_count': int,
+        },
+        'playlist_count': 3,
+        'playlist': [{
+            'info_dict': {
+                'id': 'cc325a9f-a563-41c6-bf47-516c1b506c9a',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 31204,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }, {
+            'info_dict': {
+                'id': 'd07b0a72-9493-4512-b54e-55ce468fd4b7',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 25704,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }, {
+            'info_dict': {
+                'id': '4a3bba32-78c8-422a-9432-2791aff60b42',
+                'title': 'то что не пропустил юта6',
+                'channel': 'Илья Давыдов',
+                'channel_id': '6808257',
+                'timestamp': 1694017040,
+                'upload_date': '20230906',
+                'release_timestamp': 1694017040,
+                'release_date': '20230906',
+                'modified_timestamp': 1694071178,
+                'modified_date': '20230907',
+                'like_count': int,
+                'ext': 'mp4',
+                'duration': 31867,
+                'view_count': int,
+                'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
+            },
+        }],
+    }, {
+        # single external video (youtube)
+        'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
+        'info_dict': {
+            'id': 'EXelTnve5lY',
+            'title': 'Послание Президента Федеральному Собранию | Класс народа',
+            'upload_date': '20210425',
+            'channel': 'Денис Чужой',
+            'tags': 'count:10',
+            'like_count': int,
+            'ext': 'mp4',
+            'duration': 816,
+            'view_count': int,
+            'thumbnail': r're:^https://i\.ytimg\.com/',
+            'age_limit': 0,
+            'availability': 'public',
+            'categories': list,
+            'channel_follower_count': int,
+            'channel_id': 'UCCzVNbWZfYpBfyofCCUD_0w',
+            'channel_is_verified': bool,
+            'channel_url': r're:^https://www\.youtube\.com/',
+            'comment_count': int,
+            'description': str,
+            'heatmap': 'count:100',
+            'live_status': str,
+            'playable_in_embed': bool,
+            'uploader': str,
+            'uploader_id': str,
+            'uploader_url': r're:^https://www\.youtube\.com/',
+        },
+    }]
+
+    _MP4_TYPES = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
+
+    def _extract_formats(self, player_urls, video_id):
+        formats = []
+        quality = qualities(self._MP4_TYPES)
+        for player_url in traverse_obj(player_urls, lambda _, v: url_or_none(v['url'])):
+            url = player_url['url']
+            format_type = player_url.get('type')
+            if format_type in ('hls', 'hls_live', 'live_ondemand_hls', 'live_playback_hls'):
+                formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id='hls', fatal=False))
+            elif format_type in ('dash', 'dash_live', 'live_playback_dash'):
+                formats.extend(self._extract_mpd_formats(url, video_id, mpd_id='dash', fatal=False))
+            elif format_type in self._MP4_TYPES:
+                formats.append({
+                    'url': url,
+                    'ext': 'mp4',
+                    'format_id': format_type,
+                    'quality': quality(format_type),
+                })
+            else:
+                self.report_warning(f'Unknown format type: {format_type!r}')
+        return formats
+
+    def _real_extract(self, url):
+        user, post_id = self._match_valid_url(url).group('user', 'post_id')
+        post = self._download_json(
+            f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
+            note='Downloading post data', errnote='Unable to download post data')
+
+        post_title = post.get('title')
+        if not post_title:
+            self.report_warning('Unable to extract post title. Falling back to parsing html page')
+            webpage = self._download_webpage(url, video_id=post_id)
+            post_title = self._og_search_title(webpage, default=None) or self._html_extract_title(webpage)
+
+        common_metadata = {
+            'title': post_title,
+            **traverse_obj(post, {
+                'channel': ('user', 'name', {str}),
+                'channel_id': ('user', 'id', {str_or_none}),
+                'timestamp': ('createdAt', {int_or_none}),
+                'release_timestamp': ('publishTime', {int_or_none}),
+                'modified_timestamp': ('updatedAt', {int_or_none}),
+                'tags': ('tags', ..., 'title', {str}),
+                'like_count': ('count', 'likes', {int_or_none}),
+            }),
+        }
+        entries = []
+        for item in traverse_obj(post, ('data', ..., {dict})):
+            item_type = item.get('type')
+            if item_type == 'video' and url_or_none(item.get('url')):
+                entries.append(self.url_result(item['url'], YoutubeIE))
+            elif item_type == 'ok_video':
+                video_id = item.get('id') or post_id
+                entries.append({
+                    'id': video_id,
+                    'formats': self._extract_formats(item.get('playerUrls'), video_id),
+                    **common_metadata,
+                    **traverse_obj(item, {
+                        'title': ('title', {str}),
+                        'duration': ('duration', {int_or_none}),
+                        'view_count': ('viewsCounter', {int_or_none}),
+                        'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
+                    }, get_all=False)})
+
+        if not entries:
+            raise ExtractorError('No videos found', expected=True)
+        if len(entries) == 1:
+            return entries[0]
+        return self.playlist_result(entries, post_id, post_title, **common_metadata)
@@ -33,10 +33,7 @@ class CrooksAndLiarsIE(InfoExtractor):
        webpage = self._download_webpage(
            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)

-        manifest = self._parse_json(
-            self._search_regex(
-                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
-            video_id)
+        manifest = self._search_json(r'var\s+manifest\s*=', webpage, 'manifest JSON', video_id)

        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))

@@ -1,25 +1,29 @@
 from .common import InfoExtractor
 from .nexx import NexxIE
-from ..utils import (
-    int_or_none,
-    str_or_none,
-)


 class FunkIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.|origin\.)?funk\.net/(?:channel|playlist)/[^/]+/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.funk.net/channel/ba-793/die-lustigsten-instrumente-aus-dem-internet-teil-2-1155821',
-        'md5': '8dd9d9ab59b4aa4173b3197f2ea48e81',
+        'md5': '8610449476156f338761a75391b0017d',
        'info_dict': {
            'id': '1155821',
            'ext': 'mp4',
            'title': 'Die LUSTIGSTEN INSTRUMENTE aus dem Internet - Teil 2',
-            'description': 'md5:a691d0413ef4835588c5b03ded670c1f',
+            'description': 'md5:2a03b67596eda0d1b5125c299f45e953',
            'timestamp': 1514507395,
            'upload_date': '20171229',
+            'duration': 426.0,
+            'cast': ['United Creators PMB GmbH'],
+            'thumbnail': 'https://assets.nexx.cloud/media/75/56/79/3YKUSJN1LACN0CRxL.jpg',
+            'display_id': 'die-lustigsten-instrumente-aus-dem-internet-teil-2',
+            'alt_title': 'Die LUSTIGSTEN INSTRUMENTE aus dem Internet Teil 2',
+            'season_number': 0,
+            'season': 'Season 0',
+            'episode_number': 0,
+            'episode': 'Episode 0',
        },
-
    }, {
        'url': 'https://www.funk.net/playlist/neuesteVideos/kameras-auf-dem-fusion-festival-1618699',
        'only_matching': True,
@@ -27,18 +31,10 @@ class FunkIE(InfoExtractor):

    def _real_extract(self, url):
        display_id, nexx_id = self._match_valid_url(url).groups()
-        video = self._download_json(
-            'https://www.funk.net/api/v4.0/videos/' + nexx_id, nexx_id)
        return {
            '_type': 'url_transparent',
-            'url': 'nexx:741:' + nexx_id,
+            'url': f'nexx:741:{nexx_id}',
            'ie_key': NexxIE.ie_key(),
            'id': nexx_id,
-            'title': video.get('title'),
-            'description': video.get('description'),
-            'duration': int_or_none(video.get('duration')),
-            'channel_id': str_or_none(video.get('channelId')),
            'display_id': display_id,
-            'tags': video.get('tags'),
-            'thumbnail': video.get('imageUrlLandscape'),
        }
@@ -13,7 +13,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):

    _TESTS = [{
        'url': 'https://video.lefigaro.fr/embed/figaro/video/les-francais-ne-veulent-ils-plus-travailler-suivez-en-direct-le-club-le-figaro-idees/',
-        'md5': 'e94de44cd80818084352fcf8de1ce82c',
+        'md5': 'a0c3069b7e4c4526abf0053a7713f56f',
        'info_dict': {
            'id': 'g9j7Eovo',
            'title': 'Les Français ne veulent-ils plus travailler ? Retrouvez Le Club Le Figaro Idées',
@@ -26,7 +26,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
        },
    }, {
        'url': 'https://video.lefigaro.fr/embed/figaro/video/intelligence-artificielle-faut-il-sen-mefier/',
-        'md5': '0b3f10332b812034b3a3eda1ef877c5f',
+        'md5': '319c662943dd777bab835cae1e2d73a5',
        'info_dict': {
            'id': 'LeAgybyc',
            'title': 'Intelligence artificielle : faut-il s’en méfier ?',
@@ -41,7 +41,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):

    _WEBPAGE_TESTS = [{
        'url': 'https://video.lefigaro.fr/figaro/video/suivez-en-direct-le-club-le-figaro-international-avec-philippe-gelie-9/',
-        'md5': '3972ddf2d5f8b98699f191687258e2f9',
+        'md5': '6289f9489efb969e38245f31721596fe',
        'info_dict': {
            'id': 'QChnbPYA',
            'title': 'Où en est le couple franco-allemand ? Retrouvez Le Club Le Figaro International',
@@ -55,7 +55,7 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
        },
    }, {
        'url': 'https://video.lefigaro.fr/figaro/video/la-philosophe-nathalie-sarthou-lajus-est-linvitee-du-figaro-live/',
-        'md5': '3ac0a0769546ee6be41ab52caea5d9a9',
+        'md5': 'f6df814cae53e85937621599d2967520',
        'info_dict': {
            'id': 'QJzqoNbf',
            'title': 'La philosophe Nathalie Sarthou-Lajus est l’invitée du Figaro Live',
@@ -73,7 +73,8 @@ class LeFigaroVideoEmbedIE(InfoExtractor):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

-        player_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['pageData']['playerData']
+        player_data = self._search_nextjs_data(
+            webpage, display_id)['props']['pageProps']['initialProps']['pageData']['playerData']

        return self.url_result(
            f'jwplatform:{player_data["videoId"]}', title=player_data.get('title'),
@@ -28,12 +28,24 @@ class MagellanTVIE(InfoExtractor):
            'tags': ['Investigation', 'True Crime', 'Justice', 'Europe'],
        },
        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.magellantv.com/watch/celebration-nation',
+        'info_dict': {
+            'id': 'celebration-nation',
+            'ext': 'mp4',
+            'tags': ['Art & Culture', 'Human Interest', 'Anthropology', 'China', 'History'],
+            'duration': 2640.0,
+            'title': 'Ancestors',
+        },
+        'params': {'skip_download': 'm3u8'},
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
-        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['reactContext']['video']['detail']
+        data = traverse_obj(self._search_nextjs_data(webpage, video_id), (
+            'props', 'pageProps', 'reactContext',
+            (('video', 'detail'), ('series', 'currentEpisode')), {dict}), get_all=False)
        formats, subtitles = self._extract_m3u8_formats_and_subtitles(data['jwpVideoUrl'], video_id)

        return {
@@ -32,6 +32,7 @@ class NYTimesBaseIE(InfoExtractor):
        renderedRepresentation
      }
      duration
+      firstPublished
      promotionalHeadline
      promotionalMedia {
        ... on Image {
@@ -124,6 +125,7 @@ class NYTimesBaseIE(InfoExtractor):
            'id': media_id,
            'title': data.get('promotionalHeadline'),
            'description': data.get('summary'),
+            'timestamp': parse_iso8601(data.get('firstPublished')),
            'duration': float_or_none(data.get('duration'), scale=1000),
            'creator': ', '.join(traverse_obj(data, (  # TODO: change to 'creators'
                'bylines', ..., 'renderedRepresentation', {lambda x: remove_start(x, 'By ')}))),
@@ -145,8 +147,8 @@ class NYTimesIE(NYTimesBaseIE):
            'ext': 'mp4',
            'title': 'Verbatim: What Is a Photocopier?',
            'description': 'md5:93603dada88ddbda9395632fdc5da260',
-            'timestamp': 1398631707,  # FIXME
-            'upload_date': '20140427',  # FIXME
+            'timestamp': 1398646132,
+            'upload_date': '20140428',
            'creator': 'Brett Weiner',
            'thumbnail': r're:https?://\w+\.nyt.com/images/.+\.jpg',
            'duration': 419,
@@ -310,6 +312,8 @@ class NYTimesCookingIE(NYTimesBaseIE):
            'ext': 'mp4',
            'title': 'How to Make Mac and Cheese',
            'description': 'md5:b8f2f33ec1fb7523b21367147c9594f1',
+            'timestamp': 1522950315,
+            'upload_date': '20180405',
            'duration': 9.51,
            'creator': 'Alison Roman',
            'thumbnail': r're:https?://\w+\.nyt.com/images/.*\.jpg',
@@ -21,7 +21,7 @@ from ..utils import (
 class TVPIE(InfoExtractor):
    IE_NAME = 'tvp'
    IE_DESC = 'Telewizja Polska'
-    _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:tvp(?:parlament)?\.(?:pl|info)|tvpworld\.com|swipeto\.pl)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)(?:[/?#]|$)'

    _TESTS = [{
        # TVPlayer 2 in js wrapper
@@ -514,7 +514,7 @@ class TVPVODBaseIE(InfoExtractor):

 class TVPVODVideoIE(TVPVODBaseIE):
    IE_NAME = 'tvp:vod'
-    _VALID_URL = r'https?://vod\.tvp\.pl/[a-z\d-]+,\d+/[a-z\d-]+(?<!-odcinki)(?:-odcinki,\d+/odcinek-\d+,S\d+E\d+)?,(?P<id>\d+)(?:\?[^#]+)?(?:#.+)?$'
+    _VALID_URL = r'https?://vod\.tvp\.pl/(?P<category>[a-z\d-]+,\d+)/[a-z\d-]+(?<!-odcinki)(?:-odcinki,\d+/odcinek-\d+,S\d+E\d+)?,(?P<id>\d+)/?(?:[?#]|$)'

    _TESTS = [{
        'url': 'https://vod.tvp.pl/dla-dzieci,24/laboratorium-alchemika-odcinki,309338/odcinek-24,S01E24,311357',
@@ -560,12 +560,23 @@ class TVPVODVideoIE(TVPVODBaseIE):
            'thumbnail': 're:https?://.+',
        },
        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://vod.tvp.pl/live,1/tvp-world,399731',
+        'info_dict': {
+            'id': '399731',
+            'ext': 'mp4',
+            'title': r're:TVP WORLD \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'live_status': 'is_live',
+            'thumbnail': 're:https?://.+',
+        },
    }]

    def _real_extract(self, url):
-        video_id = self._match_id(url)
+        category, video_id = self._match_valid_url(url).group('category', 'id')

-        info_dict = self._parse_video(self._call_api(f'vods/{video_id}', video_id), with_url=False)
+        is_live = category == 'live,1'
+        entity = 'lives' if is_live else 'vods'
+        info_dict = self._parse_video(self._call_api(f'{entity}/{video_id}', video_id), with_url=False)

        playlist = self._call_api(f'{video_id}/videos/playlist', video_id, query={'videoType': 'MOVIE'})

@@ -582,6 +593,8 @@ class TVPVODVideoIE(TVPVODBaseIE):
                'ext': 'ttml',
            })

+        info_dict['is_live'] = is_live
+
        return info_dict