[ie/nebula] Support --mark-watched (#13120)

Authored by: GeoffreyFrogeye

yt_dlp/extractor/espn.py

@@ -5,7 +5,6 @@ import urllib.parse
 
 from .adobepass import AdobePassIE
 from .common import InfoExtractor
-from .once import OnceIE
 from ..utils import (
     determine_ext,
     dict_get,
@@ -16,7 +15,7 @@ from ..utils import (
 )
 
 
-class ESPNIE(OnceIE):
+class ESPNIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
@@ -131,9 +130,7 @@ class ESPNIE(OnceIE):
                 return
             format_urls.add(source_url)
             ext = determine_ext(source_url)
-            if OnceIE.suitable(source_url):
+            if ext == 'smil':
-                formats.extend(self._extract_once_formats(source_url))
-            elif ext == 'smil':
                 formats.extend(self._extract_smil_formats(
                     source_url, video_id, fatal=False))
             elif ext == 'f4m':

yt_dlp/extractor/firsttv.py

@@ -2,11 +2,15 @@ import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
-    qualities,
+    join_nonempty,
+    mimetype2ext,
+    parse_qs,
     unified_strdate,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class FirstTVIE(InfoExtractor):
@@ -15,40 +19,51 @@ class FirstTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:sport)?1tv\.ru/(?:[^/?#]+/)+(?P<id>[^/?#]+)'
 
     _TESTS = [{
-        # single format
+        # single format; has item.id
-        'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
+        'url': 'https://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
-        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+        'md5': '8011ae8e88ff4150107ab9c5a8f5b659',
         'info_dict': {
             'id': '40049',
             'ext': 'mp4',
             'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
-            'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
+            'thumbnail': r're:https?://.+/.+\.jpg',
             'upload_date': '20150212',
             'duration': 2694,
         },
+        'params': {'skip_download': 'm3u8'},
     }, {
-        # multiple formats
+        # multiple formats; has item.id
-        'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
+        'url': 'https://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
         'info_dict': {
             'id': '364746',
             'ext': 'mp4',
             'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
-            'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
+            'thumbnail': r're:https?://.+/.+\.jpg',
             'upload_date': '20160407',
             'duration': 179,
             'formats': 'mincount:3',
         },
-        'params': {
+        'params': {'skip_download': 'm3u8'},
-            'skip_download': True,
-        },
     }, {
-        'url': 'http://www.1tv.ru/news/issue/2016-12-01/14:00',
+        'url': 'https://www.1tv.ru/news/issue/2016-12-01/14:00',
         'info_dict': {
             'id': '14:00',
-            'title': 'Выпуск новостей в 14:00   1 декабря 2016 года. Новости. Первый канал',
+            'title': 'Выпуск программы «Время» в 20:00   1 декабря 2016 года. Новости. Первый канал',
-            'description': 'md5:2e921b948f8c1ff93901da78ebdb1dfd',
+            'thumbnail': 'https://static.1tv.ru/uploads/photo/image/8/big/338448_big_8fc7eb236f.jpg',
         },
         'playlist_count': 13,
+    }, {
+        # has timestamp; has item.uid but not item.id
+        'url': 'https://www.1tv.ru/shows/segodnya-vecherom/vypuski/avtory-odnogo-hita-segodnya-vecherom-vypusk-ot-03-05-2025',
+        'info_dict': {
+            'id': '270411',
+            'ext': 'mp4',
+            'title': 'Авторы одного хита. Сегодня вечером. Выпуск от 03.05.2025',
+            'thumbnail': r're:https?://.+/.+\.jpg',
+            'timestamp': 1746286020,
+            'upload_date': '20250503',
+        },
+        'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'http://www.1tv.ru/shows/tochvtoch-supersezon/vystupleniya/evgeniy-dyatlov-vladimir-vysockiy-koni-priveredlivye-toch-v-toch-supersezon-fragment-vypuska-ot-06-11-2016',
         'only_matching': True,
@@ -57,96 +72,60 @@ class FirstTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _entries(self, items):
+        for item in items:
+            video_id = str(item.get('id') or item['uid'])
+
+            formats, subtitles = [], {}
+            for f in traverse_obj(item, ('sources', lambda _, v: url_or_none(v['src']))):
+                src = f['src']
+                ext = mimetype2ext(f.get('type'), default=determine_ext(src))
+                if ext == 'm3u8':
+                    fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                        src, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                elif ext == 'mpd':
+                    fmts, subs = self._extract_mpd_formats_and_subtitles(
+                        src, video_id, mpd_id='dash', fatal=False)
+                else:
+                    tbr = self._search_regex(fr'_(\d{{3,}})\.{ext}', src, 'tbr', default=None)
+                    formats.append({
+                        'url': src,
+                        'ext': ext,
+                        'format_id': join_nonempty('http', ext, tbr),
+                        'tbr': int_or_none(tbr),
+                        # quality metadata of http formats may be incorrect
+                        'quality': -10,
+                    })
+                    continue
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+
+            yield {
+                **traverse_obj(item, {
+                    'title': ('title', {str}),
+                    'thumbnail': ('poster', {url_or_none}),
+                    'timestamp': ('dvr_begin_at', {int_or_none}),
+                    'upload_date': ('date_air', {unified_strdate}),
+                    'duration': ('duration', {int_or_none}),
+                }),
+                'id': video_id,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
-        playlist_url = urllib.parse.urljoin(url, self._search_regex(
+        playlist_url = urllib.parse.urljoin(url, self._html_search_regex(
             r'data-playlist-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
             webpage, 'playlist url', group='url'))
 
-        parsed_url = urllib.parse.urlparse(playlist_url)
+        item_ids = traverse_obj(parse_qs(playlist_url), 'video_id', 'videos_ids[]', 'news_ids[]')
-        qs = urllib.parse.parse_qs(parsed_url.query)
+        items = traverse_obj(
-        item_ids = qs.get('videos_ids[]') or qs.get('news_ids[]')
+            self._download_json(playlist_url, display_id),
+            lambda _, v: v['uid'] and (str(v['uid']) in item_ids if item_ids else True))
 
-        items = self._download_json(playlist_url, display_id)
+        return self.playlist_result(
-
+            self._entries(items), display_id, self._og_search_title(webpage, default=None),
-        if item_ids:
+            thumbnail=self._og_search_thumbnail(webpage, default=None))
-            items = [
-                item for item in items
-                if item.get('uid') and str(item['uid']) in item_ids]
-        else:
-            items = [items[0]]
-
-        entries = []
-        QUALITIES = ('ld', 'sd', 'hd')
-
-        for item in items:
-            title = item['title']
-            quality = qualities(QUALITIES)
-            formats = []
-            path = None
-            for f in item.get('mbr', []):
-                src = url_or_none(f.get('src'))
-                if not src:
-                    continue
-                tbr = int_or_none(self._search_regex(
-                    r'_(\d{3,})\.mp4', src, 'tbr', default=None))
-                if not path:
-                    path = self._search_regex(
-                        r'//[^/]+/(.+?)_\d+\.mp4', src,
-                        'm3u8 path', default=None)
-                formats.append({
-                    'url': src,
-                    'format_id': f.get('name'),
-                    'tbr': tbr,
-                    'source_preference': quality(f.get('name')),
-                    # quality metadata of http formats may be incorrect
-                    'preference': -10,
-                })
-            # m3u8 URL format is reverse engineered from [1] (search for
-            # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)
-            # is taken from [2].
-            # 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted
-            # 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834
-            if not path and len(formats) == 1:
-                path = self._search_regex(
-                    r'//[^/]+/(.+?$)', formats[0]['url'],
-                    'm3u8 path', default=None)
-            if path:
-                if len(formats) == 1:
-                    m3u8_path = ','
-                else:
-                    tbrs = [str(t) for t in sorted(f['tbr'] for f in formats)]
-                    m3u8_path = '_,{},{}'.format(','.join(tbrs), '.mp4')
-                formats.extend(self._extract_m3u8_formats(
-                    f'http://balancer-vod.1tv.ru/{path}{m3u8_path}.urlset/master.m3u8',
-                    display_id, 'mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
-
-            thumbnail = item.get('poster') or self._og_search_thumbnail(webpage)
-            duration = int_or_none(item.get('duration') or self._html_search_meta(
-                'video:duration', webpage, 'video duration', fatal=False))
-            upload_date = unified_strdate(self._html_search_meta(
-                'ya:ovs:upload_date', webpage, 'upload date', default=None))
-
-            entries.append({
-                'id': str(item.get('id') or item['uid']),
-                'thumbnail': thumbnail,
-                'title': title,
-                'upload_date': upload_date,
-                'duration': int_or_none(duration),
-                'formats': formats,
-            })
-
-        title = self._html_search_regex(
-            (r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
-             r"'title'\s*:\s*'([^']+)'"),
-            webpage, 'title', default=None) or self._og_search_title(
-            webpage, default=None)
-        description = self._html_search_regex(
-            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
-            webpage, 'description', default=None) or self._html_search_meta(
-            'description', webpage, 'description', default=None)
-
-        return self.playlist_result(entries, display_id, title, description)

yt_dlp/extractor/gamespot.py

@@ -1,9 +1,9 @@
 import urllib.parse
 
-from .once import OnceIE
+from .common import InfoExtractor
 
 
-class GameSpotIE(OnceIE):
+class GameSpotIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article|review)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',

yt_dlp/extractor/iprima.py

@@ -1,3 +1,4 @@
+import json
 import re
 import time
 
@@ -6,9 +7,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     js_to_json,
-    parse_qs,
     traverse_obj,
-    urlencode_postdata,
 )
 
 
@@ -16,7 +15,6 @@ class IPrimaIE(InfoExtractor):
     _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'iprima'
-    _AUTH_ROOT = 'https://auth.iprima.cz'
     access_token = None
 
     _TESTS = [{
@@ -86,48 +84,18 @@ class IPrimaIE(InfoExtractor):
         if self.access_token:
             return
 
-        login_page = self._download_webpage(
-            f'{self._AUTH_ROOT}/oauth2/login', None, note='Downloading login page',
-            errnote='Downloading login page failed')
-
-        login_form = self._hidden_inputs(login_page)
-
-        login_form.update({
-            '_email': username,
-            '_password': password})
-
-        profile_select_html, login_handle = self._download_webpage_handle(
-            f'{self._AUTH_ROOT}/oauth2/login', None, data=urlencode_postdata(login_form),
-            note='Logging in')
-
-        # a profile may need to be selected first, even when there is only a single one
-        if '/profile-select' in login_handle.url:
-            profile_id = self._search_regex(
-                r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
-
-            login_handle = self._request_webpage(
-                f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
-                query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
-
-        code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0))
-        if not code:
-            raise ExtractorError('Login failed', expected=True)
-
-        token_request_data = {
-            'scope': 'openid+email+profile+phone+address+offline_access',
-            'client_id': 'prima_sso',
-            'grant_type': 'authorization_code',
-            'code': code,
-            'redirect_uri': f'{self._AUTH_ROOT}/sso/auth-check'}
-
         token_data = self._download_json(
-            f'{self._AUTH_ROOT}/oauth2/token', None,
+            'https://ucet.iprima.cz/api/session/create', None,
-            note='Downloading token', errnote='Downloading token failed',
+            note='Logging in', errnote='Failed to log in',
-            data=urlencode_postdata(token_request_data))
+            data=json.dumps({
+                'email': username,
+                'password': password,
+                'deviceName': 'Windows Chrome',
+            }).encode(), headers={'content-type': 'application/json'})
 
-        self.access_token = token_data.get('access_token')
+        self.access_token = token_data['accessToken']['value']
-        if self.access_token is None:
+        if not self.access_token:
-            raise ExtractorError('Getting token failed', expected=True)
+            raise ExtractorError('Failed to fetch access token')
 
     def _real_initialize(self):
         if not self.access_token:

yt_dlp/extractor/nebula.py

@@ -3,6 +3,7 @@ import json
 
 from .art19 import Art19IE
 from .common import InfoExtractor
+from ..networking import PATCHRequest
 from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
@@ -74,7 +75,7 @@ class NebulaBaseIE(InfoExtractor):
                         'app_version': '23.10.0',
                         'platform': 'ios',
                     })
-                return {'formats': fmts, 'subtitles': subs}
+                break
             except ExtractorError as e:
                 if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     self.raise_login_required()
@@ -84,6 +85,9 @@ class NebulaBaseIE(InfoExtractor):
                     continue
                 raise
 
+        self.mark_watched(content_id, slug)
+        return {'formats': fmts, 'subtitles': subs}
+
     def _extract_video_metadata(self, episode):
         channel_url = traverse_obj(
             episode, (('channel_slug', 'class_slug'), {urljoin('https://nebula.tv/')}), get_all=False)
@@ -111,6 +115,13 @@ class NebulaBaseIE(InfoExtractor):
             'uploader_url': channel_url,
         }
 
+    def _mark_watched(self, content_id, slug):
+        self._call_api(
+            PATCHRequest(f'https://content.api.nebula.app/{content_id.split(":")[0]}s/{content_id}/progress/'),
+            slug, 'Marking watched', 'Unable to mark watched', fatal=False,
+            data=json.dumps({'completed': True}).encode(),
+            headers={'content-type': 'application/json'})
+
 
 class NebulaIE(NebulaBaseIE):
     IE_NAME = 'nebula:video'
@@ -322,6 +333,7 @@ class NebulaClassIE(NebulaBaseIE):
             if not episode_url and metadata.get('premium'):
                 self.raise_login_required()
 
+            self.mark_watched(metadata['id'], slug)
             if Art19IE.suitable(episode_url):
                 return self.url_result(episode_url, Art19IE)
             return traverse_obj(metadata, {

yt_dlp/extractor/once.py

@@ -1,40 +0,0 @@
-import re
-
-from .common import InfoExtractor
-
-
-class OnceIE(InfoExtractor):  # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor
-    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/(?:ads/vmap/)?[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
-    ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
-    PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
-
-    def _extract_once_formats(self, url, http_formats_preference=None):
-        domain_id, application_id, media_item_id = re.match(
-            OnceIE._VALID_URL, url).groups()
-        formats = self._extract_m3u8_formats(
-            self.ADAPTIVE_URL_TEMPLATE % (
-                domain_id, application_id, media_item_id),
-            media_item_id, 'mp4', m3u8_id='hls', fatal=False)
-        progressive_formats = []
-        for adaptive_format in formats:
-            # Prevent advertisement from embedding into m3u8 playlist (see
-            # https://github.com/ytdl-org/youtube-dl/issues/8893#issuecomment-199912684)
-            adaptive_format['url'] = re.sub(
-                r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url'])
-            rendition_id = self._search_regex(
-                r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
-                adaptive_format['url'], 'redition id', default=None)
-            if rendition_id:
-                progressive_format = adaptive_format.copy()
-                progressive_format.update({
-                    'url': self.PROGRESSIVE_URL_TEMPLATE % (
-                        domain_id, application_id, rendition_id, media_item_id),
-                    'format_id': adaptive_format['format_id'].replace(
-                        'hls', 'http'),
-                    'protocol': 'http',
-                    'preference': http_formats_preference,
-                })
-                progressive_formats.append(progressive_format)
-        self._check_formats(progressive_formats, media_item_id)
-        formats.extend(progressive_formats)
-        return formats

yt_dlp/extractor/theplatform.py

@@ -4,7 +4,6 @@ import re
 import time
 
 from .adobepass import AdobePassIE
-from .once import OnceIE
 from ..networking import HEADRequest, Request
 from ..utils import (
     ExtractorError,
@@ -26,7 +25,7 @@ default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
-class ThePlatformBaseIE(OnceIE):
+class ThePlatformBaseIE(AdobePassIE):
     _TP_TLD = 'com'
 
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
@@ -54,16 +53,13 @@ class ThePlatformBaseIE(OnceIE):
 
         formats = []
         for _format in smil_formats:
-            if OnceIE.suitable(_format['url']):
+            media_url = _format['url']
-                formats.extend(self._extract_once_formats(_format['url']))
+            if determine_ext(media_url) == 'm3u8':
-            else:
+                hdnea2 = self._get_cookies(media_url).get('hdnea2')
-                media_url = _format['url']
+                if hdnea2:
-                if determine_ext(media_url) == 'm3u8':
+                    _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value})
-                    hdnea2 = self._get_cookies(media_url).get('hdnea2')
-                    if hdnea2:
-                        _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value})
 
-                formats.append(_format)
+            formats.append(_format)
 
         return formats, subtitles
 
@@ -129,7 +125,7 @@ class ThePlatformBaseIE(OnceIE):
         return self._parse_theplatform_metadata(info)
 
 
-class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
+class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)?|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?

yt_dlp/extractor/voxmedia.py

@@ -1,7 +1,6 @@
 import urllib.parse
 
 from .common import InfoExtractor
-from .once import OnceIE
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -10,7 +9,7 @@ from ..utils import (
 )
 
 
-class VoxMediaVolumeIE(OnceIE):
+class VoxMediaVolumeIE(InfoExtractor):
     _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P<id>[0-9a-f]{9})'
 
     def _real_extract(self, url):
@@ -57,7 +56,8 @@ class VoxMediaVolumeIE(OnceIE):
             if not provider_video_id:
                 continue
             if provider_video_type == 'brightcove':
-                info['formats'] = self._extract_once_formats(provider_video_id)
+                # TODO: Find embed example or confirm that Vox has stopped using Brightcove
+                raise ExtractorError('Vox Brightcove embeds are currently unsupported')
             else:
                 info.update({
                     '_type': 'url_transparent',
@@ -155,20 +155,6 @@ class VoxMediaIE(InfoExtractor):
             },
         }],
         'skip': 'Page no longer contain videos',
-    }, {
-        # volume embed, Brightcove Once
-        'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya',
-        'md5': '2dbc77b8b0bff1894c2fce16eded637d',
-        'info_dict': {
-            'id': '1231c973d',
-            'ext': 'mp4',
-            'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
-            'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
-            'timestamp': 1402938000,
-            'upload_date': '20140616',
-            'duration': 4114,
-        },
-        'add_ie': ['VoxMediaVolume'],
     }]
 
     def _real_extract(self, url):