[ie/xhamster] Fix extractor (#15252)

Closes #15239
Authored by: 0xvd

yt_dlp/extractor/_extractors.py

@@ -84,6 +84,7 @@ from .agora import (
 )
 from .airtv import AirTVIE
 from .aitube import AitubeKZVideoIE
+from .alibaba import AlibabaIE
 from .aliexpress import AliExpressLiveIE
 from .aljazeera import AlJazeeraIE
 from .allocine import AllocineIE

yt_dlp/extractor/alibaba.py

@@ -0,0 +1,42 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, str_or_none, url_or_none
+from ..utils.traversal import traverse_obj
+
+
+class AlibabaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?alibaba\.com/product-detail/[\w-]+_(?P<id>\d+)\.html'
+    _TESTS = [{
+        'url': 'https://www.alibaba.com/product-detail/Kids-Entertainment-Bouncer-Bouncy-Castle-Waterslide_1601271126969.html',
+        'info_dict': {
+            'id': '6000280444270',
+            'display_id': '1601271126969',
+            'ext': 'mp4',
+            'title': 'Kids Entertainment Bouncer Bouncy Castle Waterslide Juex Gonflables Commercial Inflatable Tropical Water Slide',
+            'duration': 30,
+            'thumbnail': 'https://sc04.alicdn.com/kf/Hc5bb391974454af18c7a4f91cbe4062bg.jpg_120x120.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        product_data = self._search_json(
+            r'window\.detailData\s*=', webpage, 'detail data', display_id)['globalData']['product']
+
+        return {
+            **traverse_obj(product_data, ('mediaItems', lambda _, v: v['type'] == 'video' and v['videoId'], any, {
+                'id': ('videoId', {int}, {str_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'thumbnail': ('videoCoverUrl', {url_or_none}),
+                'formats': ('videoUrl', lambda _, v: url_or_none(v['videoUrl']), {
+                    'url': 'videoUrl',
+                    'format_id': ('definition', {str_or_none}),
+                    'tbr': ('bitrate', {int_or_none}),
+                    'width': ('width', {int_or_none}),
+                    'height': ('height', {int_or_none}),
+                    'filesize': ('length', {int_or_none}),
+                }),
+            })),
+            'title': traverse_obj(product_data, ('subject', {str})),
+            'display_id': display_id,
+        }

yt_dlp/extractor/sportdeutschland.py

@@ -8,10 +8,11 @@ from ..utils import (
 
 
 class SportDeutschlandIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:player\.)?sportdeutschland\.tv/(?P<id>(?:[^/?#]+/)?[^?#/&]+)'
+    IE_NAME = 'sporteurope'
+    _VALID_URL = r'https?://(?:player\.)?sporteurope\.tv/(?P<id>(?:[^/?#]+/)?[^?#/&]+)'
     _TESTS = [{
         # Single-part video, direct link
-        'url': 'https://sportdeutschland.tv/rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
+        'url': 'https://sporteurope.tv/rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
         'md5': '35c11a19395c938cdd076b93bda54cde',
         'info_dict': {
             'id': '9f27a97d-1544-4d0b-aa03-48d92d17a03a',
@@ -19,9 +20,9 @@ class SportDeutschlandIE(InfoExtractor):
             'title': 'GFL2: Rostock Griffins vs. Elmshorn Fighting Pirates',
             'display_id': 'rostock-griffins/gfl2-rostock-griffins-vs-elmshorn-fighting-pirates',
             'channel': 'Rostock Griffins',
-            'channel_url': 'https://sportdeutschland.tv/rostock-griffins',
+            'channel_url': 'https://sporteurope.tv/rostock-griffins',
             'live_status': 'was_live',
-            'description': 'md5:60cb00067e55dafa27b0933a43d72862',
+            'description': r're:Video-Livestream des Spiels Rostock Griffins vs\. Elmshorn Fighting Pirates.+',
             'channel_id': '9635f21c-3f67-4584-9ce4-796e9a47276b',
             'timestamp': 1749913117,
             'upload_date': '20250614',
@@ -29,16 +30,16 @@ class SportDeutschlandIE(InfoExtractor):
         },
     }, {
         # Single-part video, embedded player link
-        'url': 'https://player.sportdeutschland.tv/9e9619c4-7d77-43c4-926d-49fb57dc06dc',
+        'url': 'https://player.sporteurope.tv/9e9619c4-7d77-43c4-926d-49fb57dc06dc',
         'info_dict': {
             'id': '9f27a97d-1544-4d0b-aa03-48d92d17a03a',
             'ext': 'mp4',
             'title': 'GFL2: Rostock Griffins vs. Elmshorn Fighting Pirates',
             'display_id': '9e9619c4-7d77-43c4-926d-49fb57dc06dc',
             'channel': 'Rostock Griffins',
-            'channel_url': 'https://sportdeutschland.tv/rostock-griffins',
+            'channel_url': 'https://sporteurope.tv/rostock-griffins',
             'live_status': 'was_live',
-            'description': 'md5:60cb00067e55dafa27b0933a43d72862',
+            'description': r're:Video-Livestream des Spiels Rostock Griffins vs\. Elmshorn Fighting Pirates.+',
             'channel_id': '9635f21c-3f67-4584-9ce4-796e9a47276b',
             'timestamp': 1749913117,
             'upload_date': '20250614',
@@ -47,7 +48,7 @@ class SportDeutschlandIE(InfoExtractor):
         'params': {'skip_download': True},
     }, {
         # Multi-part video
-        'url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
+        'url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
         'info_dict': {
             'id': '9f63d737-2444-4e3a-a1ea-840df73fd481',
             'display_id': 'rhine-ruhr-2025-fisu-world-university-games/volleyball-w-japan-vs-brasilien-halbfinale-2',
@@ -55,7 +56,7 @@ class SportDeutschlandIE(InfoExtractor):
             'description': 'md5:0a17da15e48a687e6019639c3452572b',
             'channel': 'Rhine-Ruhr 2025 FISU World University Games',
             'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
-            'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
+            'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
             'live_status': 'was_live',
         },
         'playlist_count': 2,
@@ -66,7 +67,7 @@ class SportDeutschlandIE(InfoExtractor):
                 'title': 'Volleyball w: Japan vs. Braslien - Halbfinale 2 Part 1',
                 'channel': 'Rhine-Ruhr 2025 FISU World University Games',
                 'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
-                'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
+                'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
                 'duration': 14773.0,
                 'timestamp': 1753085197,
                 'upload_date': '20250721',
@@ -79,16 +80,17 @@ class SportDeutschlandIE(InfoExtractor):
                 'title': 'Volleyball w: Japan vs. Braslien - Halbfinale 2 Part 2',
                 'channel': 'Rhine-Ruhr 2025 FISU World University Games',
                 'channel_id': '9f5216be-a49d-470b-9a30-4fe9df993334',
-                'channel_url': 'https://sportdeutschland.tv/rhine-ruhr-2025-fisu-world-university-games',
+                'channel_url': 'https://sporteurope.tv/rhine-ruhr-2025-fisu-world-university-games',
                 'duration': 14773.0,
                 'timestamp': 1753128421,
                 'upload_date': '20250721',
                 'live_status': 'was_live',
             },
         }],
+        'skip': '404 Not Found',
     }, {
         # Livestream
-        'url': 'https://sportdeutschland.tv/dtb/gymnastik-international-tag-1',
+        'url': 'https://sporteurope.tv/dtb/gymnastik-international-tag-1',
         'info_dict': {
             'id': '95d71b8a-370a-4b87-ad16-94680da18528',
             'ext': 'mp4',
@@ -96,7 +98,7 @@ class SportDeutschlandIE(InfoExtractor):
             'display_id': 'dtb/gymnastik-international-tag-1',
             'channel_id': '936ecef1-2f4a-4e08-be2f-68073cb7ecab',
             'channel': 'Deutscher Turner-Bund',
-            'channel_url': 'https://sportdeutschland.tv/dtb',
+            'channel_url': 'https://sporteurope.tv/dtb',
             'description': 'md5:07a885dde5838a6f0796ee21dc3b0c52',
             'live_status': 'is_live',
         },
@@ -106,9 +108,9 @@ class SportDeutschlandIE(InfoExtractor):
     def _process_video(self, asset_id, video):
         is_live = video['type'] == 'mux_live'
         token = self._download_json(
-            f'https://api.sportdeutschland.tv/api/web/personal/asset-token/{asset_id}',
+            f'https://api.sporteurope.tv/api/web/personal/asset-token/{asset_id}',
             video['id'], query={'type': video['type'], 'playback_id': video['src']},
-            headers={'Referer': 'https://sportdeutschland.tv/'})['token']
+            headers={'Referer': 'https://sporteurope.tv/'})['token']
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
             f'https://stream.mux.com/{video["src"]}.m3u8?token={token}', video['id'], live=is_live)
 
@@ -126,7 +128,7 @@ class SportDeutschlandIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         meta = self._download_json(
-            f'https://api.sportdeutschland.tv/api/stateless/frontend/assets/{display_id}',
+            f'https://api.sporteurope.tv/api/stateless/frontend/assets/{display_id}',
             display_id, query={'access_token': 'true'})
 
         info = {
@@ -139,7 +141,7 @@ class SportDeutschlandIE(InfoExtractor):
                 'channel_id': ('profile', 'id'),
                 'is_live': 'currently_live',
                 'was_live': 'was_live',
-                'channel_url': ('profile', 'slug', {lambda x: f'https://sportdeutschland.tv/{x}'}),
+                'channel_url': ('profile', 'slug', {lambda x: f'https://sporteurope.tv/{x}'}),
             }, get_all=False),
         }
 

yt_dlp/extractor/xhamster.py

@@ -1,8 +1,6 @@
-import base64
-import codecs
 import itertools
 import re
-import string
+import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
@@ -16,7 +14,6 @@ from ..utils import (
     join_nonempty,
     parse_duration,
     str_or_none,
-    try_call,
     try_get,
     unified_strdate,
     url_or_none,
@@ -32,7 +29,7 @@ class _ByteGenerator:
         try:
             self._algorithm = getattr(self, f'_algo{algo_id}')
         except AttributeError:
-            raise ExtractorError(f'Unknown algorithm ID: {algo_id}')
+            raise ExtractorError(f'Unknown algorithm ID "{algo_id}"')
         self._s = to_signed_32(seed)
 
     def _algo1(self, s):
@@ -216,32 +213,28 @@ class XHamsterIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _XOR_KEY = b'xh7999'
-
     def _decipher_format_url(self, format_url, format_id):
-        if all(char in string.hexdigits for char in format_url):
+        parsed_url = urllib.parse.urlparse(format_url)
-            byte_data = bytes.fromhex(format_url)
-            seed = int.from_bytes(byte_data[1:5], byteorder='little', signed=True)
-            byte_gen = _ByteGenerator(byte_data[0], seed)
-            return bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
 
-        cipher_type, _, ciphertext = try_call(
+        hex_string, path_remainder = self._search_regex(
-            lambda: base64.b64decode(format_url).decode().partition('_')) or [None] * 3
+            r'^/(?P<hex>[0-9a-fA-F]{12,})(?P<rem>[/,].+)$', parsed_url.path, 'url components',
-
+            default=(None, None), group=('hex', 'rem'))
-        if not cipher_type or not ciphertext:
+        if not hex_string:
-            self.report_warning(f'Skipping format "{format_id}": failed to decipher URL')
+            self.report_warning(f'Skipping format "{format_id}": unsupported URL format')
             return None
 
-        if cipher_type == 'xor':
+        byte_data = bytes.fromhex(hex_string)
-            return bytes(
+        seed = int.from_bytes(byte_data[1:5], byteorder='little', signed=True)
-                a ^ b for a, b in
-                zip(ciphertext.encode(), itertools.cycle(self._XOR_KEY))).decode()
 
-        if cipher_type == 'rot13':
+        try:
-            return codecs.decode(ciphertext, cipher_type)
+            byte_gen = _ByteGenerator(byte_data[0], seed)
+        except ExtractorError as e:
+            self.report_warning(f'Skipping format "{format_id}": {e.msg}')
+            return None
 
-        self.report_warning(f'Skipping format "{format_id}": unsupported cipher type "{cipher_type}"')
+        deciphered = bytearray(byte ^ next(byte_gen) for byte in byte_data[5:]).decode('latin-1')
-        return None
+
+        return parsed_url._replace(path=f'/{deciphered}{path_remainder}').geturl()
 
     def _fixup_formats(self, formats):
         for f in formats:
@@ -364,8 +357,11 @@ class XHamsterIE(InfoExtractor):
                                     'height': get_height(quality),
                                     'filesize': format_sizes.get(quality),
                                     'http_headers': {
-                                        'Referer': standard_url,
+                                        'Referer': urlh.url,
                                     },
+                                    # HTTP formats return "Wrong key" error even when deciphered by site JS
+                                    # TODO: Remove this when resolved on the site's end
+                                    '__needs_testing': True,
                                 })
 
             categories_list = video.get('categories')
@@ -402,7 +398,8 @@ class XHamsterIE(InfoExtractor):
                 'age_limit': age_limit if age_limit is not None else 18,
                 'categories': categories,
                 'formats': self._fixup_formats(formats),
-                '_format_sort_fields': ('res', 'proto', 'tbr'),
+                # TODO: Revert to ('res', 'proto', 'tbr') when HTTP formats problem is resolved
+                '_format_sort_fields': ('res', 'proto:m3u8', 'tbr'),
             }
 
         # Old layout fallback