Merge branch 'yt-dlp:master' into pr/6498

yt_dlp/extractor/_extractors.py

@@ -15,7 +15,6 @@ from .youtube import (  # Youtube is moved to the top to improve performance
     YoutubeSearchURLIE,
     YoutubeMusicSearchURLIE,
     YoutubeSubscriptionsIE,
-    YoutubeStoriesIE,
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeYtBeIE,
@@ -215,6 +214,7 @@ from .bild import BildIE
 from .bilibili import (
     BiliBiliIE,
     BiliBiliBangumiIE,
+    BiliBiliBangumiSeasonIE,
     BiliBiliBangumiMediaIE,
     BiliBiliSearchIE,
     BilibiliCategoryIE,
@@ -1012,6 +1012,7 @@ from .lynda import (
     LyndaCourseIE
 )
 from .m6 import M6IE
+from .magellantv import MagellanTVIE
 from .magentamusik360 import MagentaMusik360IE
 from .mailru import (
     MailRuIE,
@@ -1141,6 +1142,7 @@ from .mtv import (
 )
 from .muenchentv import MuenchenTVIE
 from .murrtube import MurrtubeIE, MurrtubeUserIE
+from .museai import MuseAIIE
 from .musescore import MuseScoreIE
 from .musicdex import (
     MusicdexSongIE,
@@ -1531,6 +1533,7 @@ from .prx import (
 )
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
+from .qdance import QDanceIE
 from .qingting import QingTingIE
 from .qqmusic import (
     QQMusicIE,
@@ -1854,6 +1857,10 @@ from .srgssr import (
     SRGSSRPlayIE,
 )
 from .srmediathek import SRMediathekIE
+from .stacommu import (
+    StacommuLiveIE,
+    StacommuVODIE,
+)
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
 from .steam import (
@@ -1866,7 +1873,6 @@ from .storyfire import (
     StoryFireSeriesIE,
 )
 from .streamable import StreamableIE
-from .streamanity import StreamanityIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streamff import StreamFFIE
@@ -2267,6 +2273,8 @@ from .vk import (
     VKIE,
     VKUserVideosIE,
     VKWallPostIE,
+    VKPlayIE,
+    VKPlayLiveIE,
 )
 from .vocaroo import VocarooIE
 from .vodlocker import VodlockerIE

yt_dlp/extractor/abc.py

@@ -12,6 +12,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     str_or_none,
+    traverse_obj,
     try_get,
     unescapeHTML,
     update_url_query,
@@ -85,6 +86,15 @@ class ABCIE(InfoExtractor):
             'uploader': 'Behind the News',
             'uploader_id': 'behindthenews',
         }
+    }, {
+        'url': 'https://www.abc.net.au/news/2023-06-25/wagner-boss-orders-troops-back-to-bases-to-avoid-bloodshed/102520540',
+        'info_dict': {
+            'id': '102520540',
+            'title': 'Wagner Group retreating from Russia, leader Prigozhin to move to Belarus',
+            'ext': 'mp4',
+            'description': 'Wagner troops leave Rostov-on-Don and\xa0Yevgeny Prigozhin will move to Belarus under a deal brokered by Belarusian President Alexander Lukashenko to end the mutiny.',
+            'thumbnail': 'https://live-production.wcms.abc-cdn.net.au/0c170f5b57f0105c432f366c0e8e267b?impolicy=wcms_crop_resize&cropH=2813&cropW=5000&xPos=0&yPos=249&width=862&height=485',
+        }
     }]
 
     def _real_extract(self, url):
@@ -107,7 +117,7 @@ class ABCIE(InfoExtractor):
                 video = True
 
         if mobj is None:
-            mobj = re.search(r'(?P<type>)"sources": (?P<json_data>\[[^\]]+\]),', webpage)
+            mobj = re.search(r'(?P<type>)"(?:sources|files|renditions)":\s*(?P<json_data>\[[^\]]+\])', webpage)
             if mobj is None:
                 mobj = re.search(
                     r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
@@ -121,7 +131,8 @@ class ABCIE(InfoExtractor):
             urls_info = self._parse_json(
                 mobj.group('json_data'), video_id, transform_source=js_to_json)
             youtube = mobj.group('type') == 'YouTube'
-            video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4'
+            video = mobj.group('type') == 'Video' or traverse_obj(
+                urls_info, (0, ('contentType', 'MIMEType')), get_all=False) == 'video/mp4'
 
         if not isinstance(urls_info, list):
             urls_info = [urls_info]

yt_dlp/extractor/abematv.py

@@ -22,80 +22,23 @@ from ..utils import (
     int_or_none,
     intlist_to_bytes,
     OnDemandPagedList,
-    request_to_url,
     time_seconds,
     traverse_obj,
     update_url_query,
 )
 
-# NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862)
 
-
-def add_opener(ydl, handler):
-    ''' Add a handler for opening URLs, like _download_webpage '''
+def add_opener(ydl, handler):  # FIXME: Create proper API in .networking
+    """Add a handler for opening URLs, like _download_webpage"""
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
-    ydl._opener.add_handler(handler)
-
-
-def remove_opener(ydl, handler):
-    '''
-    Remove handler(s) for opening URLs
-    @param handler Either handler object itself or handler type.
-    Specifying handler type will remove all handler which isinstance returns True.
-    '''
-    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
-    # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
-    opener = ydl._opener
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
-    if isinstance(handler, (type, tuple)):
-        find_cp = lambda x: isinstance(x, handler)
-    else:
-        find_cp = lambda x: x is handler
-
-    removed = []
-    for meth in dir(handler):
-        if meth in ["redirect_request", "do_open", "proxy_open"]:
-            # oops, coincidental match
-            continue
-
-        i = meth.find("_")
-        protocol = meth[:i]
-        condition = meth[i + 1:]
-
-        if condition.startswith("error"):
-            j = condition.find("_") + i + 1
-            kind = meth[j + 1:]
-            try:
-                kind = int(kind)
-            except ValueError:
-                pass
-            lookup = opener.handle_error.get(protocol, {})
-            opener.handle_error[protocol] = lookup
-        elif condition == "open":
-            kind = protocol
-            lookup = opener.handle_open
-        elif condition == "response":
-            kind = protocol
-            lookup = opener.process_response
-        elif condition == "request":
-            kind = protocol
-            lookup = opener.process_request
-        else:
-            continue
-
-        handlers = lookup.setdefault(kind, [])
-        if handlers:
-            handlers[:] = [x for x in handlers if not find_cp(x)]
-
-        removed.append(x for x in handlers if find_cp(x))
-
-    if removed:
-        for x in opener.handlers:
-            if find_cp(x):
-                x.add_parent(None)
-        opener.handlers[:] = [x for x in opener.handlers if not find_cp(x)]
+    rh = ydl._request_director.handlers['Urllib']
+    if 'abematv-license' in rh._SUPPORTED_URL_SCHEMES:
+        return
+    opener = rh._get_instance(cookiejar=ydl.cookiejar, proxies=ydl.proxies)
+    assert isinstance(opener, urllib.request.OpenerDirector)
+    opener.add_handler(handler)
+    rh._SUPPORTED_URL_SCHEMES = (*rh._SUPPORTED_URL_SCHEMES, 'abematv-license')
 
 
 class AbemaLicenseHandler(urllib.request.BaseHandler):
@@ -137,11 +80,11 @@ class AbemaLicenseHandler(urllib.request.BaseHandler):
         return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey))
 
     def abematv_license_open(self, url):
-        url = request_to_url(url)
+        url = url.get_full_url() if isinstance(url, urllib.request.Request) else url
         ticket = urllib.parse.urlparse(url).netloc
         response_data = self._get_videokey_from_ticket(ticket)
         return urllib.response.addinfourl(io.BytesIO(response_data), headers={
-            'Content-Length': len(response_data),
+            'Content-Length': str(len(response_data)),
         }, url=url, code=200)
 
 
@@ -213,10 +156,7 @@ class AbemaTVBaseIE(InfoExtractor):
             })
         AbemaTVBaseIE._USERTOKEN = user_data['token']
 
-        # don't allow adding it 2 times or more, though it's guarded
-        remove_opener(self._downloader, AbemaLicenseHandler)
         add_opener(self._downloader, AbemaLicenseHandler(self))
-
         return self._USERTOKEN
 
     def _get_media_token(self, invalidate=False, to_show=True):

yt_dlp/extractor/adn.py

@@ -6,10 +6,8 @@ import random
 
 from .common import InfoExtractor
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import (
-    compat_HTTPError,
-    compat_b64decode,
-)
+from ..compat import compat_b64decode
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ass_subtitles_timecode,
     bytes_to_intlist,
@@ -142,9 +140,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                 self._HEADERS = {'authorization': 'Bearer ' + access_token}
         except ExtractorError as e:
             message = None
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 resp = self._parse_json(
-                    e.cause.read().decode(), None, fatal=False) or {}
+                    e.cause.response.read().decode(), None, fatal=False) or {}
                 message = resp.get('message') or resp.get('code')
             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 
@@ -195,14 +193,14 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                     })
                 break
             except ExtractorError as e:
-                if not isinstance(e.cause, compat_HTTPError):
+                if not isinstance(e.cause, HTTPError):
                     raise e
 
-                if e.cause.code == 401:
+                if e.cause.status == 401:
                     # This usually goes away with a different random pkcs1pad, so retry
                     continue
 
-                error = self._parse_json(e.cause.read(), video_id)
+                error = self._parse_json(e.cause.response.read(), video_id)
                 message = error.get('message')
                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
                     self.raise_geo_restricted(msg=message)

yt_dlp/extractor/adobepass.py

@@ -2,11 +2,11 @@ import getpass
 import json
 import re
 import time
-import urllib.error
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
+from ..networking.exceptions import HTTPError
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -1394,7 +1394,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
             form_page, urlh = form_page_res
             post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
             if not re.match(r'https?://', post_url):
-                post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
+                post_url = compat_urlparse.urljoin(urlh.url, post_url)
             form_data = self._hidden_inputs(form_page)
             form_data.update(data)
             return self._download_webpage_handle(
@@ -1473,7 +1473,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     elif 'automatically signed in with' in provider_redirect_page:
                         # Seems like comcast is rolling up new way of automatically signing customers
                         oauth_redirect_url = self._html_search_regex(
-                            r'continue:\s*"(https://oauth.xfinity.com/oauth/authorize\?.+)"', provider_redirect_page,
+                            r'continue:\s*"(https://oauth\.xfinity\.com/oauth/authorize\?.+)"', provider_redirect_page,
                             'oauth redirect (signed)')
                         # Just need to process the request. No useful data comes back
                         self._download_webpage(oauth_redirect_url, video_id, 'Confirming auto login')
@@ -1619,7 +1619,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     hidden_data['history'] = 1
 
                     provider_login_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending first bookend',
+                        urlh.url, video_id, 'Sending first bookend',
                         query=hidden_data)
 
                     provider_association_redirect, urlh = post_form(
@@ -1629,7 +1629,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                         })
 
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_association_redirect, url=urlh.geturl())
+                        provider_association_redirect, url=urlh.url)
 
                     last_bookend_page, urlh = self._download_webpage_handle(
                         provider_refresh_redirect_url, video_id,
@@ -1638,7 +1638,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     hidden_data['history'] = 3
 
                     mvpd_confirm_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending final bookend',
+                        urlh.url, video_id, 'Sending final bookend',
                         query=hidden_data)
 
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1652,7 +1652,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     hidden_data['history_val'] = 1
 
                     provider_login_redirect_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending First Bookend',
+                        urlh.url, video_id, 'Sending First Bookend',
                         query=hidden_data)
 
                     provider_login_redirect_page, urlh = provider_login_redirect_page_res
@@ -1680,7 +1680,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                         })
 
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_association_redirect, url=urlh.geturl())
+                        provider_association_redirect, url=urlh.url)
 
                     last_bookend_page, urlh = self._download_webpage_handle(
                         provider_refresh_redirect_url, video_id,
@@ -1690,7 +1690,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     hidden_data['history_val'] = 3
 
                     mvpd_confirm_page_res = self._download_webpage_handle(
-                        urlh.geturl(), video_id, 'Sending Final Bookend',
+                        urlh.url, video_id, 'Sending Final Bookend',
                         query=hidden_data)
 
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1699,7 +1699,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                     # based redirect that should be followed.
                     provider_redirect_page, urlh = provider_redirect_page_res
                     provider_refresh_redirect_url = extract_redirect_url(
-                        provider_redirect_page, url=urlh.geturl())
+                        provider_redirect_page, url=urlh.url)
                     if provider_refresh_redirect_url:
                         provider_redirect_page_res = self._download_webpage_handle(
                             provider_refresh_redirect_url, video_id,
@@ -1724,7 +1724,7 @@ class AdobePassIE(InfoExtractor):  # XXX: Conventionally, base classes should en
                             'requestor_id': requestor_id,
                         }), headers=mvpd_headers)
                 except ExtractorError as e:
-                    if not mso_id and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                    if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401:
                         raise_mvpd_required()
                     raise
                 if '<pendingLogout' in session:

yt_dlp/extractor/adultswim.py

@@ -170,8 +170,10 @@ class AdultSwimIE(TurnerBaseIE):
                         continue
                     ext = determine_ext(asset_url, mimetype2ext(asset.get('mime_type')))
                     if ext == 'm3u8':
-                        info['formats'].extend(self._extract_m3u8_formats(
-                            asset_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                        fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                            asset_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        info['formats'].extend(fmts)
+                        self._merge_subtitles(subs, target=info['subtitles'])
                     elif ext == 'f4m':
                         continue
                         # info['formats'].extend(self._extract_f4m_formats(

yt_dlp/extractor/ant1newsgr.py

@@ -1,8 +1,8 @@
 import urllib.parse
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     ExtractorError,
     determine_ext,
     scale_thumbnails_to_max_format_width,
@@ -121,7 +121,7 @@ class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
         canonical_url = self._request_webpage(
             HEADRequest(url), video_id,
             note='Resolve canonical player URL',
-            errnote='Could not resolve canonical player URL').geturl()
+            errnote='Could not resolve canonical player URL').url
         _, netloc, _, _, query, _ = urllib.parse.urlparse(canonical_url)
         cid = urllib.parse.parse_qs(query)['cid'][0]
 

yt_dlp/extractor/archiveorg.py

@@ -1,16 +1,16 @@
 import json
 import re
-import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor
 from .naver import NaverBaseIE
 from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
-from ..compat import compat_HTTPError, compat_urllib_parse_unquote
+from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
 from ..utils import (
     KNOWN_EXTENSIONS,
     ExtractorError,
-    HEADRequest,
     bug_reports_message,
     clean_html,
     dict_get,
@@ -899,7 +899,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
                     video_id, note='Fetching archived video file url', expected_status=True)
             except ExtractorError as e:
                 # HTTP Error 404 is expected if the video is not saved.
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     self.raise_no_formats(
                         'The requested video is not archived, indexed, or there is an issue with web.archive.org (try again later)', expected=True)
                 else:
@@ -926,7 +926,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
         info['thumbnails'] = self._extract_thumbnails(video_id)
 
         if urlh:
-            url = compat_urllib_parse_unquote(urlh.geturl())
+            url = compat_urllib_parse_unquote(urlh.url)
             video_file_url_qs = parse_qs(url)
             # Attempt to recover any ext & format info from playback url & response headers
             format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
@@ -1052,7 +1052,7 @@ class VLiveWebArchiveIE(InfoExtractor):
             try:
                 return self._download_webpage(f'https://web.archive.org/web/{timestamp}id_/{url}', video_id, **kwargs)
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                     raise ExtractorError('Page was not archived', expected=True)
                 retry.error = e
                 continue

yt_dlp/extractor/atresplayer.py

@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -34,8 +34,8 @@ class AtresPlayerIE(InfoExtractor):
     _API_BASE = 'https://api.atresplayer.com/'
 
     def _handle_error(self, e, code):
-        if isinstance(e.cause, compat_HTTPError) and e.cause.code == code:
-            error = self._parse_json(e.cause.read(), None)
+        if isinstance(e.cause, HTTPError) and e.cause.status == code:
+            error = self._parse_json(e.cause.response.read(), None)
             if error.get('error') == 'required_registered':
                 self.raise_login_required()
             raise ExtractorError(error['error_description'], expected=True)

yt_dlp/extractor/bbc.py

@@ -2,11 +2,11 @@ import functools
 import itertools
 import json
 import re
-import urllib.error
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import compat_str, compat_urlparse
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -277,7 +277,7 @@ class BBCCoUkIE(InfoExtractor):
             post_url, None, 'Logging in', data=urlencode_postdata(login_form),
             headers={'Referer': self._LOGIN_URL})
 
-        if self._LOGIN_URL in urlh.geturl():
+        if self._LOGIN_URL in urlh.url:
             error = clean_html(get_element_by_class('form-message', response))
             if error:
                 raise ExtractorError(
@@ -388,8 +388,8 @@ class BBCCoUkIE(InfoExtractor):
                                 href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                         except ExtractorError as e:
-                            if not (isinstance(e.exc_info[1], urllib.error.HTTPError)
-                                    and e.exc_info[1].code in (403, 404)):
+                            if not (isinstance(e.exc_info[1], HTTPError)
+                                    and e.exc_info[1].status in (403, 404)):
                                 raise
                             fmts = []
                         formats.extend(fmts)
@@ -472,7 +472,7 @@ class BBCCoUkIE(InfoExtractor):
 
             return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
-            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
+            if not (isinstance(ee.cause, HTTPError) and ee.cause.status == 404):
                 raise
 
         # fallback to legacy playlist
@@ -983,7 +983,7 @@ class BBCIE(BBCCoUkIE):  # XXX: Do not subclass from concrete IE
                                     # Some playlist URL may fail with 500, at the same time
                                     # the other one may work fine (e.g.
                                     # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
-                                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
+                                    if isinstance(e.cause, HTTPError) and e.cause.status == 500:
                                         continue
                                     raise
                             if entry:

yt_dlp/extractor/bilibili.py

@@ -4,11 +4,11 @@ import hashlib
 import itertools
 import math
 import time
-import urllib.error
 import urllib.parse
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..dependencies import Cryptodome
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     GeoRestrictedError,
@@ -18,6 +18,7 @@ from ..utils import (
     float_or_none,
     format_field,
     int_or_none,
+    join_nonempty,
     make_archive_id,
     merge_dicts,
     mimetype2ext,
@@ -135,6 +136,17 @@ class BilibiliBaseIE(InfoExtractor):
         for children in map(self._get_all_children, traverse_obj(reply, ('replies', ...))):
             yield from children
 
+    def _get_episodes_from_season(self, ss_id, url):
+        season_info = self._download_json(
+            'https://api.bilibili.com/pgc/web/season/section', ss_id,
+            note='Downloading season info', query={'season_id': ss_id},
+            headers={'Referer': url, **self.geo_verification_headers()})
+
+        for entry in traverse_obj(season_info, (
+                'result', 'main_section', 'episodes',
+                lambda _, v: url_or_none(v['share_url']) and v['id'])):
+            yield self.url_result(entry['share_url'], BiliBiliBangumiIE, f'ep{entry["id"]}')
+
 
 class BiliBiliIE(BilibiliBaseIE):
     _VALID_URL = r'https?://www\.bilibili\.com/(?:video/|festival/\w+\?(?:[^#]*&)?bvid=)[aAbB][vV](?P<id>[^/?#&]+)'
@@ -403,76 +415,93 @@ class BiliBiliIE(BilibiliBaseIE):
 
 
 class BiliBiliBangumiIE(BilibiliBaseIE):
-    _VALID_URL = r'(?x)https?://www\.bilibili\.com/bangumi/play/(?P<id>(?:ss|ep)\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bilibili\.com/bangumi/play/(?P<id>ep\d+)'
 
     _TESTS = [{
-        'url': 'https://www.bilibili.com/bangumi/play/ss897',
+        'url': 'https://www.bilibili.com/bangumi/play/ep267851',
         'info_dict': {
-            'id': 'ss897',
+            'id': '267851',
             'ext': 'mp4',
-            'series': '神的记事本',
-            'season': '神的记事本',
-            'season_id': 897,
+            'series': '鬼灭之刃',
+            'series_id': '4358',
+            'season': '鬼灭之刃',
+            'season_id': '26801',
             'season_number': 1,
-            'episode': '你与旅行包',
-            'episode_number': 2,
-            'title': '神的记事本：第2话 你与旅行包',
-            'duration': 1428.487,
-            'timestamp': 1310809380,
-            'upload_date': '20110716',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'episode': '残酷',
+            'episode_id': '267851',
+            'episode_number': 1,
+            'title': '1 残酷',
+            'duration': 1425.256,
+            'timestamp': 1554566400,
+            'upload_date': '20190406',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$'
         },
-    }, {
-        'url': 'https://www.bilibili.com/bangumi/play/ep508406',
-        'only_matching': True,
+        'skip': 'According to the copyright owner\'s request, you may only watch the video after you are premium member.'
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        episode_id = video_id[2:]
         webpage = self._download_webpage(url, video_id)
 
         if '您所在的地区无法观看本片' in webpage:
             raise GeoRestrictedError('This video is restricted')
-        elif ('开通大会员观看' in webpage and '__playinfo__' not in webpage
-                or '正在观看预览，大会员免费看全片' in webpage):
+        elif '正在观看预览，大会员免费看全片' in webpage:
             self.raise_login_required('This video is for premium members only')
 
-        play_info = self._search_json(r'window\.__playinfo__\s*=', webpage, 'play info', video_id)['data']
+        headers = {'Referer': url, **self.geo_verification_headers()}
+        play_info = self._download_json(
+            'https://api.bilibili.com/pgc/player/web/v2/playurl', video_id,
+            'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
+            headers=headers)
+        premium_only = play_info.get('code') == -10403
+        play_info = traverse_obj(play_info, ('result', 'video_info', {dict})) or {}
+
         formats = self.extract_formats(play_info)
-        if (not formats and '成为大会员抢先看' in webpage
-                and play_info.get('durl') and not play_info.get('dash')):
+        if not formats and (premium_only or '成为大会员抢先看' in webpage or '开通大会员观看' in webpage):
             self.raise_login_required('This video is for premium members only')
 
-        initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', video_id)
+        bangumi_info = self._download_json(
+            'https://api.bilibili.com/pgc/view/web/season', video_id, 'Get episode details',
+            query={'ep_id': episode_id}, headers=headers)['result']
 
-        season_id = traverse_obj(initial_state, ('mediaInfo', 'season_id'))
+        episode_number, episode_info = next((
+            (idx, ep) for idx, ep in enumerate(traverse_obj(
+                bangumi_info, ('episodes', ..., {dict})), 1)
+            if str_or_none(ep.get('id')) == episode_id), (1, {}))
+
+        season_id = bangumi_info.get('season_id')
         season_number = season_id and next((
             idx + 1 for idx, e in enumerate(
-                traverse_obj(initial_state, ('mediaInfo', 'seasons', ...)))
+                traverse_obj(bangumi_info, ('seasons', ...)))
             if e.get('season_id') == season_id
         ), None)
 
+        aid = episode_info.get('aid')
+
         return {
             'id': video_id,
             'formats': formats,
-            'title': traverse_obj(initial_state, 'h1Title'),
-            'episode': traverse_obj(initial_state, ('epInfo', 'long_title')),
-            'episode_number': int_or_none(traverse_obj(initial_state, ('epInfo', 'title'))),
-            'series': traverse_obj(initial_state, ('mediaInfo', 'series')),
-            'season': traverse_obj(initial_state, ('mediaInfo', 'season_title')),
-            'season_id': season_id,
+            **traverse_obj(bangumi_info, {
+                'series': ('series', 'series_title', {str}),
+                'series_id': ('series', 'series_id', {str_or_none}),
+                'thumbnail': ('square_cover', {url_or_none}),
+            }),
+            'title': join_nonempty('title', 'long_title', delim=' ', from_dict=episode_info),
+            'episode': episode_info.get('long_title'),
+            'episode_id': episode_id,
+            'episode_number': int_or_none(episode_info.get('title')) or episode_number,
+            'season_id': str_or_none(season_id),
             'season_number': season_number,
-            'thumbnail': traverse_obj(initial_state, ('epInfo', 'cover')),
-            'timestamp': traverse_obj(initial_state, ('epInfo', 'pub_time')),
+            'timestamp': int_or_none(episode_info.get('pub_time')),
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
-            'subtitles': self.extract_subtitles(
-                video_id, initial_state, traverse_obj(initial_state, ('epInfo', 'cid'))),
-            '__post_extractor': self.extract_comments(traverse_obj(initial_state, ('epInfo', 'aid'))),
-            'http_headers': {'Referer': url, **self.geo_verification_headers()},
+            'subtitles': self.extract_subtitles(video_id, aid, episode_info.get('cid')),
+            '__post_extractor': self.extract_comments(aid),
+            'http_headers': headers,
         }
 
 
-class BiliBiliBangumiMediaIE(InfoExtractor):
+class BiliBiliBangumiMediaIE(BilibiliBaseIE):
     _VALID_URL = r'https?://www\.bilibili\.com/bangumi/media/md(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.bilibili.com/bangumi/media/md24097891',
@@ -485,16 +514,26 @@ class BiliBiliBangumiMediaIE(InfoExtractor):
     def _real_extract(self, url):
         media_id = self._match_id(url)
         webpage = self._download_webpage(url, media_id)
+        ss_id = self._search_json(
+            r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)['mediaInfo']['season_id']
 
-        initial_state = self._search_json(r'window\.__INITIAL_STATE__\s*=', webpage, 'initial_state', media_id)
-        episode_list = self._download_json(
-            'https://api.bilibili.com/pgc/web/season/section', media_id,
-            query={'season_id': initial_state['mediaInfo']['season_id']},
-            note='Downloading season info')['result']['main_section']['episodes']
+        return self.playlist_result(self._get_episodes_from_season(ss_id, url), media_id)
 
-        return self.playlist_result((
-            self.url_result(entry['share_url'], BiliBiliBangumiIE, entry['aid'])
-            for entry in episode_list), media_id)
+
+class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
+    _VALID_URL = r'(?x)https?://www\.bilibili\.com/bangumi/play/ss(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.bilibili.com/bangumi/play/ss26801',
+        'info_dict': {
+            'id': '26801'
+        },
+        'playlist_mincount': 26
+    }]
+
+    def _real_extract(self, url):
+        ss_id = self._match_id(url)
+
+        return self.playlist_result(self._get_episodes_from_season(ss_id, url), ss_id)
 
 
 class BilibiliSpaceBaseIE(InfoExtractor):
@@ -575,7 +614,7 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
                 response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
                                                playlist_id, note=f'Downloading page {page_idx}', query=query)
             except ExtractorError as e:
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 412:
                     raise ExtractorError(
                         'Request is blocked by server (412), please add cookies, wait and try later.', expected=True)
                 raise

yt_dlp/extractor/bitchute.py

@@ -2,9 +2,9 @@ import functools
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     OnDemandPagedList,
     clean_html,
     get_element_by_class,

yt_dlp/extractor/bravotv.py

@@ -1,6 +1,6 @@
 from .adobepass import AdobePassIE
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     extract_attributes,
     float_or_none,
     get_element_html_by_class,
@@ -155,7 +155,7 @@ class BravoTVIE(AdobePassIE):
             chapters = None
 
         m3u8_url = self._request_webpage(HEADRequest(
-            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
+            update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
         if 'mpeg_cenc' in m3u8_url:
             self.report_drm(video_id)
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')

yt_dlp/extractor/brightcove.py

@@ -7,10 +7,10 @@ from .adobepass import AdobePassIE
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
-    compat_HTTPError,
     compat_parse_qs,
     compat_urlparse,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     dict_get,
@@ -915,8 +915,8 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
                 json_data = self._download_json(api_url, video_id, headers=headers)
                 break
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
-                    json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
+                    json_data = self._parse_json(e.cause.response.read().decode(), video_id)[0]
                     message = json_data.get('message') or json_data['error_code']
                     if json_data.get('error_subcode') == 'CLIENT_GEO':
                         self.raise_geo_restricted(msg=message)

yt_dlp/extractor/canalplus.py

@@ -64,7 +64,7 @@ class CanalplusIE(InfoExtractor):
         #     response = self._request_webpage(
         #         HEADRequest(fmt_url), video_id,
         #         'Checking if the video is georestricted')
-        #     if '/blocage' in response.geturl():
+        #     if '/blocage' in response.url:
         #         raise ExtractorError(
         #             'The video is not available in your country',
         #             expected=True)

yt_dlp/extractor/cbsnews.py

@@ -7,9 +7,9 @@ import zlib
 from .anvato import AnvatoIE
 from .common import InfoExtractor
 from .paramountplus import ParamountPlusIE
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     UserNotLive,
     determine_ext,
     float_or_none,

yt_dlp/extractor/ceskatelevize.py

@@ -1,20 +1,20 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlparse,
-)
+from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     float_or_none,
-    sanitized_Request,
     str_or_none,
     traverse_obj,
     urlencode_postdata,
-    USER_AGENTS,
 )
 
+USER_AGENTS = {
+    'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
+}
+
 
 class CeskaTelevizeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady|zive)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
@@ -97,7 +97,7 @@ class CeskaTelevizeIE(InfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage, urlh = self._download_webpage_handle(url, playlist_id)
-        parsed_url = compat_urllib_parse_urlparse(urlh.geturl())
+        parsed_url = compat_urllib_parse_urlparse(urlh.url)
         site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
         playlist_title = self._og_search_title(webpage, default=None)
         if site_name and playlist_title:
@@ -163,16 +163,16 @@ class CeskaTelevizeIE(InfoExtractor):
         entries = []
 
         for user_agent in (None, USER_AGENTS['Safari']):
-            req = sanitized_Request(
+            req = Request(
                 'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
-            req.add_header('Content-type', 'application/x-www-form-urlencoded')
-            req.add_header('x-addr', '127.0.0.1')
-            req.add_header('X-Requested-With', 'XMLHttpRequest')
+            req.headers['Content-type'] = 'application/x-www-form-urlencoded'
+            req.headers['x-addr'] = '127.0.0.1'
+            req.headers['X-Requested-With'] = 'XMLHttpRequest'
             if user_agent:
-                req.add_header('User-Agent', user_agent)
-            req.add_header('Referer', url)
+                req.headers['User-Agent'] = user_agent
+            req.headers['Referer'] = url
 
             playlistpage = self._download_json(req, playlist_id, fatal=False)
 
@@ -183,8 +183,8 @@ class CeskaTelevizeIE(InfoExtractor):
             if playlist_url == 'error_region':
                 raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-            req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
-            req.add_header('Referer', url)
+            req = Request(compat_urllib_parse_unquote(playlist_url))
+            req.headers['Referer'] = url
 
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:

yt_dlp/extractor/cinetecamilano.py

@@ -1,6 +1,6 @@
 import json
-import urllib.error
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -40,7 +40,7 @@ class CinetecaMilanoIE(InfoExtractor):
                     'Authorization': try_get(self._get_cookies('https://www.cinetecamilano.it'), lambda x: f'Bearer {x["cnt-token"].value}') or ''
                 })
         except ExtractorError as e:
-            if ((isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 500)
+            if ((isinstance(e.cause, HTTPError) and e.cause.status == 500)
                     or isinstance(e.cause, json.JSONDecodeError)):
                 self.raise_login_required(method='cookies')
             raise

yt_dlp/extractor/ciscowebex.py

@@ -33,7 +33,7 @@ class CiscoWebexIE(InfoExtractor):
         if rcid:
             webpage = self._download_webpage(url, None, note='Getting video ID')
             url = self._search_regex(self._VALID_URL, webpage, 'redirection url', group='url')
-        url = self._request_webpage(url, None, note='Resolving final URL').geturl()
+        url = self._request_webpage(url, None, note='Resolving final URL').url
         mobj = self._match_valid_url(url)
         subdomain = mobj.group('subdomain')
         siteurl = mobj.group('siteurl_1') or mobj.group('siteurl_2')
@@ -49,7 +49,7 @@ class CiscoWebexIE(InfoExtractor):
             'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
             video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
 
-        if urlh.getcode() == 403:
+        if urlh.status == 403:
             if stream['code'] == 53004:
                 self.raise_login_required()
             if stream['code'] == 53005:
@@ -59,7 +59,7 @@ class CiscoWebexIE(InfoExtractor):
                     'This video is protected by a password, use the --video-password option', expected=True)
             raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
 
-        if urlh.getcode() == 429:
+        if urlh.status == 429:
             self.raise_login_required(
                 f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
                 method='cookies')

yt_dlp/extractor/common.py

@@ -17,15 +17,26 @@ import subprocess
 import sys
 import time
 import types
-import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
 
 from ..compat import functools  # isort: split
-from ..compat import compat_etree_fromstring, compat_expanduser, compat_os_name
+from ..compat import (
+    compat_etree_fromstring,
+    compat_expanduser,
+    compat_os_name,
+    urllib_req_to_req,
+)
 from ..cookies import LenientSimpleCookie
 from ..downloader.f4m import get_base_url, remove_encrypted_media
+from ..downloader.hls import HlsFD
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import (
+    HTTPError,
+    IncompleteRead,
+    network_exceptions,
+)
 from ..utils import (
     IDENTITY,
     JSON_LD_RE,
@@ -34,7 +45,6 @@ from ..utils import (
     FormatSorter,
     GeoRestrictedError,
     GeoUtils,
-    HEADRequest,
     LenientJSONDecoder,
     Popen,
     RegexNotFoundError,
@@ -60,7 +70,6 @@ from ..utils import (
     js_to_json,
     mimetype2ext,
     netrc_from_content,
-    network_exceptions,
     orderedSet,
     parse_bitrate,
     parse_codecs,
@@ -70,7 +79,6 @@ from ..utils import (
     parse_resolution,
     sanitize_filename,
     sanitize_url,
-    sanitized_Request,
     smuggle_url,
     str_or_none,
     str_to_int,
@@ -82,8 +90,6 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
-    update_Request,
-    update_url_query,
     url_basename,
     url_or_none,
     urlhandle_detect_ext,
@@ -224,7 +230,8 @@ class InfoExtractor:
                                  width : height ratio as float.
                     * no_resume  The server does not support resuming the
                                  (HTTP or RTMP) download. Boolean.
-                    * has_drm    The format has DRM and cannot be downloaded. Boolean
+                    * has_drm    True if the format has DRM and cannot be downloaded.
+                                 'maybe' if the format may have DRM and has to be tested before download.
                     * extra_param_to_segment_url  A query string to append to each
                                  fragment's URL, or to update each existing query string
                                  with. Only applied by the native HLS/DASH downloaders.
@@ -726,7 +733,7 @@ class InfoExtractor:
             e.ie = e.ie or self.IE_NAME,
             e.traceback = e.traceback or sys.exc_info()[2]
             raise
-        except http.client.IncompleteRead as e:
+        except IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occurred.', cause=e, video_id=self.get_temp_id(url))
@@ -785,20 +792,25 @@ class InfoExtractor:
 
     @staticmethod
     def __can_accept_status_code(err, expected_status):
-        assert isinstance(err, urllib.error.HTTPError)
+        assert isinstance(err, HTTPError)
         if expected_status is None:
             return False
         elif callable(expected_status):
-            return expected_status(err.code) is True
+            return expected_status(err.status) is True
         else:
-            return err.code in variadic(expected_status)
+            return err.status in variadic(expected_status)
 
     def _create_request(self, url_or_request, data=None, headers=None, query=None):
         if isinstance(url_or_request, urllib.request.Request):
-            return update_Request(url_or_request, data=data, headers=headers, query=query)
-        if query:
-            url_or_request = update_url_query(url_or_request, query)
-        return sanitized_Request(url_or_request, data, headers or {})
+            self._downloader.deprecation_warning(
+                'Passing a urllib.request.Request to _create_request() is deprecated. '
+                'Use yt_dlp.networking.common.Request instead.')
+            url_or_request = urllib_req_to_req(url_or_request)
+        elif not isinstance(url_or_request, Request):
+            url_or_request = Request(url_or_request)
+
+        url_or_request.update(data=data, headers=headers, query=query)
+        return url_or_request
 
     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None, expected_status=None):
         """
@@ -834,14 +846,9 @@ class InfoExtractor:
         try:
             return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query))
         except network_exceptions as err:
-            if isinstance(err, urllib.error.HTTPError):
+            if isinstance(err, HTTPError):
                 if self.__can_accept_status_code(err, expected_status):
-                    # Retain reference to error to prevent file object from
-                    # being closed before it can be read. Works around the
-                    # effects of <https://bugs.python.org/issue15002>
-                    # introduced in Python 3.4.1.
-                    err.fp._error = err
-                    return err.fp
+                    return err.response
 
             if errnote is False:
                 return False
@@ -973,11 +980,11 @@ class InfoExtractor:
         if prefix is not None:
             webpage_bytes = prefix + webpage_bytes
         if self.get_param('dump_intermediate_pages', False):
-            self.to_screen('Dumping request to ' + urlh.geturl())
+            self.to_screen('Dumping request to ' + urlh.url)
             dump = base64.b64encode(webpage_bytes).decode('ascii')
             self._downloader.to_screen(dump)
         if self.get_param('write_pages'):
-            filename = self._request_dump_filename(urlh.geturl(), video_id)
+            filename = self._request_dump_filename(urlh.url, video_id)
             self.to_screen(f'Saving request to {filename}')
             with open(filename, 'wb') as outf:
                 outf.write(webpage_bytes)
@@ -1035,7 +1042,7 @@ class InfoExtractor:
                              fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
             if self.get_param('load_pages'):
                 url_or_request = self._create_request(url_or_request, data, headers, query)
-                filename = self._request_dump_filename(url_or_request.full_url, video_id)
+                filename = self._request_dump_filename(url_or_request.url, video_id)
                 self.to_screen(f'Loading request from {filename}')
                 try:
                     with open(filename, 'rb') as dumpf:
@@ -1109,7 +1116,7 @@ class InfoExtractor:
         while True:
             try:
                 return self.__download_webpage(url_or_request, video_id, note, errnote, None, fatal, *args, **kwargs)
-            except http.client.IncompleteRead as e:
+            except IncompleteRead as e:
                 try_count += 1
                 if try_count >= tries:
                     raise e
@@ -1806,7 +1813,7 @@ class InfoExtractor:
             return []
 
         manifest, urlh = res
-        manifest_url = urlh.geturl()
+        manifest_url = urlh.url
 
         return self._parse_f4m_formats(
             manifest, manifest_url, video_id, preference=preference, quality=quality, f4m_id=f4m_id,
@@ -1965,7 +1972,7 @@ class InfoExtractor:
             return [], {}
 
         m3u8_doc, urlh = res
-        m3u8_url = urlh.geturl()
+        m3u8_url = urlh.url
 
         return self._parse_m3u8_formats_and_subtitles(
             m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
@@ -1979,11 +1986,7 @@ class InfoExtractor:
             errnote=None, fatal=True, data=None, headers={}, query={},
             video_id=None):
         formats, subtitles = [], {}
-
-        has_drm = re.search('|'.join([
-            r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
-            r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
-        ]), m3u8_doc)
+        has_drm = HlsFD._has_drm(m3u8_doc)
 
         def format_url(url):
             return url if re.match(r'^https?://', url) else urllib.parse.urljoin(m3u8_url, url)
@@ -2247,7 +2250,7 @@ class InfoExtractor:
             return [], {}
 
         smil, urlh = res
-        smil_url = urlh.geturl()
+        smil_url = urlh.url
 
         namespace = self._parse_smil_namespace(smil)
 
@@ -2270,7 +2273,7 @@ class InfoExtractor:
             return {}
 
         smil, urlh = res
-        smil_url = urlh.geturl()
+        smil_url = urlh.url
 
         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
 
@@ -2462,7 +2465,7 @@ class InfoExtractor:
             return []
 
         xspf, urlh = res
-        xspf_url = urlh.geturl()
+        xspf_url = urlh.url
 
         return self._parse_xspf(
             xspf, playlist_id, xspf_url=xspf_url,
@@ -2533,7 +2536,7 @@ class InfoExtractor:
             return [], {}
 
         # We could have been redirected to a new url when we retrieved our mpd file.
-        mpd_url = urlh.geturl()
+        mpd_url = urlh.url
         mpd_base_url = base_url(mpd_url)
 
         return self._parse_mpd_formats_and_subtitles(
@@ -2919,7 +2922,7 @@ class InfoExtractor:
         if ism_doc is None:
             return [], {}
 
-        return self._parse_ism_formats_and_subtitles(ism_doc, urlh.geturl(), ism_id)
+        return self._parse_ism_formats_and_subtitles(ism_doc, urlh.url, ism_id)
 
     def _parse_ism_formats_and_subtitles(self, ism_doc, ism_url, ism_id=None):
         """

yt_dlp/extractor/crackle.py

@@ -4,7 +4,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -113,7 +113,7 @@ class CrackleIE(InfoExtractor):
                     errnote='Unable to download media JSON')
             except ExtractorError as e:
                 # 401 means geo restriction, trying next country
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     continue
                 raise
 

yt_dlp/extractor/crunchyroll.py

@@ -1,7 +1,7 @@
 import base64
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -27,11 +27,24 @@ class CrunchyrollBaseIE(InfoExtractor):
     _AUTH_HEADERS = None
     _API_ENDPOINT = None
     _BASIC_AUTH = None
-    _QUERY = {}
+    _CLIENT_ID = ('cr_web', 'noaihdevm_6iyg0a8l0q')
+    _LOCALE_LOOKUP = {
+        'ar': 'ar-SA',
+        'de': 'de-DE',
+        '': 'en-US',
+        'es': 'es-419',
+        'es-es': 'es-ES',
+        'fr': 'fr-FR',
+        'it': 'it-IT',
+        'pt-br': 'pt-BR',
+        'pt-pt': 'pt-PT',
+        'ru': 'ru-RU',
+        'hi': 'hi-IN',
+    }
 
     @property
     def is_logged_in(self):
-        return self._get_cookies(self._BASE_URL).get('etp_rt')
+        return bool(self._get_cookies(self._BASE_URL).get('etp_rt'))
 
     def _perform_login(self, username, password):
         if self.is_logged_in:
@@ -62,49 +75,49 @@ class CrunchyrollBaseIE(InfoExtractor):
         if not self.is_logged_in:
             raise ExtractorError('Login succeeded but did not set etp_rt cookie')
 
-    def _update_query(self, lang):
-        if lang in CrunchyrollBaseIE._QUERY:
-            return
-
-        webpage = self._download_webpage(
-            f'{self._BASE_URL}/{lang}', None, note=f'Retrieving main page (lang={lang or None})')
-
-        initial_state = self._search_json(r'__INITIAL_STATE__\s*=', webpage, 'initial state', None)
-        CrunchyrollBaseIE._QUERY[lang] = traverse_obj(initial_state, {
-            'locale': ('localization', 'locale'),
-        }) or None
-
-        if CrunchyrollBaseIE._BASIC_AUTH:
-            return
-
-        app_config = self._search_json(r'__APP_CONFIG__\s*=', webpage, 'app config', None)
-        cx_api_param = app_config['cxApiParams']['accountAuthClientId' if self.is_logged_in else 'anonClientId']
-        self.write_debug(f'Using cxApiParam={cx_api_param}')
-        CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
-
     def _update_auth(self):
         if CrunchyrollBaseIE._AUTH_HEADERS and CrunchyrollBaseIE._AUTH_REFRESH > time_seconds():
             return
 
-        assert CrunchyrollBaseIE._BASIC_AUTH, '_update_query needs to be called at least one time beforehand'
+        if not CrunchyrollBaseIE._BASIC_AUTH:
+            cx_api_param = self._CLIENT_ID[self.is_logged_in]
+            self.write_debug(f'Using cxApiParam={cx_api_param}')
+            CrunchyrollBaseIE._BASIC_AUTH = 'Basic ' + base64.b64encode(f'{cx_api_param}:'.encode()).decode()
+
         grant_type = 'etp_rt_cookie' if self.is_logged_in else 'client_id'
-        auth_response = self._download_json(
-            f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
-            headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
+        try:
+            auth_response = self._download_json(
+                f'{self._BASE_URL}/auth/v1/token', None, note=f'Authenticating with grant_type={grant_type}',
+                headers={'Authorization': CrunchyrollBaseIE._BASIC_AUTH}, data=f'grant_type={grant_type}'.encode())
+        except ExtractorError as error:
+            if isinstance(error.cause, HTTPError) and error.cause.status == 403:
+                raise ExtractorError(
+                    'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                    'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
+                    'and your browser\'s User-Agent (with --user-agent)', expected=True)
+            raise
 
         CrunchyrollBaseIE._AUTH_HEADERS = {'Authorization': auth_response['token_type'] + ' ' + auth_response['access_token']}
         CrunchyrollBaseIE._AUTH_REFRESH = time_seconds(seconds=traverse_obj(auth_response, ('expires_in', {float_or_none}), default=300) - 10)
 
+    def _locale_from_language(self, language):
+        config_locale = self._configuration_arg('metadata', ie_key=CrunchyrollBetaIE, casesense=True)
+        return config_locale[0] if config_locale else self._LOCALE_LOOKUP.get(language)
+
     def _call_base_api(self, endpoint, internal_id, lang, note=None, query={}):
-        self._update_query(lang)
         self._update_auth()
 
         if not endpoint.startswith('/'):
             endpoint = f'/{endpoint}'
 
+        query = query.copy()
+        locale = self._locale_from_language(lang)
+        if locale:
+            query['locale'] = locale
+
         return self._download_json(
             f'{self._BASE_URL}{endpoint}', internal_id, note or f'Calling API: {endpoint}',
-            headers=CrunchyrollBaseIE._AUTH_HEADERS, query={**CrunchyrollBaseIE._QUERY[lang], **query})
+            headers=CrunchyrollBaseIE._AUTH_HEADERS, query=query)
 
     def _call_api(self, path, internal_id, lang, note='api', query={}):
         if not path.startswith(f'/content/v2/{self._API_ENDPOINT}/'):
@@ -114,7 +127,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             result = self._call_base_api(
                 path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
         except ExtractorError as error:
-            if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
+            if isinstance(error.cause, HTTPError) and error.cause.status == 404:
                 return None
             raise
 
@@ -206,7 +219,7 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
     IE_NAME = 'crunchyroll'
     _VALID_URL = r'''(?x)
         https?://(?:beta\.|www\.)?crunchyroll\.com/
-        (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
+        (?:(?P<lang>\w{2}(?:-\w{2})?)/)?
         watch/(?!concert|musicvideo)(?P<id>\w+)'''
     _TESTS = [{
         # Premium only
@@ -304,7 +317,7 @@ class CrunchyrollBetaIE(CrunchyrollCmsBaseIE):
         },
         'playlist_mincount': 5,
     }, {
-        'url': 'https://www.crunchyroll.com/watch/GY2P1Q98Y',
+        'url': 'https://www.crunchyroll.com/de/watch/GY2P1Q98Y',
         'only_matching': True,
     }, {
         'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
@@ -490,8 +503,21 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?crunchyroll\.com/
         (?P<lang>(?:\w{2}(?:-\w{2})?/)?)
-        watch/(?P<type>concert|musicvideo)/(?P<id>\w{10})'''
+        watch/(?P<type>concert|musicvideo)/(?P<id>\w+)'''
     _TESTS = [{
+        'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'MV5B02C79',
+            'display_id': 'egaono-hana',
+            'title': 'Egaono Hana',
+            'track': 'Egaono Hana',
+            'artist': 'Goose house',
+            'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$',
+            'genre': ['J-Pop'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
         'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C',
         'info_dict': {
             'ext': 'mp4',
@@ -519,11 +545,14 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE):
         },
         'params': {'skip_download': 'm3u8'},
     }, {
-        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'url': 'https://www.crunchyroll.com/de/watch/musicvideo/MV5B02C79/egaono-hana',
         'only_matching': True,
     }, {
         'url': 'https://www.crunchyroll.com/watch/concert/MC2E2AC135/live-is-smile-always-364joker-at-yokohama-arena',
         'only_matching': True,
+    }, {
+        'url': 'https://www.crunchyroll.com/watch/musicvideo/MV88BB7F2C/crossing-field',
+        'only_matching': True,
     }]
     _API_ENDPOINT = 'music'
 

yt_dlp/extractor/cultureunplugged.py

@@ -1,10 +1,8 @@
 import time
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    HEADRequest,
-)
+from ..networking import HEADRequest
+from ..utils import int_or_none
 
 
 class CultureUnpluggedIE(InfoExtractor):

yt_dlp/extractor/dacast.py

@@ -1,9 +1,9 @@
 import hashlib
 import re
 import time
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     classproperty,
@@ -105,7 +105,7 @@ class DacastVODIE(DacastBaseIE):
                 formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
             except ExtractorError as e:
                 # CDN will randomly respond with 403
-                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                     retry.error = e
                     continue
                 raise

yt_dlp/extractor/dailymotion.py

@@ -3,7 +3,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -68,9 +68,9 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
                         None, 'Downloading Access Token',
                         data=urlencode_postdata(data))['access_token']
                 except ExtractorError as e:
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                    if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                         raise ExtractorError(self._parse_json(
-                            e.cause.read().decode(), xid)['error_description'], expected=True)
+                            e.cause.response.read().decode(), xid)['error_description'], expected=True)
                     raise
                 self._set_dailymotion_cookie('access_token' if username else 'client_token', token)
             self._HEADERS['Authorization'] = 'Bearer ' + token

yt_dlp/extractor/discovery.py

@@ -3,8 +3,8 @@ import string
 
 from .discoverygo import DiscoveryGoBaseIE
 from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError
-from ..compat import compat_HTTPError
 
 
 class DiscoveryIE(DiscoveryGoBaseIE):
@@ -100,9 +100,9 @@ class DiscoveryIE(DiscoveryGoBaseIE):
                 self._API_BASE_URL + 'streaming/video/' + video_id,
                 display_id, 'Downloading streaming JSON metadata', headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+            if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
                 e_description = self._parse_json(
-                    e.cause.read().decode(), display_id)['description']
+                    e.cause.response.read().decode(), display_id)['description']
                 if 'resource not available for country' in e_description:
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 if 'Authorized Networks' in e_description:

yt_dlp/extractor/dplay.py

@@ -2,7 +2,7 @@ import json
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -39,7 +39,7 @@ class DPlayBaseIE(InfoExtractor):
         return f'Bearer {token}'
 
     def _process_errors(self, e, geo_countries):
-        info = self._parse_json(e.cause.read().decode('utf-8'), None)
+        info = self._parse_json(e.cause.response.read().decode('utf-8'), None)
         error = info['errors'][0]
         error_code = error.get('code')
         if error_code == 'access.denied.geoblocked':
@@ -87,7 +87,7 @@ class DPlayBaseIE(InfoExtractor):
                     'include': 'images,primaryChannel,show,tags'
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 self._process_errors(e, geo_countries)
             raise
         video_id = video['data']['id']
@@ -99,7 +99,7 @@ class DPlayBaseIE(InfoExtractor):
             streaming = self._download_video_playback_info(
                 disco_base, video_id, headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self._process_errors(e, geo_countries)
             raise
         for format_dict in streaming:

yt_dlp/extractor/eagleplatform.py

@@ -2,7 +2,7 @@ import functools
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -111,8 +111,8 @@ class EaglePlatformIE(InfoExtractor):
             response = super(EaglePlatformIE, self)._download_json(
                 url_or_request, video_id, *args, **kwargs)
         except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError):
-                response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
+            if isinstance(ee.cause, HTTPError):
+                response = self._parse_json(ee.cause.response.read().decode('utf-8'), video_id)
                 self._handle_error(response)
             raise
         return response

yt_dlp/extractor/eitb.py

@@ -1,10 +1,6 @@
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import float_or_none, int_or_none, parse_iso8601
 
 
 class EitbIE(InfoExtractor):
@@ -54,7 +50,7 @@ class EitbIE(InfoExtractor):
 
         hls_url = media.get('HLS_SURL')
         if hls_url:
-            request = sanitized_Request(
+            request = Request(
                 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
                 headers={'Referer': url})
             token_data = self._download_json(

yt_dlp/extractor/eporner.py

@@ -52,7 +52,7 @@ class EpornerIE(InfoExtractor):
 
         webpage, urlh = self._download_webpage_handle(url, display_id)
 
-        video_id = self._match_id(urlh.geturl())
+        video_id = self._match_id(urlh.url)
 
         hash = self._search_regex(
             r'hash\s*[:=]\s*["\']([\da-f]{32})', webpage, 'hash')

yt_dlp/extractor/ettutv.py

@@ -41,7 +41,7 @@ class EttuTvIE(InfoExtractor):
                 'device': 'desktop',
             })
 
-        stream_response = self._download_json(player_settings['streamAccess'], video_id, data={})
+        stream_response = self._download_json(player_settings['streamAccess'], video_id, data=b'')
 
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
             stream_response['data']['stream'], video_id, 'mp4')

yt_dlp/extractor/facebook.py

@@ -8,6 +8,8 @@ from ..compat import (
     compat_str,
     compat_urllib_parse_unquote,
 )
+from ..networking import Request
+from ..networking.exceptions import network_exceptions
 from ..utils import (
     ExtractorError,
     clean_html,
@@ -19,11 +21,10 @@ from ..utils import (
     int_or_none,
     js_to_json,
     merge_dicts,
-    network_exceptions,
     parse_count,
     parse_qs,
     qualities,
-    sanitized_Request,
+    str_or_none,
     traverse_obj,
     try_get,
     url_or_none,
@@ -90,16 +91,16 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '274175099429670',
             'ext': 'mp4',
-            'title': 'Asif Nawab Butt',
-            'description': 'Asif Nawab Butt',
+            'title': 'Asif',
+            'description': '',
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
             'thumbnail': r're:^https?://.*',
+            'uploader_id': 'pfbid04scW44U4P9iTyLZAGy8y8W3pR3i2VugvHCimiRudUAVbN3MPp9eXBaYFcgVworZwl',
+            'duration': 131.03,
+            'concurrent_view_count': int,
         },
-        'expected_warnings': [
-            'title'
-        ]
     }, {
         'note': 'Video with DASH manifest',
         'url': 'https://www.facebook.com/video.php?v=957955867617029',
@@ -151,7 +152,7 @@ class FacebookIE(InfoExtractor):
         # have 1080P, but only up to 720p in swf params
         # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
-        'md5': '3f3798adb2b73423263e59376f1f5eb7',
+        'md5': 'ca63897a90c9452efee5f8c40d080e25',
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
@@ -162,6 +163,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'CNN',
             'thumbnail': r're:^https?://.*',
             'view_count': int,
+            'uploader_id': '100059479812265',
+            'concurrent_view_count': int,
+            'duration': 44.478,
         },
     }, {
         # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
@@ -170,12 +174,16 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '1417995061575415',
             'ext': 'mp4',
-            'title': 'Ukrainian Scientists Worldwide | Довгоочікуване відео',
+            'title': 'Довгоочікуване відео | By Yaroslav - Facebook',
             'description': 'Довгоочікуване відео',
-            'timestamp': 1486648771,
+            'timestamp': 1486648217,
             'upload_date': '20170209',
             'uploader': 'Yaroslav Korpan',
-            'uploader_id': '100000948048708',
+            'uploader_id': 'pfbid029y8j22EwH3ikeqgH3SEP9G3CAi9kmWKgXJJG9s5geV7mo3J2bvURqHCdgucRgAyhl',
+            'concurrent_view_count': int,
+            'thumbnail': r're:^https?://.*',
+            'view_count': int,
+            'duration': 11736.446,
         },
         'params': {
             'skip_download': True,
@@ -192,9 +200,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'La Guía Del Varón',
             'thumbnail': r're:^https?://.*',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'Requires logging in',
     }, {
         # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
@@ -208,9 +214,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Elisabeth Ahtn',
             'uploader_id': '100013949973717',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'Requires logging in',
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -252,7 +256,11 @@ class FacebookIE(InfoExtractor):
             'timestamp': 1527084179,
             'upload_date': '20180523',
             'uploader': 'ESL One Dota 2',
-            'uploader_id': '234218833769558',
+            'uploader_id': '100066514874195',
+            'duration': 4524.212,
+            'view_count': int,
+            'thumbnail': r're:^https?://.*',
+            'concurrent_view_count': int,
         },
         'params': {
             'skip_download': True,
@@ -262,8 +270,17 @@ class FacebookIE(InfoExtractor):
         'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/',
         'info_dict': {
             'id': '106560053808006',
+            'ext': 'mp4',
+            'title': 'Josef',
+            'thumbnail': r're:^https?://.*',
+            'concurrent_view_count': int,
+            'uploader_id': 'pfbid02gXHbDwxumkaKJQaTGUf3znYfYzTuidGEWawiramNx4YamSj2afwYSRkpcjtHtMRJl',
+            'timestamp': 1549275572,
+            'duration': 3.413,
+            'uploader': 'Josef Novak',
+            'description': '',
+            'upload_date': '20190204',
         },
-        'playlist_count': 2,
     }, {
         # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/watch/?v=647537299265662',
@@ -276,6 +293,7 @@ class FacebookIE(InfoExtractor):
             'id': '10157667649866271',
         },
         'playlist_count': 3,
+        'skip': 'Requires logging in',
     }, {
         # data.nodes[].comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://m.facebook.com/Alliance.Police.Department/posts/4048563708499330',
@@ -319,7 +337,7 @@ class FacebookIE(InfoExtractor):
     }
 
     def _perform_login(self, username, password):
-        login_page_req = sanitized_Request(self._LOGIN_URL)
+        login_page_req = Request(self._LOGIN_URL)
         self._set_cookie('facebook.com', 'locale', 'en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
@@ -340,8 +358,8 @@ class FacebookIE(InfoExtractor):
             'timezone': '-60',
             'trynum': '1',
         }
-        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request = Request(self._LOGIN_URL, urlencode_postdata(login_form))
+        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         try:
             login_results = self._download_webpage(request, None,
                                                    note='Logging in', errnote='unable to fetch login page')
@@ -367,8 +385,8 @@ class FacebookIE(InfoExtractor):
                 'h': h,
                 'name_action_selected': 'dont_save',
             }
-            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
-            check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+            check_req = Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+            check_req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
@@ -497,6 +515,13 @@ class FacebookIE(InfoExtractor):
                 entries = []
 
                 def parse_graphql_video(video):
+                    v_id = video.get('videoId') or video.get('id') or video_id
+                    reel_info = traverse_obj(
+                        video, ('creation_story', 'short_form_video_context', 'playback_video', {dict}))
+                    if reel_info:
+                        video = video['creation_story']
+                        video['owner'] = traverse_obj(video, ('short_form_video_context', 'video_owner'))
+                        video.update(reel_info)
                     formats = []
                     q = qualities(['sd', 'hd'])
                     for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
@@ -513,15 +538,15 @@ class FacebookIE(InfoExtractor):
                                 'url': playable_url,
                             })
                     extract_dash_manifest(video, formats)
-                    v_id = video.get('videoId') or video.get('id') or video_id
                     info = {
                         'id': v_id,
                         'formats': formats,
                         'thumbnail': traverse_obj(
                             video, ('thumbnailImage', 'uri'), ('preferred_thumbnail', 'image', 'uri')),
-                        'uploader_id': try_get(video, lambda x: x['owner']['id']),
-                        'timestamp': int_or_none(video.get('publish_time')),
-                        'duration': float_or_none(video.get('playable_duration_in_ms'), 1000),
+                        'uploader_id': traverse_obj(video, ('owner', 'id', {str_or_none})),
+                        'timestamp': traverse_obj(video, 'publish_time', 'creation_time', expected_type=int_or_none),
+                        'duration': (float_or_none(video.get('playable_duration_in_ms'), 1000)
+                                     or float_or_none(video.get('length_in_second'))),
                     }
                     process_formats(info)
                     description = try_get(video, lambda x: x['savable_description']['text'])
@@ -782,18 +807,18 @@ class FacebookReelIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://www.facebook.com/reel/1195289147628387',
-        'md5': 'c4ff9a7182ff9ff7d6f7a83603bae831',
+        'md5': 'f13dd37f2633595982db5ed8765474d3',
         'info_dict': {
             'id': '1195289147628387',
             'ext': 'mp4',
-            'title': 'md5:9f5b142921b2dc57004fa13f76005f87',
-            'description': 'md5:24ea7ef062215d295bdde64e778f5474',
-            'uploader': 'Beast Camp Training',
-            'uploader_id': '1738535909799870',
-            'duration': 9.536,
-            'thumbnail': r're:^https?://.*',
+            'title': 'md5:b05800b5b1ad56c0ca78bd3807b6a61e',
+            'description': 'md5:22f03309b216ac84720183961441d8db',
+            'uploader': 'md5:723e6cb3091241160f20b3c5dc282af1',
+            'uploader_id': '100040874179269',
+            'duration': 9.579,
+            'timestamp': 1637502609,
             'upload_date': '20211121',
-            'timestamp': 1637502604,
+            'thumbnail': r're:^https?://.*',
         }
     }]
 

yt_dlp/extractor/fc2.py

@@ -3,11 +3,11 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
 from ..dependencies import websockets
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     WebSocketsWrapper,
     js_to_json,
-    sanitized_Request,
     traverse_obj,
     update_url_query,
     urlencode_postdata,
@@ -57,7 +57,7 @@ class FC2IE(InfoExtractor):
         }
 
         login_data = urlencode_postdata(login_form_strs)
-        request = sanitized_Request(
+        request = Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +66,7 @@ class FC2IE(InfoExtractor):
             return False
 
         # this is also needed
-        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
+        login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
         self._download_webpage(
             login_redir, None, note='Login redirect', errnote='Login redirect failed')
 

yt_dlp/extractor/filmon.py

@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_HTTPError,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     qualities,
     strip_or_none,
@@ -40,8 +38,8 @@ class FilmOnIE(InfoExtractor):
                 'https://www.filmon.com/api/vod/movie?id=%s' % video_id,
                 video_id)['response']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                errmsg = self._parse_json(e.cause.read().decode(), video_id)['reason']
+            if isinstance(e.cause, HTTPError):
+                errmsg = self._parse_json(e.cause.response.read().decode(), video_id)['reason']
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
             raise
 
@@ -124,8 +122,8 @@ class FilmOnChannelIE(InfoExtractor):
             channel_data = self._download_json(
                 'http://www.filmon.com/api-v2/channel/' + channel_id, channel_id)['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                errmsg = self._parse_json(e.cause.read().decode(), channel_id)['message']
+            if isinstance(e.cause, HTTPError):
+                errmsg = self._parse_json(e.cause.response.read().decode(), channel_id)['message']
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
             raise
 

yt_dlp/extractor/fox.py

@@ -3,10 +3,10 @@ import uuid
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_HTTPError,
     compat_str,
     compat_urllib_parse_unquote,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -68,9 +68,9 @@ class FOXIE(InfoExtractor):
                 'https://api3.fox.com/v2.0/' + path,
                 video_id, data=data, headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 entitlement_issues = self._parse_json(
-                    e.cause.read().decode(), video_id)['entitlementIssues']
+                    e.cause.response.read().decode(), video_id)['entitlementIssues']
                 for e in entitlement_issues:
                     if e.get('errorCode') == 1005:
                         raise ExtractorError(
@@ -123,8 +123,8 @@ class FOXIE(InfoExtractor):
         try:
             m3u8_url = self._download_json(release_url, video_id)['playURL']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                error = self._parse_json(e.cause.read().decode(), video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                error = self._parse_json(e.cause.response.read().decode(), video_id)
                 if error.get('exception') == 'GeoLocationBlocked':
                     self.raise_geo_restricted(countries=['US'])
                 raise ExtractorError(error['description'], expected=True)

yt_dlp/extractor/foxsports.py

@@ -1,6 +1,7 @@
 from .common import InfoExtractor
 from .uplynk import UplynkPreplayIE
-from ..utils import HEADRequest, float_or_none, make_archive_id, smuggle_url
+from ..networking import HEADRequest
+from ..utils import float_or_none, make_archive_id, smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
@@ -35,7 +36,7 @@ class FoxSportsIE(InfoExtractor):
                 'x-api-key': 'cf289e299efdfa39fb6316f259d1de93',
             })
         preplay_url = self._request_webpage(
-            HEADRequest(data['url']), video_id, 'Fetching preplay URL').geturl()
+            HEADRequest(data['url']), video_id, 'Fetching preplay URL').url
 
         return {
             '_type': 'url_transparent',

yt_dlp/extractor/fujitv.py

@@ -1,5 +1,5 @@
-from ..utils import HEADRequest
 from .common import InfoExtractor
+from ..networking import HEADRequest
 
 
 class FujiTVFODPlus7IE(InfoExtractor):

yt_dlp/extractor/funimation.py

@@ -3,7 +3,7 @@ import re
 import string
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -46,8 +46,8 @@ class FunimationBaseIE(InfoExtractor):
                 }))
             FunimationBaseIE._TOKEN = data['token']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read().decode(), None)['error']
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)['error']
                 raise ExtractorError(error, expected=True)
             raise
 

yt_dlp/extractor/gdcvault.py

@@ -2,13 +2,8 @@ import re
 
 from .common import InfoExtractor
 from .kaltura import KalturaIE
-from ..utils import (
-    HEADRequest,
-    remove_start,
-    sanitized_Request,
-    smuggle_url,
-    urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import remove_start, smuggle_url, urlencode_postdata
 
 
 class GDCVaultIE(InfoExtractor):
@@ -138,8 +133,8 @@ class GDCVaultIE(InfoExtractor):
             'password': password,
         }
 
-        request = sanitized_Request(login_url, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request = Request(login_url, urlencode_postdata(login_form))
+        request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
         self._download_webpage(logout_url, display_id, 'Logging out')
@@ -163,7 +158,7 @@ class GDCVaultIE(InfoExtractor):
             video_url = 'http://www.gdcvault.com' + direct_url
             # resolve the url so that we can detect the correct extension
             video_url = self._request_webpage(
-                HEADRequest(video_url), video_id).geturl()
+                HEADRequest(video_url), video_id).url
 
             return {
                 'id': video_id,

yt_dlp/extractor/generic.py

@@ -2431,7 +2431,7 @@ class GenericIE(InfoExtractor):
             'Accept-Encoding': 'identity',
             **smuggled_data.get('http_headers', {})
         })
-        new_url = full_response.geturl()
+        new_url = full_response.url
         url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
         if new_url != extract_basic_auth(url)[0]:
             self.report_following_redirect(new_url)
@@ -2529,12 +2529,12 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(
                     self._parse_xspf(
                         doc, video_id, xspf_url=url,
-                        xspf_base_url=full_response.geturl()),
+                        xspf_base_url=full_response.url),
                     video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'], info_dict['subtitles'] = self._parse_mpd_formats_and_subtitles(
                     doc,
-                    mpd_base_url=full_response.geturl().rpartition('/')[0],
+                    mpd_base_url=full_response.url.rpartition('/')[0],
                     mpd_url=url)
                 self._extra_manifest_info(info_dict, url)
                 self.report_detected('DASH manifest')
@@ -2562,7 +2562,7 @@ class GenericIE(InfoExtractor):
         self._downloader.write_debug('Looking for embeds')
         embeds = list(self._extract_embeds(original_url, webpage, urlh=full_response, info_dict=info_dict))
         if len(embeds) == 1:
-            return {**info_dict, **embeds[0]}
+            return merge_dicts(embeds[0], info_dict)
         elif embeds:
             return self.playlist_result(embeds, **info_dict)
         raise UnsupportedError(url)
@@ -2572,7 +2572,7 @@ class GenericIE(InfoExtractor):
         info_dict = types.MappingProxyType(info_dict)  # Prevents accidental mutation
         video_id = traverse_obj(info_dict, 'display_id', 'id') or self._generic_id(url)
         url, smuggled_data = unsmuggle_url(url, {})
-        actual_url = urlh.geturl() if urlh else url
+        actual_url = urlh.url if urlh else url
 
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448)

yt_dlp/extractor/globo.py

@@ -8,8 +8,8 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     ExtractorError,
     float_or_none,
     orderedSet,

yt_dlp/extractor/googledrive.py

@@ -5,7 +5,9 @@ from ..compat import compat_parse_qs
 from ..utils import (
     ExtractorError,
     determine_ext,
+    extract_attributes,
     get_element_by_class,
+    get_element_html_by_id,
     int_or_none,
     lowercase_escape,
     try_get,
@@ -34,6 +36,7 @@ class GoogleDriveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Big Buck Bunny.mp4',
             'duration': 45,
+            'thumbnail': 'https://drive.google.com/thumbnail?id=0ByeS4oOUV-49Zzh4R1J6R09zazQ',
         }
     }, {
         # video can't be watched anonymously due to view count limit reached,
@@ -207,10 +210,10 @@ class GoogleDriveIE(InfoExtractor):
                 'export': 'download',
             })
 
-        def request_source_file(source_url, kind):
+        def request_source_file(source_url, kind, data=None):
             return self._request_webpage(
                 source_url, video_id, note='Requesting %s file' % kind,
-                errnote='Unable to request %s file' % kind, fatal=False)
+                errnote='Unable to request %s file' % kind, fatal=False, data=data)
         urlh = request_source_file(source_url, 'source')
         if urlh:
             def add_source_format(urlh):
@@ -225,7 +228,7 @@ class GoogleDriveIE(InfoExtractor):
                     # Using original URLs may result in redirect loop due to
                     # google.com's cookies mistakenly used for googleusercontent.com
                     # redirect URLs (see #23919).
-                    'url': urlh.geturl(),
+                    'url': urlh.url,
                     'ext': determine_ext(title, 'mp4').lower(),
                     'format_id': 'source',
                     'quality': 1,
@@ -237,14 +240,10 @@ class GoogleDriveIE(InfoExtractor):
                     urlh, url, video_id, note='Downloading confirmation page',
                     errnote='Unable to confirm download', fatal=False)
                 if confirmation_webpage:
-                    confirm = self._search_regex(
-                        r'confirm=([^&"\']+)', confirmation_webpage,
-                        'confirmation code', default=None)
-                    if confirm:
-                        confirmed_source_url = update_url_query(source_url, {
-                            'confirm': confirm,
-                        })
-                        urlh = request_source_file(confirmed_source_url, 'confirmed source')
+                    confirmed_source_url = extract_attributes(
+                        get_element_html_by_id('download-form', confirmation_webpage) or '').get('action')
+                    if confirmed_source_url:
+                        urlh = request_source_file(confirmed_source_url, 'confirmed source', data=b'')
                         if urlh and urlh.headers.get('Content-Disposition'):
                             add_source_format(urlh)
                     else:

yt_dlp/extractor/hketv.py

@@ -126,7 +126,7 @@ class HKETVIE(InfoExtractor):
             # If we ever wanted to provide the final resolved URL that
             # does not require cookies, albeit with a shorter lifespan:
             #     urlh = self._downloader.urlopen(file_url)
-            #     resolved_url = urlh.geturl()
+            #     resolved_url = urlh.url
             label = fmt.get('label')
             h = self._FORMAT_HEIGHTS.get(label)
             w = h * width // height if h and width and height else None

yt_dlp/extractor/hotnewhiphop.py

@@ -1,11 +1,7 @@
 from .common import InfoExtractor
 from ..compat import compat_b64decode
-from ..utils import (
-    ExtractorError,
-    HEADRequest,
-    sanitized_Request,
-    urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import ExtractorError, urlencode_postdata
 
 
 class HotNewHipHopIE(InfoExtractor):
@@ -36,9 +32,9 @@ class HotNewHipHopIE(InfoExtractor):
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
-        r = sanitized_Request(
+        r = Request(
             'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
-        r.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
         mkd = self._download_json(
             r, video_id, note='Requesting media key',
             errnote='Could not download media key')
@@ -50,7 +46,7 @@ class HotNewHipHopIE(InfoExtractor):
         req = self._request_webpage(
             redirect_req, video_id,
             note='Resolving final URL', errnote='Could not resolve final URL')
-        video_url = req.geturl()
+        video_url = req.url
         if video_url.endswith('.html'):
             raise ExtractorError('Redirect failed')
 

yt_dlp/extractor/hotstar.py

@@ -6,7 +6,8 @@ import time
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -233,7 +234,7 @@ class HotStarIE(HotStarBaseIE):
                         'height': int_or_none(playback_set.get('height')),
                     }]
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                     geo_restricted = True
                 continue
 

yt_dlp/extractor/hrti.py

@@ -1,13 +1,13 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking import Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
     parse_age_limit,
-    sanitized_Request,
     try_get,
 )
 
@@ -42,7 +42,7 @@ class HRTiBaseIE(InfoExtractor):
             'application_version': self._APP_VERSION
         }
 
-        req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
+        req = Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
         req.get_method = lambda: 'PUT'
 
         resources = self._download_json(
@@ -73,8 +73,8 @@ class HRTiBaseIE(InfoExtractor):
                 self._login_url, None, note='Logging in', errnote='Unable to log in',
                 data=json.dumps(auth_data).encode('utf-8'))
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
-                auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 406:
+                auth_info = self._parse_json(e.cause.response.read().encode('utf-8'), None)
             else:
                 raise
 

yt_dlp/extractor/ign.py

@@ -1,8 +1,9 @@
 import re
-import urllib.error
+import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -27,9 +28,9 @@ class IGNBaseIE(InfoExtractor):
         try:
             return self._call_api(slug)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                 e.cause.args = e.cause.args or [
-                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                    e.cause.response.url, e.cause.status, e.cause.reason]
                 raise ExtractorError(
                     'Content not found: expired?', cause=e.cause,
                     expected=True)
@@ -226,7 +227,7 @@ class IGNVideoIE(IGNBaseIE):
             parsed_url._replace(path=parsed_url.path.rsplit('/', 1)[0] + '/embed'))
 
         webpage, urlh = self._download_webpage_handle(embed_url, video_id)
-        new_url = urlh.geturl()
+        new_url = urlh.url
         ign_url = compat_parse_qs(
             urllib.parse.urlparse(new_url).query).get('url', [None])[-1]
         if ign_url:
@@ -323,14 +324,14 @@ class IGNArticleIE(IGNBaseIE):
         try:
             return self._call_api(slug)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError):
+            if isinstance(e.cause, HTTPError):
                 e.cause.args = e.cause.args or [
-                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
-                if e.cause.code == 404:
+                    e.cause.response.url, e.cause.status, e.cause.reason]
+                if e.cause.status == 404:
                     raise ExtractorError(
                         'Content not found: expired?', cause=e.cause,
                         expected=True)
-                elif e.cause.code == 503:
+                elif e.cause.status == 503:
                     self.report_warning(error_to_compat_str(e.cause))
                     return
             raise

yt_dlp/extractor/imggaming.py

@@ -1,7 +1,7 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -52,9 +52,9 @@ class ImgGamingBaseIE(InfoExtractor):
             return self._call_api(
                 stream_path, media_id)['playerUrlCallback']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 raise ExtractorError(
-                    self._parse_json(e.cause.read().decode(), media_id)['messages'][0],
+                    self._parse_json(e.cause.response.read().decode(), media_id)['messages'][0],
                     expected=True)
             raise
 

yt_dlp/extractor/instagram.py

@@ -3,9 +3,9 @@ import itertools
 import json
 import re
 import time
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     decode_base_n,
@@ -442,7 +442,7 @@ class InstagramIE(InstagramBaseIE):
             shared_data = self._search_json(
                 r'window\._sharedData\s*=', webpage, 'shared data', video_id, fatal=False) or {}
 
-            if shared_data and self._LOGIN_URL not in urlh.geturl():
+            if shared_data and self._LOGIN_URL not in urlh.url:
                 media.update(traverse_obj(
                     shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
                     ('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {})
@@ -589,7 +589,7 @@ class InstagramPlaylistBaseIE(InstagramBaseIE):
                 except ExtractorError as e:
                     # if it's an error caused by a bad query, and there are
                     # more GIS templates to try, ignore it and keep trying
-                    if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+                    if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                         if gis_tmpl != gis_tmpls[-1]:
                             continue
                     raise

yt_dlp/extractor/iprima.py

@@ -81,7 +81,7 @@ class IPrimaIE(InfoExtractor):
             note='Logging in')
 
         # a profile may need to be selected first, even when there is only a single one
-        if '/profile-select' in login_handle.geturl():
+        if '/profile-select' in login_handle.url:
             profile_id = self._search_regex(
                 r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
 
@@ -89,7 +89,7 @@ class IPrimaIE(InfoExtractor):
                 f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
                 query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
 
-        code = traverse_obj(login_handle.geturl(), ({parse_qs}, 'code', 0))
+        code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0))
         if not code:
             raise ExtractorError('Login failed', expected=True)
 

yt_dlp/extractor/iqiyi.py

@@ -527,7 +527,7 @@ class IqIE(InfoExtractor):
         if player_js_cache:
             return player_js_cache
         webpack_js_url = self._proto_relative_url(self._search_regex(
-            r'<script src="((?:https?)?//stc.iqiyipic.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
+            r'<script src="((?:https?:)?//stc\.iqiyipic\.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
         webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
         webpack_map = self._search_json(
             r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,

yt_dlp/extractor/kakao.py

@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -101,8 +101,8 @@ class KakaoIE(InfoExtractor):
                     cdn_api_base, video_id, query=query,
                     note='Downloading video URL for profile %s' % profile_name)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                    resp = self._parse_json(e.cause.read().decode(), video_id)
+                if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                    resp = self._parse_json(e.cause.response.read().decode(), video_id)
                     if resp.get('code') == 'GeoBlocked':
                         self.raise_geo_restricted()
                 raise

yt_dlp/extractor/kick.py

@@ -1,7 +1,6 @@
 from .common import InfoExtractor
-
+from ..networking import HEADRequest
 from ..utils import (
-    HEADRequest,
     UserNotLive,
     float_or_none,
     merge_dicts,
@@ -30,7 +29,7 @@ class KickBaseIE(InfoExtractor):
 
 
 class KickIE(KickBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w_]+)'
+    _VALID_URL = r'https?://(?:www\.)?kick\.com/(?!(?:video|categories|search|auth)(?:[/?#]|$))(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://kick.com/yuppy',
         'info_dict': {

yt_dlp/extractor/kuwo.py

@@ -91,7 +91,7 @@ class KuwoIE(KuwoBaseIE):
         webpage, urlh = self._download_webpage_handle(
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
-        if song_id not in urlh.geturl() or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+        if song_id not in urlh.url or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
             raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(

yt_dlp/extractor/la7.py

@@ -1,13 +1,8 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    HEADRequest,
-    int_or_none,
-    parse_duration,
-    unified_strdate,
-)
+from ..networking import HEADRequest
+from ..utils import float_or_none, int_or_none, parse_duration, unified_strdate
 
 
 class LA7IE(InfoExtractor):

yt_dlp/extractor/lbry.py

@@ -3,9 +3,9 @@ import json
 import urllib.parse
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     OnDemandPagedList,
     UnsupportedError,
     determine_ext,
@@ -266,7 +266,7 @@ class LBRYIE(LBRYBaseIE):
             # HEAD request returns redirect response to m3u8 URL if available
             final_url = self._request_webpage(
                 HEADRequest(streaming_url), display_id, headers=headers,
-                note='Downloading streaming redirect url info').geturl()
+                note='Downloading streaming redirect url info').url
 
         elif result.get('value_type') == 'stream':
             claim_id, is_live = result['signing_channel']['claim_id'], True

yt_dlp/extractor/lecturio.py

@@ -25,7 +25,7 @@ class LecturioBaseIE(InfoExtractor):
             self._LOGIN_URL, None, 'Downloading login popup')
 
         def is_logged(url_handle):
-            return self._LOGIN_URL not in url_handle.geturl()
+            return self._LOGIN_URL not in url_handle.url
 
         # Already logged in
         if is_logged(urlh):

yt_dlp/extractor/lego.py

@@ -1,7 +1,7 @@
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -75,7 +75,7 @@ class LEGOIE(InfoExtractor):
                     'videoId': '%s_%s' % (uuid.UUID(video_id), locale),
                 }, headers=self.geo_verification_headers())
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 451:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 451:
                 self.raise_geo_restricted(countries=countries)
             raise
 

yt_dlp/extractor/limelight.py

@@ -1,7 +1,7 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -69,8 +69,8 @@ class LimelightBaseIE(InfoExtractor):
                 item_id, 'Downloading PlaylistService %s JSON' % method,
                 fatal=fatal, headers=headers)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                error = self._parse_json(e.cause.read().decode(), item_id)['detail']['contentAccessPermission']
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                error = self._parse_json(e.cause.response.read().decode(), item_id)['detail']['contentAccessPermission']
                 if error == 'CountryDisabled':
                     self.raise_geo_restricted()
                 raise ExtractorError(error, expected=True)

yt_dlp/extractor/linuxacademy.py

@@ -2,11 +2,8 @@ import json
 import random
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_b64decode,
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_b64decode, compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -107,7 +104,7 @@ class LinuxAcademyIE(InfoExtractor):
             'sso': 'true',
         })
 
-        login_state_url = urlh.geturl()
+        login_state_url = urlh.url
 
         try:
             login_page = self._download_webpage(
@@ -119,8 +116,8 @@ class LinuxAcademyIE(InfoExtractor):
                     'Referer': login_state_url,
                 })
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read(), None)
                 message = error.get('description') or error['code']
                 raise ExtractorError(
                     '%s said: %s' % (self.IE_NAME, message), expected=True)
@@ -137,7 +134,7 @@ class LinuxAcademyIE(InfoExtractor):
             })
 
         access_token = self._search_regex(
-            r'access_token=([^=&]+)', urlh.geturl(),
+            r'access_token=([^=&]+)', urlh.url,
             'access token', default=None)
         if not access_token:
             access_token = self._parse_json(

yt_dlp/extractor/magellantv.py

@@ -0,0 +1,50 @@
+from .common import InfoExtractor
+from ..utils import parse_age_limit, parse_duration, traverse_obj
+
+
+class MagellanTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?magellantv\.com/(?:watch|video)/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.magellantv.com/watch/my-dads-on-death-row?type=v',
+        'info_dict': {
+            'id': 'my-dads-on-death-row',
+            'ext': 'mp4',
+            'title': 'My Dad\'s On Death Row',
+            'description': 'md5:33ba23b9f0651fc4537ed19b1d5b0d7a',
+            'duration': 3780.0,
+            'age_limit': 14,
+            'tags': ['Justice', 'Reality', 'United States', 'True Crime'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.magellantv.com/video/james-bulger-the-new-revelations',
+        'info_dict': {
+            'id': 'james-bulger-the-new-revelations',
+            'ext': 'mp4',
+            'title': 'James Bulger: The New Revelations',
+            'description': 'md5:7b97922038bad1d0fe8d0470d8a189f2',
+            'duration': 2640.0,
+            'age_limit': 0,
+            'tags': ['Investigation', 'True Crime', 'Justice', 'Europe'],
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['reactContext']['video']['detail']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(data['jwpVideoUrl'], video_id)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('metadata', 'description', {str}),
+                'duration': ('duration', {parse_duration}),
+                'age_limit': ('ratingCategory', {parse_age_limit}),
+                'tags': ('tags', ..., {str}),
+            }),
+        }

yt_dlp/extractor/mailru.py

@@ -1,6 +1,7 @@
 import itertools
 import json
 import re
+import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
@@ -140,17 +141,15 @@ class MailRuIE(InfoExtractor):
                 'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
                 video_id, 'Downloading video JSON')
 
-        headers = {}
-
         video_key = self._get_cookies('https://my.mail.ru').get('video_key')
-        if video_key:
-            headers['Cookie'] = 'video_key=%s' % video_key.value
 
         formats = []
         for f in video_data['videos']:
             video_url = f.get('url')
             if not video_url:
                 continue
+            if video_key:
+                self._set_cookie(urllib.parse.urlparse(video_url).hostname, 'video_key', video_key.value)
             format_id = f.get('key')
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None)) if format_id else None
@@ -158,7 +157,6 @@ class MailRuIE(InfoExtractor):
                 'url': video_url,
                 'format_id': format_id,
                 'height': height,
-                'http_headers': headers,
             })
 
         meta_data = video_data['meta']

yt_dlp/extractor/mediasite.py

@@ -171,7 +171,7 @@ class MediasiteIE(InfoExtractor):
         query = mobj.group('query')
 
         webpage, urlh = self._download_webpage_handle(url, resource_id)  # XXX: add UrlReferrer?
-        redirect_url = urlh.geturl()
+        redirect_url = urlh.url
 
         # XXX: might have also extracted UrlReferrer and QueryString from the html
         service_path = compat_urlparse.urljoin(redirect_url, self._html_search_regex(

yt_dlp/extractor/megatvcom.py

@@ -1,14 +1,14 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
-    ExtractorError,
     extract_attributes,
     get_element_by_class,
     get_element_html_by_id,
-    HEADRequest,
     parse_qs,
     unescapeHTML,
     unified_timestamp,
@@ -160,5 +160,5 @@ class MegaTVComEmbedIE(MegaTVComBaseIE):
         canonical_url = self._request_webpage(
             HEADRequest(canonical_url), video_id,
             note='Resolve canonical URL',
-            errnote='Could not resolve canonical URL').geturl()
+            errnote='Could not resolve canonical URL').url
         return self.url_result(canonical_url, MegaTVComIE.ie_key(), video_id)

yt_dlp/extractor/mgtv.py

@@ -1,9 +1,9 @@
 import base64
 import time
-import urllib.error
 import uuid
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -86,8 +86,8 @@ class MGTVIE(InfoExtractor):
                     'type': 'pch5'
                 }, headers=self.geo_verification_headers())['data']
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
-                error = self._parse_json(e.cause.read().decode(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)
                 if error.get('code') == 40005:
                     self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
                 raise ExtractorError(error['msg'], expected=True)

yt_dlp/extractor/minds.py

@@ -106,7 +106,7 @@ class MindsIE(MindsBaseIE):
         if poster:
             urlh = self._request_webpage(poster, video_id, fatal=False)
             if urlh:
-                thumbnail = urlh.geturl()
+                thumbnail = urlh.url
 
         return {
             'id': video_id,

yt_dlp/extractor/miomio.py

@@ -2,12 +2,8 @@ import random
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import (
-    xpath_text,
-    int_or_none,
-    ExtractorError,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError, int_or_none, xpath_text
 
 
 class MioMioIE(InfoExtractor):
@@ -61,7 +57,7 @@ class MioMioIE(InfoExtractor):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        vid_config_request = sanitized_Request(
+        vid_config_request = Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
             headers=http_headers)
 

yt_dlp/extractor/mtv.py

@@ -2,16 +2,15 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..networking import HEADRequest, Request
 from ..utils import (
     ExtractorError,
+    RegexNotFoundError,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
-    HEADRequest,
     int_or_none,
     join_nonempty,
-    RegexNotFoundError,
-    sanitized_Request,
     strip_or_none,
     timeconvert,
     try_get,
@@ -51,15 +50,15 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
-        req = sanitized_Request(webpage_url)
+        req = Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
-        req.add_header('User-Agent', 'curl/7')
+        req.headers['User-Agent'] = 'curl/7'
         webpage = self._download_webpage(req, mtvn_id,
                                          'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
         req = HEADRequest(metrics_url)
         response = self._request_webpage(req, mtvn_id, 'Resolving url')
-        url = response.geturl()
+        url = response.url
         # Transform the url to get the best quality:
         url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
         return [{'url': url, 'ext': 'mp4'}]

yt_dlp/extractor/museai.py

@@ -0,0 +1,112 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    float_or_none,
+    int_or_none,
+    js_to_json,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class MuseAIIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?muse\.ai/(?:v|embed)/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://muse.ai/embed/YdTWvUW',
+        'md5': 'f994f9a38be1c3aaf9e37cbd7d76fe7c',
+        'info_dict': {
+            'id': 'YdTWvUW',
+            'ext': 'mp4',
+            'title': '2023-05-28-Grabien-1941111 (1)',
+            'description': '',
+            'uploader': 'Today News Africa',
+            'uploader_id': 'TodayNewsAfrica',
+            'upload_date': '20230528',
+            'timestamp': 1685285044,
+            'duration': 1291.3,
+            'view_count': int,
+            'availability': 'public',
+        },
+    }, {
+        'url': 'https://muse.ai/v/gQ4gGAA-0756',
+        'md5': '52dbfc78e865e56dc19a1715badc35e8',
+        'info_dict': {
+            'id': 'gQ4gGAA',
+            'ext': 'mp4',
+            'title': '0756',
+            'description': 'md5:0ca1483f9aac423e9a96ad00bb3a0785',
+            'uploader': 'Aerial.ie',
+            'uploader_id': 'aerial',
+            'upload_date': '20210306',
+            'timestamp': 1615072842,
+            'duration': 21.4,
+            'view_count': int,
+            'availability': 'public',
+        },
+    }]
+    _WEBPAGE_TESTS = [{
+        'url': 'https://muse.ai/docs',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'id': 'docs',
+            'title': 'muse.ai | docs',
+            'description': 'md5:6c0293431481582739c82ee8902687fa',
+            'age_limit': 0,
+            'thumbnail': 'https://muse.ai/static/imgs/poster-img-docs.png',
+        },
+        'params': {'allowed_extractors': ['all', '-html5']},
+    }]
+    _EMBED_REGEX = [r'<iframe[^>]*\bsrc=["\'](?P<url>https://muse\.ai/embed/\w+)']
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        yield from super()._extract_embed_urls(url, webpage)
+        for embed_id in re.findall(r'<script>[^<]*\bMusePlayer\(\{[^}<]*\bvideo:\s*["\'](\w+)["\']', webpage):
+            yield f'https://muse.ai/embed/{embed_id}'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://muse.ai/embed/{video_id}', video_id)
+        data = self._search_json(
+            r'player\.setData\(', webpage, 'player data', video_id, transform_source=js_to_json)
+
+        source_url = data['url']
+        if not url_or_none(source_url):
+            raise ExtractorError('Unable to extract video URL')
+
+        formats = [{
+            'url': source_url,
+            'format_id': 'source',
+            'quality': 1,
+            **traverse_obj(data, {
+                'ext': ('filename', {determine_ext}),
+                'width': ('width', {int_or_none}),
+                'height': ('height', {int_or_none}),
+                'filesize': ('size', {int_or_none}),
+            }),
+        }]
+        if source_url.endswith('/data'):
+            base_url = f'{source_url[:-5]}/videos'
+            formats.extend(self._extract_m3u8_formats(
+                f'{base_url}/hls.m3u8', video_id, m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_mpd_formats(
+                f'{base_url}/dash.mpd', video_id, mpd_id='dash', fatal=False))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(data, {
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'duration': ('duration', {float_or_none}),
+                'timestamp': ('tcreated', {int_or_none}),
+                'uploader': ('owner_name', {str}),
+                'uploader_id': ('owner_username', {str}),
+                'view_count': ('views', {int_or_none}),
+                'age_limit': ('mature', {lambda x: 18 if x else None}),
+                'availability': ('visibility', {lambda x: x if x in ('private', 'unlisted') else 'public'}),
+            }),
+        }

yt_dlp/extractor/nbc.py

@@ -6,9 +6,9 @@ from .common import InfoExtractor
 from .theplatform import ThePlatformIE, default_ns
 from .adobepass import AdobePassIE
 from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     RegexNotFoundError,
     UserNotLive,
     clean_html,

yt_dlp/extractor/nebula.py

@@ -1,8 +1,8 @@
 import itertools
 import json
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start
 
 _BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
@@ -48,7 +48,7 @@ class NebulaBaseIE(InfoExtractor):
             return inner_call()
         except ExtractorError as exc:
             # if 401 or 403, attempt credential re-auth and retry
-            if exc.cause and isinstance(exc.cause, urllib.error.HTTPError) and exc.cause.code in (401, 403):
+            if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.status in (401, 403):
                 self.to_screen(f'Reauthenticating to Nebula and retrying, because last {auth_type} call resulted in error {exc.cause.code}')
                 self._perform_login()
                 return inner_call()

yt_dlp/extractor/neteasemusic.py

@@ -11,6 +11,7 @@ from random import randint
 from .common import InfoExtractor
 from ..aes import aes_ecb_encrypt, pkcs7_padding
 from ..compat import compat_urllib_parse_urlencode
+from ..networking import Request
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
@@ -18,7 +19,6 @@ from ..utils import (
     float_or_none,
     int_or_none,
     intlist_to_bytes,
-    sanitized_Request,
     try_get,
 )
 
@@ -146,8 +146,8 @@ class NetEaseMusicBaseIE(InfoExtractor):
         return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
-        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
-        req.add_header('Referer', self._API_BASE)
+        req = Request('%s%s' % (self._API_BASE, endpoint))
+        req.headers['Referer'] = self._API_BASE
         return self._download_json(req, video_id, note)
 
 

yt_dlp/extractor/niconico.py

@@ -8,10 +8,8 @@ import time
 from urllib.parse import urlparse
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..compat import (
-    compat_HTTPError,
-)
 from ..dependencies import websockets
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -396,7 +394,7 @@ class NiconicoIE(InfoExtractor):
             webpage, handle = self._download_webpage_handle(
                 'https://www.nicovideo.jp/watch/' + video_id, video_id)
             if video_id.startswith('so'):
-                video_id = self._match_id(handle.geturl())
+                video_id = self._match_id(handle.url)
 
             api_data = self._parse_json(self._html_search_regex(
                 'data-api-data="([^"]+)"', webpage,
@@ -407,9 +405,9 @@ class NiconicoIE(InfoExtractor):
                     'https://www.nicovideo.jp/api/watch/v3/%s?_frontendId=6&_frontendVersion=0&actionTrackId=AAAAAAAAAA_%d' % (video_id, round(time.time() * 1000)), video_id,
                     note='Downloading API JSON', errnote='Unable to fetch data')['data']
             except ExtractorError:
-                if not isinstance(e.cause, compat_HTTPError):
+                if not isinstance(e.cause, HTTPError):
                     raise
-                webpage = e.cause.read().decode('utf-8', 'replace')
+                webpage = e.cause.response.read().decode('utf-8', 'replace')
                 error_msg = self._html_search_regex(
                     r'(?s)<section\s+class="(?:(?:ErrorMessage|WatchExceptionPage-message)\s*)+">(.+?)</section>',
                     webpage, 'error reason', default=None)
@@ -742,7 +740,7 @@ class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
         try:
             mylist = self._call_api(list_id, 'list', {'pageSize': 1})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self.raise_login_required('You have to be logged in to get your history')
             raise
         return self.playlist_result(self._entries(list_id), list_id, **self._parse_owner(mylist))
@@ -951,8 +949,8 @@ class NiconicoLiveIE(InfoExtractor):
             'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
         })
 
-        hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
-        cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
+        hostname = remove_start(urlparse(urlh.url).hostname, 'sp.')
+        cookies = try_get(urlh.url, self._downloader._calc_cookies)
         latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
         if latency not in self._KNOWN_LATENCY:
             latency = 'high'

yt_dlp/extractor/njpwworld.py

@@ -51,7 +51,7 @@ class NJPWWorldIE(InfoExtractor):
             data=urlencode_postdata({'login_id': username, 'pw': password}),
             headers={'Referer': 'https://front.njpwworld.com/auth'})
         # /auth/login will return 302 for successful logins
-        if urlh.geturl() == self._LOGIN_URL:
+        if urlh.url == self._LOGIN_URL:
             self.report_warning('unable to login')
             return False
 

yt_dlp/extractor/nosvideo.py

@@ -1,9 +1,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import Request
 from ..utils import (
     ExtractorError,
-    sanitized_Request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -36,8 +36,8 @@ class NosVideoIE(InfoExtractor):
             'op': 'download1',
             'method_free': 'Continue to Video',
         }
-        req = sanitized_Request(url, urlencode_postdata(fields))
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+        req = Request(url, urlencode_postdata(fields))
+        req.headers['Content-type'] = 'application/x-www-form-urlencoded'
         webpage = self._download_webpage(req, video_id,
                                          'Downloading download page')
         if re.search(self._FILE_DELETED_REGEX, webpage) is not None:

yt_dlp/extractor/nowness.py

@@ -4,10 +4,8 @@ from .brightcove import (
 )
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError
 
 
 class NownessBaseIE(InfoExtractor):
@@ -40,7 +38,7 @@ class NownessBaseIE(InfoExtractor):
 
     def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-        request = sanitized_Request(
+        request = Request(
             'http://api.nowness.com/api/' + request_path % display_id,
             headers={
                 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',

yt_dlp/extractor/nrk.py

@@ -3,7 +3,8 @@ import random
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -148,7 +149,7 @@ class NRKIE(NRKBaseIE):
             try:
                 return self._call_api(f'playback/{item}/program/{video_id}', video_id, item, query=query)
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                     return self._call_api(f'playback/{item}/{video_id}', video_id, item, query=query)
                 raise
 

yt_dlp/extractor/odkmedia.py

@@ -1,7 +1,7 @@
 import json
-import urllib.error
 
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     GeoRestrictedError,
@@ -74,8 +74,8 @@ class OnDemandChinaEpisodeIE(InfoExtractor):
                 f'https://odkmedia.io/odc/api/v2/playback/{video_info["id"]}/', display_id,
                 headers={'Authorization': '', 'service-name': 'odc'})
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError):
-                error_data = self._parse_json(e.cause.read(), display_id)['detail']
+            if isinstance(e.cause, HTTPError):
+                error_data = self._parse_json(e.cause.response.read(), display_id)['detail']
                 raise GeoRestrictedError(error_data)
 
         formats, subtitles = [], {}

yt_dlp/extractor/odnoklassniki.py

@@ -7,9 +7,9 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     float_or_none,
     int_or_none,
     qualities,
@@ -448,7 +448,7 @@ class OdnoklassnikiIE(InfoExtractor):
         json_data = self._parse_json(unescapeHTML(json_data), video_id) or {}
 
         redirect_url = self._request_webpage(HEADRequest(
-            json_data['videoSrc']), video_id, 'Requesting download URL').geturl()
+            json_data['videoSrc']), video_id, 'Requesting download URL').url
         self._clear_cookies(redirect_url)
 
         return {

yt_dlp/extractor/orf.py

@@ -2,11 +2,11 @@ import functools
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     clean_html,
     determine_ext,
     float_or_none,
-    HEADRequest,
     InAdvancePagedList,
     int_or_none,
     join_nonempty,

yt_dlp/extractor/owncloud.py

@@ -44,7 +44,7 @@ class OwnCloudIE(InfoExtractor):
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
         if re.search(r'<label[^>]+for="password"', webpage):
-            webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
+            webpage = self._verify_video_password(webpage, urlh.url, video_id)
 
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs.get('filename')

yt_dlp/extractor/packtpub.py

@@ -1,10 +1,7 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    # compat_str,
-    compat_HTTPError,
-)
+from ..networking.exceptions import HTTPError
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -54,8 +51,8 @@ class PacktPubIE(PacktPubBaseIE):
                     'password': password,
                 }).encode())['data']['access']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
-                message = self._parse_json(e.cause.read().decode(), None)['message']
+            if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 404):
+                message = self._parse_json(e.cause.response.read().decode(), None)['message']
                 raise ExtractorError(message, expected=True)
             raise
 
@@ -70,7 +67,7 @@ class PacktPubIE(PacktPubBaseIE):
                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
                 'Downloading JSON video', headers=headers)['data']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
                 self.raise_login_required('This video is locked')
             raise
 

yt_dlp/extractor/patreon.py

@@ -1,22 +1,22 @@
 import itertools
-from urllib.error import HTTPError
 
 from .common import InfoExtractor
 from .vimeo import VimeoIE
-
 from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
 from ..utils import (
+    KNOWN_EXTENSIONS,
+    ExtractorError,
     clean_html,
     determine_ext,
-    ExtractorError,
     int_or_none,
-    KNOWN_EXTENSIONS,
     mimetype2ext,
     parse_iso8601,
     str_or_none,
     traverse_obj,
     try_get,
     url_or_none,
+    urljoin,
 )
 
 
@@ -37,9 +37,9 @@ class PatreonBaseIE(InfoExtractor):
                 item_id, note='Downloading API JSON' if not note else note,
                 query=query, fatal=fatal, headers=headers)
         except ExtractorError as e:
-            if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.headers.get('Content-Type')) != 'json':
+            if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.response.headers.get('Content-Type')) != 'json':
                 raise
-            err_json = self._parse_json(self._webpage_read_content(e.cause, None, item_id), item_id, fatal=False)
+            err_json = self._parse_json(self._webpage_read_content(e.cause.response, None, item_id), item_id, fatal=False)
             err_message = traverse_obj(err_json, ('errors', ..., 'detail'), get_all=False)
             if err_message:
                 raise ExtractorError(f'Patreon said: {err_message}', expected=True)
@@ -404,8 +404,8 @@ class PatreonCampaignIE(PatreonBaseIE):
             posts_json = self._call_api('posts', campaign_id, query=params, note='Downloading posts page %d' % page)
 
             cursor = traverse_obj(posts_json, ('meta', 'pagination', 'cursors', 'next'))
-            for post in posts_json.get('data') or []:
-                yield self.url_result(url_or_none(traverse_obj(post, ('attributes', 'patreon_url'))), 'Patreon')
+            for post_url in traverse_obj(posts_json, ('data', ..., 'attributes', 'patreon_url')):
+                yield self.url_result(urljoin('https://www.patreon.com/', post_url), PatreonIE)
 
             if cursor is None:
                 break

yt_dlp/extractor/peloton.py

@@ -3,7 +3,7 @@ import re
 import urllib.parse
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -83,8 +83,8 @@ class PelotonIE(InfoExtractor):
                 }).encode(),
                 headers={'Content-Type': 'application/json', 'User-Agent': 'web'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                json_string = self._webpage_read_content(e.cause, None, video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                json_string = self._webpage_read_content(e.cause.response, None, video_id)
                 res = self._parse_json(json_string, video_id)
                 raise ExtractorError(res['message'], expected=res['message'] == 'Login failed')
             else:
@@ -96,8 +96,8 @@ class PelotonIE(InfoExtractor):
                 'https://api.onepeloton.com/api/subscription/stream', video_id, note='Downloading token',
                 data=json.dumps({}).encode(), headers={'Content-Type': 'application/json'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_string = self._webpage_read_content(e.cause, None, video_id)
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                json_string = self._webpage_read_content(e.cause.response, None, video_id)
                 res = self._parse_json(json_string, video_id)
                 raise ExtractorError(res['message'], expected=res['message'] == 'Stream limit reached')
             else:
@@ -109,7 +109,7 @@ class PelotonIE(InfoExtractor):
         try:
             self._start_session(video_id)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 self._login(video_id)
                 self._start_session(video_id)
             else:

yt_dlp/extractor/piapro.py

@@ -12,17 +12,22 @@ from ..utils import (
 
 class PiaproIE(InfoExtractor):
     _NETRC_MACHINE = 'piapro'
-    _VALID_URL = r'https?://piapro\.jp/t/(?P<id>\w+)/?'
+    _VALID_URL = r'https?://piapro\.jp/(?:t|content)/(?P<id>\w+)/?'
     _TESTS = [{
         'url': 'https://piapro.jp/t/NXYR',
-        'md5': 'a9d52f27d13bafab7ee34116a7dcfa77',
+        'md5': 'f7c0f760913fb1d44a1c45a4af793909',
         'info_dict': {
             'id': 'NXYR',
             'ext': 'mp3',
             'uploader': 'wowaka',
             'uploader_id': 'wowaka',
             'title': '裏表ラバーズ',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'http://www.nicovideo.jp/watch/sm8082467',
+            'duration': 189.0,
+            'timestamp': 1251785475,
+            'thumbnail': r're:^https?://.*\.(?:png|jpg)$',
+            'upload_date': '20090901',
+            'view_count': int,
         }
     }, {
         'note': 'There are break lines in description, mandating (?s) flag',
@@ -34,8 +39,16 @@ class PiaproIE(InfoExtractor):
             'title': '青に溶けた風船 / 初音ミク',
             'description': 'md5:d395a9bd151447631a5a1460bc7f9132',
             'uploader': 'シアン・キノ',
+            'duration': 229.0,
+            'timestamp': 1644030039,
+            'upload_date': '20220205',
+            'view_count': int,
+            'thumbnail': r're:^https?://.*\.(?:png|jpg)$',
             'uploader_id': 'cyankino',
         }
+    }, {
+        'url': 'https://piapro.jp/content/hcw0z3a169wtemz6',
+        'only_matching': True
     }]
 
     _login_status = False
@@ -56,7 +69,7 @@ class PiaproIE(InfoExtractor):
         if urlh is False:
             login_ok = False
         else:
-            parts = compat_urlparse.urlparse(urlh.geturl())
+            parts = compat_urlparse.urlparse(urlh.url)
             if parts.path != '/':
                 login_ok = False
         if not login_ok:

yt_dlp/extractor/pladform.py

@@ -78,7 +78,7 @@ class PladformIE(InfoExtractor):
                 expected=True)
 
         if not video:
-            targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').geturl()
+            targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').url
             if targetUrl == url:
                 raise ExtractorError('Can\'t parse page')
             return self.url_result(targetUrl)

yt_dlp/extractor/platzi.py

@@ -36,7 +36,7 @@ class PlatziBaseIE(InfoExtractor):
             headers={'Referer': self._LOGIN_URL})
 
         # login succeeded
-        if 'platzi.com/login' not in urlh.geturl():
+        if 'platzi.com/login' not in urlh.url:
             return
 
         login_error = self._webpage_read_content(

yt_dlp/extractor/playplustv.py

@@ -1,13 +1,9 @@
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
-from ..utils import (
-    clean_html,
-    ExtractorError,
-    int_or_none,
-    PUTRequest,
-)
+from ..networking import PUTRequest
+from ..networking.exceptions import HTTPError
+from ..utils import ExtractorError, clean_html, int_or_none
 
 
 class PlayPlusTVIE(InfoExtractor):
@@ -47,9 +43,9 @@ class PlayPlusTVIE(InfoExtractor):
         try:
             self._token = self._download_json(req, None)['token']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
                 raise ExtractorError(self._parse_json(
-                    e.cause.read(), None)['errorMessage'], expected=True)
+                    e.cause.response.read(), None)['errorMessage'], expected=True)
             raise
 
         self._profile = self._call_api('Profiles')['list'][0]['_id']

yt_dlp/extractor/pornhub.py

@@ -3,11 +3,12 @@ import itertools
 import math
 import operator
 import re
-import urllib.request
 
 from .common import InfoExtractor
 from .openload import PhantomJSwrapper
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking import Request
+from ..networking.exceptions import HTTPError
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -46,8 +47,8 @@ class PornHubBaseIE(InfoExtractor):
                 r'document\.cookie\s*=\s*["\']RNKEY=',
                 r'document\.location\.reload\(true\)')):
             url_or_request = args[0]
-            url = (url_or_request.get_full_url()
-                   if isinstance(url_or_request, urllib.request.Request)
+            url = (url_or_request.url
+                   if isinstance(url_or_request, Request)
                    else url_or_request)
             phantom = PhantomJSwrapper(self, required_version='2.0')
             phantom.get(url, html=webpage)
@@ -61,6 +62,7 @@ class PornHubBaseIE(InfoExtractor):
     def _set_age_cookies(self, host):
         self._set_cookie(host, 'age_verified', '1')
         self._set_cookie(host, 'accessAgeDisclaimerPH', '1')
+        self._set_cookie(host, 'accessAgeDisclaimerUK', '1')
         self._set_cookie(host, 'accessPH', '1')
 
     def _login(self, host):
@@ -602,7 +604,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
                 base_url, item_id, note, query={'page': num})
 
         def is_404(e):
-            return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
+            return isinstance(e.cause, HTTPError) and e.cause.status == 404
 
         base_url = url
         has_page = page is not None

yt_dlp/extractor/puhutv.py

@@ -1,8 +1,6 @@
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -72,7 +70,7 @@ class PuhuTVIE(InfoExtractor):
                 display_id, 'Downloading video JSON',
                 headers=self.geo_verification_headers())
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_geo_restricted()
             raise
 

yt_dlp/extractor/qdance.py

@@ -0,0 +1,150 @@
+import json
+import time
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    jwt_decode_hs256,
+    str_or_none,
+    traverse_obj,
+    try_call,
+    url_or_none,
+)
+
+
+class QDanceIE(InfoExtractor):
+    _NETRC_MACHINE = 'qdance'
+    _VALID_URL = r'https?://(?:www\.)?q-dance\.com/network/(?:library|live)/(?P<id>\d+)'
+    _TESTS = [{
+        'note': 'vod',
+        'url': 'https://www.q-dance.com/network/library/146542138',
+        'info_dict': {
+            'id': '146542138',
+            'ext': 'mp4',
+            'title': 'Sound Rush [LIVE] | Defqon.1 Weekend Festival 2022 | Friday | RED',
+            'display_id': 'sound-rush-live-v3-defqon-1-weekend-festival-2022-friday-red',
+            'description': 'Relive Defqon.1 - Primal Energy 2022 with the sounds of Sound Rush LIVE at the RED on Friday! 🔥',
+            'season': 'Defqon.1 Weekend Festival 2022',
+            'season_id': '31840632',
+            'series': 'Defqon.1',
+            'series_id': '31840378',
+            'thumbnail': 'https://images.q-dance.network/1674829540-20220624171509-220624171509_delio_dn201093-2.jpg',
+            'availability': 'premium_only',
+            'duration': 1829,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'note': 'livestream',
+        'url': 'https://www.q-dance.com/network/live/149170353',
+        'info_dict': {
+            'id': '149170353',
+            'ext': 'mp4',
+            'title': r're:^Defqon\.1 2023 - Friday - RED',
+            'display_id': 'defqon-1-2023-friday-red',
+            'description': 'md5:3c73fbbd4044e578e696adfc64019163',
+            'season': 'Defqon.1 Weekend Festival 2023',
+            'season_id': '141735599',
+            'series': 'Defqon.1',
+            'series_id': '31840378',
+            'thumbnail': 'https://images.q-dance.network/1686849069-area-thumbs_red.png',
+            'availability': 'subscriber_only',
+            'live_status': 'is_live',
+            'channel_id': 'qdancenetwork.video_149170353',
+        },
+        'skip': 'Completed livestream',
+    }]
+
+    _access_token = None
+    _refresh_token = None
+
+    def _call_login_api(self, data, note='Logging in'):
+        login = self._download_json(
+            'https://members.id-t.com/api/auth/login', None, note, headers={
+                'content-type': 'application/json',
+                'brand': 'qdance',
+                'origin': 'https://www.q-dance.com',
+                'referer': 'https://www.q-dance.com/',
+            }, data=json.dumps(data, separators=(',', ':')).encode(),
+            expected_status=lambda x: True)
+
+        tokens = traverse_obj(login, ('data', {
+            '_id-t-accounts-token': ('accessToken', {str}),
+            '_id-t-accounts-refresh': ('refreshToken', {str}),
+            '_id-t-accounts-id-token': ('idToken', {str}),
+        }))
+
+        if not tokens.get('_id-t-accounts-token'):
+            error = ': '.join(traverse_obj(login, ('error', ('code', 'message'), {str})))
+            if 'validation_error' not in error:
+                raise ExtractorError(f'Q-Dance API said "{error}"')
+            msg = 'Invalid username or password' if 'email' in data else 'Refresh token has expired'
+            raise ExtractorError(msg, expected=True)
+
+        for name, value in tokens.items():
+            self._set_cookie('.q-dance.com', name, value)
+
+    def _perform_login(self, username, password):
+        self._call_login_api({'email': username, 'password': password})
+
+    def _real_initialize(self):
+        cookies = self._get_cookies('https://www.q-dance.com/')
+        self._refresh_token = try_call(lambda: cookies['_id-t-accounts-refresh'].value)
+        self._access_token = try_call(lambda: cookies['_id-t-accounts-token'].value)
+        if not self._access_token:
+            self.raise_login_required()
+
+    def _get_auth(self):
+        if (try_call(lambda: jwt_decode_hs256(self._access_token)['exp']) or 0) <= int(time.time() - 120):
+            if not self._refresh_token:
+                raise ExtractorError(
+                    'Cannot refresh access token, login with yt-dlp or refresh cookies in browser')
+            self._call_login_api({'refreshToken': self._refresh_token}, note='Refreshing access token')
+            self._real_initialize()
+
+        return {'Authorization': self._access_token}
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nuxt_data(webpage, video_id, traverse=('data', 0, 'data'))
+
+        def extract_availability(level):
+            level = int_or_none(level) or 0
+            return self._availability(
+                needs_premium=(level >= 20), needs_subscription=(level >= 15), needs_auth=True)
+
+        info = traverse_obj(data, {
+            'title': ('title', {str.strip}),
+            'description': ('description', {str.strip}),
+            'display_id': ('slug', {str}),
+            'thumbnail': ('thumbnail', {url_or_none}),
+            'duration': ('durationInSeconds', {int_or_none}, {lambda x: x or None}),
+            'availability': ('subscription', 'level', {extract_availability}),
+            'is_live': ('type', {lambda x: x.lower() == 'live'}),
+            'artist': ('acts', ..., {str}),
+            'series': ('event', 'title', {str.strip}),
+            'series_id': ('event', 'id', {str_or_none}),
+            'season': ('eventEdition', 'title', {str.strip}),
+            'season_id': ('eventEdition', 'id', {str_or_none}),
+            'channel_id': ('pubnub', 'channelName', {str}),
+        })
+
+        stream = self._download_json(
+            f'https://dc9h6qmsoymbq.cloudfront.net/api/content/videos/{video_id}/url',
+            video_id, headers=self._get_auth(), expected_status=401)
+
+        m3u8_url = traverse_obj(stream, ('data', 'url', {url_or_none}))
+        if not m3u8_url and traverse_obj(stream, ('error', 'code')) == 'unauthorized':
+            raise ExtractorError('Your account does not have access to this content', expected=True)
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, fatal=False, live=True) if m3u8_url else []
+        if not formats:
+            self.raise_no_formats('No active streams found', expected=bool(info.get('is_live')))
+
+        return {
+            **info,
+            'id': video_id,
+            'formats': formats,
+        }

yt_dlp/extractor/radiko.py

@@ -41,7 +41,7 @@ class RadikoBaseIE(InfoExtractor):
                 'x-radiko-device': 'pc',
                 'x-radiko-user': 'dummy_user',
             })
-        auth1_header = auth1_handle.info()
+        auth1_header = auth1_handle.headers
 
         auth_token = auth1_header['X-Radiko-AuthToken']
         kl = int(auth1_header['X-Radiko-KeyLength'])

yt_dlp/extractor/radiocanada.py

@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -74,8 +74,8 @@ class RadioCanadaIE(InfoExtractor):
             return self._download_json(
                 'https://services.radio-canada.ca/media/' + path, video_id, query=query)
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 422):
-                data = self._parse_json(e.cause.read().decode(), None)
+            if isinstance(e.cause, HTTPError) and e.cause.status in (401, 422):
+                data = self._parse_json(e.cause.response.read().decode(), None)
                 error = data.get('error_description') or data['errorMessage']['text']
                 raise ExtractorError(error, expected=True)
             raise

yt_dlp/extractor/rcs.py

@@ -1,9 +1,9 @@
 import re
 
 from .common import InfoExtractor
+from ..networking import HEADRequest
 from ..utils import (
     ExtractorError,
-    HEADRequest,
     base_url,
     clean_html,
     extract_attributes,

yt_dlp/extractor/rcti.py

@@ -3,7 +3,7 @@ import random
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     dict_get,
     ExtractorError,
@@ -186,7 +186,7 @@ class RCTIPlusIE(RCTIPlusBaseIE):
         try:
             formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_geo_restricted(countries=['ID'], metadata_available=True)
             else:
                 raise e

yt_dlp/extractor/recurbate.py

@@ -1,6 +1,5 @@
-import urllib.error
-
 from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
 from ..utils import ExtractorError, merge_dicts
 
 
@@ -25,7 +24,7 @@ class RecurbateIE(InfoExtractor):
         try:
             webpage = self._download_webpage(url, video_id)
         except ExtractorError as e:
-            if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
             raise
         token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')

yt_dlp/extractor/redbulltv.py

@@ -1,5 +1,5 @@
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
 from ..utils import (
     float_or_none,
     ExtractorError,
@@ -68,9 +68,9 @@ class RedBullTVIE(InfoExtractor):
                 headers={'Authorization': token}
             )
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 404:
                 error_message = self._parse_json(
-                    e.cause.read().decode(), video_id)['error']
+                    e.cause.response.read().decode(), video_id)['error']
                 raise ExtractorError('%s said: %s' % (
                     self.IE_NAME, error_message), expected=True)
             raise

yt_dlp/extractor/redgifs.py

@@ -1,8 +1,8 @@
 import functools
-import urllib
 
 from .common import InfoExtractor
 from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -82,7 +82,7 @@ class RedGifsBaseInfoExtractor(InfoExtractor):
                     f'https://api.redgifs.com/v2/{ep}', video_id, headers=headers, *args, **kwargs)
                 break
             except ExtractorError as e:
-                if first_attempt and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+                if first_attempt and isinstance(e.cause, HTTPError) and e.cause.status == 401:
                     del self._API_HEADERS['authorization']  # refresh the token
                     continue
                 raise

yt_dlp/extractor/regiotv.py

@@ -1,10 +1,6 @@
 from .common import InfoExtractor
-
-from ..utils import (
-    sanitized_Request,
-    xpath_text,
-    xpath_with_ns,
-)
+from ..networking import Request
+from ..utils import xpath_text, xpath_with_ns
 
 
 class RegioTVIE(InfoExtractor):
@@ -33,7 +29,7 @@ class RegioTVIE(InfoExtractor):
 
         SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'
 
-        request = sanitized_Request(
+        request = Request(
             'http://v.telvi.de/',
             SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
         video_data = self._download_xml(request, video_id, 'Downloading video XML')

yt_dlp/extractor/rokfin.py

@@ -245,7 +245,7 @@ class RokfinIE(InfoExtractor):
             f'{self._AUTH_BASE}/token', None,
             note='getting access credentials', errnote='error getting access credentials',
             data=urlencode_postdata({
-                'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.geturl()).fragment).get('code')[0],
+                'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.url).fragment).get('code')[0],
                 'client_id': 'web',
                 'grant_type': 'authorization_code',
                 'redirect_uri': 'https://rokfin.com/silent-check-sso.html'
@@ -269,7 +269,7 @@ class RokfinIE(InfoExtractor):
 
         json_string, urlh = self._download_webpage_handle(
             url_or_request, video_id, headers=headers, query=query, expected_status=401)
-        if not auth_token or urlh.code != 401 or refresh_token is None:
+        if not auth_token or urlh.status != 401 or refresh_token is None:
             return self._parse_json(json_string, video_id)
 
         self._access_mgmt_tokens = self._download_json(