From db96683cf12194fdf09d4762cdcf0780df067003 Mon Sep 17 00:00:00 2001 From: nixxo Date: Thu, 15 Dec 2022 16:46:37 +0100 Subject: [PATCH 1/6] [core] _VALID_URLS implementation - implemented _VALID_URLS extractor property - improvement suggested by @Grub4k --- devscripts/make_lazy_extractors.py | 2 +- yt_dlp/extractor/common.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index c502bdf896..e763f5484f 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -14,7 +14,7 @@ from devscripts.utils import get_filename_args, read_file, write_file NO_ATTR = object() STATIC_CLASS_PROPERTIES = [ - 'IE_NAME', '_ENABLED', '_VALID_URL', # Used for URL matching + 'IE_NAME', '_ENABLED', '_VALID_URL', '_VALID_URLS', # Used for URL matching '_WORKING', 'IE_DESC', '_NETRC_MACHINE', 'SEARCH_KEY', # Used for --extractor-descriptions 'age_limit', # Used for --age-limit (evaluated) '_RETURN_TYPE', # Accessed in CLI only with instance (evaluated) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 3910c55adb..11fc959207 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -447,7 +447,7 @@ class InfoExtractor: Subclasses of this should also be added to the list of extractors and - should define a _VALID_URL regexp and, re-define the _real_extract() and + should define a _VALID_URL regexp (or a list of _VALID_URLS) and, re-define the _real_extract() and (optionally) _real_initialize() methods. Subclasses may also override suitable() if necessary, but ensure the function @@ -508,6 +508,7 @@ class InfoExtractor: IE_DESC = None SEARCH_KEY = None _VALID_URL = None + _VALID_URLS = [] _EMBED_REGEX = [] def _login_hint(self, method=NO_DEFAULT, netrc=None): @@ -534,6 +535,13 @@ class InfoExtractor: def _match_valid_url(cls, url): if cls._VALID_URL is False: return None + + if cls._VALID_URLS: + if '_VALID_URLS_RE' not in cls.__dict__: + cls._VALID_URLS_RE = tuple(map(re.compile, cls._VALID_URLS)) + return next(filter(None, ( + valid_url_re.match(url) for valid_url_re in cls._VALID_URLS_RE)), None) + # This does not use has/getattr intentionally - we want to know whether # we have cached the regexp for *this* class, whereas getattr would also # match the superclass From 74b5d3479497a54cebef34bdcf790935e93e92d0 Mon Sep 17 00:00:00 2001 From: nixxo Date: Fri, 16 Dec 2022 11:18:27 +0100 Subject: [PATCH 2/6] [extractor/la7] testing implementation of new _VALID_URLS property --- yt_dlp/extractor/la7.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py index 36bfaf5c30..5eeca98097 100644 --- a/yt_dlp/extractor/la7.py +++ b/yt_dlp/extractor/la7.py @@ -12,10 +12,11 @@ from ..utils import ( class LA7IE(InfoExtractor): IE_NAME = 'la7.it' - _VALID_URL = r'''(?x)https?://(?: - (?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video|news)/| - tg\.la7\.it/repliche-tgla7\?id= - )(?P.+)''' + _VALID_URLS = [ + r'https?://(?:www\.)?la7\.it/[^/]+/(?:rivedila7|video|news)/.+-(?P\d{5,})', + r'https?://tg\.la7\.it/repliche-tgla7\?id=(?P\d{5,})', + r'https?://tg\.la7\.it(?:/[^/]+)+-(?P\d{5,})' + ] _TESTS = [{ # single quality video @@ -44,7 +45,7 @@ class LA7IE(InfoExtractor): 'formats': 'count:8', }, }, { - 'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077', + 'url': 'https://tg.la7.it/repliche-tgla7?id=464601', 'only_matching': True, }] _HOST = 'https://awsvodpkg.iltrovatore.it' From 7b93fb5ddccfaed77b986f3f0aa6b60f7c25245b Mon Sep 17 00:00:00 2001 From: nixxo Date: Fri, 16 Dec 2022 11:20:03 +0100 Subject: [PATCH 3/6] [extractor/rai] using new _VALID_URLS property - implemented _VALID_URLS property - deleted a couple of subclasses - added '_old_archive_ids' to manage back-compatibility with removed subclasses (by @pukkandan) --- yt_dlp/extractor/_extractors.py | 2 - yt_dlp/extractor/rai.py | 94 ++++++++++++++++----------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index a12328f04a..1677d38e25 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1479,10 +1479,8 @@ from .radlive import ( ) from .rai import ( RaiPlayIE, - RaiPlayLiveIE, RaiPlayPlaylistIE, RaiPlaySoundIE, - RaiPlaySoundLiveIE, RaiPlaySoundPlaylistIE, RaiNewsIE, RaiSudtirolIE, diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py index cab12cc214..a84d63fa0a 100644 --- a/yt_dlp/extractor/rai.py +++ b/yt_dlp/extractor/rai.py @@ -16,6 +16,7 @@ from ..utils import ( HEADRequest, int_or_none, join_nonempty, + make_archive_id, parse_duration, remove_start, strip_or_none, @@ -234,7 +235,10 @@ class RaiBaseIE(InfoExtractor): class RaiPlayIE(RaiBaseIE): - _VALID_URL = rf'(?Phttps?://(?:www\.)?raiplay\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)' + _VALID_URLS = [ + rf'(?Phttps?://(?:www\.)?raiplay\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)', + r'(?Phttps?://(?:www\.)?raiplay\.it/dirette/(?P[^/?#&]+))', + ] _TESTS = [{ 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', 'md5': '8970abf8caf8aef4696e7b1f2adfc696', @@ -282,6 +286,25 @@ class RaiPlayIE(RaiBaseIE): 'timestamp': 1637318940, 'upload_date': '20211119', }, + }, { + # live stream + 'url': 'https://www.raiplay.it/dirette/rainews24', + 'info_dict': { + 'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c', + 'display_id': 'rainews24', + 'ext': 'mp4', + 'title': r're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', + 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497', + 'uploader': 'Rai News 24', + 'creator': 'Rai News 24', + 'is_live': True, + 'live_status': 'is_live', + 'upload_date': '20090502', + 'timestamp': 1241276220, + }, + 'params': { + 'skip_download': True, + }, }, { 'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?', 'only_matching': True, @@ -336,6 +359,7 @@ class RaiPlayIE(RaiBaseIE): return { 'id': remove_start(media.get('id'), 'ContentItem-') or video_id, 'display_id': video_id, + '_old_archive_ids': [make_archive_id('RaiPlayLive', video_id)] if not re.match(RaiBaseIE._UUID_RE, video_id) else None, 'title': title, 'alt_title': strip_or_none(alt_title or None), 'description': media.get('description'), @@ -355,29 +379,6 @@ class RaiPlayIE(RaiBaseIE): } -class RaiPlayLiveIE(RaiPlayIE): # XXX: Do not subclass from concrete IE - _VALID_URL = r'(?Phttps?://(?:www\.)?raiplay\.it/dirette/(?P[^/?#&]+))' - _TESTS = [{ - 'url': 'http://www.raiplay.it/dirette/rainews24', - 'info_dict': { - 'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c', - 'display_id': 'rainews24', - 'ext': 'mp4', - 'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', - 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497', - 'uploader': 'Rai News 24', - 'creator': 'Rai News 24', - 'is_live': True, - 'live_status': 'is_live', - 'upload_date': '20090502', - 'timestamp': 1241276220, - }, - 'params': { - 'skip_download': True, - }, - }] - - class RaiPlayPlaylistIE(InfoExtractor): _VALID_URL = r'(?Phttps?://(?:www\.)?raiplay\.it/programmi/(?P[^/?#&]+))(?:/(?P[^?#&]+))?' _TESTS = [{ @@ -440,7 +441,10 @@ class RaiPlayPlaylistIE(InfoExtractor): class RaiPlaySoundIE(RaiBaseIE): - _VALID_URL = rf'(?Phttps?://(?:www\.)?raiplaysound\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)' + _VALID_URLS = [ + rf'(?Phttps?://(?:www\.)?raiplaysound\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)', + r'(?Phttps?://(?:www\.)?raiplaysound\.it/(?P[^/?#&]+)$)', + ] _TESTS = [{ 'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html', 'md5': '8970abf8caf8aef4696e7b1f2adfc696', @@ -462,6 +466,23 @@ class RaiPlaySoundIE(RaiBaseIE): 'params': { 'skip_download': True, }, + }, { + 'url': 'https://www.raiplaysound.it/radio2', + 'info_dict': { + 'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44', + 'display_id': 'radio2', + 'ext': 'mp4', + 'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+', + 'thumbnail': r're:https://www\.raiplaysound\.it/dl/img/.+png', + 'uploader': 'rai radio 2', + 'series': 'Rai Radio 2', + 'creator': 'raiplaysound', + 'is_live': True, + 'live_status': 'is_live', + }, + 'params': { + 'skip_download': 'live', + }, }] def _real_extract(self, url): @@ -488,6 +509,7 @@ class RaiPlaySoundIE(RaiBaseIE): **info, 'id': uid or audio_id, 'display_id': audio_id, + '_old_archive_ids': [make_archive_id('RaiPlaySoundLive', audio_id)] if not re.match(RaiBaseIE._UUID_RE, audio_id) else None, 'title': traverse_obj(media, 'title', 'episode_title'), 'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none), 'description': media.get('description'), @@ -503,28 +525,6 @@ class RaiPlaySoundIE(RaiBaseIE): } -class RaiPlaySoundLiveIE(RaiPlaySoundIE): # XXX: Do not subclass from concrete IE - _VALID_URL = r'(?Phttps?://(?:www\.)?raiplaysound\.it/(?P[^/?#&]+)$)' - _TESTS = [{ - 'url': 'https://www.raiplaysound.it/radio2', - 'info_dict': { - 'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44', - 'display_id': 'radio2', - 'ext': 'mp4', - 'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+', - 'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png', - 'uploader': 'rai radio 2', - 'series': 'Rai Radio 2', - 'creator': 'raiplaysound', - 'is_live': True, - 'live_status': 'is_live', - }, - 'params': { - 'skip_download': 'live', - }, - }] - - class RaiPlaySoundPlaylistIE(InfoExtractor): _VALID_URL = r'(?Phttps?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P[^/?#&]+))(?:/(?P[^?#&]+))?' _TESTS = [{ From 42d0fba2bf8e89bba266e5c59410dd6b1866de16 Mon Sep 17 00:00:00 2001 From: nixxo Date: Fri, 30 Dec 2022 23:55:32 +0100 Subject: [PATCH 4/6] Implemented solution suggested by @dirkf using variadic --- devscripts/lazy_load_template.py | 1 + devscripts/make_lazy_extractors.py | 2 +- yt_dlp/extractor/common.py | 14 ++++---------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index c8815e01bc..6f52165c5c 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -6,6 +6,7 @@ from ..utils import ( age_restricted, bug_reports_message, classproperty, + variadic, write_string, ) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index e763f5484f..c502bdf896 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -14,7 +14,7 @@ from devscripts.utils import get_filename_args, read_file, write_file NO_ATTR = object() STATIC_CLASS_PROPERTIES = [ - 'IE_NAME', '_ENABLED', '_VALID_URL', '_VALID_URLS', # Used for URL matching + 'IE_NAME', '_ENABLED', '_VALID_URL', # Used for URL matching '_WORKING', 'IE_DESC', '_NETRC_MACHINE', 'SEARCH_KEY', # Used for --extractor-descriptions 'age_limit', # Used for --age-limit (evaluated) '_RETURN_TYPE', # Accessed in CLI only with instance (evaluated) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 11fc959207..9586745744 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -447,7 +447,7 @@ class InfoExtractor: Subclasses of this should also be added to the list of extractors and - should define a _VALID_URL regexp (or a list of _VALID_URLS) and, re-define the _real_extract() and + should define a _VALID_URL regexp (a single string or a list) and, re-define the _real_extract() and (optionally) _real_initialize() methods. Subclasses may also override suitable() if necessary, but ensure the function @@ -508,7 +508,6 @@ class InfoExtractor: IE_DESC = None SEARCH_KEY = None _VALID_URL = None - _VALID_URLS = [] _EMBED_REGEX = [] def _login_hint(self, method=NO_DEFAULT, netrc=None): @@ -536,18 +535,13 @@ class InfoExtractor: if cls._VALID_URL is False: return None - if cls._VALID_URLS: - if '_VALID_URLS_RE' not in cls.__dict__: - cls._VALID_URLS_RE = tuple(map(re.compile, cls._VALID_URLS)) - return next(filter(None, ( - valid_url_re.match(url) for valid_url_re in cls._VALID_URLS_RE)), None) - # This does not use has/getattr intentionally - we want to know whether # we have cached the regexp for *this* class, whereas getattr would also # match the superclass if '_VALID_URL_RE' not in cls.__dict__: - cls._VALID_URL_RE = re.compile(cls._VALID_URL) - return cls._VALID_URL_RE.match(url) + cls._VALID_URL_RE = tuple(map(re.compile, variadic(cls._VALID_URL))) + return next(filter(None, ( + valid_url_re.match(url) for valid_url_re in cls._VALID_URL_RE)), None) @classmethod def suitable(cls, url): From 518d585eb72e95a701509bff1c2a54ba982755d9 Mon Sep 17 00:00:00 2001 From: nixxo Date: Fri, 30 Dec 2022 23:56:09 +0100 Subject: [PATCH 5/6] _VALID_URLS > _VALID_URL --- yt_dlp/extractor/la7.py | 2 +- yt_dlp/extractor/rai.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py index 5eeca98097..c178de59a2 100644 --- a/yt_dlp/extractor/la7.py +++ b/yt_dlp/extractor/la7.py @@ -12,7 +12,7 @@ from ..utils import ( class LA7IE(InfoExtractor): IE_NAME = 'la7.it' - _VALID_URLS = [ + _VALID_URL = [ r'https?://(?:www\.)?la7\.it/[^/]+/(?:rivedila7|video|news)/.+-(?P\d{5,})', r'https?://tg\.la7\.it/repliche-tgla7\?id=(?P\d{5,})', r'https?://tg\.la7\.it(?:/[^/]+)+-(?P\d{5,})' diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py index a84d63fa0a..527b036151 100644 --- a/yt_dlp/extractor/rai.py +++ b/yt_dlp/extractor/rai.py @@ -235,7 +235,7 @@ class RaiBaseIE(InfoExtractor): class RaiPlayIE(RaiBaseIE): - _VALID_URLS = [ + _VALID_URL = [ rf'(?Phttps?://(?:www\.)?raiplay\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)', r'(?Phttps?://(?:www\.)?raiplay\.it/dirette/(?P[^/?#&]+))', ] @@ -441,7 +441,7 @@ class RaiPlayPlaylistIE(InfoExtractor): class RaiPlaySoundIE(RaiBaseIE): - _VALID_URLS = [ + _VALID_URL = [ rf'(?Phttps?://(?:www\.)?raiplaysound\.it/.+?-(?P{RaiBaseIE._UUID_RE}))\.(?:html|json)', r'(?Phttps?://(?:www\.)?raiplaysound\.it/(?P[^/?#&]+)$)', ] From a73ef8c77671ff08742dad337716392ad3c96568 Mon Sep 17 00:00:00 2001 From: nixxo Date: Thu, 12 Jan 2023 21:37:07 +0100 Subject: [PATCH 6/6] Improved documentation for _VALID_URL Co-authored-by: dirkf --- yt_dlp/extractor/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 9586745744..a4483576e7 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -447,8 +447,11 @@ class InfoExtractor: Subclasses of this should also be added to the list of extractors and - should define a _VALID_URL regexp (a single string or a list) and, re-define the _real_extract() and - (optionally) _real_initialize() methods. + should define _VALID_URL as a regexp, or list (or tuple) of regexps, and + re-define the _real_extract() and (optionally) _real_initialize() methods. + The list syntax for _VALID_URL simplifies matching disparate URL patterns + where conditional groups or subclass IEs would be needed to target the + different locations of the ID field with a single regexp. Subclasses may also override suitable() if necessary, but ensure the function signature is preserved and that this function imports everything it needs