feat(multimodal): re-implement audio/video support on main

Re-implements PR #2908's generalized multimodal support on the
post-FSM main branch:

- Audio input: detects WAV/MP3/OGG/FLAC via magic bytes, sends
  input_audio blocks to compatible providers, falls back to
  [audio: path] placeholder when unsupported.
- Video input: sends video_url data-URI blocks to compatible
  providers, falls back to [video: path] placeholder.
- InputLimitsConfig: count limits (images/audios/videos) and byte
  limits per media type.
- AgentDefaults: pattern-matched vision_models, audio_models,
  video_models with supports_*() helpers.
- Provider retry: strips all media types (image_url, input_audio,
  video_url) on non-transient errors and retries once.
- Feishu: extracts media tags from post messages.
- Anthropic & OpenAI Responses converters handle audio/video.

Tests: 17 new multimodal tests + existing suite passes.
This commit is contained in:
chengyongru
2026-05-20 11:47:01 +08:00
parent 1391aa3d57
commit 0e754d2591
11 changed files with 625 additions and 71 deletions
+22 -2
View File
@@ -27,10 +27,11 @@ def test_extract_post_content_supports_post_wrapper_shape() -> None:
}
}
text, image_keys = _extract_post_content(payload)
text, image_keys, media_items = _extract_post_content(payload)
assert text == "日报 完成"
assert image_keys == ["img_1"]
assert media_items == []
def test_extract_post_content_keeps_direct_shape_behavior() -> None:
@@ -45,10 +46,29 @@ def test_extract_post_content_keeps_direct_shape_behavior() -> None:
],
}
text, image_keys = _extract_post_content(payload)
text, image_keys, media_items = _extract_post_content(payload)
assert text == "Daily report"
assert image_keys == ["img_a", "img_b"]
assert media_items == []
def test_extract_post_content_extracts_media_tags() -> None:
payload = {
"title": "Video",
"content": [
[
{"tag": "text", "text": "see this"},
{"tag": "media", "file_key": "vid_1"},
]
],
}
text, image_keys, media_items = _extract_post_content(payload)
assert text == "Video see this"
assert image_keys == []
assert media_items == [{"tag": "media", "file_key": "vid_1"}]
def test_register_optional_event_keeps_builder_when_method_missing() -> None: