fix(channels): harden message split boundaries

This commit is contained in:
Xubin Ren
2026-09-04 01:39:41 +08:00
parent 8509432dcf
commit b04aeeac0e
3 changed files with 85 additions and 5 deletions
@@ -393,6 +393,14 @@ def test_partition_styles_keeps_offset_in_indented_chunk():
assert _partition_styles(plain, chunks, ["11:4:BOLD"]) == [[], ["4:4:BOLD"]]
def test_partition_styles_keeps_offset_after_crlf_boundary():
plain = "header\r\n code"
chunks = split_message(plain, 10)
assert chunks == ["header", " code"]
assert _partition_styles(plain, chunks, ["12:4:BOLD"]) == [[], ["4:4:BOLD"]]
def test_partition_styles_long_message_preserves_chunk_one_styles():
"""A bold span deep in the message must follow the message into chunk 1."""
# Two ~30-char paragraphs separated by a blank line, then **tail**.
+35 -5
View File
@@ -633,6 +633,7 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
return [content]
if len(content) <= max_len:
return [content]
original_content = content
chunks: list[str] = []
while content:
if len(content) <= max_len:
@@ -642,8 +643,14 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
cut = content[:max_len]
# Consume only the newline itself so indentation starts the next chunk.
newline_pos = cut.rfind("\n")
if newline_pos > 0:
chunks.append(content[:newline_pos])
if newline_pos >= 0:
# Exclude both bytes of a CRLF boundary from the emitted chunk.
line_end = newline_pos
if line_end > 0 and content[line_end - 1] == "\r":
line_end -= 1
chunk = content[:line_end]
if chunk.strip():
chunks.append(chunk)
content = content[newline_pos + 1 :]
continue
@@ -652,11 +659,31 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
space_pos = cut.rfind(" ")
if space_pos > 0 and cut[:space_pos].strip():
chunks.append(content[:space_pos])
content = content[space_pos:].lstrip()
content = content[space_pos:].lstrip(" \t")
# A space boundary may sit immediately before a line break. Drop
# that delimiter too, without stripping the next line's indent.
if content.startswith("\r\n"):
content = content[2:]
elif content.startswith("\n"):
content = content[1:]
continue
chunks.append(content[:max_len])
# Do not split between the two code points of a CRLF delimiter.
if cut.endswith("\r") and content[max_len : max_len + 1] == "\n":
chunk = cut[:-1]
if chunk.strip():
chunks.append(chunk)
content = content[max_len + 1 :]
continue
chunk = content[:max_len]
if chunk.strip():
chunks.append(chunk)
content = content[max_len:]
if not chunk.strip():
# Keep any remaining indentation so the final non-blank chunk can
# retain as much of it as the channel limit permits.
continue
# A delimiter can sit immediately after the hard-break boundary. Keep
# ordinary space trimming, but consume only the newline so indentation
# on the following line is preserved.
@@ -665,7 +692,10 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
content = content[2:]
elif content.startswith("\n"):
content = content[1:]
return chunks
# Preserve the historical non-empty-input contract for callers that take
# the first chunk directly. This fallback is only reachable for content
# made entirely of whitespace.
return chunks or [original_content[:max_len]]
def build_assistant_message(
+42
View File
@@ -39,6 +39,42 @@ def test_split_message_preserves_indentation_when_newline_is_at_hard_break():
]
def test_split_message_handles_crlf_before_hard_break():
content = "header\r\n indented code"
assert split_message(content, max_len=18) == ["header", " indented code"]
assert split_message("abcdefg\r\n code", max_len=8) == [
"abcdefg",
" code",
]
def test_split_message_preserves_indent_after_space_then_newline_boundary():
content = "abcdef \n code"
assert split_message(content, max_len=7) == ["abcdef", " cod", "e"]
assert split_message(content.replace("\n", "\r\n"), max_len=7) == [
"abcdef",
" cod",
"e",
]
def test_split_message_drops_blank_chunks_from_long_indentation():
content = "head\n" + " " * 20 + "x"
chunks = split_message(content, max_len=8)
assert chunks == ["head", " x"]
assert all(chunk.strip() for chunk in chunks)
def test_split_message_drops_whitespace_only_line_at_boundary():
content = " \nhello world"
assert split_message(content, max_len=8) == ["hello", "world"]
def test_split_message_drops_whitespace_only_tail_after_hard_break():
prefix = "abcdefgh"
@@ -46,6 +82,12 @@ def test_split_message_drops_whitespace_only_tail_after_hard_break():
assert split_message(prefix + " ", max_len=8) == [prefix]
def test_split_message_keeps_one_chunk_for_all_whitespace_input():
content = " " * 10
assert split_message(content, max_len=4) == [" " * 4]
def test_split_message_nonpositive_maxlen_returns_unsplit():
content = "alpha beta gamma delta"