mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-09-04 02:01:48 +03:00
fix(channels): harden message split boundaries
This commit is contained in:
@@ -393,6 +393,14 @@ def test_partition_styles_keeps_offset_in_indented_chunk():
|
||||
assert _partition_styles(plain, chunks, ["11:4:BOLD"]) == [[], ["4:4:BOLD"]]
|
||||
|
||||
|
||||
def test_partition_styles_keeps_offset_after_crlf_boundary():
|
||||
plain = "header\r\n code"
|
||||
chunks = split_message(plain, 10)
|
||||
|
||||
assert chunks == ["header", " code"]
|
||||
assert _partition_styles(plain, chunks, ["12:4:BOLD"]) == [[], ["4:4:BOLD"]]
|
||||
|
||||
|
||||
def test_partition_styles_long_message_preserves_chunk_one_styles():
|
||||
"""A bold span deep in the message must follow the message into chunk 1."""
|
||||
# Two ~30-char paragraphs separated by a blank line, then **tail**.
|
||||
|
||||
@@ -633,6 +633,7 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
||||
return [content]
|
||||
if len(content) <= max_len:
|
||||
return [content]
|
||||
original_content = content
|
||||
chunks: list[str] = []
|
||||
while content:
|
||||
if len(content) <= max_len:
|
||||
@@ -642,8 +643,14 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
||||
cut = content[:max_len]
|
||||
# Consume only the newline itself so indentation starts the next chunk.
|
||||
newline_pos = cut.rfind("\n")
|
||||
if newline_pos > 0:
|
||||
chunks.append(content[:newline_pos])
|
||||
if newline_pos >= 0:
|
||||
# Exclude both bytes of a CRLF boundary from the emitted chunk.
|
||||
line_end = newline_pos
|
||||
if line_end > 0 and content[line_end - 1] == "\r":
|
||||
line_end -= 1
|
||||
chunk = content[:line_end]
|
||||
if chunk.strip():
|
||||
chunks.append(chunk)
|
||||
content = content[newline_pos + 1 :]
|
||||
continue
|
||||
|
||||
@@ -652,11 +659,31 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
||||
space_pos = cut.rfind(" ")
|
||||
if space_pos > 0 and cut[:space_pos].strip():
|
||||
chunks.append(content[:space_pos])
|
||||
content = content[space_pos:].lstrip()
|
||||
content = content[space_pos:].lstrip(" \t")
|
||||
# A space boundary may sit immediately before a line break. Drop
|
||||
# that delimiter too, without stripping the next line's indent.
|
||||
if content.startswith("\r\n"):
|
||||
content = content[2:]
|
||||
elif content.startswith("\n"):
|
||||
content = content[1:]
|
||||
continue
|
||||
|
||||
chunks.append(content[:max_len])
|
||||
# Do not split between the two code points of a CRLF delimiter.
|
||||
if cut.endswith("\r") and content[max_len : max_len + 1] == "\n":
|
||||
chunk = cut[:-1]
|
||||
if chunk.strip():
|
||||
chunks.append(chunk)
|
||||
content = content[max_len + 1 :]
|
||||
continue
|
||||
|
||||
chunk = content[:max_len]
|
||||
if chunk.strip():
|
||||
chunks.append(chunk)
|
||||
content = content[max_len:]
|
||||
if not chunk.strip():
|
||||
# Keep any remaining indentation so the final non-blank chunk can
|
||||
# retain as much of it as the channel limit permits.
|
||||
continue
|
||||
# A delimiter can sit immediately after the hard-break boundary. Keep
|
||||
# ordinary space trimming, but consume only the newline so indentation
|
||||
# on the following line is preserved.
|
||||
@@ -665,7 +692,10 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
||||
content = content[2:]
|
||||
elif content.startswith("\n"):
|
||||
content = content[1:]
|
||||
return chunks
|
||||
# Preserve the historical non-empty-input contract for callers that take
|
||||
# the first chunk directly. This fallback is only reachable for content
|
||||
# made entirely of whitespace.
|
||||
return chunks or [original_content[:max_len]]
|
||||
|
||||
|
||||
def build_assistant_message(
|
||||
|
||||
@@ -39,6 +39,42 @@ def test_split_message_preserves_indentation_when_newline_is_at_hard_break():
|
||||
]
|
||||
|
||||
|
||||
def test_split_message_handles_crlf_before_hard_break():
|
||||
content = "header\r\n indented code"
|
||||
|
||||
assert split_message(content, max_len=18) == ["header", " indented code"]
|
||||
assert split_message("abcdefg\r\n code", max_len=8) == [
|
||||
"abcdefg",
|
||||
" code",
|
||||
]
|
||||
|
||||
|
||||
def test_split_message_preserves_indent_after_space_then_newline_boundary():
|
||||
content = "abcdef \n code"
|
||||
|
||||
assert split_message(content, max_len=7) == ["abcdef", " cod", "e"]
|
||||
assert split_message(content.replace("\n", "\r\n"), max_len=7) == [
|
||||
"abcdef",
|
||||
" cod",
|
||||
"e",
|
||||
]
|
||||
|
||||
|
||||
def test_split_message_drops_blank_chunks_from_long_indentation():
|
||||
content = "head\n" + " " * 20 + "x"
|
||||
|
||||
chunks = split_message(content, max_len=8)
|
||||
|
||||
assert chunks == ["head", " x"]
|
||||
assert all(chunk.strip() for chunk in chunks)
|
||||
|
||||
|
||||
def test_split_message_drops_whitespace_only_line_at_boundary():
|
||||
content = " \nhello world"
|
||||
|
||||
assert split_message(content, max_len=8) == ["hello", "world"]
|
||||
|
||||
|
||||
def test_split_message_drops_whitespace_only_tail_after_hard_break():
|
||||
prefix = "abcdefgh"
|
||||
|
||||
@@ -46,6 +82,12 @@ def test_split_message_drops_whitespace_only_tail_after_hard_break():
|
||||
assert split_message(prefix + " ", max_len=8) == [prefix]
|
||||
|
||||
|
||||
def test_split_message_keeps_one_chunk_for_all_whitespace_input():
|
||||
content = " " * 10
|
||||
|
||||
assert split_message(content, max_len=4) == [" " * 4]
|
||||
|
||||
|
||||
def test_split_message_nonpositive_maxlen_returns_unsplit():
|
||||
content = "alpha beta gamma delta"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user