mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-09-04 10:11:46 +03:00
fix(channels): harden message split boundaries
This commit is contained in:
@@ -393,6 +393,14 @@ def test_partition_styles_keeps_offset_in_indented_chunk():
|
|||||||
assert _partition_styles(plain, chunks, ["11:4:BOLD"]) == [[], ["4:4:BOLD"]]
|
assert _partition_styles(plain, chunks, ["11:4:BOLD"]) == [[], ["4:4:BOLD"]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_partition_styles_keeps_offset_after_crlf_boundary():
|
||||||
|
plain = "header\r\n code"
|
||||||
|
chunks = split_message(plain, 10)
|
||||||
|
|
||||||
|
assert chunks == ["header", " code"]
|
||||||
|
assert _partition_styles(plain, chunks, ["12:4:BOLD"]) == [[], ["4:4:BOLD"]]
|
||||||
|
|
||||||
|
|
||||||
def test_partition_styles_long_message_preserves_chunk_one_styles():
|
def test_partition_styles_long_message_preserves_chunk_one_styles():
|
||||||
"""A bold span deep in the message must follow the message into chunk 1."""
|
"""A bold span deep in the message must follow the message into chunk 1."""
|
||||||
# Two ~30-char paragraphs separated by a blank line, then **tail**.
|
# Two ~30-char paragraphs separated by a blank line, then **tail**.
|
||||||
|
|||||||
@@ -633,6 +633,7 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
|||||||
return [content]
|
return [content]
|
||||||
if len(content) <= max_len:
|
if len(content) <= max_len:
|
||||||
return [content]
|
return [content]
|
||||||
|
original_content = content
|
||||||
chunks: list[str] = []
|
chunks: list[str] = []
|
||||||
while content:
|
while content:
|
||||||
if len(content) <= max_len:
|
if len(content) <= max_len:
|
||||||
@@ -642,8 +643,14 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
|||||||
cut = content[:max_len]
|
cut = content[:max_len]
|
||||||
# Consume only the newline itself so indentation starts the next chunk.
|
# Consume only the newline itself so indentation starts the next chunk.
|
||||||
newline_pos = cut.rfind("\n")
|
newline_pos = cut.rfind("\n")
|
||||||
if newline_pos > 0:
|
if newline_pos >= 0:
|
||||||
chunks.append(content[:newline_pos])
|
# Exclude both bytes of a CRLF boundary from the emitted chunk.
|
||||||
|
line_end = newline_pos
|
||||||
|
if line_end > 0 and content[line_end - 1] == "\r":
|
||||||
|
line_end -= 1
|
||||||
|
chunk = content[:line_end]
|
||||||
|
if chunk.strip():
|
||||||
|
chunks.append(chunk)
|
||||||
content = content[newline_pos + 1 :]
|
content = content[newline_pos + 1 :]
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -652,11 +659,31 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
|||||||
space_pos = cut.rfind(" ")
|
space_pos = cut.rfind(" ")
|
||||||
if space_pos > 0 and cut[:space_pos].strip():
|
if space_pos > 0 and cut[:space_pos].strip():
|
||||||
chunks.append(content[:space_pos])
|
chunks.append(content[:space_pos])
|
||||||
content = content[space_pos:].lstrip()
|
content = content[space_pos:].lstrip(" \t")
|
||||||
|
# A space boundary may sit immediately before a line break. Drop
|
||||||
|
# that delimiter too, without stripping the next line's indent.
|
||||||
|
if content.startswith("\r\n"):
|
||||||
|
content = content[2:]
|
||||||
|
elif content.startswith("\n"):
|
||||||
|
content = content[1:]
|
||||||
continue
|
continue
|
||||||
|
|
||||||
chunks.append(content[:max_len])
|
# Do not split between the two code points of a CRLF delimiter.
|
||||||
|
if cut.endswith("\r") and content[max_len : max_len + 1] == "\n":
|
||||||
|
chunk = cut[:-1]
|
||||||
|
if chunk.strip():
|
||||||
|
chunks.append(chunk)
|
||||||
|
content = content[max_len + 1 :]
|
||||||
|
continue
|
||||||
|
|
||||||
|
chunk = content[:max_len]
|
||||||
|
if chunk.strip():
|
||||||
|
chunks.append(chunk)
|
||||||
content = content[max_len:]
|
content = content[max_len:]
|
||||||
|
if not chunk.strip():
|
||||||
|
# Keep any remaining indentation so the final non-blank chunk can
|
||||||
|
# retain as much of it as the channel limit permits.
|
||||||
|
continue
|
||||||
# A delimiter can sit immediately after the hard-break boundary. Keep
|
# A delimiter can sit immediately after the hard-break boundary. Keep
|
||||||
# ordinary space trimming, but consume only the newline so indentation
|
# ordinary space trimming, but consume only the newline so indentation
|
||||||
# on the following line is preserved.
|
# on the following line is preserved.
|
||||||
@@ -665,7 +692,10 @@ def split_message(content: str, max_len: int = 2000) -> list[str]:
|
|||||||
content = content[2:]
|
content = content[2:]
|
||||||
elif content.startswith("\n"):
|
elif content.startswith("\n"):
|
||||||
content = content[1:]
|
content = content[1:]
|
||||||
return chunks
|
# Preserve the historical non-empty-input contract for callers that take
|
||||||
|
# the first chunk directly. This fallback is only reachable for content
|
||||||
|
# made entirely of whitespace.
|
||||||
|
return chunks or [original_content[:max_len]]
|
||||||
|
|
||||||
|
|
||||||
def build_assistant_message(
|
def build_assistant_message(
|
||||||
|
|||||||
@@ -39,6 +39,42 @@ def test_split_message_preserves_indentation_when_newline_is_at_hard_break():
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_message_handles_crlf_before_hard_break():
|
||||||
|
content = "header\r\n indented code"
|
||||||
|
|
||||||
|
assert split_message(content, max_len=18) == ["header", " indented code"]
|
||||||
|
assert split_message("abcdefg\r\n code", max_len=8) == [
|
||||||
|
"abcdefg",
|
||||||
|
" code",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_message_preserves_indent_after_space_then_newline_boundary():
|
||||||
|
content = "abcdef \n code"
|
||||||
|
|
||||||
|
assert split_message(content, max_len=7) == ["abcdef", " cod", "e"]
|
||||||
|
assert split_message(content.replace("\n", "\r\n"), max_len=7) == [
|
||||||
|
"abcdef",
|
||||||
|
" cod",
|
||||||
|
"e",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_message_drops_blank_chunks_from_long_indentation():
|
||||||
|
content = "head\n" + " " * 20 + "x"
|
||||||
|
|
||||||
|
chunks = split_message(content, max_len=8)
|
||||||
|
|
||||||
|
assert chunks == ["head", " x"]
|
||||||
|
assert all(chunk.strip() for chunk in chunks)
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_message_drops_whitespace_only_line_at_boundary():
|
||||||
|
content = " \nhello world"
|
||||||
|
|
||||||
|
assert split_message(content, max_len=8) == ["hello", "world"]
|
||||||
|
|
||||||
|
|
||||||
def test_split_message_drops_whitespace_only_tail_after_hard_break():
|
def test_split_message_drops_whitespace_only_tail_after_hard_break():
|
||||||
prefix = "abcdefgh"
|
prefix = "abcdefgh"
|
||||||
|
|
||||||
@@ -46,6 +82,12 @@ def test_split_message_drops_whitespace_only_tail_after_hard_break():
|
|||||||
assert split_message(prefix + " ", max_len=8) == [prefix]
|
assert split_message(prefix + " ", max_len=8) == [prefix]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_message_keeps_one_chunk_for_all_whitespace_input():
|
||||||
|
content = " " * 10
|
||||||
|
|
||||||
|
assert split_message(content, max_len=4) == [" " * 4]
|
||||||
|
|
||||||
|
|
||||||
def test_split_message_nonpositive_maxlen_returns_unsplit():
|
def test_split_message_nonpositive_maxlen_returns_unsplit():
|
||||||
content = "alpha beta gamma delta"
|
content = "alpha beta gamma delta"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user