diff --git a/nanobot/channels/signal/tests/test_signal_markdown.py b/nanobot/channels/signal/tests/test_signal_markdown.py index 33a4d7495..ce298f043 100644 --- a/nanobot/channels/signal/tests/test_signal_markdown.py +++ b/nanobot/channels/signal/tests/test_signal_markdown.py @@ -393,6 +393,14 @@ def test_partition_styles_keeps_offset_in_indented_chunk(): assert _partition_styles(plain, chunks, ["11:4:BOLD"]) == [[], ["4:4:BOLD"]] +def test_partition_styles_keeps_offset_after_crlf_boundary(): + plain = "header\r\n code" + chunks = split_message(plain, 10) + + assert chunks == ["header", " code"] + assert _partition_styles(plain, chunks, ["12:4:BOLD"]) == [[], ["4:4:BOLD"]] + + def test_partition_styles_long_message_preserves_chunk_one_styles(): """A bold span deep in the message must follow the message into chunk 1.""" # Two ~30-char paragraphs separated by a blank line, then **tail**. diff --git a/nanobot/utils/helpers.py b/nanobot/utils/helpers.py index 1832d0663..1e4ac3e74 100644 --- a/nanobot/utils/helpers.py +++ b/nanobot/utils/helpers.py @@ -633,6 +633,7 @@ def split_message(content: str, max_len: int = 2000) -> list[str]: return [content] if len(content) <= max_len: return [content] + original_content = content chunks: list[str] = [] while content: if len(content) <= max_len: @@ -642,8 +643,14 @@ def split_message(content: str, max_len: int = 2000) -> list[str]: cut = content[:max_len] # Consume only the newline itself so indentation starts the next chunk. newline_pos = cut.rfind("\n") - if newline_pos > 0: - chunks.append(content[:newline_pos]) + if newline_pos >= 0: + # Exclude both bytes of a CRLF boundary from the emitted chunk. + line_end = newline_pos + if line_end > 0 and content[line_end - 1] == "\r": + line_end -= 1 + chunk = content[:line_end] + if chunk.strip(): + chunks.append(chunk) content = content[newline_pos + 1 :] continue @@ -652,11 +659,31 @@ def split_message(content: str, max_len: int = 2000) -> list[str]: space_pos = cut.rfind(" ") if space_pos > 0 and cut[:space_pos].strip(): chunks.append(content[:space_pos]) - content = content[space_pos:].lstrip() + content = content[space_pos:].lstrip(" \t") + # A space boundary may sit immediately before a line break. Drop + # that delimiter too, without stripping the next line's indent. + if content.startswith("\r\n"): + content = content[2:] + elif content.startswith("\n"): + content = content[1:] continue - chunks.append(content[:max_len]) + # Do not split between the two code points of a CRLF delimiter. + if cut.endswith("\r") and content[max_len : max_len + 1] == "\n": + chunk = cut[:-1] + if chunk.strip(): + chunks.append(chunk) + content = content[max_len + 1 :] + continue + + chunk = content[:max_len] + if chunk.strip(): + chunks.append(chunk) content = content[max_len:] + if not chunk.strip(): + # Keep any remaining indentation so the final non-blank chunk can + # retain as much of it as the channel limit permits. + continue # A delimiter can sit immediately after the hard-break boundary. Keep # ordinary space trimming, but consume only the newline so indentation # on the following line is preserved. @@ -665,7 +692,10 @@ def split_message(content: str, max_len: int = 2000) -> list[str]: content = content[2:] elif content.startswith("\n"): content = content[1:] - return chunks + # Preserve the historical non-empty-input contract for callers that take + # the first chunk directly. This fallback is only reachable for content + # made entirely of whitespace. + return chunks or [original_content[:max_len]] def build_assistant_message( diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 9d4b2d4fd..0a7a10338 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -39,6 +39,42 @@ def test_split_message_preserves_indentation_when_newline_is_at_hard_break(): ] +def test_split_message_handles_crlf_before_hard_break(): + content = "header\r\n indented code" + + assert split_message(content, max_len=18) == ["header", " indented code"] + assert split_message("abcdefg\r\n code", max_len=8) == [ + "abcdefg", + " code", + ] + + +def test_split_message_preserves_indent_after_space_then_newline_boundary(): + content = "abcdef \n code" + + assert split_message(content, max_len=7) == ["abcdef", " cod", "e"] + assert split_message(content.replace("\n", "\r\n"), max_len=7) == [ + "abcdef", + " cod", + "e", + ] + + +def test_split_message_drops_blank_chunks_from_long_indentation(): + content = "head\n" + " " * 20 + "x" + + chunks = split_message(content, max_len=8) + + assert chunks == ["head", " x"] + assert all(chunk.strip() for chunk in chunks) + + +def test_split_message_drops_whitespace_only_line_at_boundary(): + content = " \nhello world" + + assert split_message(content, max_len=8) == ["hello", "world"] + + def test_split_message_drops_whitespace_only_tail_after_hard_break(): prefix = "abcdefgh" @@ -46,6 +82,12 @@ def test_split_message_drops_whitespace_only_tail_after_hard_break(): assert split_message(prefix + " ", max_len=8) == [prefix] +def test_split_message_keeps_one_chunk_for_all_whitespace_input(): + content = " " * 10 + + assert split_message(content, max_len=4) == [" " * 4] + + def test_split_message_nonpositive_maxlen_returns_unsplit(): content = "alpha beta gamma delta"