mirror/nanobot
1
0
Fork 0
mirror of https://github.com/HKUDS/nanobot.git synced 2026-08-06 17:38:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: centralize thinking tag patterns

Browse Source 
maintainer edit: derive thinking tag regexes and streaming partial prefixes from one tag list so future aliases only need one entry while preserving legacy self-closing think/thought behavior.
This commit is contained in:
chengyongru 2026-06-24 15:24:41 +08:00 committed by Xubin Ren
parent 98dd883ce8
commit 1e22932313
1 changed files with 38 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
nanobot/utils/helpers.py
View File

@ -65,6 +65,27 @@ def _estimate_tools_tokens(
return token_count return token_count
def _tag_regex(tags: tuple[str, ...]) -> str:
return rf"(?:{'|'.join(re.escape(tag) for tag in tags)})"
_THINKING_TAGS = ("think", "thinking", "thought")
_INLINE_SELF_CLOSING_PRESERVED_TAGS = frozenset({"think", "thought"})
_INLINE_SELF_CLOSING_THINKING_TAGS = tuple(
tag for tag in _THINKING_TAGS if tag not in _INLINE_SELF_CLOSING_PRESERVED_TAGS
)
_THINKING_TAG = _tag_regex(_THINKING_TAGS)
_INLINE_SELF_CLOSING_THINKING_TAG = _tag_regex(_INLINE_SELF_CLOSING_THINKING_TAGS)
_THINKING_TAG_PREFIX = "|".join(
sorted(
{re.escape(tag[:i]) for tag in _THINKING_TAGS for i in range(1, len(tag) + 1)},
key=len,
reverse=True,
)
)
_PARTIAL_THINKING_TAG = rf"</?(?:{_THINKING_TAG_PREFIX})>?"
def strip_think(text: str) -> str: def strip_think(text: str) -> str:
"""Remove thinking blocks, unclosed trailing tags, and tokenizer-level """Remove thinking blocks, unclosed trailing tags, and tokenizer-level
template leaks occasionally emitted by some models (notably Gemma 4's template leaks occasionally emitted by some models (notably Gemma 4's
@ -92,38 +113,28 @@ def strip_think(text: str) -> str:
assistant discusses the tokens themselves. assistant discusses the tokens themselves.
""" """
# Well-formed blocks first. # Well-formed blocks first.
text = re.sub(r"<think>[\s\S]*?</think>", "", text) text = re.sub(rf"<(?P<tag>{_THINKING_TAG})>[\s\S]*?</(?P=tag)>", "", text)
text = re.sub(r"^\s*<think>[\s\S]*$", "", text) text = re.sub(rf"^\s*<{_THINKING_TAG}>[\s\S]*$", "", text)
text = re.sub(r"<thinking>[\s\S]*?</thinking>", "", text)
text = re.sub(r"^\s*<thinking>[\s\S]*$", "", text)
text = re.sub(r"<thought>[\s\S]*?</thought>", "", text)
text = re.sub(r"^\s*<thought>[\s\S]*$", "", text)
# Self-closing `<thinking/>` is an empty marker, not user-visible text. # Self-closing `<thinking/>` is an empty marker, not user-visible text.
text = re.sub(r"^\s*<thinking/>\s*", "", text) text = re.sub(rf"^\s*<{_INLINE_SELF_CLOSING_THINKING_TAG}/>\s*", "", text)
text = re.sub(r"\s*<thinking/>\s*$", "", text) text = re.sub(rf"\s*<{_INLINE_SELF_CLOSING_THINKING_TAG}/>\s*$", "", text)
# Malformed opening tags: `<think` / `<thinking` / `<thought` where the next char is # Malformed opening tags: `<think` / `<thinking` / `<thought` where the next char is
# NOT one that could continue a valid tag / identifier name. Explicitly # NOT one that could continue a valid tag / identifier name. Explicitly
# listing ASCII tag-name chars (letters, digits, `_`, `-`, `:`) plus # listing ASCII tag-name chars (letters, digits, `_`, `-`, `:`) plus
# `>` / `/` — we can't use `\w` here because in Python's default # `>` / `/` — we can't use `\w` here because in Python's default
# Unicode regex mode it matches CJK characters too, which would defeat # Unicode regex mode it matches CJK characters too, which would defeat
# the primary fix for `<think广场…` leaks. # the primary fix for `<think广场…` leaks.
text = re.sub(r"<think(?![A-Za-z0-9_\-:>/])", "", text) text = re.sub(rf"<{_THINKING_TAG}(?![A-Za-z0-9_\-:>/])", "", text)
text = re.sub(r"<thinking(?![A-Za-z0-9_\-:>/])", "", text)
text = re.sub(r"<thought(?![A-Za-z0-9_\-:>/])", "", text)
# Edge-only orphan closing tags (start or end of text). # Edge-only orphan closing tags (start or end of text).
text = re.sub(r"^\s*</think>\s*", "", text) text = re.sub(rf"^\s*</{_THINKING_TAG}>\s*", "", text)
text = re.sub(r"\s*</think>\s*$", "", text) text = re.sub(rf"\s*</{_THINKING_TAG}>\s*$", "", text)
text = re.sub(r"^\s*</thinking>\s*", "", text)
text = re.sub(r"\s*</thinking>\s*$", "", text)
text = re.sub(r"^\s*</thought>\s*", "", text)
text = re.sub(r"\s*</thought>\s*$", "", text)
# Edge-only channel markers (harmony / Gemma 4 variant leaks). # Edge-only channel markers (harmony / Gemma 4 variant leaks).
text = re.sub(r"^\s*<\|?channel\|?>\s*", "", text) text = re.sub(r"^\s*<\|?channel\|?>\s*", "", text)
# Stream chunks may end in the middle of a control tag. Strip only known # Stream chunks may end in the middle of a control tag. Strip only known
# control-token prefixes at the very end. # control-token prefixes at the very end.
partial_control_tag = ( partial_control_tag = (
r"</?(?:t|th|thi|thin|think|thinki|thinkin|thinking|tho|thou|thoug|though|thought)>?" rf"{_PARTIAL_THINKING_TAG}|"
r"|<\|?(?:c|ch|cha|chan|chann|channe|channel)(?:\|?>?)?" r"<\|?(?:c|ch|cha|chan|chann|channe|channel)(?:\|?>?)?"
) )
text = re.sub(rf"(?:{partial_control_tag})$", "", text) text = re.sub(rf"(?:{partial_control_tag})$", "", text)
text = re.sub(r"^\s*<\|?$", "", text) text = re.sub(r"^\s*<\|?$", "", text)
@ -134,15 +145,12 @@ def strip_reasoning_tags(text: object) -> str:
"""Remove wrapper tags from text that is already known to be reasoning.""" """Remove wrapper tags from text that is already known to be reasoning."""
if not isinstance(text, str): if not isinstance(text, str):
return "" return ""
partial_reasoning_tag = ( text = re.sub(rf"^\s*(?:{_PARTIAL_THINKING_TAG})$", "", text)
r"</?(?:t|th|thi|thin|think|thinki|thinkin|thinking|tho|thou|thoug|though|thought)>?" text = re.sub(rf"^\s*<{_THINKING_TAG}/>\s*", "", text)
) text = re.sub(rf"\s*<{_THINKING_TAG}/>\s*$", "", text)
text = re.sub(rf"^\s*(?:{partial_reasoning_tag})$", "", text) text = re.sub(rf"^\s*<{_THINKING_TAG}>\s*", "", text)
text = re.sub(r"^\s*<(?:think|thinking|thought)/>\s*", "", text) text = re.sub(rf"\s*</{_THINKING_TAG}>\s*$", "", text)
text = re.sub(r"\s*<(?:think|thinking|thought)/>\s*$", "", text) text = re.sub(rf"\s*(?:{_PARTIAL_THINKING_TAG})$", "", text)
text = re.sub(r"^\s*<(?:think|thinking|thought)>\s*", "", text)
text = re.sub(r"\s*</(?:think|thinking|thought)>\s*$", "", text)
text = re.sub(rf"\s*(?:{partial_reasoning_tag})$", "", text)
return text.strip() return text.strip()
@ -154,12 +162,8 @@ def extract_think(text: str) -> tuple[str | None, str]:
text but not surfaced — :func:`strip_think` handles that case. text but not surfaced — :func:`strip_think` handles that case.
""" """
parts: list[str] = [] parts: list[str] = []
for m in re.finditer(r"<think>([\s\S]*?)</think>", text): for m in re.finditer(rf"<(?P<tag>{_THINKING_TAG})>([\s\S]*?)</(?P=tag)>", text):
parts.append(m.group(1).strip()) parts.append(m.group(2).strip())
for m in re.finditer(r"<thinking>([\s\S]*?)</thinking>", text):
parts.append(m.group(1).strip())
for m in re.finditer(r"<thought>([\s\S]*?)</thought>", text):
parts.append(m.group(1).strip())
thinking = "\n\n".join(parts) if parts else None thinking = "\n\n".join(parts) if parts else None
return thinking, strip_think(text) return thinking, strip_think(text)