feat(dream): single-phase consolidation (cherry-pick from feat/single-phase-dream)

This commit is contained in:
chengyongru 2026-05-26 00:38:04 +08:00
parent 418cb23da2
commit 8dd35c373a
16 changed files with 1189 additions and 743 deletions

View File

@ -8,6 +8,7 @@ import os
import time
from contextlib import AsyncExitStack, nullcontext, suppress
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum, auto
from pathlib import Path
from typing import TYPE_CHECKING, Any, Awaitable, Callable
@ -19,7 +20,13 @@ from nanobot.agent import model_presets as preset_helpers
from nanobot.agent.autocompact import AutoCompact
from nanobot.agent.context import ContextBuilder
from nanobot.agent.hook import AgentHook, CompositeHook
from nanobot.agent.memory import Consolidator, Dream
from nanobot.agent.memory import (
_STALE_THRESHOLD_DAYS,
Consolidator,
Dream,
_estimate_tokens,
_strip_skip_lines,
)
from nanobot.agent.progress_hook import AgentProgressHook
from nanobot.agent.runner import _MAX_INJECTIONS_PER_TURN, AgentRunner, AgentRunSpec
from nanobot.agent.subagent import SubagentManager
@ -34,6 +41,8 @@ from nanobot.config.schema import AgentDefaults, ModelPresetConfig
from nanobot.providers.base import LLMProvider
from nanobot.providers.factory import ProviderSnapshot
from nanobot.session.goal_state import (
GOAL_STATE_KEY,
goal_state_runtime_lines,
runner_wall_llm_timeout_s,
)
from nanobot.session.manager import Session, SessionManager
@ -47,7 +56,11 @@ from nanobot.utils.helpers import image_placeholder_text
from nanobot.utils.helpers import truncate_text as truncate_text_fn
from nanobot.utils.image_generation_intent import image_generation_prompt
from nanobot.utils.llm_runtime import LLMRuntime
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
from nanobot.utils.prompt_templates import _TEMPLATES_ROOT, render_template
from nanobot.utils.runtime import (
EMPTY_FINAL_RESPONSE_MESSAGE,
SUSTAINED_GOAL_CONTINUE_PROMPT,
)
if TYPE_CHECKING:
from nanobot.config.schema import (
@ -191,6 +204,7 @@ class AgentLoop:
model_preset: str | None = None,
preset_snapshot_loader: preset_helpers.PresetSnapshotLoader | None = None,
runtime_model_publisher: Callable[[str, str | None], None] | None = None,
dream_model_override: str | None = None,
):
from nanobot.config.schema import ToolsConfig
@ -203,6 +217,7 @@ class AgentLoop:
self._preset_snapshot_loader = preset_snapshot_loader
self._runtime_model_publisher = runtime_model_publisher
self._provider_signature = provider_signature
self._dream_model_override = dream_model_override
self._default_selection_signature = preset_helpers.default_selection_signature(provider_signature)
self.workspace = workspace
self.model = model or provider.get_default_model()
@ -310,6 +325,7 @@ class AgentLoop:
self._active_preset: str | None = None
if model_preset:
self.set_model_preset(model_preset, publish_update=False)
self._configure_dream()
self._register_default_tools()
self._runtime_vars: dict[str, Any] = {}
self._current_iteration: int = 0
@ -369,6 +385,7 @@ class AgentLoop:
model_preset=defaults.model_preset,
provider_snapshot_loader=provider_snapshot_loader,
preset_snapshot_loader=preset_snapshot_loader,
dream_model_override=config.agents.defaults.dream.model_override,
**extra,
)
@ -394,7 +411,7 @@ class AgentLoop:
self.runner.provider = provider
self.subagents.set_provider(provider, model)
self.consolidator.set_provider(provider, model, context_window_tokens)
self.dream.set_provider(provider, model)
self._configure_dream()
self._provider_signature = snapshot.signature
if publish_update and self._runtime_model_publisher is not None:
self._runtime_model_publisher(
@ -403,6 +420,20 @@ class AgentLoop:
)
logger.info("Runtime model switched for next turn: {} -> {}", old_model, model)
def _configure_dream(self) -> None:
"""Apply dream.model_override, resolving preset names if needed."""
if not self._dream_model_override:
self.dream.set_provider(self.provider, self.model)
return
if self._dream_model_override in self.model_presets:
snapshot = self._build_model_preset_snapshot(self._dream_model_override)
self.dream.set_provider(snapshot.provider, snapshot.model)
return
# Raw model name fallback — same provider, different model
self.dream.set_provider(self.provider, self._dream_model_override)
def _refresh_provider_snapshot(self) -> None:
if self._provider_snapshot_loader is None:
return
@ -1006,6 +1037,28 @@ class AgentLoop:
msg.chat_id.split(":", 1) if ":" in msg.chat_id else ("cli", msg.chat_id)
)
logger.info("Processing system message from {}", msg.sender_id)
if msg.sender_id == "dream":
session_key = "system:dream"
session = self.sessions.get_or_create(session_key)
session.metadata["is_dream"] = True
# Capture trigger source on first batch so _dream_finalize_commit
# can notify the user who ran /dream (cron-triggered runs have no trigger).
if "_dream_trigger_channel" not in session.metadata:
trigger_ch = msg.metadata.get("trigger_channel")
trigger_ci = msg.metadata.get("trigger_chat_id")
if trigger_ch and trigger_ci:
session.metadata["_dream_trigger_channel"] = trigger_ch
session.metadata["_dream_trigger_chat_id"] = trigger_ci
if not sustained_goal_active(session.metadata):
session.metadata[GOAL_STATE_KEY] = {
"status": "active",
"objective": "Dream: consolidate unprocessed memory backlog into MEMORY.md, SOUL.md, USER.md",
"started_at": datetime.now().isoformat(),
}
self.sessions.save(session)
await self._process_dream_batch(session, msg)
await self._dream_finalize_commit(session)
return None
key = msg.session_key_override or f"{channel}:{chat_id}"
session = self.sessions.get_or_create(key)
if self._restore_runtime_checkpoint(session):
@ -1082,6 +1135,204 @@ class AgentLoop:
metadata=outbound_metadata,
)
async def _process_dream_batch(self, session: Session, msg: InboundMessage) -> None:
"""Process the full Dream backlog in batches within a single invocation."""
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
# System prompt caching with mtime invalidation
template_path = _TEMPLATES_ROOT / "agent" / "dream.md"
cached_prompt = session.metadata.get("_dream_system_prompt")
cached_mtime = session.metadata.get("_dream_system_prompt_mtime")
current_mtime = template_path.stat().st_mtime if template_path.exists() else None
if cached_prompt is None or cached_mtime != current_mtime:
skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md"
workspace = self.dream.store.workspace
cached_prompt = render_template(
"agent/dream.md",
strip=True,
skill_creator_path=str(skill_creator_path),
soul_path=str(workspace / "SOUL.md"),
user_path=str(workspace / "USER.md"),
memory_path=str(workspace / "memory" / "MEMORY.md"),
stale_threshold_days=_STALE_THRESHOLD_DAYS,
)
session.metadata["_dream_system_prompt"] = cached_prompt
session.metadata["_dream_system_prompt_mtime"] = current_mtime
while True:
last_cursor = self.dream.store.get_last_dream_cursor()
entries = self.dream.store.read_unprocessed_history(since_cursor=last_cursor)
if not entries:
return
batch = entries[: self.dream.max_batch_size]
logger.info(
"Dream: processing {}/{} entries (cursor {}{})",
len(batch), len(entries), last_cursor, batch[-1]["cursor"],
)
# Build history text — cap each entry and strip [skip] lines
history_text = "\n".join(
f"[{e['timestamp']}] "
f"{truncate_text_fn(_strip_skip_lines(e['content']), self.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}"
for e in batch
)
# Current file contents + per-line age annotations
current_date = datetime.now().strftime("%Y-%m-%d")
annotate = self.dream.annotate_line_ages
raw_memory = self.dream.store.read_memory() or "(empty)"
raw_soul = self.dream.store.read_soul() or "(empty)"
raw_user = self.dream.store.read_user() or "(empty)"
annotated_memory = (
self.dream._annotate_with_ages(raw_memory, "memory/MEMORY.md")
if annotate else raw_memory
)
annotated_soul = (
self.dream._annotate_with_ages(raw_soul, "SOUL.md")
if annotate else raw_soul
)
annotated_user = (
self.dream._annotate_with_ages(raw_user, "USER.md")
if annotate else raw_user
)
current_memory = truncate_text_fn(annotated_memory, self.dream._MEMORY_FILE_MAX_CHARS)
current_soul = truncate_text_fn(annotated_soul, self.dream._SOUL_FILE_MAX_CHARS)
current_user = truncate_text_fn(annotated_user, self.dream._USER_FILE_MAX_CHARS)
file_context = (
f"## Current Date\n{current_date}\n\n"
f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n"
f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n"
f"## Current USER.md ({len(current_user)} chars)\n{current_user}"
)
existing_skills = self.dream._list_existing_skills()
skills_section = ""
if existing_skills:
skills_section = (
"\n\n## Existing Skills\n"
+ "\n".join(f"- {s}" for s in existing_skills)
)
user_prompt = f"## Conversation History\n{history_text}\n\n{file_context}{skills_section}"
logger.info("Dream prompt: {} chars, ~{} tokens", len(user_prompt), _estimate_tokens(user_prompt))
messages: list[dict[str, Any]] = [
{"role": "system", "content": cached_prompt},
{"role": "user", "content": user_prompt},
]
t_start = time.perf_counter()
try:
result = await self.dream._runner.run(AgentRunSpec(
initial_messages=messages,
tools=self.dream._tools,
model=self.dream.model,
max_iterations=self.dream.max_iterations,
max_tool_result_chars=self.dream.max_tool_result_chars,
context_window_tokens=self.context_window_tokens,
fail_on_tool_error=False,
))
elapsed = time.perf_counter() - t_start
logger.info(
"Dream run complete in {:.1f}s: stop_reason={}, tool_events={}",
elapsed, result.stop_reason, len(result.tool_events),
)
except Exception:
elapsed = time.perf_counter() - t_start
logger.exception("Dream run failed after {:.1f}s", elapsed)
result = None
# Build changelog from tool events
changelog: list[str] = []
if result and result.tool_events:
for event in result.tool_events:
if event.get("status") == "ok":
changelog.append(f"{event['name']}: {event['detail']}")
success = result is not None and result.stop_reason == "completed"
if success:
new_cursor = batch[-1]["cursor"]
self.dream.store.set_last_dream_cursor(new_cursor)
session.metadata.setdefault("_dream_changelog", []).extend(changelog)
self.sessions.save(session)
logger.info(
"Dream done: {} change(s), cursor advanced to {}",
len(changelog), new_cursor,
)
else:
reason = result.stop_reason if result else "exception"
logger.warning(
"Dream incomplete ({}): cursor NOT advanced, stopping",
reason,
)
return
self.dream.store.compact_history()
# Persist session record for debugging / visualization
record = {
"timestamp": datetime.now().isoformat(),
"batch": {
"from_cursor": last_cursor,
"to_cursor": batch[-1]["cursor"],
"count": len(batch),
},
"prompt_chars": len(user_prompt),
"elapsed_seconds": elapsed,
"stop_reason": result.stop_reason,
"usage": result.usage,
"tool_events": result.tool_events,
"changelog": changelog,
"commit_sha": None,
"messages": result.messages,
}
self.dream.store.write_dream_session(record)
session.metadata["_dream_last_record"] = record
async def _dream_finalize_commit(self, session: Session) -> None:
"""Collapse accumulated changelog into a single git commit, clear caches, and complete the goal."""
changelog = session.metadata.pop("_dream_changelog", [])
sha = None
if changelog and self.dream.store.git.is_initialized():
ts = datetime.now().strftime("%Y-%m-%d %H:%M")
summary = f"dream: {ts}, {len(changelog)} change(s)"
commit_msg = f"{summary}\n\n" + "\n".join(changelog)
sha = self.dream.store.git.auto_commit(commit_msg)
if sha:
logger.info("Dream commit: {}", sha)
record = session.metadata.pop("_dream_last_record", None)
if record and sha:
record["commit_sha"] = sha
self.dream.store.write_dream_session(record)
session.metadata.pop("_dream_system_prompt", None)
session.metadata.pop("_dream_system_prompt_mtime", None)
trigger_channel = session.metadata.pop("_dream_trigger_channel", None)
trigger_chat_id = session.metadata.pop("_dream_trigger_chat_id", None)
goal = session.metadata.get(GOAL_STATE_KEY)
if isinstance(goal, dict) and goal.get("status") == "active":
session.metadata[GOAL_STATE_KEY] = {
**goal,
"status": "completed",
"completed_at": datetime.now().isoformat(),
"recap": f"Memory backlog consolidated ({len(changelog)} change(s)).",
}
self.sessions.save(session)
session.metadata["_dream_finalized"] = True
# Notify the user who triggered /dream
if trigger_channel and trigger_chat_id:
content = f"Dream completed: {len(changelog)} change(s) committed."
if not changelog:
content = "Dream: nothing to process."
await self.bus.publish_outbound(OutboundMessage(
channel=trigger_channel,
chat_id=trigger_chat_id,
content=content,
))
async def _process_message(
self,
msg: InboundMessage,

View File

@ -6,6 +6,7 @@ import asyncio
import json
import os
import re
import time
import weakref
from contextlib import suppress
from datetime import datetime
@ -15,7 +16,7 @@ from typing import TYPE_CHECKING, Any, Callable, Iterator
import tiktoken
from loguru import logger
from nanobot.agent.runner import AgentRunner, AgentRunSpec
from nanobot.agent.runner import AgentRunner
from nanobot.agent.tools.registry import ToolRegistry
from nanobot.session.manager import Session
from nanobot.utils.gitstore import GitStore
@ -33,6 +34,20 @@ if TYPE_CHECKING:
from nanobot.providers.base import LLMProvider
from nanobot.session.manager import SessionManager
# Cache the tiktoken encoding to avoid repeated instantiation on every
# truncate/encode call. Encoding objects are thread-safe and reusable.
try:
_TIKTOKEN_ENC = tiktoken.get_encoding("cl100k_base")
except Exception: # pragma: no cover
_TIKTOKEN_ENC = None
def _estimate_tokens(text: str) -> int:
"""Approximate token count for a text string."""
if _TIKTOKEN_ENC is not None:
return len(_TIKTOKEN_ENC.encode(text))
return len(text) // 4
# ---------------------------------------------------------------------------
# MemoryStore — pure file I/O layer
@ -400,6 +415,26 @@ class MemoryStore:
def set_last_dream_cursor(self, cursor: int) -> None:
self._dream_cursor_file.write_text(str(cursor), encoding="utf-8")
def write_dream_session(self, data: dict[str, Any]) -> None:
"""Atomic overwrite of the latest Dream run record."""
path = self.memory_dir / ".dream_session.json"
tmp_path = path.with_suffix(".tmp")
try:
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_path, path)
with suppress(PermissionError):
fd = os.open(str(path.parent), os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
except BaseException:
tmp_path.unlink(missing_ok=True)
raise
# -- message formatting utility ------------------------------------------
@staticmethod
@ -618,19 +653,21 @@ class Consolidator:
"""Available input token budget for consolidation LLM."""
return self.context_window_tokens - self.max_completion_tokens - self._SAFETY_BUFFER
def _truncate_to_token_budget(self, text: str) -> str:
"""Truncate text so it fits within the consolidation LLM's token budget."""
budget = self._input_token_budget
def _truncate_to_token_budget(self, text: str, reserve_tokens: int = 0) -> str:
"""Truncate text so it fits within the consolidation LLM's token budget.
reserve_tokens: additional tokens to reserve for dedup context or other
overhead that will be appended after truncation.
"""
budget = self._input_token_budget - reserve_tokens
if budget <= 0:
return truncate_text(text, _RAW_ARCHIVE_MAX_CHARS)
try:
enc = tiktoken.get_encoding("cl100k_base")
tokens = enc.encode(text)
if _TIKTOKEN_ENC is not None:
tokens = _TIKTOKEN_ENC.encode(text)
if len(tokens) <= budget:
return text
return enc.decode(tokens[:budget]) + "\n... (truncated)"
except Exception:
return truncate_text(text, budget * 4)
return _TIKTOKEN_ENC.decode(tokens[:budget]) + "\n... (truncated)"
return truncate_text(text, budget * 4)
async def archive(self, messages: list[dict]) -> str | None:
"""Summarize messages via LLM and append to history.jsonl.
@ -639,9 +676,53 @@ class Consolidator:
"""
if not messages:
return None
t_start = time.perf_counter()
try:
formatted = MemoryStore._format_messages(messages)
formatted = self._truncate_to_token_budget(formatted)
logger.debug(
"Consolidator: {} messages, formatted={} chars",
len(messages), len(formatted),
)
# Inject current memory context for dedup-aware summarization.
memory_preview = self.store.read_memory()[:4000]
user_preview = self.store.read_user()[:2000]
dedup_context = ""
if memory_preview:
dedup_context += f"\n\n## Current MEMORY.md (for dedup)\n{memory_preview}"
if user_preview:
dedup_context += f"\n\n## Current USER.md (for dedup)\n{user_preview}"
reserve_tokens = 0
if dedup_context:
if _TIKTOKEN_ENC is not None:
reserve_tokens = len(_TIKTOKEN_ENC.encode(dedup_context)) + 100
else:
reserve_tokens = len(dedup_context) // 4 + 100
if self._input_token_budget <= reserve_tokens:
logger.warning(
"Consolidator: dedup_context ({} tokens) exceeds budget ({}), dropping it",
reserve_tokens, self._input_token_budget,
)
dedup_context = ""
reserve_tokens = 0
else:
logger.debug(
"Consolidator: dedup_context={} chars, reserve_tokens={}",
len(dedup_context), reserve_tokens,
)
formatted_before = len(formatted)
formatted = self._truncate_to_token_budget(
formatted, reserve_tokens=reserve_tokens
)
if len(formatted) < formatted_before:
logger.warning(
"Consolidator: truncated formatted messages from {} to {} chars",
formatted_before, len(formatted),
)
response = await self.provider.chat_with_retry(
model=self.model,
messages=[
@ -652,18 +733,31 @@ class Consolidator:
strip=True,
),
},
{"role": "user", "content": formatted},
{"role": "user", "content": formatted + dedup_context},
],
tools=None,
tool_choice=None,
)
elapsed = time.perf_counter() - t_start
if response.finish_reason == "error":
logger.warning(
"Consolidator LLM error after {:.1f}s: {}",
elapsed, response.content,
)
raise RuntimeError(f"LLM returned error: {response.content}")
summary = response.content or "[no summary]"
logger.info(
"Consolidator: {} entries -> {} chars summary in {:.1f}s",
len(messages), len(summary), elapsed,
)
self.store.append_history(summary, max_chars=_ARCHIVE_SUMMARY_MAX_CHARS)
return summary
except Exception:
logger.warning("Consolidation LLM call failed, raw-dumping to history")
elapsed = time.perf_counter() - t_start
logger.warning(
"Consolidation LLM call failed after {:.1f}s, raw-dumping to history",
elapsed,
)
self.store.raw_archive(messages)
return None
@ -851,35 +945,44 @@ class Consolidator:
# Single source of truth for the staleness threshold used in _annotate_with_ages
# *and* in the Phase 1 prompt template (passed as `stale_threshold_days`).
# *and* in the system prompt template (passed as `stale_threshold_days`).
# Keep code and prompt aligned — if you bump this, the LLM's instruction string
# updates automatically.
_STALE_THRESHOLD_DAYS = 14
_SKIP_LINE_RE = re.compile(r"^\s*-\s*\[skip\]\s*.*$", re.MULTILINE | re.IGNORECASE)
def _strip_skip_lines(text: str) -> str:
"""Remove lines marked [skip] from history content."""
lines = text.splitlines()
kept = [line for line in lines if not _SKIP_LINE_RE.match(line)]
return "\n".join(kept)
class Dream:
"""Two-phase memory processor: analyze history.jsonl, then edit files via AgentRunner.
"""Single-phase memory processor: analyze history.jsonl and edit files via AgentRunner.
Phase 1 produces an analysis summary (plain LLM call).
Phase 2 delegates to AgentRunner with read_file / edit_file tools so the
LLM can make targeted, incremental edits instead of replacing entire files.
Delegates to AgentRunner with read_file / edit_file tools so the LLM can
analyze conversation history, extract facts, deduplicate, and make targeted
incremental edits all in a single agent run.
"""
# Caps on prompt-bound inputs so Dream's LLM calls never exceed the model's
# context window just because a file (or a legacy large history entry) grew
# unexpectedly. Each file still appears in full via read_file when the agent
# needs it in Phase 2 — these caps only bound the Phase 1/2 prompt preview.
_MEMORY_FILE_MAX_CHARS = 32_000
_SOUL_FILE_MAX_CHARS = 16_000
_USER_FILE_MAX_CHARS = 16_000
_HISTORY_ENTRY_PREVIEW_MAX_CHARS = 4_000
# needs it — these caps only bound the prompt preview.
_MEMORY_FILE_MAX_CHARS = 16_000
_SOUL_FILE_MAX_CHARS = 4_000
_USER_FILE_MAX_CHARS = 4_000
_HISTORY_ENTRY_PREVIEW_MAX_CHARS = 2_000
def __init__(
self,
store: MemoryStore,
provider: LLMProvider,
model: str,
max_batch_size: int = 20,
max_batch_size: int = 5,
max_iterations: int = 10,
max_tool_result_chars: int = 16_000,
annotate_line_ages: bool = True,
@ -890,9 +993,9 @@ class Dream:
self.max_batch_size = max_batch_size
self.max_iterations = max_iterations
self.max_tool_result_chars = max_tool_result_chars
# Kill switch for the git-blame-based per-line age annotation in Phase 1.
# Default True keeps the #3212 behavior; set False to feed MEMORY.md raw
# (e.g. if a specific LLM reacts poorly to the `← Nd` suffix).
# Kill switch for the git-blame-based per-line age annotation in the prompt.
# Default True keeps the #3212 behavior; set False to feed all memory
# files raw (e.g. if a specific LLM reacts poorly to the `← Nd` suffix).
self.annotate_line_ages = annotate_line_ages
self._runner = AgentRunner(provider)
self._tools = self._build_tools()
@ -907,6 +1010,7 @@ class Dream:
def _build_tools(self) -> ToolRegistry:
"""Build a minimal tool registry for the Dream agent."""
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
from nanobot.agent.tools.apply_patch import ApplyPatchTool
from nanobot.agent.tools.file_state import FileStates
from nanobot.agent.tools.filesystem import EditFileTool, ReadFileTool, WriteFileTool
@ -924,6 +1028,7 @@ class Dream:
file_states=file_states,
))
tools.register(EditFileTool(workspace=workspace, allowed_dir=workspace, file_states=file_states))
tools.register(ApplyPatchTool(workspace=workspace, allowed_dir=workspace, file_states=file_states))
# write_file resolves relative paths from workspace root, but can only
# write under skills/ so the prompt can safely use skills/<name>/SKILL.md.
skills_dir = workspace / "skills"
@ -961,8 +1066,8 @@ class Dream:
# -- main entry ----------------------------------------------------------
def _annotate_with_ages(self, content: str) -> str:
"""Append per-line age suffixes to MEMORY.md content.
def _annotate_with_ages(self, content: str, file_path: str = "memory/MEMORY.md") -> str:
"""Append per-line age suffixes to file content.
Each non-blank line whose age exceeds ``_STALE_THRESHOLD_DAYS`` gets a
suffix like `` 30d`` indicating days since last modification.
@ -970,9 +1075,7 @@ class Dream:
annotate fails, or the line count doesn't match the age count
(which can happen with an uncommitted working-tree edit better to
skip annotation than to tag the wrong line).
SOUL.md and USER.md are never annotated.
"""
file_path = "memory/MEMORY.md"
try:
ages = self.store.git.line_ages(file_path)
except Exception:
@ -1007,156 +1110,3 @@ class Dream:
result += "\n"
return result
async def run(self) -> bool:
"""Process unprocessed history entries. Returns True if work was done."""
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
last_cursor = self.store.get_last_dream_cursor()
entries = self.store.read_unprocessed_history(since_cursor=last_cursor)
if not entries:
return False
batch = entries[: self.max_batch_size]
logger.info(
"Dream: processing {} entries (cursor {}{}), batch={}",
len(entries), last_cursor, batch[-1]["cursor"], len(batch),
)
# Build history text for LLM — cap each entry so a legacy oversized
# record (e.g. pre-#3412 raw_archive dump) can't blow up the prompt.
history_text = "\n".join(
f"[{e['timestamp']}] "
f"{truncate_text(e['content'], self._HISTORY_ENTRY_PREVIEW_MAX_CHARS)}"
for e in batch
)
# Current file contents + per-line age annotations (MEMORY.md only).
# Each file is capped in the *prompt preview* only; Phase 2 still sees
# the full file via the read_file tool.
current_date = datetime.now().strftime("%Y-%m-%d")
raw_memory = self.store.read_memory() or "(empty)"
annotated_memory = (
self._annotate_with_ages(raw_memory)
if self.annotate_line_ages
else raw_memory
)
current_memory = truncate_text(annotated_memory, self._MEMORY_FILE_MAX_CHARS)
current_soul = truncate_text(
self.store.read_soul() or "(empty)", self._SOUL_FILE_MAX_CHARS,
)
current_user = truncate_text(
self.store.read_user() or "(empty)", self._USER_FILE_MAX_CHARS,
)
file_context = (
f"## Current Date\n{current_date}\n\n"
f"## Current MEMORY.md ({len(current_memory)} chars)\n{current_memory}\n\n"
f"## Current SOUL.md ({len(current_soul)} chars)\n{current_soul}\n\n"
f"## Current USER.md ({len(current_user)} chars)\n{current_user}"
)
# Phase 1: Analyze (no skills list — dedup is Phase 2's job)
phase1_prompt = (
f"## Conversation History\n{history_text}\n\n{file_context}"
)
try:
phase1_response = await self.provider.chat_with_retry(
model=self.model,
messages=[
{
"role": "system",
"content": render_template(
"agent/dream_phase1.md",
strip=True,
stale_threshold_days=_STALE_THRESHOLD_DAYS,
),
},
{"role": "user", "content": phase1_prompt},
],
tools=None,
tool_choice=None,
)
analysis = phase1_response.content or ""
logger.debug("Dream Phase 1 analysis ({} chars): {}", len(analysis), analysis[:500])
except Exception:
logger.exception("Dream Phase 1 failed")
return False
# Phase 2: Delegate to AgentRunner with read_file / edit_file
existing_skills = self._list_existing_skills()
skills_section = ""
if existing_skills:
skills_section = (
"\n\n## Existing Skills\n"
+ "\n".join(f"- {s}" for s in existing_skills)
)
phase2_prompt = f"## Analysis Result\n{analysis}\n\n{file_context}{skills_section}"
tools = self._tools
skill_creator_path = BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md"
messages: list[dict[str, Any]] = [
{
"role": "system",
"content": render_template(
"agent/dream_phase2.md",
strip=True,
skill_creator_path=str(skill_creator_path),
),
},
{"role": "user", "content": phase2_prompt},
]
try:
result = await self._runner.run(AgentRunSpec(
initial_messages=messages,
tools=tools,
model=self.model,
max_iterations=self.max_iterations,
max_tool_result_chars=self.max_tool_result_chars,
fail_on_tool_error=False,
))
logger.debug(
"Dream Phase 2 complete: stop_reason={}, tool_events={}",
result.stop_reason, len(result.tool_events),
)
for ev in (result.tool_events or []):
logger.info("Dream tool_event: name={}, status={}, detail={}", ev.get("name"), ev.get("status"), ev.get("detail", "")[:200])
except Exception:
logger.exception("Dream Phase 2 failed")
result = None
# Build changelog from tool events
changelog: list[str] = []
if result and result.tool_events:
for event in result.tool_events:
if event["status"] == "ok":
changelog.append(f"{event['name']}: {event['detail']}")
# Only advance cursor on successful completion to prevent silent loss
if result and result.stop_reason == "completed":
new_cursor = batch[-1]["cursor"]
self.store.set_last_dream_cursor(new_cursor)
logger.info(
"Dream done: {} change(s), cursor advanced to {}",
len(changelog), new_cursor,
)
else:
reason = result.stop_reason if result else "exception"
logger.warning(
"Dream incomplete ({}): cursor NOT advanced, will retry next cron cycle",
reason,
)
self.store.compact_history()
# Git auto-commit (only when there are actual changes)
if changelog and self.store.git.is_initialized():
ts = batch[-1]["timestamp"]
summary = f"dream: {ts}, {len(changelog)} change(s)"
commit_msg = f"{summary}\n\n{analysis.strip()}"
sha = self.store.git.auto_commit(commit_msg)
if sha:
logger.info("Dream commit: {}", sha)
return True

View File

@ -762,7 +762,7 @@ def _run_gateway(
)
from nanobot.agent.loop import UNIFIED_SESSION_KEY
from nanobot.bus.events import OutboundMessage
from nanobot.bus.events import InboundMessage, OutboundMessage
def _channel_session_key(channel: str, chat_id: str) -> str:
return (
@ -810,13 +810,13 @@ def _run_gateway(
# Set cron callback (needs agent)
async def on_cron_job(job: CronJob) -> str | None:
"""Execute a cron job through the agent."""
# Dream is an internal job — run directly, not through the agent loop.
if job.name == "dream":
try:
await agent.dream.run()
logger.info("Dream cron job completed")
except Exception:
logger.exception("Dream cron job failed")
await bus.publish_inbound(InboundMessage(
channel="system",
sender_id="dream",
chat_id="dream",
content="",
))
return None
from nanobot.utils.evaluator import evaluate_response
@ -1027,8 +1027,6 @@ def _run_gateway(
await server.serve_forever()
# Register Dream system job (always-on, idempotent on restart)
dream_cfg = config.agents.defaults.dream
if dream_cfg.model_override:
agent.dream.model = dream_cfg.model_override
agent.dream.max_batch_size = dream_cfg.max_batch_size
agent.dream.max_iterations = dream_cfg.max_iterations
agent.dream.annotate_line_ages = dream_cfg.annotate_line_ages

View File

@ -299,30 +299,22 @@ async def cmd_model(ctx: CommandContext) -> OutboundMessage:
async def cmd_dream(ctx: CommandContext) -> OutboundMessage:
"""Manually trigger a Dream consolidation run."""
import time
from nanobot.bus.events import InboundMessage
loop = ctx.loop
msg = ctx.msg
async def _run_dream():
t0 = time.monotonic()
try:
did_work = await loop.dream.run()
elapsed = time.monotonic() - t0
if did_work:
content = f"Dream completed in {elapsed:.1f}s."
else:
content = "Dream: nothing to process."
except Exception as e:
elapsed = time.monotonic() - t0
content = f"Dream failed after {elapsed:.1f}s: {e}"
await loop.bus.publish_outbound(OutboundMessage(
channel=msg.channel, chat_id=msg.chat_id, content=content,
))
asyncio.create_task(_run_dream())
await ctx.loop.bus.publish_inbound(InboundMessage(
channel="system",
sender_id="dream",
chat_id="dream",
content="",
metadata={
"trigger_channel": ctx.msg.channel,
"trigger_chat_id": ctx.msg.chat_id,
},
))
return OutboundMessage(
channel=msg.channel, chat_id=msg.chat_id, content="Dreaming...",
channel=ctx.msg.channel,
chat_id=ctx.msg.chat_id,
content="Dream started. It will process memory backlog and report when done.",
)
@ -355,6 +347,18 @@ def _format_changed_files(diff: str) -> str:
def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None = None) -> str:
files_line = _format_changed_files(diff)
msg_lines = commit.message.splitlines() if commit.message else []
msg_summary = msg_lines[0] if msg_lines else ""
msg_body = []
in_body = False
for line in msg_lines[1:]:
if not in_body:
if not line:
in_body = True
continue
msg_body.append(line)
body_text = "\n".join(msg_body).strip()
lines = [
"## Dream Update",
"",
@ -362,8 +366,12 @@ def _format_dream_log_content(commit, diff: str, *, requested_sha: str | None =
"",
f"- Commit: `{commit.sha}`",
f"- Time: {commit.timestamp}",
f"- Changed files: {files_line}",
]
if msg_summary:
lines.append(f"- Summary: {msg_summary}")
lines.append(f"- Changed files: {files_line}")
if body_text:
lines.extend(["", "### Analysis", "", body_text])
if diff:
lines.extend([
"",
@ -389,7 +397,8 @@ def _format_dream_restore_list(commits: list) -> str:
"",
]
for c in commits:
lines.append(f"- `{c.sha}` {c.timestamp} - {c.message.splitlines()[0]}")
summary = c.message.splitlines()[0] if c.message else "(no message)"
lines.append(f"- `{c.sha}` {c.timestamp} - {summary}")
lines.extend([
"",
"Preview a version with `/dream-log <sha>` before restoring it.",

View File

@ -52,13 +52,12 @@ class DreamConfig(Base):
model_override: str | None = Field(
default=None,
validation_alias=AliasChoices("modelOverride", "model", "model_override"),
) # Optional Dream-specific model override
max_batch_size: int = Field(default=20, ge=1) # Max history entries per run
# Bumped from 10 to 15 in #3212 (exp002: +30% dedup, no accuracy loss; >15 plateaus).
max_iterations: int = Field(default=15, ge=1) # Max tool calls per Phase 2
# Per-line git-blame age annotation in Phase 1 prompt (see #3212). Default
# on — set to False to feed MEMORY.md raw if a specific LLM reacts poorly
# to the `← Nd` suffix or you want deterministic, git-independent prompts.
) # Optional Dream-specific model override. Supports preset names (resolved against model_presets) or raw model identifiers.
max_batch_size: int = Field(default=5, ge=1) # Max history entries per run
max_iterations: int = Field(default=15, ge=1) # Max tool calls per Dream run
# Per-line git-blame age annotation in the Dream prompt (see #3212). Default
# on — set to False to feed all memory files raw if a specific LLM reacts
# poorly to the `← Nd` suffix or you want deterministic, git-independent prompts.
annotate_line_ages: bool = True
def build_schedule(self, timezone: str) -> CronSchedule:

View File

@ -34,3 +34,5 @@ Examples (replace `keyword`):
- **Do NOT edit SOUL.md, USER.md, or MEMORY.md.** They are automatically managed by Dream.
- If you notice outdated information, it will be corrected when Dream runs next.
- Users can view Dream's activity with the `/dream-log` command.
- Dream runs as a `system` session inside the AgentLoop, triggered by the `/dream` command or cron. Each turn processes one batch; if backlog remains, Dream automatically chains additional turns until complete. All changes are committed in a single git commit.
- Dream can use a different model than the main agent via `agents.defaults.dream.modelOverride`. Supports preset names or raw model identifiers.

View File

@ -1,13 +1,27 @@
Extract key facts from this conversation. Only output items matching these categories, skip everything else:
- User facts: personal info, preferences, stated opinions, habits
- Decisions: choices made, conclusions reached
- Solutions: working approaches discovered through trial and error, especially non-obvious methods that succeeded after failed attempts
- Events: plans, deadlines, notable occurrences
- Preferences: communication style, tool preferences
Extract key facts from this conversation. For each fact, annotate its memory attributes.
Priority: user corrections and preferences > solutions > decisions > events > environment facts. The most valuable memory prevents the user from having to repeat themselves.
Only SNIP facts deserve a non-[skip] mark:
- Signal: would the user need to repeat this if forgotten?
- Novel: not already in MEMORY.md or USER.md (check context below)
- Important: prevents rework or captures preferences / rules
- Persistent: still relevant after 2 weeks
Skip: code patterns derivable from source, git history, or anything already captured in existing memory.
Output one fact per line in this format:
- [mark] fact content
Marks (choose the best match):
- [permanent] Core preferences, personal traits, habits — never becomes stale
- [durable] Technical discoveries, project knowledge, config details — valid for months
- [ephemeral] Active task state, temporary decisions — may change in weeks
- [correction] Correction to a previous memory — must state what it replaces
- [skip] Does not meet SNIP criteria — still written to history.jsonl for audit, but Dream will ignore it
Categories to capture: people/roles, decisions/rationale, solutions, events/dates, preferences.
Decisions must include their motivation.
Write densely. Prefer 'X=A, Y=B' over separate bullets for tightly coupled facts.
Priority: user corrections > decisions with rationale > solutions > specific events > general context.
Output in the same language as the input conversation.
CRITICAL: Never drop person names, team names, or project names.
Skip: code patterns derivable from source, git history, or anything already in existing memory.
Output as concise bullet points, one fact per line. No preamble, no commentary.
If nothing noteworthy happened, output: (nothing)

View File

@ -0,0 +1,58 @@
Update memory files by analyzing conversation history and editing files directly.
Prune before adding — removing stale content is as important as adding new facts.
## File routing
Do NOT guess paths. Route each fact to its canonical file:
| File | Full path | Content |
|------|------|---------|
| SOUL.md | `{{ soul_path }}` | Agent behavior, guardrails, tone, interaction patterns |
| USER.md | `{{ user_path }}` | Personal info, preferences, habits, work context, communication style |
| MEMORY.md | `{{ memory_path }}` | Technical knowledge, project context, infrastructure, accounts |
| SKILL.md | `skills/<name>/SKILL.md` | Reusable workflow templates ([SKILL] entries only) |
Cross-boundary rule: no technical configs in USER.md, no user facts in SOUL.md, no preferences in MEMORY.md. If a fact fits multiple files, keep the most specific copy and remove the rest.
## Delete-or-keep
**Always delete:**
- Same fact at multiple locations — keep canonical copy only
- Merged/closed PR notes, resolved incidents, superseded info
- Verbose entries restatable in fewer words
- Overlapping or nested sections covering the same topic
**Likely delete** (apply judgment):
- Same fact at different detail levels — keep most complete version only
- Debugging steps unlikely to recur
- Ephemeral facts past their useful life
- Tool/service details documented upstream
- Lines with ``← Nd`` where N>{{ stale_threshold_days }} — closer review, not automatic removal
**Never delete:**
- User preferences and personality traits (permanent regardless of age)
- Active project context still referenced in conversations
- Behavioral rules in SOUL.md
When removing: prefer deleting individual items over entire sections.
## Fact extraction
- Atomic facts: "has a cat named Luna" not "discussed pet care"
- Corrections: edit the existing entry, don't append a new one
- Capture confirmed approaches the user validated
## Skill discovery & creation
Flag [SKILL] only when ALL are true: repeatable workflow appeared 2+ times, involves clear steps (not vague preferences), substantial enough for its own instruction set. Check existing skills to avoid redundancy.
For [SKILL] entries:
- Use write_file to create skills/<name>/SKILL.md; read_file `{{ skill_creator_path }}` for format reference
- YAML frontmatter (name, description), under 2000 words: when to use, steps, output format, example
- Do NOT overwrite existing skills — if overlapping, merge delta into the existing skill
- Skills are instruction sets, not code. Keep concrete values in MEMORY.md; skills use placeholders
## Editing
- Default tool: apply_patch. Use edit_file only for small exact replacements.
- File contents provided below — no read_file needed for initial edits.
- Batch all changes into a single apply_patch call. Surgical edits only.
- dry_run=true to preview. If nothing to update, stop without calling tools.
Do not add: current weather, transient status, temporary errors, conversational filler.

View File

@ -1,40 +0,0 @@
You have TWO equally important tasks:
1. Extract new facts from conversation history
2. Deduplicate existing memory files — find and flag redundant, overlapping, or stale content even if NOT mentioned in history
Output one line per finding:
[FILE] atomic fact (not already in memory)
[FILE-REMOVE] reason for removal
[SKILL] kebab-case-name: one-line description of the reusable pattern
Files: USER (identity, preferences), SOUL (bot behavior, tone), MEMORY (knowledge, project context)
Rules:
- Atomic facts: "has a cat named Luna" not "discussed pet care"
- Corrections: [USER] location is Tokyo, not Osaka
- Capture confirmed approaches the user validated
Deduplication — scan ALL memory files for these redundancy patterns:
- Same fact stated in multiple places (e.g., "communicates in Chinese" in both USER.md and multiple MEMORY.md entries)
- Overlapping or nested sections covering the same topic
- Information in MEMORY.md that is already captured in USER.md or SOUL.md (MEMORY.md should not duplicate permanent-file content)
- Verbose entries that can be condensed without losing information
For each duplicate found, output [FILE-REMOVE] for the less authoritative copy (prefer keeping facts in their canonical location)
Staleness — MEMORY.md lines may have a ``← Nd`` suffix showing days since last modification:
- SOUL.md and USER.md have no age annotations — they are permanent, only update with corrections
- Age only indicates when content was last touched, not whether it should be removed
- Use content judgment: user habits/preferences/personality traits are permanent regardless of age
- Only prune content that is objectively outdated: passed events, resolved tracking, superseded approaches
- Lines with ``← Nd`` (N>{{ stale_threshold_days }}) deserve closer review but are NOT automatically removable
- When removing: prefer deleting individual items over entire sections
Skill discovery — flag [SKILL] when ALL of these are true:
- A specific, repeatable workflow appeared 2+ times in the conversation history
- It involves clear steps (not vague preferences like "likes concise answers")
- It is substantial enough to warrant its own instruction set (not trivial like "read a file")
- Do not worry about duplicates — the next phase will check against existing skills
Do not add: current weather, transient status, temporary errors, conversational filler.
[SKIP] if nothing needs updating.

View File

@ -1,37 +0,0 @@
Update memory files based on the analysis below.
- [FILE] entries: add the described content to the appropriate file
- [FILE-REMOVE] entries: delete the corresponding content from memory files
- [SKILL] entries: create a new skill under skills/<name>/SKILL.md using write_file
## File paths (relative to workspace root)
- SOUL.md
- USER.md
- memory/MEMORY.md
- skills/<name>/SKILL.md (for [SKILL] entries only)
Do NOT guess paths.
## Editing rules
- Edit directly — file contents provided below, no read_file needed
- Use exact text as old_text, include surrounding blank lines for unique match
- Batch changes to the same file into one edit_file call
- For deletions: section header + all bullets as old_text, new_text empty
- Surgical edits only — never rewrite entire files
- If nothing to update, stop without calling tools
## Skill creation rules (for [SKILL] entries)
- Use write_file to create skills/<name>/SKILL.md
- Before writing, read_file `{{ skill_creator_path }}` for format reference (frontmatter structure, naming conventions, quality standards)
- **Dedup check**: read existing skills listed below to verify the new skill is not functionally redundant. Skip creation if an existing skill already covers the same workflow.
- Include YAML frontmatter with name and description fields
- Keep SKILL.md under 2000 words — concise and actionable
- Include: when to use, steps, output format, at least one example
- Do NOT overwrite existing skills — skip if the skill directory already exists
- Reference specific tools the agent has access to (read_file, write_file, exec, web_search, etc.)
- Skills are instruction sets, not code — do not include implementation code
## Quality
- Every line must carry standalone value
- Concise bullets under clear headers
- When reducing (not deleting): keep essential facts, drop verbose details
- If uncertain whether to delete, keep but add "(verify currency)"

View File

@ -19,7 +19,8 @@ class CommitInfo:
def format(self, diff: str = "") -> str:
"""Format this commit for display, optionally with a diff."""
header = f"## {self.message.splitlines()[0]}\n`{self.sha}` — {self.timestamp}\n"
summary = self.message.splitlines()[0] if self.message else "(no message)"
header = f"## {summary}\n`{self.sha}` — {self.timestamp}\n"
if diff:
return f"{header}\n```diff\n{diff}\n```"
return f"{header}\n(no file changes)"

View File

@ -1,19 +1,32 @@
"""Tests for the Dream class — two-phase memory consolidation via AgentRunner."""
"""Tests for Dream driven through AgentLoop._process_system_message."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from nanobot.agent.memory import Dream, MemoryStore
from nanobot.agent.loop import AgentLoop
from nanobot.agent.runner import AgentRunResult
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
from nanobot.bus.events import InboundMessage
from nanobot.bus.queue import MessageBus
from nanobot.utils.gitstore import LineAge
def _provider(default_model: str, max_tokens: int = 123) -> MagicMock:
provider = MagicMock()
provider.get_default_model.return_value = default_model
provider.generation = SimpleNamespace(
max_tokens=max_tokens, temperature=0.1, reasoning_effort=None
)
return provider
@pytest.fixture
def store(tmp_path):
from nanobot.agent.memory import MemoryStore
s = MemoryStore(tmp_path)
s.write_soul("# Soul\n- Helpful")
s.write_user("# User\n- Developer")
@ -23,9 +36,7 @@ def store(tmp_path):
@pytest.fixture
def mock_provider():
p = MagicMock()
p.chat_with_retry = AsyncMock()
return p
return _provider("test-model")
@pytest.fixture
@ -34,10 +45,16 @@ def mock_runner():
@pytest.fixture
def dream(store, mock_provider, mock_runner):
d = Dream(store=store, provider=mock_provider, model="test-model", max_batch_size=5)
d._runner = mock_runner
return d
def loop(tmp_path, mock_provider, mock_runner):
loop = AgentLoop(
bus=MessageBus(),
provider=mock_provider,
workspace=tmp_path,
model="test-model",
context_window_tokens=1000,
)
loop.dream._runner = mock_runner
return loop
def _make_run_result(
@ -56,254 +73,418 @@ def _make_run_result(
)
class TestDreamRun:
async def test_noop_when_no_unprocessed_history(self, dream, mock_provider, mock_runner, store):
"""Dream should not call LLM when there's nothing to process."""
result = await dream.run()
assert result is False
mock_provider.chat_with_retry.assert_not_called()
class TestDreamAgentLoopIntegration:
async def test_completes_goal_state_after_full_backlog(self, loop, mock_runner, store):
"""Goal should be completed after processing all backlog in internal loop."""
for i in range(6):
store.append_history(f"event {i}")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
session = loop.sessions.get_or_create("system:dream")
goal = session.metadata.get("goal_state")
assert isinstance(goal, dict)
assert goal["status"] == "completed"
assert store.get_last_dream_cursor() == 6
async def test_completes_goal_state_on_finish(self, loop, mock_runner, store):
"""Goal should be marked completed when backlog is fully processed."""
store.append_history("event 1")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
session = loop.sessions.get_or_create("system:dream")
goal = session.metadata.get("goal_state")
assert goal["status"] == "completed"
assert "completed_at" in goal
assert "recap" in goal
async def test_noop_when_no_unprocessed_history(self, loop, mock_runner):
"""Dream should not call runner when there's nothing to process."""
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
result = await loop._process_system_message(msg)
assert result is None
mock_runner.run.assert_not_called()
async def test_calls_runner_for_unprocessed_entries(self, dream, mock_provider, mock_runner, store):
async def test_calls_runner_for_unprocessed_entries(self, loop, mock_runner, store):
"""Dream should call AgentRunner when there are unprocessed history entries."""
store.append_history("User prefers dark mode")
mock_provider.chat_with_retry.return_value = MagicMock(content="New fact")
mock_runner.run = AsyncMock(return_value=_make_run_result(
tool_events=[{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}],
))
result = await dream.run()
assert result is True
mock_runner.run = AsyncMock(
return_value=_make_run_result(
tool_events=[
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
],
)
)
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
mock_runner.run.assert_called_once()
spec = mock_runner.run.call_args[0][0]
assert spec.max_iterations == 10
assert spec.fail_on_tool_error is False
async def test_advances_dream_cursor(self, dream, mock_provider, mock_runner, store):
async def test_advances_dream_cursor(self, loop, mock_runner, store):
"""Dream should advance the cursor after processing."""
store.append_history("event 1")
store.append_history("event 2")
mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new")
mock_runner.run = AsyncMock(return_value=_make_run_result())
await dream.run()
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
assert store.get_last_dream_cursor() == 2
async def test_compacts_processed_history(self, dream, mock_provider, mock_runner, store):
async def test_compacts_processed_history(self, loop, mock_runner, store):
"""Dream should compact history after processing."""
store.append_history("event 1")
store.append_history("event 2")
store.append_history("event 3")
mock_provider.chat_with_retry.return_value = MagicMock(content="Nothing new")
mock_runner.run = AsyncMock(return_value=_make_run_result())
await dream.run()
# After Dream, cursor is advanced and 3, compact keeps last max_history_entries
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
entries = store.read_unprocessed_history(since_cursor=0)
assert all(e["cursor"] > 0 for e in entries)
async def test_skill_phase_uses_builtin_skill_creator_path(self, dream, mock_provider, mock_runner, store):
"""Dream should point skill creation guidance at the builtin skill-creator template."""
async def test_processes_full_backlog_in_one_call(self, loop, mock_runner, store):
"""Backlog larger than max_batch_size should be fully processed in one call."""
for i in range(12):
store.append_history(f"event {i}")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
assert store.get_last_dream_cursor() == 12
assert mock_runner.run.call_count == 3 # 5 + 5 + 2
async def test_single_git_commit_for_multi_batch(self, loop, mock_runner, store):
"""Multi-batch run should collapse into exactly one git commit."""
store.git.init()
store.git.auto_commit("initial")
for i in range(12):
store.append_history(f"event {i}")
mock_runner.run = AsyncMock(
return_value=_make_run_result(
tool_events=[
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
],
)
)
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
commits = store.git.log()
dream_commits = [c for c in commits if c.message.startswith("dream:")]
assert len(dream_commits) == 1
async def test_system_prompt_cached(self, loop, mock_runner, store):
"""Batches within one run should reuse cached system prompt when template mtime unchanged."""
for i in range(6):
store.append_history(f"event {i}")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
# Two batches (5 + 1), both should use the same cached prompt
assert mock_runner.run.call_count == 2
first_prompt = mock_runner.run.call_args_list[0][0][0].initial_messages[0]["content"]
second_prompt = mock_runner.run.call_args_list[1][0][0].initial_messages[0]["content"]
assert second_prompt is first_prompt
async def test_noop_when_empty_backlog(self, loop, mock_runner, store):
"""Empty backlog should not advance cursor or create a commit."""
store.git.init()
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
assert store.get_last_dream_cursor() == 0
commits = store.git.log()
assert len([c for c in commits if c.message.startswith("dream:")]) == 0
class TestDreamPrompt:
async def test_prompt_contains_mece_rules(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
system_prompt = spec.initial_messages[0]["content"]
assert "Do NOT guess paths" in system_prompt
assert "SOUL.md" in system_prompt
assert "USER.md" in system_prompt
assert "MEMORY.md" in system_prompt
async def test_skill_phase_uses_builtin_skill_creator_path(self, loop, mock_runner, store):
store.append_history("Repeated workflow one")
store.append_history("Repeated workflow two")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKILL] test-skill: test description")
mock_runner.run = AsyncMock(return_value=_make_run_result())
await dream.run()
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
system_prompt = spec.initial_messages[0]["content"]
expected = str(BUILTIN_SKILLS_DIR / "skill-creator" / "SKILL.md")
assert expected in system_prompt
async def test_skill_write_tool_accepts_workspace_relative_skill_path(self, dream, store):
"""Dream skill creation should allow skills/<name>/SKILL.md relative to workspace root."""
write_tool = dream._tools.get("write_file")
assert write_tool is not None
result = await write_tool.execute(
path="skills/test-skill/SKILL.md",
content="---\nname: test-skill\ndescription: Test\n---\n",
async def test_system_prompt_uses_threshold_from_template_var(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
assert "Successfully wrote" in result
assert (store.workspace / "skills" / "test-skill" / "SKILL.md").exists()
async def test_phase1_prompt_includes_line_age_annotations(self, dream, mock_provider, mock_runner, store):
"""Phase 1 prompt should have per-line age suffixes in MEMORY.md when git is available."""
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
# Init git so line_ages works
store.git.init()
store.git.auto_commit("initial memory state")
await dream.run()
# The MEMORY.md section should not crash and should contain the memory content
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
assert "## Current MEMORY.md" in user_msg
async def test_phase1_annotates_only_memory_not_soul_or_user(self, dream, mock_provider, mock_runner, store):
"""SOUL.md and USER.md should never have age annotations — they are permanent."""
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
store.git.init()
store.git.auto_commit("initial state")
await dream.run()
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
# The ← suffix should only appear in MEMORY.md section
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
soul_section = user_msg.split("## Current SOUL.md")[1].split("## Current USER.md")[0]
user_section = user_msg.split("## Current USER.md")[1]
# SOUL and USER should not contain age arrows
assert "\u2190" not in soul_section
assert "\u2190" not in user_section
async def test_phase1_prompt_works_without_git(self, dream, mock_provider, mock_runner, store):
"""Phase 1 should work fine even if git is not initialized (no age annotations)."""
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
await dream.run()
# Should still succeed — just without age annotations
mock_provider.chat_with_retry.assert_called_once()
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
assert "## Current MEMORY.md" in user_msg
async def test_phase1_prompt_carries_age_suffix_for_stale_lines(
self, dream, mock_provider, mock_runner, store,
):
"""End-to-end: ages >14d must appear verbatim in the LLM prompt, ages ≤14d must not."""
# MEMORY.md fixture has 2 non-blank lines ("# Memory" and "- Project X active").
# Inject four ages to cover threshold boundaries: >14 suffix, ==14 no suffix, <14 no suffix.
store.write_memory("# Memory\n- Project X active\n- fresh item\n- edge case line")
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
fake_ages = [
LineAge(age_days=30), # "# Memory" → should get ← 30d
LineAge(age_days=20), # "- Project X..." → should get ← 20d
LineAge(age_days=14), # "- fresh item" → ==14, threshold is strictly >14, no suffix
LineAge(age_days=5), # "- edge case..." → no suffix
]
with patch.object(store.git, "line_ages", return_value=fake_ages):
await dream.run()
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
assert "\u2190 30d" in memory_section
assert "\u2190 20d" in memory_section
assert "\u2190 14d" not in memory_section
assert "\u2190 5d" not in memory_section
async def test_phase1_skips_annotation_when_disabled(
self, dream, mock_provider, mock_runner, store,
):
"""`annotate_line_ages=False` must bypass the git lookup entirely and keep MEMORY.md raw."""
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
dream.annotate_line_ages = False
# line_ages must be bypassed entirely — verify with a spy rather than a
# raising side_effect, because _annotate_with_ages catches Exception
# (which swallows AssertionError) and would hide an accidental call.
with patch.object(store.git, "line_ages") as mock_line_ages:
await dream.run()
mock_line_ages.assert_not_called()
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
assert "\u2190" not in user_msg
async def test_phase1_skips_annotation_on_line_ages_length_mismatch(
self, dream, mock_provider, mock_runner, store,
):
"""If ages length != lines length (dirty working tree), skip annotation instead of mis-tagging."""
# MEMORY.md has 2 non-blank lines but we hand back only 1 age → mismatch.
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
with patch.object(store.git, "line_ages", return_value=[LineAge(age_days=999)]):
await dream.run()
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
# No age arrow at all — we refused to annotate rather than tag the wrong line.
assert "\u2190" not in memory_section
async def test_phase1_prompt_uses_threshold_from_template_var(
self, dream, mock_provider, mock_runner, store,
):
"""System prompt should reference the stale-threshold constant, not a hardcoded 14."""
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
await dream.run()
system_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][0]["content"]
# The template renders with stale_threshold_days=14 → LLM must see "N>14"
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
system_msg = spec.initial_messages[0]["content"]
assert "N>14" in system_msg
class TestDreamPromptCaps:
"""Dream's Phase 1/2 prompt must not be poisoned by a legacy oversized
history entry or a runaway MEMORY.md. Without caps, a single pre-#3412
raw_archive dump in history.jsonl would make every subsequent Dream run
exceed the context window and silently advance the cursor past real work.
"""
async def test_phase1_caps_huge_memory_file(
self, dream, mock_provider, mock_runner, store,
):
"""A MEMORY.md much larger than _MEMORY_FILE_MAX_CHARS must be truncated
in the prompt preview (full content is still reachable via read_file)."""
store.write_memory("M" * (dream._MEMORY_FILE_MAX_CHARS * 5))
async def test_caps_huge_memory_file(self, loop, mock_runner, store):
store.write_memory("M" * (loop.dream._MEMORY_FILE_MAX_CHARS * 5))
store.append_history("some event")
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
"## Current SOUL.md"
)[0]
assert len(memory_section) < loop.dream._MEMORY_FILE_MAX_CHARS + 500
await dream.run()
user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0]
assert len(memory_section) < dream._MEMORY_FILE_MAX_CHARS + 500
async def test_phase1_caps_huge_history_entry(
self, dream, mock_provider, mock_runner, store,
):
"""A legacy oversized history entry (e.g. pre-#3412 raw_archive dump)
must not explode the Phase 1 prompt each entry is capped in the
preview, even though the JSONL record itself stays full-size."""
# Bypass the append_history cap by writing directly, simulating a
# record that was written by an older nanobot build before any caps.
async def test_caps_huge_history_entry(self, loop, mock_runner, store):
store.history_file.write_text(
json.dumps({
"cursor": 1,
"timestamp": "2026-04-01 10:00",
"content": "H" * (dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8),
}) + "\n",
json.dumps(
{
"cursor": 1,
"timestamp": "2026-04-01 10:00",
"content": "H" * (loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS * 8),
}
)
+ "\n",
encoding="utf-8",
)
mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
history_section = user_msg.split("## Conversation History\n")[1].split(
"\n\n## Current Date"
)[0]
assert len(history_section) < loop.dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500
await dream.run()
user_msg = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"]
history_section = user_msg.split("## Conversation History\n")[1].split("\n\n## Current Date")[0]
assert len(history_section) < dream._HISTORY_ENTRY_PREVIEW_MAX_CHARS + 500
class TestDreamTools:
def test_apply_patch_tool_registered(self, loop):
tool = loop.dream._tools.get("apply_patch")
assert tool is not None
class TestDreamCaps:
def test_batch_size_default_is_5(self):
from nanobot.config.schema import DreamConfig
assert DreamConfig().max_batch_size == 5
def test_memory_cap_is_16k(self, loop):
assert loop.dream._MEMORY_FILE_MAX_CHARS == 16_000
class TestDreamSkipFiltering:
async def test_skip_entries_removed_from_prompt(self, loop, mock_runner, store):
store.append_history("- [skip] greeting\n- [permanent] User prefers dark mode")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
assert "User prefers dark mode" in user_msg
assert "[skip]" not in user_msg
assert "greeting" not in user_msg
class TestDreamAgeAnnotations:
async def test_prompt_includes_line_age_annotations(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
store.git.init()
store.git.auto_commit("initial memory state")
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
assert "## Current MEMORY.md" in user_msg
async def test_annotates_only_memory_not_soul_or_user(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
store.git.init()
store.git.auto_commit("initial state")
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
soul_section = user_msg.split("## Current SOUL.md")[1].split(
"## Current USER.md"
)[0]
user_section = user_msg.split("## Current USER.md")[1]
assert "" not in soul_section
assert "" not in user_section
async def test_prompt_works_without_git(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
mock_runner.run.assert_called_once()
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
assert "## Current MEMORY.md" in user_msg
async def test_prompt_carries_age_suffix_for_stale_lines(self, loop, mock_runner, store):
store.write_memory(
"# Memory\n- Project X active\n- fresh item\n- edge case line"
)
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
fake_ages = [
LineAge(age_days=30),
LineAge(age_days=20),
LineAge(age_days=14),
LineAge(age_days=5),
]
with patch.object(loop.dream.store.git, "line_ages", return_value=fake_ages):
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
"## Current SOUL.md"
)[0]
assert "← 30d" in memory_section
assert "← 20d" in memory_section
assert "← 14d" not in memory_section
assert "← 5d" not in memory_section
async def test_skips_annotation_when_disabled(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
loop.dream.annotate_line_ages = False
with patch.object(loop.dream.store.git, "line_ages") as mock_line_ages:
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
mock_line_ages.assert_not_called()
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
assert "" not in user_msg
async def test_skips_annotation_on_line_ages_length_mismatch(self, loop, mock_runner, store):
store.append_history("some event")
mock_runner.run = AsyncMock(return_value=_make_run_result())
with patch.object(
loop.dream.store.git, "line_ages", return_value=[LineAge(age_days=999)]
):
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
spec = mock_runner.run.call_args[0][0]
user_msg = spec.initial_messages[1]["content"]
memory_section = user_msg.split("## Current MEMORY.md")[1].split(
"## Current SOUL.md"
)[0]
assert "" not in memory_section
class TestDreamSessionPersistence:
async def test_writes_session_on_success(self, loop, mock_runner, store):
store.append_history("event one")
store.append_history("event two")
mock_runner.run = AsyncMock(
return_value=_make_run_result(
tool_events=[
{"name": "edit_file", "status": "ok", "detail": "memory/MEMORY.md"}
],
)
)
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
session_path = store.memory_dir / ".dream_session.json"
assert session_path.exists()
data = json.loads(session_path.read_text(encoding="utf-8"))
assert data["batch"]["from_cursor"] == 0
assert data["batch"]["to_cursor"] == 2
assert data["batch"]["count"] == 2
assert data["stop_reason"] == "completed"
assert data["changelog"] == ["edit_file: memory/MEMORY.md"]
assert "timestamp" in data
assert "elapsed_seconds" in data
assert "messages" in data
async def test_no_session_record_on_failure(self, loop, mock_runner, store):
"""Failed batch should not write a session record (cursor stays put for retry)."""
store.append_history("event one")
mock_runner.run = AsyncMock(side_effect=RuntimeError("LLM error"))
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
session_path = store.memory_dir / ".dream_session.json"
assert not session_path.exists()
assert store.get_last_dream_cursor() == 0
async def test_session_contains_full_messages(self, loop, mock_runner, store):
store.append_history("event one")
messages = [
{"role": "system", "content": "you are a memory bot"},
{"role": "user", "content": "history here"},
{"role": "assistant", "content": "I will edit MEMORY.md"},
]
result = _make_run_result()
result.messages = messages
mock_runner.run = AsyncMock(return_value=result)
msg = InboundMessage(
channel="system", sender_id="dream", chat_id="dream", content=""
)
await loop._process_system_message(msg)
session_path = store.memory_dir / ".dream_session.json"
data = json.loads(session_path.read_text(encoding="utf-8"))
assert data["messages"] == messages
assert data["prompt_chars"] > 0
assert data["commit_sha"] is None

125
tests/agent/test_memory.py Normal file
View File

@ -0,0 +1,125 @@
"""Tests for memory system: Consolidator, token estimation, truncation."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from nanobot.agent.memory import _TIKTOKEN_ENC, Consolidator, MemoryStore, _estimate_tokens
@pytest.fixture
def store(tmp_path):
s = MemoryStore(tmp_path)
s.write_soul("# Soul\n- Helpful")
s.write_user("# User\n- Developer")
s.write_memory("# Memory\n- Project X active")
return s
@pytest.fixture
def mock_provider():
p = MagicMock()
p.chat_with_retry = AsyncMock()
p.generation.max_tokens = 4096
return p
@pytest.fixture
def mock_sessions():
return MagicMock()
@pytest.fixture
def mock_build_messages():
return MagicMock(return_value=[])
@pytest.fixture
def mock_get_tool_definitions():
return MagicMock(return_value=[])
@pytest.fixture
def consolidator(store, mock_provider, mock_sessions, mock_build_messages, mock_get_tool_definitions):
return Consolidator(
store=store,
provider=mock_provider,
model="test-model",
sessions=mock_sessions,
context_window_tokens=128_000,
build_messages=mock_build_messages,
get_tool_definitions=mock_get_tool_definitions,
)
class TestEstimateTokens:
def test_estimate_tokens_returns_positive(self):
assert _estimate_tokens("hello world") > 0
def test_estimate_tokens_english_approximate(self):
# English is roughly 1 token per 4 chars as fallback
text = "a " * 100
if _TIKTOKEN_ENC is not None:
expected = len(_TIKTOKEN_ENC.encode(text))
else:
expected = len(text) // 4
assert _estimate_tokens(text) == expected
class TestTruncateToTokenBudget:
def test_reserve_tokens_reduces_budget(self, consolidator):
long_text = "word " * 200_000
# Without reserve, more text survives
no_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=0)
with_reserve = consolidator._truncate_to_token_budget(long_text, reserve_tokens=500)
assert len(with_reserve) < len(no_reserve)
def test_reserve_tokens_zero_default(self, consolidator):
text = "hello world"
result = consolidator._truncate_to_token_budget(text)
assert result == text
class TestConsolidatorPrompt:
def test_prompt_contains_snip(self):
from nanobot.utils.prompt_templates import render_template
text = render_template("agent/consolidator_archive.md", strip=True)
assert "SNIP" in text
assert "[permanent]" in text
assert "[skip]" in text
class TestConsolidatorArchive:
async def test_archive_injects_dedup_context(self, consolidator, mock_provider, store):
store.write_memory("- User prefers dark mode")
store.write_user("- Developer")
messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}]
mock_provider.chat_with_retry.return_value = MagicMock(
content="(nothing)", finish_reason="stop"
)
await consolidator.archive(messages)
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs["messages"][1]["content"]
assert "## Current MEMORY.md (for dedup)" in user_msg
assert "User prefers dark mode" in user_msg
assert "## Current USER.md (for dedup)" in user_msg
assert "Developer" in user_msg
async def test_archive_skips_dedup_when_budget_exhausted(self, consolidator, mock_provider, store):
# Shrink token budget so dedup context (always capped at ~6000 chars)
# exceeds the available room.
consolidator.context_window_tokens = 6_000
store.write_memory("word " * 10_000)
messages = [{"role": "user", "content": "hello", "timestamp": "2026-01-01 10:00"}]
mock_provider.chat_with_retry.return_value = MagicMock(
content="(nothing)", finish_reason="stop"
)
await consolidator.archive(messages)
call_args = mock_provider.chat_with_retry.call_args
user_msg = call_args.kwargs["messages"][1]["content"]
# Should not contain dedup context when budget is exhausted
assert "## Current MEMORY.md (for dedup)" not in user_msg

View File

@ -292,3 +292,95 @@ def test_from_config_static_preset_loader_does_not_enable_hot_reload(tmp_path) -
loop = AgentLoop.from_config(config)
assert loop._provider_snapshot_loader is None
assert loop._preset_snapshot_loader is not None
class TestDreamModelOverride:
def test_dream_follows_main_when_no_override(self, tmp_path) -> None:
provider = _provider("base-model")
loop = AgentLoop(
bus=MessageBus(),
provider=provider,
workspace=tmp_path,
model="base-model",
context_window_tokens=1000,
)
assert loop.dream.model == "base-model"
assert loop.dream.provider is provider
def test_dream_raw_model_override(self, tmp_path) -> None:
provider = _provider("base-model")
loop = AgentLoop(
bus=MessageBus(),
provider=provider,
workspace=tmp_path,
model="base-model",
context_window_tokens=1000,
dream_model_override="custom-model-v2",
)
assert loop.dream.model == "custom-model-v2"
assert loop.dream.provider is provider
def test_dream_preset_override(self, tmp_path) -> None:
cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048)
preset = ModelPresetConfig(
model="openai/gpt-4.1-mini",
provider="openai",
max_tokens=2048,
context_window_tokens=128_000,
)
loop = AgentLoop(
bus=MessageBus(),
provider=_provider("base-model"),
workspace=tmp_path,
model="base-model",
context_window_tokens=1000,
model_presets={"cheap": preset},
dream_model_override="cheap",
preset_snapshot_loader=lambda _name: ProviderSnapshot(
provider=cheap_provider,
model=preset.model,
context_window_tokens=preset.context_window_tokens,
signature=("cheap", preset.model),
),
)
assert loop.dream.model == "openai/gpt-4.1-mini"
assert loop.dream.provider is cheap_provider
assert loop.dream._runner.provider is cheap_provider
def test_dream_override_survives_main_preset_switch(self, tmp_path) -> None:
base_provider = _provider("base-model")
fast_provider = _provider("openai/gpt-4.1", max_tokens=4096)
cheap_provider = _provider("openai/gpt-4.1-mini", max_tokens=2048)
loop = AgentLoop(
bus=MessageBus(),
provider=base_provider,
workspace=tmp_path,
model="base-model",
context_window_tokens=1000,
model_presets={
"fast": ModelPresetConfig(model="openai/gpt-4.1"),
"cheap": ModelPresetConfig(model="openai/gpt-4.1-mini"),
},
dream_model_override="cheap",
preset_snapshot_loader=lambda name: ProviderSnapshot(
provider=fast_provider if name == "fast" else cheap_provider,
model="openai/gpt-4.1" if name == "fast" else "openai/gpt-4.1-mini",
context_window_tokens=32_768 if name == "fast" else 128_000,
signature=(name, "model"),
),
)
# Initially dream is on cheap
assert loop.dream.model == "openai/gpt-4.1-mini"
assert loop.dream.provider is cheap_provider
# Switch main preset to fast
loop.set_model_preset("fast")
# Main agent should be on fast
assert loop.model == "openai/gpt-4.1"
assert loop.provider is fast_provider
# Dream should still be on cheap override
assert loop.dream.model == "openai/gpt-4.1-mini"
assert loop.dream.provider is cheap_provider
assert loop.dream._runner.provider is cheap_provider

View File

@ -113,6 +113,52 @@ async def test_dream_restore_lists_versions_with_next_steps() -> None:
assert "Restore a version with `/dream-restore <sha>`." in out.content
@pytest.mark.asyncio
async def test_dream_log_shows_summary_and_analysis() -> None:
commit = CommitInfo(
sha="abcd1234",
message="dream: 2026-04-04, 2 change(s)\n\n[ADD] fact A →USER\n[REMOVE] old fact",
timestamp="2026-04-04 12:00",
)
diff = (
"diff --git a/SOUL.md b/SOUL.md\n"
"--- a/SOUL.md\n"
"+++ b/SOUL.md\n"
"@@ -1 +1 @@\n"
"-old\n"
"+new\n"
)
git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)})
out = await cmd_dream_log(_make_ctx("/dream-log", git))
assert "## Dream Update" in out.content
assert "- Summary: dream: 2026-04-04, 2 change(s)" in out.content
assert "### Analysis" in out.content
assert "[ADD] fact A →USER" in out.content
assert "[REMOVE] old fact" in out.content
@pytest.mark.asyncio
async def test_dream_log_with_empty_commit_message() -> None:
commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00")
diff = (
"diff --git a/SOUL.md b/SOUL.md\n"
"--- a/SOUL.md\n"
"+++ b/SOUL.md\n"
"@@ -1 +1 @@\n"
"-old\n"
"+new\n"
)
git = _FakeGit(commits=[commit], diff_map={commit.sha: (commit, diff)})
out = await cmd_dream_log(_make_ctx("/dream-log", git))
assert "## Dream Update" in out.content
assert "- Summary:" not in out.content
assert "### Analysis" not in out.content
@pytest.mark.asyncio
async def test_dream_restore_success_mentions_files_and_followup() -> None:
commit = CommitInfo(sha="abcd1234", message="dream: latest", timestamp="2026-04-04 12:00")

View File

@ -1,216 +1,13 @@
"""Tests for GitStore — line_ages() and core git operations."""
import subprocess
import time
from datetime import datetime, timedelta, timezone
from unittest.mock import patch
import pytest
from nanobot.utils.gitstore import GitStore
from nanobot.utils.gitstore import CommitInfo
@pytest.fixture
def git(tmp_path):
"""Create an initialized GitStore with tracked MEMORY.md."""
g = GitStore(tmp_path, tracked_files=["MEMORY.md", "SOUL.md"])
g.init()
return g
class TestCommitInfo:
def test_format_with_empty_message(self):
commit = CommitInfo(sha="abcd1234", message="", timestamp="2026-04-04 12:00")
result = commit.format()
assert "(no message)" in result or "## " in result
class TestLineAges:
def test_returns_empty_when_not_initialized(self, tmp_path):
"""line_ages should return [] if the git repo is not initialized."""
git = GitStore(tmp_path, tracked_files=["MEMORY.md"])
assert git.line_ages("MEMORY.md") == []
def test_returns_empty_for_missing_file(self, git):
"""line_ages should return [] for a file that doesn't exist."""
assert git.line_ages("SOUL.md") == []
def test_returns_empty_for_empty_file(self, git, tmp_path):
"""line_ages should return [] for an empty tracked file."""
(tmp_path / "SOUL.md").write_text("", encoding="utf-8")
git.auto_commit("empty soul")
assert git.line_ages("SOUL.md") == []
def test_one_age_per_line(self, git, tmp_path):
"""line_ages should return one entry per line in the file."""
content = "# Memory\n\n## Section A\n- item 1\n"
(tmp_path / "MEMORY.md").write_text(content, encoding="utf-8")
git.auto_commit("initial")
ages = git.line_ages("MEMORY.md")
assert len(ages) == len(content.splitlines())
def test_fresh_lines_have_age_zero(self, git, tmp_path):
"""Lines committed today should have age_days=0."""
(tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8")
git.auto_commit("initial")
ages = git.line_ages("MEMORY.md")
assert all(a.age_days == 0 for a in ages)
def test_age_differentiates_across_days(self, git, tmp_path):
"""Lines committed today should show correct age when 'now' is mocked forward."""
(tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8")
git.auto_commit("initial")
future_now = datetime.now(tz=timezone.utc) + timedelta(days=30)
with patch("nanobot.utils.gitstore.datetime") as mock_dt:
mock_dt.now.return_value = future_now
mock_dt.fromtimestamp = datetime.fromtimestamp
ages = git.line_ages("MEMORY.md")
assert len(ages) == 2
assert all(a.age_days == 30 for a in ages)
def test_annotate_failure_returns_empty(self, tmp_path):
"""If annotate fails, line_ages should return [] gracefully."""
git = GitStore(tmp_path, tracked_files=["MEMORY.md"])
# Don't init — annotate will fail
assert git.line_ages("MEMORY.md") == []
def test_partial_edit_only_updates_changed_lines(self, git, tmp_path):
"""Only modified lines should reflect the new commit's timestamp."""
(tmp_path / "MEMORY.md").write_text(
"# Memory\n\n## A\n- old\n\n## B\n- keep\n", encoding="utf-8"
)
git.auto_commit("commit1")
time.sleep(1.1)
# Only modify section A
(tmp_path / "MEMORY.md").write_text(
"# Memory\n\n## A\n- new\n\n## B\n- keep\n", encoding="utf-8"
)
git.auto_commit("commit2")
ages = git.line_ages("MEMORY.md")
lines = (tmp_path / "MEMORY.md").read_text(encoding="utf-8").splitlines()
# All lines are from today, but verify line-level tracking works
assert len(ages) == len(lines)
# "- new" line and "- keep" line both age=0 (same day), but
# the key point is we get per-line results
assert len(ages) == 7
class TestNestedRepoProtection:
"""Regression tests for GitHub issue #2980: nested repo protection."""
def test_init_refuses_inside_git_repo(self, tmp_path):
"""init() should detect it's inside an existing git repo and refuse."""
project = tmp_path / "project"
project.mkdir()
(project / ".git").mkdir()
workspace = project / "workspace"
workspace.mkdir()
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is False
assert not (workspace / ".git").is_dir()
def test_init_preserves_existing_gitignore(self, tmp_path):
"""init() should preserve existing .gitignore entries and append new ones."""
workspace = tmp_path / "workspace"
workspace.mkdir()
existing = "*.pyc\n__pycache__/\n"
(workspace / ".gitignore").write_text(existing, encoding="utf-8")
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is True
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
assert "*.pyc" in gitignore
assert "__pycache__/" in gitignore
assert "!MEMORY.md" in gitignore
assert "!.gitignore" in gitignore
def test_init_no_gitignore_creates_new(self, tmp_path):
"""init() should create .gitignore with Dream content when none exists."""
workspace = tmp_path / "workspace"
workspace.mkdir()
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is True
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
expected = g._build_gitignore()
assert gitignore == expected
def test_init_gitignore_merge_idempotent(self, tmp_path):
"""init() should not duplicate Dream entries already in .gitignore."""
workspace = tmp_path / "workspace"
workspace.mkdir()
# Pre-existing .gitignore that already has some Dream entries
existing = "*.pyc\n/*\n!MEMORY.md\n"
(workspace / ".gitignore").write_text(existing, encoding="utf-8")
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is True
gitignore = (workspace / ".gitignore").read_text(encoding="utf-8")
# No duplicate lines
lines = gitignore.splitlines()
assert lines.count("/*") == 1
assert lines.count("!MEMORY.md") == 1
# Existing entry preserved, new Dream entries appended
assert "*.pyc" in gitignore
assert "!.gitignore" in gitignore
def test_init_outside_git_repo_works_normally(self, tmp_path):
"""init() should succeed and create .git when not inside a git repo."""
workspace = tmp_path / "workspace"
workspace.mkdir()
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is True
assert (workspace / ".git").is_dir()
def test_init_refuses_inside_git_worktree(self, tmp_path):
"""init() should refuse when the parent checkout is a git worktree."""
repo = tmp_path / "repo"
repo.mkdir()
subprocess.run(["git", "init", "-q", str(repo)], check=True)
(repo / "README.md").write_text("x\n", encoding="utf-8")
subprocess.run(["git", "-C", str(repo), "add", "README.md"], check=True)
subprocess.run(
[
"git",
"-C",
str(repo),
"-c",
"user.name=test",
"-c",
"user.email=test@example.com",
"commit",
"-q",
"-m",
"init",
],
check=True,
)
subprocess.run(["git", "-C", str(repo), "branch", "wt-branch"], check=True)
worktree = tmp_path / "worktree"
subprocess.run(
["git", "-C", str(repo), "worktree", "add", "-q", str(worktree), "wt-branch"],
check=True,
)
assert (worktree / ".git").is_file()
workspace = worktree / "workspace"
workspace.mkdir()
g = GitStore(workspace, tracked_files=["MEMORY.md"])
result = g.init()
assert result is False
assert not (workspace / ".git").exists()
def test_format_with_message(self):
commit = CommitInfo(sha="abcd1234", message="dream: update", timestamp="2026-04-04 12:00")
result = commit.format()
assert "dream: update" in result